diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index 69f4c570e4..cae94dd6ba 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -1308,10 +1308,11 @@ export async function quarantineSeededWorktreeExecutionState( .update(agents) .set({ runtimeConfig: normalized.runtimeConfig, - status: nextStatus, updatedAt: new Date(), }) .where(eq(agents.id, agent.id)); + if (nextStatus !== agent.status) await tx.update(agents).set({ status: "idle", updatedAt: new Date() }) + .where(and(eq(agents.id, agent.id), eq(agents.lifecycleState, "ready"))); } } diff --git a/doc/AGENT-ARTIFACTS.md b/doc/AGENT-ARTIFACTS.md index 6a451b5f3f..1d97ee4ae8 100644 --- a/doc/AGENT-ARTIFACTS.md +++ b/doc/AGENT-ARTIFACTS.md @@ -13,6 +13,17 @@ remote workspace. Supply a workspace-relative `contentRef`, basename `filename`, The tool verifies the file, stores an attachment and artifact work product, and binds it to the response. Generic API tools and a legacy API key are unnecessary. +The completion gate permits a narrow sentence-bound exception for one requested +file when the following sentence explicitly qualifies it, for example: +`Write memory.txt. This is personal memory, not a task deliverable.` or +`Write internal-proof.txt. This is an internal verification file, not a deliverable.` +Each qualifier applies to one file in the immediately preceding request sentence. +Separate file requests can each qualify when each has its own following qualifier. +Multiple files requested in one sentence, unqualified report requests, and explicit +attachment/download requests still need verified accessible publication evidence. Saying only “for internal verification” +does not establish this exception. Expected denied native writes in a negative +test are separately recognized as denied attempts, not requested deliverables. + Wait for the receipt. It includes `attachmentId`, `contentPath`, and `downloadPath`, along with the existing command, revision, entity references, and disposition. Reuse the original key after an ambiguous result. A receipt diff --git a/doc/AGENT-LIFECYCLE.md b/doc/AGENT-LIFECYCLE.md new file mode 100644 index 0000000000..87033f8d56 --- /dev/null +++ b/doc/AGENT-LIFECYCLE.md @@ -0,0 +1,264 @@ +# Agent lifecycle + +The agent lifecycle module controls hiring, pause, resume, and termination. +Its public entry point is `server/src/modules/agent-lifecycle/index.ts`. +Use its commands to create an agent or change its lifecycle state. +Do not pass a database transaction to a command. +Each hire or transition command owns its transaction. + +## Application use + +Application code imports the configured factory from `services/agent-lifecycle.ts`. +This service supplies the module dependencies. Use the root database connection. +Apply the existing caller authorization checks before you call a command. + +```ts +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; + +const lifecycle = createAgentLifecycle(db); +await lifecycle.updateConfiguration(agentId, { title: "Engineer" }); +await lifecycle.pauseAgent(agentId); +const state = await lifecycle.get(agentId); +``` + +The pause command records the request. The worker completes the external steps. +Read `lifecycleState` to check progress. +Configuration updates accept only the fields in `AGENT_CONFIGURATION_FIELDS`. +They reject identity, company, accounting totals, timestamp, and lifecycle fields. +The public update schema also rejects spend totals. Budget limit changes remain +permitted through `budgetMonthlyCents`. + +Do not import module adapters or construct module dependencies in a caller. +The company deletion service uses the separate deletion entry point. +The listed configuration workflows use the restricted transaction service. +These are the only transaction integrations. They are described below. + +## States + +| State | Meaning | Next state | +| --- | --- | --- | +| `pending_approval` | The hire needs board approval. | `preparing` or `rejected` | +| `preparing` | Required plugins prepare the agent. | `verifying` | +| `verifying` | The host tests the saved harness configuration. | `ready` | +| `ready` | The lifecycle permits work. | `pausing` or `terminating` | +| `pausing` | The host stops work. Required plugins stop their resources. | `paused` | +| `paused` | A pause hold prevents work. | `resuming` or `terminating` | +| `resuming` | Required plugins start their resources. | The state saved before pause | +| `terminating` | The host revokes keys and stops work. | `cleaning_up` | +| `cleaning_up` | Required plugins remove their resources. | `terminated` | +| `terminated` | Termination is complete. | None | +| `rejected` | The board rejected the hire. | None | + +Termination can start before preparation or verification completes. +An old result cannot complete a newer operation. +Deletion requires completed termination or a rejected hire. +The existing restrictions for built-in agents and accounting still apply. + +## Commands and transactions + +`requestHire` writes the agent record before preparation starts. +An approved hire starts preparation after the transaction commits. +A proposed hire waits for `approveHire`. +`approveHire` and `rejectHire` accept an agent ID or `{ approvalId }`. +Both forms use the same hire decision path. An agent ID also resolves its open +hire approval. The approval and the agent change commit together. +The existing company approval rules still apply. + +`pauseAgent`, `resumeAgent`, and `terminateAgent` record the requested change. +They do not wait for external resource operations. +`retry` makes a failed step available for another attempt. +`reconcilePolicyHolds` updates budget and company holds for the specified company. +The background policy check processes at most 100 eligible agents each minute. +It continues from the last agent ID and returns to the start after the last page. +Pending hires and agents in termination or a final state are excluded. +An explicit company or budget change still updates all affected agents immediately. +`updateAndTransition` applies a configuration change and a lifecycle command +in one transaction. A failed command does not save the configuration change. + +The onboarding and invitation services own their feature workflows. +They commit the hire through `requestHire` before they save the other records. +A separate database transaction holds only the workflow lock. +It prevents concurrent requests for the same workflow. +The hire command commits through its own connection and transaction. +A failure in the lock transaction cannot undo a committed hire. +If a later step fails, a retry uses the existing agent. +The onboarding revision is saved only after its content and audit record commit. + +The approval service owns comments, revisions, and general approval records. +The lifecycle module resolves a hire approval and changes the agent in one +transaction. It retains the original credential owner when it approves a hire. + +The company deletion service owns the company data cascade. +It requests agent termination before it starts that cascade. +It locks the company in the deletion transaction and removes dependent records. +It then calls `agentLifecycleCompanyDeletion.deleteCompanyData(tx, companyId)` +through the module's separate `company-deletion.ts` entry point. +This operation implements the shared `CompanyDeletionParticipant` interface. +It locks the agent rows, checks their states, and deletes them. +All agents must be terminated or rejected. Otherwise, it throws an error. +The error rolls back all database changes in the deletion transaction. +A concurrent hire makes deletion fail or waits until deletion ends. +The module's delete statement also restricts removal to terminated or rejected agents. +`purgeAgent` deletes a single agent. It owns the transaction and checks the agent +state before it calls the service to remove dependent records. +The service checks accounting holds. The module then deletes the agent record. +An error rolls back both operations. Neither path permits incomplete termination. + +The ordinary agent service owns reads, permissions, and keys. +The configuration service owns validation and configuration revisions. +The lifecycle module owns the agent configuration write. +Credential checks remain in the agent credential service. +Secret proposal approval saves the secret before it applies the agent binding. +If the binding fails, the secret remains approved and the binding remains pending. +A retry uses the approved secret. It does not create another secret. +The binding transaction checks authorization, expiry, and the current reporting +structure again before it changes the agent. +The hire service owns names, appearance, permissions, credential binding, +connection installs, identity setup, and primary-agent selection. +The module calls the hire service before and after the agent record write. +These operations use the lifecycle transaction. A failed operation rolls back +the hire and its dependent records. +The module owns the initial state, approval transition, and lifecycle events. +The service in `services/agent-lifecycle.ts` supplies these integrations to the +module. The module does not import services or routes. Shared record queries +and validation functions are in `lib/`. They do not call lifecycle commands. +The application supplies the worker driver at startup. +`updateConfiguration` owns the transaction for an ordinary configuration update. +The module locks the agent. The configuration service validates and prepares the +patch. The module saves it and invalidates an old verification result when +execution configuration changes during setup. The service then updates dependent +records and saves the configuration revision in the same transaction. +There is no separate invalidation operation for callers. +Configuration writes reject caller-supplied lifecycle fields. +For `updateAndTransition`, lifecycle owns the transaction and calls the +same configuration operation. Both changes commit together. + +Some workflows must save an agent reference and their own records together. +The module exposes `configuration.ts` for these transactions. Only +`services/agent-configuration-transaction.ts` can import it. +This service supplies the module integrations. It cannot save a configuration +without the module's invalidation check. +Its permitted callers are secret binding approval, Connection adoption, runner +pairing, avatar updates, instruction revisions, and skill reassignment. +The boundary check lists these callers. Ordinary callers use the root command. +Budget changes always use a root command so enforcement follows the commit. +The transaction integration cannot approve, pause, resume, or terminate an agent. +It does not start a worker before the caller commits. The periodic scan recovers +pending setup work after the commit. + +Manual, budget, and company pause holds are independent. +A manual resume does not remove a budget or company hold. +Restoring a company does not remove a manual hold. +Budget and company policy still prevent admission while the pause work runs. + +The legacy `status` field remains available to existing clients. +It shows `paused` during preparation, verification, pause, and resume. +It shows `terminated` as soon as termination starts. +Execution code can change this field only when the lifecycle is `ready`. +Use `lifecycleState` to distinguish the steps. +This change does not change the user interface. + +## Required plugin work + +A plugin can declare `agentLifecycle: true` in its manifest. +It must also request `agents.lifecycle.manage`. +The plugin implements `onAgentLifecycle` in the SDK. +This method is separate from event delivery and event acknowledgment. + +The host selects the required plugins before the first lifecycle step. +`setRequiredPlugins` stores their IDs on the agent. +The host is not a plugin ID. Each operation records host completion and +completed plugin IDs separately. The host step must finish before plugin calls. +A disabled or removed plugin cannot silently release an existing requirement. +Restore that plugin to complete the operation. +A later plugin installation does not change an existing selection. + +Each request contains these fields: + +- `companyId` and `agentId` identify the agent. +- `phase` identifies the lifecycle step. +- `operationId` identifies the operation. +- `version` identifies the state revision. + +Return the same `operationId` and `version` with `status: "complete"` or +`status: "pending"`. +Return `complete` only after the requested effect is complete. +Return `pending` while external work continues. +Throw an error when the step fails. +The host stores a fixed error message. It does not store the provider error text. +Server logs identify the failed step, phase, operation, and plugin when available. +They include only known error codes. Unknown codes become `unclassified`. +They do not include provider messages, stacks, or configuration values. +A failure before the agent is read can have only a step and agent ID. +A scan failure has no agent ID. + +Calls can repeat after a timeout or a server restart. +A plugin must make repeated calls safe. +It must retain the highest version for each company and agent. +It must reject an older request after it accepts a newer version. +It must also prevent an older operation from creating resources after cleanup. +Serialize external effects or remove resources from a late completion. +Do not treat a host timeout as cancellation of the external effect. + +## Verification and recovery + +The harness verification service uses the existing environment-test code. +It owns test configuration, credential selection, and test result checks. +It tests the saved agent configuration and the selected environment. +A failed login or an incomplete probe cannot complete verification. +The lifecycle module receives completion, pending work, or failure through its +driver. It owns the state transition and has no harness test rules. +The current UI can also run a test before the hire. +Some adapters can charge for these tests. +It uses the saved responsible user for managed credentials. +The host does not grant new credential access for this test. +A connection pool selects an account for a separate lifecycle test operation. +An external controller must complete its existing readiness test. + +Task requests can enter the durable run queue during preparation, verification, +and resume. Execution waits for readiness. Queue recovery does not require +periodic agent heartbeats to be enabled. + +A cancelled run record does not prove that execution stopped. +The host also checks process ownership, controller leases, and environment cleanup. +Required plugin cleanup cannot start while these checks are incomplete. + +The worker starts after a committed command. +A periodic scan recovers work after a process stops. +A database lease permits one active attempt for each agent. +The worker renews that lease during external calls. +Each result must match the current revision and lease owner. +Shutdown waits for active work. + +The agent API returns `lifecycleState`, `lifecycleVersion`, and `lifecycleError`. +A board user can retry through `POST /api/agents/:id/lifecycle/retry`. +The route applies the existing company access checks. + +## Existing installations + +Migration `0320_daily_onslaught.sql` maps the current status of each agent. +Active execution states become `ready`. +Paused, pending, and terminated agents retain their current meaning. +The migration retains each existing pause reason as a hold. +It does not provision external resources or record a successful harness test. +Resource backfill is a separate task. + +Run the normal database migration process before starting the new server. +Stop older server processes during the change. +An older process can write the legacy status without the new lifecycle checks. + +`pnpm check:module-boundaries` checks the module imports and agent writes. +Tests and migration fixtures can write records directly. +Production creation and lifecycle state writes must use the lifecycle module. +Production record deletion must also use the lifecycle module. +Execution configuration writes must use the module. The offline worktree seed +command can disable timers in a copied database before the server starts. + +## Extension rules + +Add a lifecycle command to the module entry point. Put state transition rules in +`domain/policy.ts`. Put database operations in the module adapters. +Keep harness and provider rules in services. Supply external work through the +worker driver. Report completion only after that work is complete. +Add a test for a failed or repeated operation. Run `pnpm check:module-boundaries` +to check the import and write boundaries. diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 7799ead39e..a90cd32862 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -681,3 +681,46 @@ and are atomically deleted before code exchange. The `code_verifier` column hold this non-secret binding for this namespace; Slack bot installation does not use PKCE. Removal and manual recovery invalidate outstanding attempts under the same credential-mutation lease used by configuration. + + +## Transaction-aware delivery work signals + +Five existing delivery queues use process-local work signals to avoid empty +polling. Register work **before** writing a queue row, on that row's transaction: + +```ts +await db.transaction(async tx => { + await signalDatabaseWork(tx, "queue-topic"); + await tx.insert(existingQueueTable).values(row); +}); +``` + +`createDb` instruments transaction callbacks and nested savepoints. Subscribers +receive intent immediately and settlement only after the outer transaction +finishes. Caller-owned transactions therefore need no extra post-commit wrapper. +Dedicated clients from `withDedicatedDbConnection` share the owner's signal +scope. No schema changes, triggers, dedicated listener connection, or periodic +queries are installed by this API. + +The first registered write in a transaction fetches `pg_current_xact_id()`. +Awaiting registration is required: a failure at that point must prevent the +queue write. Unrelated transactions add no queries. If the transaction rejects, +the coordinator probes `pg_xact_status(xid)` through the root pool before scanning +the queue. An in-progress transaction remains unresolved even when the queue +currently looks empty. Committed/aborted results permit reconciliation; NULL +means PostgreSQL has discarded an old, no-longer-active transaction's outcome. +The existing durable queue supplies the work in either case. Probe failures +retain the idle hold and schedule another attempt. Work signals never replay +queries or convert a failed database operation into success. + +This uses PostgreSQL's transaction-information functions (this path requires +PostgreSQL 14+; embedded PostgreSQL uses 18). See the +[PostgreSQL transaction information documentation](https://www.postgresql.org/docs/18/functions-info.html#FUNCTIONS-PG-SNAPSHOT). +The process-local intent survives coordinator replacement, but not process exit. +Startup scans recover committed queue rows. Independent DB clients, other +processes, late transactions from an old process, and database failover need an +explicit ownership/wake protocol; these signals are not cross-process messaging. +Use the same root client for in-process writers. New queue insertion paths must +register before writing, or they can remain unseen until the next startup or +another notification. Tests should verify both the producer and its outer +transaction boundary. diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 318fe32102..35fcb16bc1 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1250,9 +1250,9 @@ It owns bounded run projections, database encoding checks, task session reads an writes, explicit resumes, session compaction, and usage/billing helpers. `createHeartbeatRunState(db)` binds these operations without doing database work during construction. The encoding-check cache belongs to each factory instance. -`heartbeat.ts` keeps run execution, session-goal recovery, cost accounting writes, -and status transitions, and re-exports the existing public helpers. Keep session -policy changes separate from run orchestration changes. +`heartbeat.ts` keeps run execution and session-goal recovery, and re-exports the +existing public helpers. Keep session policy changes separate from run +orchestration changes. Retry scheduling is in `server/src/services/heartbeat/retries.ts`. It owns bounded retry schedules, connection and workspace contention deferrals, shared-workspace @@ -1281,6 +1281,25 @@ its process-wide execution and wakeup promise sets. Forwarding callbacks preserv construction order for retry and recovery services. Keep queue policy changes separate from this extraction and from adapter execution changes. +Run lifecycle handling is in `server/src/services/heartbeat/run-lifecycle.ts`. +It owns status transitions, run events and progress, liveness classification, +completion handoffs, issue-comment finalization, and runtime/cost settlement. +`createHeartbeatLifecycle` binds the database and explicit reporting, recovery, +and wakeup callbacks without starting work. The service supplies terminal +reporting and forwarding callbacks preserve construction order for the other +heartbeat modules. Existing public helpers remain re-exported by `heartbeat.ts`. +Keep lifecycle policy changes separate from this extraction and adapter execution. + +Run cancellation and cleanup are in `server/src/services/heartbeat/run-control.ts`. +It owns Stop, agent and budget cancellation, environment lease release, and +resumption of saved comments after execution stops. `createHeartbeatRunControl` +binds the database, lifecycle and admission callbacks, and shared executor +barriers without starting work. The service retains the process-wide execution +and cancellation maps so scheduler and route instances observe the same owners. +Forwarding callbacks preserve construction order for budget, retry, recovery, +and queue services. Existing public helpers remain re-exported by `heartbeat.ts`. +Keep cancellation policy changes separate from this extraction and execution. + ## Wake Context Delivery Built-in adapters deliver wake context through the run prompt, including structured @@ -1428,6 +1447,14 @@ the updated sandbox image with the matching runner qualification changes. ### Native runner restart recovery +Custom source-mode HTTP launchers should create their listener with +`createPaperclipHttpServer` from `server/src/http/server.ts`, passing the selected +API origin as `apiUrl`. This registers the native Runner WebSocket transport +before execution is admitted. Plain Express `app.listen()` or Node +`createServer(app)` alone does not initialize that transport and native runs +fail with `runner_prp_websocket_server_not_configured`. Browser live events still +use their separate authenticated setup. + Project discovery through `list_projects` returns up to 50 compact summaries. It uses `GET /api/companies/:companyId/projects?view=summary&limit=50&cursor=...`; the cursor is optional. The database reads bounded summary projections. Agent diff --git a/doc/agent-files.md b/doc/agent-files.md index d0fce516eb..5dc4d18070 100644 --- a/doc/agent-files.md +++ b/doc/agent-files.md @@ -37,6 +37,16 @@ workspace: their independent agent copy therefore lives under the excluded `.paperclip-runtime/agent-files///` area. It is not included in task workspace sync, Git staging, or task deliverables. +Sandbox transfer scratch lives separately under +`.paperclip-runtime/paperclip-runner/agent-file-transfers///`. +Workspace-staging transfer scratch stays outside `AGENT_HOME`, including +recovery after controller restart. Remote warm Codex checkpoints may temporarily +stage changed-file payloads under `AGENT_HOME/.paperclip-runtime/checkpoint-*`; +these are checkpoint scratch, not managed personal files. Cleanup removes +only the original materialization's agent directory and transfer scratch +through its owned lease. A later warm turn keeps that original materialization +identity; it does not grant collection or cleanup of another run's files. + Claude CLI runs keep the working-copy location in each run's prompt, separate from the cached system instructions. A new copy path alone does not reset the task session. Each turn names the current copy for relative file references; diff --git a/doc/architecture/runner-copilot-capabilities.md b/doc/architecture/runner-copilot-capabilities.md index ceddfb1b7d..f9d97b997a 100644 --- a/doc/architecture/runner-copilot-capabilities.md +++ b/doc/architecture/runner-copilot-capabilities.md @@ -6,7 +6,17 @@ It binds master's shared protocol-validation sources without changing the executable or supplying new paid qualification. Historical checkpoints below retain their original identities and outcomes. -Historical source candidate (2026-10-01): **Copilot profile v12 is unqualified**. +Historical source candidate (2026-10-02): **Copilot profile v13 is unqualified**. +Pi's explicit reasoning-mode support changed four shared transport and schema +source files included in Copilot's execution identity. Version 13 records those +source hashes with digest +`sha256:3ff08fbe76fe4549c9eb01e8794428d8909c65c151d775220f2ec111d9e6f7c1`. +The Copilot executable closure, model and provider behavior are unchanged. The +historical v12 declaration is preserved; v12 warm sessions are rejected by the +current candidate. This identity update supplies no new paid qualification. +Fresh runtime packages and local/Daytona qualification remain required. + +Historical source candidate (2026-10-01): **Copilot profile v12 was unqualified**. Receipt v2 preserves the original `inputSha256` and separately captures `normalizedInputSha256` from the same invocation's validated outgoing body. For the native sidecar, only a successful, correlated `tool.resolve` commits diff --git a/doc/architecture/runner-pi-capabilities.md b/doc/architecture/runner-pi-capabilities.md index 7876c5cba4..6046bfb223 100644 --- a/doc/architecture/runner-pi-capabilities.md +++ b/doc/architecture/runner-pi-capabilities.md @@ -22,7 +22,115 @@ for the exact source/image evidence and merge status. All dated checkpoints below describe historical qualification attempts. Their failures and profile numbers are retained; they are not current release gates. -## Historical Pi 1.0 candidate (2026-10-02, profile v12) +## User-selected models (2026-10-06) + +Native Pi accepts any explicit provider/model ID. The settings builder, server, +release profile, and Rust runner do not restrict selection to a qualification +model. Pi must acknowledge the exact selection before a prompt and on recovery; +unavailable models fail clearly without selecting a substitute. Runtime package, +version, command, permission, and process-ownership checks remain enforced. + +Bind the selected provider's credentials in the agent environment, for example +`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, or `OPENROUTER_API_KEY`. +Custom models use `PAPERCLIP_PI_PROVIDERS`, an explicit JSON object in Pi's +`models.json` providers format. The runner writes it into the private Pi home, +forwards explicitly bound credential references, and includes its digest in +recovery identity. Credential commands and control-plane credential references +are rejected. General AWS IAM keys are excluded; Bedrock uses `AWS_BEARER_TOKEN_BEDROCK`. Choose a thinking level supported by the selected model (`off` +for a model without reasoning). + +The DeepSeek model IDs throughout the historical qualification results below +identify those attempts; they are not a product allowlist. Those results do not +qualify this merge or additional models. + + +## Historical Pi 1.0 candidate (2026-10-02, profile v13) + +Pi remains pinned to **`@earendil-works/pi-coding-agent@1.0.0`**, with +`pi-acp@0.0.33`, ACPX `0.13.1`, and Node `24.21.0`. Profile 13 adds explicit +reasoning-mode selection and rejects previous profile identities. It is a new +qualification candidate; profile-12 results below remain historical evidence. + +Pi's new field also changes four shared files covered by Copilot's source +identity. The subsequent live callback snapshot changes the shared sidecar again, +so Copilot now uses still-unqualified profile v14 without changing its executable +or model. Pi's profile-13 identity and native closure +remain unchanged. Final artifacts must bind the corrected source; the initial +ARM startup and 61 contract passes alone do not establish production readiness. + +The normal Pi configuration records `piThinkingLevel` as `off`, `low`, `high`, +or `max`. New configurations and qualification cases explicitly select `low`. +The runner binds that setting to the session identity and requires native +confirmation of the effective level before sending the prompt, including after +session restoration. An unsupported level, absent acknowledgement, or effective +level different from the requested value must fail admission. Changing the +level requires a new compatible session; a warm session cannot silently retain +its previous setting. + +The wrapper obtains available levels from Pi's native +`get_available_thinking_levels` RPC, exposes them through ACP session modes and +the `thought_level` configuration option, and confirms changes with native +`get_state`. This avoids advertising the static `minimal`, `medium`, or `xhigh` +aliases when the selected model does not support them. For the pinned DeepSeek +model, Pi 1.0 supports `off`, `low`, `high`, and `max`. Pi's implicit `medium` +default resolves to `high`; Pi 0.84.2 also resolved that default to `high`, so +this is a configuration gap rather than evidence of a new Pi 1.0 default. + +The [provider-free wire proof](../../packages/paperclip-runner/test-fixtures/pi-acp/thinking-modes.v13.darwin-arm64.json) +records actual Pi 1.0 request bodies changing from `high` to `low` through ACP, +truthful `max` restoration, blocked non-loopback connections, and owned-process +retirement. Synthetic responses establish propagation, not paid qualification. + +The paid profile-12 hello passed in the actual installed application. Its first +question continuation failed at the unchanged 120-second native deadline after +1,019 reasoning deltas and no emitted tool call or question. The required tool +was present with its exact schema. This establishes model activity without the +required interaction; it does not prove the reasoning setting caused the timeout. +The test remains failed. Profile-13 deterministic, local paid, Runner, native +control, and Daytona results must be established independently. + +| Capability | Native / ACP surface | Paperclip surface | Remaining boundary | +| --- | --- | --- | --- | +| Supported thinking levels | `get_available_thinking_levels`; ACP available session modes and `thought_level` choices | Pi reasoning-level configuration | Model selection is explicit and unrestricted; historical qualification used the recorded model and level. | +| Set and verify thinking level | Native `set_thinking_level`, then `get_state`; ACP `session/set_mode` / config option | Saved agent setting, typed runner input, effective-mode admission | Live setting changes during an active turn are not exposed; a new compatible session is required. | +| Recover selected level | Native restored state plus explicit effective-level verification | Mode-bound durable session identity | Old profiles and missing or mismatched mode identity cannot reuse a warm session. | +| Provider notice severity and pricing provenance | `paperclip/pi_notice` and canonical `provider.notice.recorded` | Retained run event, usage metadata, and safe summary with severity in the activity row | Summary/severity rendering has deterministic coverage; paid visual verification of the rebuilt UI remains pending. Pricing estimates remain distinct from provider billing receipts. | + +## Historical profile-13 qualification checkpoint (2026-10-02) + +The installed candidate from source `eac50643213d0902a93a5384bae2f1d6c065c9f2` +uses Pi 1.0.0, the explicit OpenRouter DeepSeek model above, and verified `low` +reasoning. Paid local Product E2E passes cover hello, semantic question/answer +continuation, all four native question forms, and human permission denial with +no file side effect. The paid extended Runner `get-task-context` case also +passes through the packaged runner against its authenticated, seeded test +control plane. Runner protocol evidence does not qualify the Product UI. + +Three local attempts remain failed: file-edit validation, pending-input +controller restart, and three-turn warm continuation. The first two exposed +assertion gaps: streamed command output supplies the exit receipt, and a +pre-start submission may precede assignment of the native turn ID. Narrow +fixture repairs have positive and negative coverage; neither failed attempt +has been regraded and both require fresh live runs. + +The warm attempt completed its first turn and edited files on its second, but +its final semantic tool stayed pending until the existing deadline. OpenRouter +metadata joins the exact native session and records the final response as +cancelled with no finish reason. This supports an unfinished upstream response +at cancellation; it does not establish why the response stopped progressing. +A real pinned SDK/wrapper replay proves both restored-session completion with +the new agent home and completion contract, and safe cancellation when complete +tool JSON arrives without the provider's stream finish marker. The latter +executes no semantic mutation. No timeout or runtime policy was changed to make +that replay pass. + +Fresh builds must incorporate the subsequent launch, UI, and fixture fixes. +macOS Intel startup, the remaining local and Runner cases, Linux/Daytona, and +final source-wide checks remain qualification gates. The provider remains +unqualified until that evidence is complete. API-key usage observations cover +the capped qualification campaign; native pricing estimates are not bills. + +## Historical Pi 1.0 profile-v12 checkpoint (2026-10-02) The candidate now pins **`@earendil-works/pi-coding-agent@1.0.0`** with `pi-acp@0.0.33`, ACPX `0.13.1`, and portable Node `24.21.0`. Pi v11 and older @@ -97,6 +205,76 @@ Pi's published CLI mode union is `text | json | rpc`; no native ACP mode is present in this release. The reviewed `pi-acp` wrapper remains necessary. +## Explicit Linux companion setup for a Mac controller + +A Mac controller needs independently verified Linux runner bytes for Daytona. +Its host Pi installation is not a Linux provider pack. The operator imports a +release companion once, before launching agents: + +```sh +paperclipai runtime import-remote /path/to/release/linux-x64 --sha256 MANIFEST_SHA256 +``` + +Obtain `MANIFEST_SHA256` from the trusted release channel, separately from the +copied directory. The directory contains `companion.json`, +`bin/paperclip-runnerd`, and the complete `provider-pack/`. Import verifies the +installed server's exact build commit, qualified Pi profile, Linux x64 ELF, +manifest digest, and every file, mode, directory and contained symbolic link. +External hardlinks, escaping links, extra or modified files, and unsafe modes +are rejected. This command performs no network requests or model calls and does +not run the Linux executable on the Mac. + +The destination is the private `remote-companions/linux-x64` directory beneath +the actual installed `@paperclipai/server` package. It must be writable by the +operator, outside the task workspace, and protected with the same host access +controls as the server installation. It is not an agent workspace or a remote +image's self-reported authority. An existing different or damaged installation +is never replaced automatically. Stop all runs before an operator removes or +upgrades that cache. Verification and copying use asynchronous 1 MiB chunks, +yield between chunks, and enforce the same full inventory and ten-minute bound. +Directory entries must retain owner read/write/search permissions so failed imports +can drain their owned cleanup. SIGINT/SIGTERM are deferred through owned filesystem +phases and drain cleanup; +an uncatchable kill or host failure leaves the import unadmitted until the +operator resolves its retained lock or incomplete directory. Do not move the +cache into a task workspace to work around permissions. + +Normal remote Pi resolves this authority without binary or pack environment +overrides. The controller re-verifies its complete inventory and uses the exact +Linux daemon identity. Existing remote verification compares image bytes against +that local authority; a mismatch takes the existing verified upload path. +Controller restart and warm-session recovery retain the original Runner artifact +identity checks. Cursor, Copilot and explicit operator overrides keep their +existing admission rules. + +For release maintainers, assemble the Linux daemon and default provider pack +for the **same final commit** as the public server/CLI packages. A reused daemon +requires complete native source/config/protocol input equality and retains its +original compiler/build provenance; the manifest does not claim recompilation. In a fresh +release directory place those outputs at `bin/paperclip-runnerd` and +`provider-pack/`, preserving the pack's relative links and modes. Then run the +source-owned manifest generator after building the matching server: + +```sh +node scripts/create-runner-remote-companion.mjs /path/to/release/linux-x64 FINAL_SOURCE_SHA +``` + +Publish the complete companion directory (or distribute it through the normal +trusted release channel) together with the printed manifest SHA256 and the +source/profile/platform provenance. The import command accepts an already +extracted directory, not an archive or URL. The release companion must remain +available with that release; a short-lived qualification artifact is not a +release distribution. No release publication is implied by the tests here. + +Build/publish the Daytona image from those same daemon/pack outputs and configure +its immutable OCI digest through the ordinary Daytona environment `image` field. +The image digest and companion manifest are output metadata, not new source +constants, so publication does not require another source commit. The final live +proof must use the installed public CLI, this import command, and ordinary image +configuration without E2E remote binary/provider-pack overrides. That installed +and paid proof remains pending. + + Historical v10 qualification checkpoint (2026-09-30): **Pi profile v10 remains unqualified.** The capped-key/login prerequisite remains blocked. Native USD is unknown. The optional private budget helper is not integrated; Cursor's account-cycle cap does not establish Pi's provider spending bound. V10 keeps the native wrapper and closures unchanged, binds the shared ACPX patch under a new digest, and rejects v9 sessions. Fresh exact-runtime admission and final controller verification remain pending. Historical v9 checkpoint (2026-09-30): **Pi profile v9 remains unqualified**. Current runtime source is `5b8e4454ef0bf12d0bb068c2e41d8c9df9356a1c`; controller/Product harness source is `40064d28522de25fea85c1297f35b41bb8a8897a`. Runtime builds for macOS ARM64/x64 and Linux x64 are complete. No paid profile-v9 pass is claimed. A credential with a verifiable spend limit is still needed for the remaining paid qualification. The optional transport-budget candidate is frozen on a separate branch and is not integrated or a live spending guarantee. See the [comparative capability report](runner-rich-acp-capabilities.md) for current qualification gates and the field audit. The dated observations below retain their original profile identities. @@ -209,8 +387,8 @@ addressed by v4. Version 2 hello completion and every failure remain retained. The wider local and Linux x64 Daytona matrix remains pending; this document does not promote the candidate to a qualified production runtime. -The runner pins `pi-acp@0.0.33` and -`@earendil-works/pi-coding-agent@0.84.2`. The candidate model is +At this historical pre-1.0 checkpoint, the runner pinned `pi-acp@0.0.33` and +`@earendil-works/pi-coding-agent@0.84.2`. The candidate model was `openrouter/deepseek/deepseek-v4-flash-0731`; only an explicitly bound OpenRouter credential may reach this profile. `patches/pi-acp@0.0.33.patch` repairs the ACP wrapper. `pi-runtime-extension.ts` supplies the runner-owned semantic bridge and @@ -985,3 +1163,62 @@ that ID unchanged into the shared MCP dedupe boundary. This is a further integration defect. The shared dedupe guard and canonical grader remain intact. Both runs lack terminal usage; measured billing is $0.002440082 and cleanup passes. Restart and refreshed hello are held until this defect is repaired. + +## Explicit host installation + +For a published local Paperclip installation, run `paperclipai runtime setup pi` +with the same installed CLI and account that owns the server package. This is an +explicit download and verification step; npm installation and agent launch never +perform it automatically. It installs only this host's supported platform +(macOS ARM64, macOS x64, or Linux x64), using the source-pinned Node archive, npm +lock, wrapper patch and complete Pi closure. Node 24, npm, git, tar and the normal +platform dependency inspector (`otool` or `ldd`) must be available. The server +package must be writable by the installing account. Explicit setup preserves +`HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, their lowercase equivalents, +`SSL_CERT_FILE`, `SSL_CERT_DIR` and +`NODE_EXTRA_CA_CERTS` for public downloads, and enables Node's environment proxy +handling. The provisioner keeps the same closed allowlist. Instance settings, +provider credentials, user npm configuration, `HOME`, `NODE_OPTIONS` and +`NODE_PATH` are excluded; TLS certificate validation stays enabled. + +The public server carries a self-contained setup tool and small pinned inputs in +`dist/vendor/paperclip-runner/cli`; the installed host closure lives in that +server package's `provider-assets/pi/`. Setup validates an existing +closure again before accepting it. A corrupt existing installation is left +untouched and rejected; reinstall the same Paperclip release into a clean package +location and repeat setup. Concurrent setup is rejected. Cancellation drains the +current bounded download/build command before removing its private staging tree; +allow that cleanup to complete before trying again. + +Then select Pi with an explicit provider/model ID and bind its provider credential +through the normal agent environment configuration. Setup itself makes no model request. +A missing host closure produces explicit setup guidance. Daytona uses the +separately built and verified Linux provider pack in its runner image; running +local setup does not install or qualify a remote image. Published-tar local and +Daytona startup evidence must bind the final installation candidate, with no +candidate qualification flags, before a production-readiness claim. + + +## Cold process admission budget + +Pi cold `session.open`, including process replacement and durable reopen, has a +60-second admission budget. The controller shares one deadline across its cold +open/recovery command and provider identity barriers (`session.open`, replacement +`run.attach`, and replacement `runner.drain`). A live adopted runner keeps the +ordinary 30-second waits. Rust gives only Pi's sidecar `session.open` request the +60-second budget; initialize, attach, turn start and other sidecar commands stay +at 30 seconds. Inner ACP handshake (30 seconds), post-admission verification +(8 seconds), cancellation (2 seconds), and close (7 seconds) are unchanged. + +Timeout still poisons and retires the Rust sidecar process group. Closing the +controller during its startup wait rejects that wait and uses the existing owned +process cleanup. Callers must await transport close in their cleanup path. +This is trusted controller timing policy, outside the immutable Pi wrapper/profile +closure; it does not change profile 13, runtime 1.0.0, or the explicit model. + +The credential-free closed-startup regression requires the real +`session_ensure_failed` rejection within 60 seconds, with no prompt, no provider +identity and confirmed cleanup. Its child watchdog is 80 seconds (20 seconds for +cleanup after admission), inside a 90-second test timeout. Failed evidence stays +in its reported temporary directory. These margins do not extend production +command deadlines or qualify a target platform without a fresh run. diff --git a/doc/architecture/runner-rich-acp-capabilities.md b/doc/architecture/runner-rich-acp-capabilities.md index c1334de585..c1e6e61260 100644 --- a/doc/architecture/runner-rich-acp-capabilities.md +++ b/doc/architecture/runner-rich-acp-capabilities.md @@ -22,6 +22,100 @@ for the exact source/image evidence and merge status. All dated checkpoints below describe historical qualification attempts. Their failures and profile numbers are retained; they are not current release gates. +## Historical Pi 1.0 qualification checkpoint — October 2, 2026 + +Pi uses `@earendil-works/pi-coding-agent@1.0.0`, `pi-acp@0.0.33`, and ACPX +`0.13.1`. Profile **13** adds explicit, native-acknowledged reasoning selection +for the exact model `openrouter/deepseek/deepseek-v4-flash-0731`. New Pi +configuration and qualification cases select `low`. This candidate is not yet +production qualified. Cursor v10 and Copilot v13 remain gated. + +The installed profile-13 candidate at `eac50643213d0902a93a5384bae2f1d6c065c9f2` +has paid local passes for hello, semantic question continuation, all four native +question forms, and human permission denial without a file side effect. Its +paid extended Runner `get-task-context` case passes against the authenticated +seeded test control plane. That protocol case does not qualify the Product UI. +File validation, pending-input controller restart, and three-turn warm +continuation remain failed and unregraded. Assertion repairs and an actual +SDK/wrapper replay now cover the identified boundaries; fresh live proof is +still required. See the [historical Pi checkpoint](runner-pi-capabilities.md#historical-profile-13-qualification-checkpoint-2026-10-02) +for the request-stream diagnosis and remaining platform gates. + +Copilot's declaration hashes the shared sidecar, sidecar protocol, direct driver, +and generated schema bundle. Pi's reasoning field changed those four files, so +Copilot's pending declaration advances to v13 with digest +`sha256:3ff08fbe76fe4549c9eb01e8794428d8909c65c151d775220f2ec111d9e6f7c1`. +Its executable, distribution closure, model, permissions, and qualification +status are unchanged. The v12 declaration remains historical, and v12 warm +sessions are rejected. This source-identity repair supplies no Copilot paid proof. + +The first profile-13 source (`f5c5fde38`) built a fresh ARM daemon and Pi pack, +then passed original startup and all 61 native/ACP contract tests without model +calls. Its full Runner suite found three missing-mode test fixtures and the +stale Copilot declaration. Those failures are retained. The corrected final +source still requires fresh platform builds, complete checks, and paid tests. + +Profile 12 passed one paid local hello through the actual installed public +CLI/server and normal runtime setup. The next question-continuation case failed +at the unchanged 120-second native deadline: Pi emitted 1,019 reasoning deltas, +but no tool call or question. The tool and schema were present. One native turn +was submitted; the two visible start rows were canonical and mirrored activity, +not evidence of two provider requests. Both attempts completed owned-process, +IPC, temporary-root, installed-inventory, and post-run accounting checks. + +The dedicated key recorded $0.000510742 for the hello and $0.004995065 for the +failed question. Including the earlier profile-11 failure ($0.000142518), its +observed total was $0.005648325 at that reconciliation. These are observed key +usage changes, not native per-request bills. No failed result has been regraded. + +Diagnosis found that the runner did not select the ACP reasoning mode. The +pinned model's implicit Pi `medium` setting resolves to `high`; this behavior +also existed in Pi 0.84.2. Pi 1.0 exposes a genuine `low` level. Profile 13 carries +an explicit setting through configuration, transport, and durable identity, +and requires the native effective level to match before a prompt is sent. +It does not increase the timeout. The timed-out HTTP body was not captured, +so source-derived default mapping does not establish the cause of that failure. +Fresh qualification must use the new profile and exact final build. + +Historical source `4c58da84fd86a0cd7bc1d952987b031786fc6b2b` passed full Runner +verification and normal Intel startup at 28.295 seconds under the original +30-second limit, plus all 48 native/ACP contract tests. The Intel result has +only 1.705 seconds of margin and does not establish broad cold-start reliability. +Its fresh Linux daemon, Pi pack, and image were built and inventoried; live +Daytona qualification remained pending. The public Mac installation reproduced +the complete Pi dependency closure using npm 11.19.0 from the verified Node +archive. None of these artifacts qualifies profile 13. + +The same source's full repository check failed six stale test assertions and +an `ETXTBSY` binary-staging error; the downstream Grok check then lacked built +server output. Typecheck, token gates, and Product E2E typecheck/unit checks +passed. The stale assertions have targeted repairs. A Linux process owner was +not captured for `ETXTBSY`; two exact native fixtures later passed on ARM with +all owned daemon handles closed, which does not resolve the Linux failure. +The next complete source run retains bounded executable-holder evidence around +check boundaries. A later run on obsolete harness source was canceled by the +lead and is retained as incomplete, not successful verification. + +Normal Daytona setup has an explicit operator command to import a verified +Linux companion into the installed server. It binds the trusted manifest to the +installed source, Pi profile, daemon, and complete provider-pack inventory, then +uses ordinary remote execution without test-only binary or pack overrides. +The final installed import, image publication, and paid proof remain required. + +The dedicated Pi test key has a $5 lifetime OpenRouter-credit limit. Its BYOK +charges are outside that provider-enforced limit, so qualification additionally +uses the reviewed no-BYOK operational policy, fresh account/key evidence, one +paid case at a time, and usage monitoring. The $5 reservation is counted once +within the combined $100 campaign budget. Native price estimates are never +substituted for provider bills. + +The [Pi capability inventory](runner-pi-capabilities.md#historical-pi-10-candidate-2026-10-02-profile-v13) +records native interfaces, ACP mappings, user surfaces, and explicit gaps. +The comparison below remains the supported-surface map; old observations retain +their exact historical profile identities. Final source checks and profile-13 +local Product, Runner protocol, native-control, and Daytona qualification remain +pending. + Historical source checkpoint (2026-10-01): **Cursor v10, Copilot v12 and Pi v10 remain unqualified**. Copilot receipt v2 distinguishes original provider arguments from the validated outgoing completion input. The sidecar commits @@ -73,7 +167,7 @@ as `25303cf84953985b961b95f89aff0bdb864b2d65`, from head unchanged; this adds no paid proof. Those definitions remain the historical Cursor v8 checkpoint and require a separate reviewed Cursor v9 update. -Current checkpoint (2026-09-30): **Cursor v9, Copilot v8 and Pi v10 remain unqualified.** The frozen runtime is `5c69b69ef2aeab8d8a367b25a8d894cc5308befa`; controller candidate is `a8df2064d68f40fbf4dec670c4b8478c4b1b1b3f`. All profiles bind shared ACPX patch SHA-256 `bd5393058a218040d217fa85449d59a6f30507de54cd645bf0ef21422823f85e`. No profile is promoted and no prior grade is changed. +Historical checkpoint (2026-09-30): **Cursor v9, Copilot v8 and Pi v10 remain unqualified.** The frozen runtime is `5c69b69ef2aeab8d8a367b25a8d894cc5308befa`; controller candidate is `a8df2064d68f40fbf4dec670c4b8478c4b1b1b3f`. All profiles bind shared ACPX patch SHA-256 `bd5393058a218040d217fa85449d59a6f30507de54cd645bf0ef21422823f85e`. No profile is promoted and no prior grade is changed. Current-profile protocol eval definitions merged in [paperclip-evals #36](https://github.com/paperclipai/paperclip-evals/pull/36) as `d987357461933baca0d4c10cc081f40e7eae5c1b`: 136 deterministic tests and 21 validated cells. These definitions do not supply paid qualification evidence. @@ -683,22 +777,22 @@ deterministic evidence are mapped separately after the comparison. | Capability | Codex app-server benchmark | Cursor ACP candidate | Copilot ACP candidate | Pi ACP candidate | | --- | --- | --- | --- | --- | -| Exact model | Selected and reported model | Explicit ID required; exact echo, paid semantic protocol and five local semantic product cases passed | Explicit ID required; exact `gpt-5.6-luna` echo, paid semantic protocol and local completion/file/plan/restart passed | Exact `openrouter/deepseek/deepseek-v4-flash-0731`; model echo and v2/v3 paid Product completion/file/question cases passed; v5 native controls passed | +| Exact model | Selected and reported model | Explicit ID required; exact echo, paid semantic protocol and five local semantic product cases passed | Explicit ID required; exact `gpt-5.6-luna` echo, paid semantic protocol and local completion/file/plan/restart passed | Exact `openrouter/deepseek/deepseek-v4-flash-0731`, native-verified `low`; profile-13 paid hello, semantic continuation, native questions and human denial pass; file/restart/warm failures remain open | | Text and tools | Typed thread/turn/item events | Standard ACP updates; child activity kept separate | Standard ACP plus opt-in native session events | Wrapper text/tool updates and owned tool gates | -| Active steering | Dedicated `turn/steer` | Concurrent prompt replaces/cancels, so it is not steering | Concurrent prompt replaces/cancels, so it is not steering | Owned `pi/steer` requires handshake, exact active turn and acknowledgment; real active-turn probe passed | -| Queued follow-up | Product continuation controls | Controller can schedule a later prompt; native queue not established | Native pending-message activity exists; no qualified ACP queue responder | Owned `pi/follow_up`, separately named and ordered; real queued marker followed the steered current response | +| Active steering | Dedicated `turn/steer` | Concurrent prompt replaces/cancels, so it is not steering | Concurrent prompt replaces/cancels, so it is not steering | Owned `pi/steer` requires handshake, exact active turn and acknowledgment; historical probe passed, profile-13 paid proof pending | +| Queued follow-up | Product continuation controls | Controller can schedule a later prompt; native queue not established | Native pending-message activity exists; no qualified ACP queue responder | Owned `pi/follow_up`, separately named and ordered; historical marker-order proof, profile-13 paid proof pending | | Cancellation | Typed interrupt and process lifecycle | ACP cancel; paid command cleanup pending | ACP cancel; attached async native settlement passes offline. Explicit detached work is rejected before side effects; it is not supported background settlement | Native abort; wrapper waits for `agent_settled` and treats provider errors as failure | | Session continuity | Read/load/history/fork and durable identity | Session load/list observed; paid semantic warm continuation passed; native history replay unverified and fork absent in tested methods | Session load plus native history events; semantic pending-question controller recovery passed | Private Pi JSONL mapping/load; native RPC `fork`, `clone`, `get_fork_messages` are not mapped through this ACP wrapper; unresolved UI promises cannot survive provider death | | Questions | Typed input requests and response correlation | `cursor/ask_question`, option identity and multiple selection preserved | Native ask-user capability exists, but pinned ACP does not wire its responder; do not display a false answerable form | `select`, `confirm`, `input`, `editor` through typed form elicitation | | Permissions | Durable typed approvals | Standard ACP permission options; denied shell write had no observed side effects; separately labeled exact-correlation assessment | Standard ACP; real native reject_once prevented marker creation; session decision scope inspected | Native pre-tool gate; allow once, exact-operation session grant, deny; paths rechecked after wait | | Plans | Typed plan and collaboration mode | `cursor/create_plan` includes full plan and revision-bound accept/reject/cancel; todo activity separate | Native plan events displayed; native plan-decision callback not exposed in ACP | No native structured plan event; authenticated Paperclip planning tools available | -| Authenticated tools | Runner bridge and governed operations | ACP HTTP MCP binding; paid context/history reads passed | ACP HTTP MCP binding; paid context read passed | Owned extension registers exact bound MCP tools; four authenticated semantic reads succeeded in a paid partial run; no ambient servers | +| Authenticated tools | Runner bridge and governed operations | ACP HTTP MCP binding; paid context/history reads passed | ACP HTTP MCP binding; paid context read passed | Owned extension registers exact bound MCP tools; profile-13 paid Product completion/questions and seeded Runner context pass; no ambient servers | | Delegation | Typed agent roles and lifecycle | Opt-in subagent lifecycle, nested ownership and bounded child activity; never parent transcript flattening | Native session details retain agent/model fields with unknown originating turn; typed turn-owned delegation is not inferred | No built-in ACP delegation protocol; arbitrary extensions are excluded | | Files/diffs | Typed file changes and artifact references | Standard tool changes plus validated image references | Correlated ACP tool events remain available; native session file/workspace details are session notices, not attributed task file changes | Wrapper-retained read/write/edit diffs; common typed/UI projection remains partial; semantic artifact tools | | Images/artifacts | Typed references and registered work products | Existing contained files only, provenance, `registered:false` | Native session artifact details are bounded notices with no inferred turn ownership or automatic registration | Image/resource tool blocks preserved by wrapper; dedicated artifact channel absent | | Image/attachment input | Typed input conversion | ACP advertises images, but the runner turn converter currently forwards text only | ACP advertises images/embedded context, but the runner turn converter currently forwards text only | Native Pi/ACP image input exists, but the runner turn converter currently forwards text only | | Usage | Per-request receipt and model context | Pinned ACP omitted receipts on denied and successful turns; account UI confirms included usage separately | ACP/native tokens retained; account UI confirms included credits separately, without a per-run USD receipt | Assistant-message and compaction token receipts; dollar cost is a catalog pricing estimate, never authoritative billing | -| Config/model changes | Typed configurable controls | Agent/Plan/Ask setup is bound to native configuration acknowledgements and recovery; unsolicited mode drift fails admission. Rich parameterized model picker remains unused | Model/reasoning/mode options exist; runtime policy remains authoritative | Exact pinned candidate model; arbitrary slash commands/config/extensions disabled | +| Config/model changes | Typed configurable controls | Agent/Plan/Ask setup is bound to native configuration acknowledgements and recovery; unsolicited mode drift fails admission. Rich parameterized model picker remains unused | Model/reasoning/mode options exist; runtime policy remains authoritative | Exact model; native available levels and set/get-state confirmation map through ACP mode/config to typed `piThinkingLevel`, recovery identity and UI setup. Active-turn mode changes and arbitrary slash commands/extensions remain disabled | | Reconnect/restart | Durable controller replay and qualified provider restoration | Semantic pending question survived a real controller restart; exact native callback restoration remains unverified | Semantic pending question survived controller restart with the same interaction and provider session; native callback restoration remains unverified | Wrapper explicitly advertises live-process-only pending-input recovery | | Shared capability | User-visible surface | Inspectable implementation / deterministic evidence | @@ -763,7 +857,7 @@ configuration defaults to full auto; it never overrides read-only task policy. ## Distribution and isolation Cursor pins `2026.09.26-dd393fe`; Copilot pins `1.0.88`; Pi pins wrapper `0.0.33`, -runtime `1.0.0` (Pi profile v12), portable Node and its full npm lock. Native distribution hashes +runtime `1.0.0` (Pi profile v13), portable Node and its full npm lock. Native distribution hashes cover macOS ARM64, macOS x64 and Linux x64. Source-owned closure pins remain separate from profile declaration digests. Native admission reads held files, verifies every admitted byte, creates a private immutable snapshot and retains @@ -792,8 +886,8 @@ qualification gates, not claims that a JavaScript path check confines a shell. | Priority | Exposed but unused, partial, or unverified | Reason and next proof | | --- | --- | --- | -| P0 | Remaining paid product cases on local and Daytona | Current Cursor7/Copilot7/Pi9 require authenticated local and Daytona evidence on the exact current runtime. Historical Cursor4 passed hello/question/plan/restart but its latest file marker failed despite correct bytes. Copilot4 passed all five basic local journeys and Daytona hello. Pi7 passed native questions but continuation exposed the final-message bug repaired in8. The semaphore blocker is resolved and Cursor's approved on-demand cap is $25. Native interaction, denial, settlement, platform and complete cost/cleanup gates remain open; no old profile pass qualifies new bytes. | -| P0 | Pi fresh-profile Product verification | Historical v4 plan startup failed with unavailable Postgres semaphores. Historical v6 attempts exposed profile admission and verified-runtime startup faults, repaired without raising deadlines. Preserve every failed attempt and run unchanged canonical cases against the final build; do not remove unrelated IPC objects. | +| P0 | Remaining paid product cases on local and Daytona | Current Cursor10/Copilot13/Pi13 require authenticated local and Daytona evidence on the exact current runtime. The current Pi checkpoint above supersedes older profile coverage. Historical Cursor4 passed hello/question/plan/restart but its latest file marker failed despite correct bytes. Copilot4 passed all five basic local journeys and Daytona hello. Pi7 passed native questions but continuation exposed the final-message bug repaired in8. The semaphore blocker is resolved and Cursor's approved on-demand cap is $25. Native interaction, denial, settlement, platform and complete cost/cleanup gates remain open; no old profile pass qualifies new bytes. | +| P0 | Pi profile-13 Product and platform verification | Four paid local cases and one seeded Runner case pass. File validation, pending-input restart and warm continuation remain failed; assertion repairs do not regrade them. Finish the remaining local, Runner/native-control, Intel startup and Linux/Daytona cases on fresh final builds. Earlier semaphore and v6 startup faults remain historical evidence; never remove unrelated IPC objects. | | P0 | Copilot native ask-user and plan-decision callbacks | Pinned ACP does not install native responders. Prove no blocking request is exposed, or add a qualified responder/wrapper; never swallow the request. | | P0 | Broader Copilot denial and background settlement qualification | Pinned 1.0.88 and isolated 1.0.89 settle attached async commands, but deliberately detached work can finish after end_turn. The earlier paid detached success depended on that model waiting and is not general settlement evidence. Profiles v3/v4 reject explicit detached admission before effects; governed long-lived background work remains unsupported. Denial and process-death recovery pass offline; paid local/Daytona proof remains required. | | P0 | Cursor native question availability | Explicit Luna and Composer native-only Plan-mode probes admitted the selected model and trusted instructions but emitted no question callback. The vendor-patch reversal and actual MCP-context constructor proof identify no local native-tool removal. Effective native tool selection is beyond the remote AgentService boundary; seek upstream catalog/availability evidence. No further blind model grid or semantic-question substitution qualifies this feature. | @@ -808,7 +902,7 @@ qualification gates, not claims that a JavaScript path check confines a shell. | P1 | User attachments and image prompting | `AcpxRuntimeTurnInput` and the common runtime adapter currently forward text only, despite underlying image-input support. Implement validated attachment-to-ACP content conversion and model-specific capability admission, then qualify real local/Daytona image prompts. This is an implementation gap as well as a live-verification gap. | | P1 | Copilot native session event attribution | `_session_event` omits an originating turn. Preserve bounded event fields as session-scoped notices with `turnAttribution: unknown`; typed delegation, artifacts and compaction need an explicit native correlation contract before turn-owned projection. Standard correlated ACP events remain separate. | | P1 | Copilot session-store files and export/artifact URIs | Provider paths are not task-workspace paths. Add a separately authorized export flow with validated bytes and provenance; do not resolve arbitrary URLs or auto-register. | -| P1 | Pi native fork/history/export interfaces | Pinned Pi 0.84.2 native RPC exposes `fork(entryId)`, `clone`, `get_fork_messages` and `export_html`. The wrapper does not map them to runner controls. Add durable branch lineage for fork/clone and an authorized, contained artifact flow for HTML export before exposing them; do not label these native capabilities absent. | +| P1 | Pi native fork/history/export interfaces | Pi native RPC exposes `fork(entryId)`, `clone`, `get_fork_messages` and `export_html`; these remain available in the pinned 1.0.0 release. The wrapper does not map them to runner controls. Add durable branch lineage for fork/clone and an authorized, contained artifact flow for HTML export before exposing them; do not label these native capabilities absent. | | P1 | Complete usage/billing provenance | Missing cache fields remain unknown. Pi price estimates are displayed separately. Budget qualification requires actual spend coverage, not an estimate presented as a bill. | | P1 | Fork/history and richer configuration controls | Cursor session mode now has explicit admission-bound setup. Arbitrary session discovery/forking, mid-turn configuration changes and the parameterized model picker still need company-scoped controls and durable lineage. | | P1 | Exact pending-request restoration after process death | Session transcript restoration does not restore callbacks. Expire unresolved requests unless a provider proves exact restoration. | @@ -816,7 +910,7 @@ qualification gates, not claims that a JavaScript path check confines a shell. | P2 | Remaining Copilot native diagnostic/config/account events | The provider inventory records every event and field, its projection or reason for omission. Preserve bounded useful context; avoid credentials, raw environment or unbounded blobs. | | P2 | Pi empty or whitespace-only submitted input/editor text | Canonical required text rejects blank responses; optional blank responses are omitted. Defaults can be empty and nonblank text preserves boundary whitespace. Add explicit empty-text semantics across TypeScript, Rust and UI before claiming complete native empty-answer parity. | | P2 | Pi native extension surfaces and unsupported slash commands | Arbitrary extensions/templates/themes may execute ambient code. Only reviewed runner-owned capabilities are admitted; structured native plan/goals are not fabricated. Native fork/clone/export are separate unmapped capabilities above. | -| P2 | Pi status/widget/title/editor and session/configuration notifications | `setStatus`, `setWidget`, `setTitle` and `set_editor_text` have no UI projection and are unused by the owned extension. Native session-name and thinking-level events also lack a dedicated projection. Add reviewed bounded notice schemas and governed configuration controls before exposing these fields; `notify` and interactive input already have separate bridges. | +| P2 | Pi status/widget/title/editor and session/configuration notifications | `setStatus`, `setWidget`, `setTitle` and `set_editor_text` have no UI projection and are unused by the owned extension. Native session-name and thinking-level notifications also lack a dedicated projection, although selected thinking level is now handled through ACP mode/config and native confirmation. Provider notice summary/severity has deterministic UI coverage, with rebuilt paid visual proof pending. Add bounded schemas for the remaining fields; `notify` and interactive input already have separate bridges. | | P2 | Pi invocation and history provenance metadata | The v4 wrapper retains `nativeToolCallId`, `modelIteration`, `historyMessageIndex` and `identityScope` in private ACP-wire metadata. Closed common tool and permission projections omit them from the UI. Normalized IDs still correlate live tool, MCP and permission events. Add bounded, redacted display-only provenance fields and parity tests before surfacing the native metadata. | | P2 | Conditional native-plan follow-up fields | Cursor's optional rejection-reason field also appears for accept/cancel. The current question renderer has no conditional fields; add conditional presentation without changing the revision-bound decision receipt. | | P2 | Cursor command exit code projection | The paid file case preserved native `exitCode: 0` inside output text, while the canonical command field remained null. Normalize a typed, correlated exit code without parsing arbitrary prose; current independent file assertions do not prove this field. | diff --git a/doc/grok-native-runner.md b/doc/grok-native-runner.md index 1cdac172eb..dfd2096ea2 100644 --- a/doc/grok-native-runner.md +++ b/doc/grok-native-runner.md @@ -110,7 +110,7 @@ single successful browser task do not establish that qualification. ## Remote verification Do not run Docker on a developer laptop when using remote verification. The -`Docker Runner check` workflow offers maintainer-authorized manual EC2 image +`Docker Runner check` workflow offers maintainer-authorized manual Linux image builds and broad source checks without provider credentials. It records the source revision, resolved lock digest and immutable image reference. Paid Product E2E remains behind the protected default-branch workflow and environment. diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md new file mode 100644 index 0000000000..cc7df120ed --- /dev/null +++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md @@ -0,0 +1,867 @@ +# Instance sleep and scheduler inventory + +Date: 2026-10-08 + +Status: Inventory with first delivery-coordinator implementation; full idle sleep remains open + +Code baseline: `0ac194450` + +Paperclip should release idle database connections and allow inactive customer +instances to sleep without losing scheduled work, incoming events, accounting, +or recovery. Thousands of individually hosted instances make empty polling a +material cost even when each query is cheap. + +**Architectural direction: triage each loop before choosing its replacement.** +Work caused by a committed change or incoming event should run from that event. +Work caused by the passage of time belongs in one in-process, instance-wide +scheduler. A later external-waker adapter publishes only the stack's next wake +time. Routine definitions and execution decisions remain inside the stack. + +Immediate delivery does not have to pass through the scheduler. Its delayed +retry does. The same service can contain both categories; classify its work +paths rather than treating its current polling loop as one indivisible job. + +The database already closes idle pooled connections after 60 seconds and opens +new connections on demand. The primary work is eliminating unnecessary queries +and coordinating background work, connections, and hosting sleep. + +## First slice: transaction-aware delivery notifications and shared scheduling + +Updated: 2026-10-09. This implementation covers five existing durable queues: + +- [x] JOB-04: feedback exports register intent before the export write. +- [x] JOB-22: completion receipts register intent inside their transaction, including caller-owned and nested transactions. No extra post-commit action is needed from callers. +- [x] JOB-23: connection continuations register intent before enqueue. +- [x] JOB-25: question answers register intent before enqueue. +- [x] JOB-24: tool receipts and review resolution register intent before enqueue. +- [x] `createDb` centrally defers delivery wakes until the outer transaction settles. Dedicated child pools share the owning client's signal scope. +- [x] One scheduler owns pending-work and failure retries. Empty, settled queues have no recurring deadline or database scan. +- [x] Unresolved local writes prevent idle admission. Recovery probes PostgreSQL's transaction status before scanning the existing queue and clears the hold once both are settled. +- [x] Tests cover nested commits, rollback, late commit/abort after a lost client response, concurrent writes during a sweep, worker replacement, restart recovery, drain/standby, and shutdown. +- [ ] Prove the complete instance writer-admission and idle/resume boundary across all other work. +- [ ] Migrate other actual scheduled work and expose the whole instance's earliest deadline to an external waker. +- [ ] Verify actual app and database sleep on the hosting providers. + +Existing delivery rows remain authoritative; no tables or migrations are added. +A producer adds one awaited `notifyDeliveryWork(tx, queue)` before its write. +`createDb().transaction()` and nested savepoints are instrumented centrally. The +first opted-in write fetches the outer transaction's XID; unrelated transactions +add no SQL. A rejected transaction with a known XID is reconciled with +`pg_xact_status` before the queue scan. An empty scan cannot discharge an +in-progress transaction. Committed, aborted, or aged-out XIDs can be discharged +after scanning. A failed status query retains the hold and a retry deadline. + +The scheduler retains startup sweeps, retry deadlines while durable work exists, +and retries after failures. Idle drain permits transaction and queue checks but +does not start deliveries; warm standby suppresses both. There is no +unconditional 60-second recovery pass. +Feedback flushes remain serialized and cancellable, with a 30-second upload +limit. Vote routes return after saving rather than waiting for upload. Tool +receipts awaiting review or execution still retain their retry cadence. + +**Scope:** notifications and unresolved-XID tracking are process-local to a +`createDb` owner and its dedicated child pools. They do not observe direct SQL, +an independently created client, or another process. Such writers need an +explicit wake/reconciliation integration before this mechanism can support +them. Startup scans recover already committed durable rows; they do not prove +that transactions still running in an old process have finished. Cross-process +handoff/fencing is separate work, as is PostgreSQL failover during uncertainty. + +This removes these five empty polling loops; it does not establish whole-stack +sleep. `nextWakeAt` covers only registered queues. Database connection teardown, +other background loops, full idle reconciliation, and external wake registration +remain separate work. + +Implementation: `packages/db/src/work-signals.ts`, +`server/src/services/work-scheduler.ts`, `delivery-work-coordinator.ts`, and +`delivery-work-notifications.ts`. + +## Using this checklist + +Items have stable IDs so they can become separate issues or PRs. All boxes start +unchecked: an existing mechanism or a recommendation is not proof that the +sleep journey has passed. Mark an item complete only after its completion +criteria have evidence, and append the issue/PR and verification result to it. +Record deferrals with a reason and the capability that must remain awake. + +Source links identify files in the audited revision. Function names and stated +cadences are navigation aids; recheck them when implementing a later revision. +Cadences describe ordinary claimed-instance operation. Warm-standby, idle-drain, +configuration, or feature gates can suppress individual paths. This inventory +does not establish the actual deployed fleet configuration. + +## Triage before implementation + +Use this question for every operation: **If no new request or event arrives, +does this operation still need to happen at a particular future time?** + +- **Yes: clock-driven.** Register that deadline with the internal scheduler. + Examples are a routine at 09:00, a retry after backoff, and a session expiry. +- **No: event-driven or on demand.** Invoke it from the owning committed mutation, + verified external event, or user request. An empty queue is not a scheduled + task. Examples are delivering a newly committed answer and refreshing a UI + after an issue changes. + +Many existing sweeps combine both. Separate their immediate dispatch from their +retry/timeout/recovery paths. Persistent transports and active-operation +keepalives are lifecycle requirements, not idle business schedules; their +ownership and shutdown need separate treatment. + +The tables below are a first-pass mechanism triage grounded in the inventory. +They select where to investigate first, not permission to delete a recovery +path. Detailed checklist items remain the implementation and verification record. + +### Work that genuinely needs a clock + +| Checklist items | Reason time matters | What belongs in the scheduler | +| --- | --- | --- | +| JOB-08 Agent timer heartbeats and issue monitors | The user explicitly requested future checks. | The next enabled heartbeat/monitor deadline; none when unconfigured. | +| JOB-09 Routine cron triggers | A routine must run even without new incoming activity. | Existing `nextRunAt`, retaining catch-up and timezone semantics. | +| JOB-07 Plugin scheduled jobs | A plugin may declare real recurring work. | Declared job deadlines; plugin load/configuration events update registration. | +| JOB-29 Backups | A backup guarantee has a time requirement. | The next backup if this instance owns it; otherwise the explicit external backup owner. | +| JOB-18, JOB-19 Login and setup expiry | An abandoned session/resource must expire even if nobody returns. | Actual expiry and failed-cleanup retry times, only while relevant sessions exist. | +| JOB-11, JOB-21 Decision and secret/token expiry | Decision deadlines and some deletion/retention obligations are time-based. | Required expiry/scrub deadlines. Check validity synchronously on use regardless of whether physical cleanup ran. | +| JOB-10 Decision retention | An unchanged item can become old enough to archive. | The next actual retention deadline; immediate change handling and notification delivery are event-driven. | +| JOB-12, JOB-13 External objects and status cards | Some configured freshness/update policies require future work. | Only time-based refresh/evaluation deadlines; reactive updates use events. | +| JOB-27, JOB-28 Retries and watchdogs | A retry delay or absence of expected progress can itself require action. | Concrete retry, lease-expiry, monitor, or watchdog deadlines for existing work. | + +### Work whose normal path should stop polling + +| Checklist items | Preferred normal trigger | Clock-driven remainder or qualification | +| --- | --- | --- | +| JOB-04 Feedback exports | Committed export enqueue. | Backoff after delivery failure and durable recovery of outstanding exports. No empty five-second scan. | +| JOB-22 Chat completion delivery | Committed task completion/update. An existing postcommit fast path is already present. | Delivery retries and recovery; the current fast path is best-effort, so prove replacement reliability before removing its sweep. | +| JOB-23 Connection continuation delivery | Committed connection availability/change and pending continuation enqueue. | Failed-delivery retry/claim recovery. | +| JOB-25 Question response delivery | Committed user answer. | Retry/claim expiry while delivery is outstanding. | +| JOB-24 Tool action reviews and deliveries | Committed approval/rejection or action enqueue. | Review expiry and delivery retries. | +| JOB-26 Cost accounting | Arrival or durable capture of a usage receipt/result. | Recovery of unresolved receipts/reservations and failed writes. Accounting debt cannot be dropped. | +| JOB-05 MCP and Dot events | Committed source event or subscription change. | Delivery retries and required expiry cleanup. | +| JOB-01 Chat reconciliation | Incoming message, committed publication, or endpoint configuration change. | Split its lanes: delivery retry and missed-event recovery are temporal; socket ownership is NET work. | +| JOB-02 Email reconciliation | Verified webhook, outgoing send, or endpoint configuration change. | Retry and provider catch-up guarantees; socket reception and catch-up are NET-01/NET-02. | +| JOB-03 Browser-use cleanup | Browser/session lifecycle change. | Register actual idle/expiry/reconciliation deadlines while sessions exist; active leases remain protected. | +| JOB-06 Execution-control reconciliation | Execution state transitions and committed control requests. | Timeout, lease-expiry, abandoned-operation, and failed-delivery recovery. Split the lanes before changing cadence. | +| JOB-14 GitHub events and continuity | Verified webhook and connection lifecycle change where supported. | Credential deadlines; bounded polling only for providers/events without adequate push or replay guarantees. | +| JOB-15 Merge confirmations | Verified merge event for an outstanding candidate. | Bounded fallback only while candidates exist and webhook/replay coverage is insufficient. | +| JOB-16 Terminal workspaces | Committed terminal task/tree transition. | Cooldown, failed-cleanup retry, and recovery of outstanding cleanup. | +| JOB-17 Sandbox cleanup | Resource release or failed acquire/release. | Retry/backoff and provider/fleet orphan recovery. | +| JOB-20 Connection health | Use of the connection or configuration change. | Proactive checks only where a defined freshness guarantee requires them. | +| JOB-10, JOB-11 Decision effects and notifications | Committed decision/relevant source mutation. | Expiry/retention stays in the clock-driven category. | +| JOB-12, JOB-13 Reactive external objects/cards | Verified provider event or relevant local mutation. | Configured time-based refresh stays in the clock-driven category. | +| JOB-27, JOB-28 Queued runs, assignments, and dependencies | Committed enqueue, assignment, terminal result, or dependency transition. | Time-based recovery/watchdogs stay in the clock-driven category. | +| AUX-01, AUX-03 Telemetry and plugin log flushing | First buffered entry activates a bounded flush. | Batch delay and failed-delivery retry while nonempty; no empty heartbeat. | +| AUX-04 Import spools and runtime status expiry | Spool creation/completion, startup, or access to an expiring in-memory value. | Schedule only required file-retention/cleanup deadlines. Lazy expiration can handle noncontractual memory cleanup. | +| UI-01, UI-02 Stable UI data | Existing live events, initial read, foreground return, and reconnect reconciliation. | Preserve bounded fallback during a genuine realtime outage or active operation. Reconcile subscription gaps because live events are not replayed. | +| UI-03, UI-04, UI-06 Recovery, app updates, and setup | Foreground/online/user activity and bounded setup state. | Client-side retry/update deadlines where needed; these do not become tenant server jobs. | + +JOB-30 is removal of the aggregate polling wrapper after its constituents are +handled, not a new scheduled job. DB-01–DB-03 concern the connection pool; +DB-04/DB-05 concern probes. NET-01–NET-07, UI-05, and AUX-05 concern transport or +plugin lifetime. AUX-02/AUX-06 require checking actual exporter/SDK traffic. +SCH, WAKE, SAFE, HOST, and TEST items are infrastructure, policy, or verification +work rather than business jobs. This keeps every inventory item accounted for +without making the scheduler responsible for every source of activity. + +### Qualification before removing a poll + +- [ ] **TRI-01 — Identify the real trigger and every producer.** For each + event-driven candidate, list the committed mutations/external events that + create work, including alternate API paths and recovery writers. **Complete + when:** every supported producer has an owned dispatch path and the service + needs no scan to discover normal new work. + +- [ ] **TRI-02 — Separate first delivery from delayed recovery.** Record which + paths are immediate, which have a concrete due time, and which depend on an + external provider's push capabilities. **Complete when:** each migrated lane + has an explicit trigger and its remaining deadlines are enumerated. + +- [ ] **TRI-03 — Prove durable delivery before retiring its backstop.** A + best-effort callback after commit is not a durable queue consumer. Cover + commit-before-signal crashes, lost signals while the process stays alive, + handler failures, concurrent producers, and restart. **Complete when:** an + outstanding record cannot become invisible indefinitely. Retain a bounded, + explicitly justified reconciliation path until this is proven; document its + wake cost rather than assuming startup recovery covers every failure. + +- [ ] **TRI-04 — Order work by avoidable idle traffic and migration risk.** + Record expected idle SQL/network reduction and the smallest focused proof for + the next item. **Complete when:** the next PR has one clear boundary and does + not require the entire scheduler or all connector lifecycles to be finished. + +### Initial implementation shortlist + +1. **DB-04/DB-05: health and probe separation.** A self-contained source of + avoidable demand, independent of building a scheduler. Verify actual callers + so the new endpoint is used. +2. **JOB-04: event-driven feedback exports.** A contained first dispatch + migration. Triage its enqueue, failed-export behavior, and recovery first; + put only actual retry/batch deadlines in scheduling infrastructure. +3. **JOB-22/JOB-25: completion and answer delivery.** Reuse existing postcommit + paths where possible, after proving that durable delivery can replace the + sweep rather than merely supplement it. +4. **JOB-01/JOB-02/JOB-03: frequent empty chat, email, and browser scans.** Large + baseline benefit, but split the component lanes and preserve configured + transports, expiry, and recovery. Resource absence must be invalidated when + a new resource is committed. +5. **SCH with JOB-09: routine scheduling as the first true clock-driven slice.** + Existing `nextRunAt` makes the real scheduler requirement concrete. Add other + temporal paths and then the external-waker adapter. + +This order supersedes starting by routing the feedback polling loop through a +generic scheduler. Event-driven improvements can ship before that scheduler is +complete. The resulting service still needs an owned way to retry failures. + +## Provider behavior and cost model + +| Layer | Documented behavior | Consequence | +| --- | --- | --- | +| Neon | Normally suspends after five minutes without active queries. Idle-in-transaction connections count as active; ordinary idle connections do not necessarily block suspension. New connection requests can reset inactivity. | Stop empty SQL polling and connection churn; do not force-close active transactions. | +| Railway | Detailed Serverless documentation specifies five minutes without outbound packets, sampled so actual sleep is roughly 5–10 minutes. Private-network traffic and responses count. Enabling Serverless requires a deployment to affect the container. | Every recurring source of packets matters. Even a DB-free endpoint can keep the app awake when repeatedly called externally. | +| Unikraft | `on` waits for connections/HTTP requests to finish; `idle` can suspend inactive TCP connections. Active HTTP requests prevent suspension. Stateful mode resumes a snapshot. | Quiesce connections deliberately and explicitly protect background work from premature suspension. | + +Provider sources: [Neon compute lifecycle](https://neon.com/docs/introduction/compute-lifecycle), +[Neon compute management](https://neon.com/docs/manage/endpoints/), +[Railway Serverless](https://docs.railway.com/deployments/serverless), +[Unikraft scale to zero](https://unikraft.com/docs/features/scale-to-zero). + +Neon usage is reported in CU-seconds. The transcript's five-minute concern is +primarily an idle-timeout cost, not a universal billing quantum. With a +five-minute timeout, a trivial hourly check creates roughly an 8.3% active-time +floor before useful work; checking every five minutes can leave compute +continuously active. This illustration assumes the default timeout and excludes +other traffic, processing time, and storage charges. Verify actual endpoint +settings and the applicable plan. See [Neon usage calculations](https://neon.com/docs/introduction/usage-calculations). + +## 1. Internal scheduler for true deadlines + +- [ ] **SCH-01 — Establish the instance-wide scheduler boundary.** Route + application-owned delayed jobs, recurring jobs, retry deadlines, and required + expiry/cleanup/maintenance deadlines through one scheduler shared across all + companies. Immediate event/queue dispatch remains in its owning service; + running-work tracking is shared with sleep admission. Keep domain handlers and + their authorization/atomic claims in their existing services. **Complete + when:** one scheduling interface supports true temporal obligations, company + scope remains explicit, and self-hosted operation requires no external service. + +- [ ] **SCH-02 — Model work and deadlines explicitly.** Track a stable job/lane + identity, company scope where applicable, due time, pending/running state, + retry policy, and whether running work blocks sleep. Accept running-work + information from event-driven handlers without forcing their dispatch through + the scheduler. Distinguish scheduled + work, immediate queued work, waiting for an external event, and passive + configuration/history. **Complete when:** an empty lane has no recurring SQL + check, and the scheduler can explain outstanding work and the earliest due + time without querying the database on every timer tick. + +- [ ] **SCH-03 — Reconstruct state and handle committed changes.** Seed the + scheduler from existing durable domain records on startup; use committed + mutations to register, reschedule, cancel, or activate work. In-memory hints + cannot be the sole record of an obligation. Define how all supported writers + invalidate hints; initially retain the existing single-process boundary. + **Complete when:** restart and a crash between commit and notification recover + work, and creating an earlier deadline updates the scheduler immediately. + +- [ ] **SCH-04 — Implement local scheduling and catch-up.** Arm the nearest + necessary local timer and run due handlers with bounded concurrency and + per-lane overlap protection. Preserve each domain's timezone, missed-run, + coalescing, priority, and retry semantics. **Complete when:** delayed callbacks, + clock changes, restart, and duplicate activation cannot silently lose work or + bypass existing execution claims. + +- [ ] **SCH-05 — Expose an aggregate next wake time.** Compute the earliest + required deadline across the whole instance, including retries, cleanup, + accounting, and maintenance. Represent no scheduled work as no alarm; represent + unknown obligations as a sleep blocker. **Complete when:** adding, moving, + removing, or completing the earliest item correctly changes the aggregate. + +- [ ] **SCH-06 — Integrate running-work ownership and drain.** Reuse + [task admission](../../server/src/services/task-admission.ts), + [idle admission](../../server/src/middleware/idle-admission.ts), and the + [existing sleep protocol](../idle-sleep-safety.md). Track handler completion + even after an HTTP response or client disconnect. **Complete when:** draining + prevents new scheduled admissions and waits for accepted work without losing + fire-and-forget work or mutations racing the final check. + +- [ ] **SCH-07 — Classify transport and SDK timer exceptions.** Business + scheduling belongs to the scheduler. Protocol pings, socket deadlines, and + active-operation lease renewals can remain local to their transports, but + must have explicit start/stop ownership and participate in sleep blocking. + Browser timers use a separate foreground/activity policy. **Complete when:** + exceptions are inventoried and owned; no hidden independent timer can renew + an idle connector indefinitely after its scheduler lane is stopped. + +- [ ] **SCH-08 — Make scheduler migration incremental and observable.** Migrate + each inventory item below independently. Report pending work, next deadline, + active holds, and unknown/unmigrated blockers through local diagnostics. + Avoid periodic SQL or remote reporting merely to observe inactivity. + **Complete when:** an operator can explain why a stack is awake and a migrated + lane can be rolled back without duplicate processing. Preserve the telemetry + review rules if any first-party event contract changes. + +## 2. External waker adapter later + +The minimal external model is `(stack, nextWakeTime)`. Operational delivery may +also carry a generation/idempotency value and delivery status. It must not +require copying every routine, agent configuration, or permission into Cloud. +The internal scheduler remains authoritative about what to execute. + +```mermaid +flowchart LR + D[Temporal obligations and retry deadlines] --> S[Instance scheduler] + S --> L[Local timer while awake] + S -->|stack, nextWakeTime, generation| W[External waker adapter] + W --> G[Wake-aware ingress or provider wake] + U[User request or webhook] --> G + G -->|Reconcile due deadlines on wake| S + G -->|Verified request or event| H + L --> H[Existing domain handlers] + S --> H +``` + +- [ ] **WAKE-01 — Define the adapter and local-only behavior.** Support + setting/replacing and cancelling the next alarm. No external adapter is + required to use the internal scheduler. **Complete when:** timed work remains + local and the instance remains awake if no durable external wake is available; + introducing the internal scheduler alone never authorizes sleeping through a + deadline. + +- [ ] **WAKE-02 — Make alarm publication durable and acknowledge it before sleep.** + Persist publication intent/generation or use an equivalent recoverable + handshake. Recheck that the acknowledged alarm still covers the earliest + deadline before sleeping. **Complete when:** failure, lost responses, and a + concurrently moved-earlier deadline keep the instance awake until covered; + local scheduling continues during external-waker outages. + +- [ ] **WAKE-03 — Handle duplicate, stale, delayed, and missed wake delivery.** + Fence stale generations, retry failed delivery, reconcile due work after + wake, and retain domain-level atomic claims. **Complete when:** a repeated + doorbell does not duplicate work, and missed deadlines follow each domain's + explicit catch-up policy. Do not claim exactly-once network delivery. + +- [ ] **WAKE-04 — Serialize ingress, sleep, deploy, and wake.** Buffer or retry + incoming requests while draining; authenticate external control and preserve + webhook verification and acknowledgement deadlines. The host must bind its + final stop decision to the correct process/provider generation. **Complete + when:** requests arriving during sleep entry are neither lost nor admitted + into a process being stopped. Multi-replica support requires shared fencing. + +- [ ] **WAKE-05 — Implement and qualify provider adapters.** Evaluate Unikraft + scheduled `start` operations and a shared durable delayed-delivery mechanism + for Railway. Verify that provider resume actually invokes due-work processing, + including stateful resume without process startup. **Complete when:** an + isolated instance wakes for its earliest deadline and no waker polls sleeping + tenant databases. See [Unikraft scheduled wake-ups](https://unikraft.com/docs/features/cron-jobs). + +- [ ] **WAKE-06 — Protect active background execution on the host.** Connect + active-work holds to the provider's sleep controls. Unikraft exposes a + reference-counted disable mechanism; account for its signal delay. **Complete + when:** startup, transactions, agent control, finalization, and cleanup cannot + be suspended halfway through accepted work. See [Unikraft application sleep controls](https://unikraft.com/docs/tutorials/scale-to-zero-triggers). + +- [ ] **WAKE-07 — Validate wake concentration and cost.** Capacity-test concurrent + scheduled wakes, apply bounded dispatch, and use jitter only where semantics + permit. **Complete when:** scheduler backlog/delivery delay is observable and + a midnight burst does not silently skip customer jobs. Avoid periodically + waking all tenants just to discover whether any work exists. + +## 3. Database and health + +- [ ] **DB-01 — Qualify the existing idle pool behavior.** + [client.ts](../../packages/db/src/client.ts) already defaults + `DATABASE_IDLE_TIMEOUT_SECONDS` to 60; `0` disables reaping. Keep the existing + Drizzle handle and lazy reconnection; evaluate 30 seconds after pollers become + quiet. **Complete when:** idle connections disappear and a subsequent read, + write, and transaction work after actual provider suspension without per-route + lifecycle code. Existing tests are in + [client.test.ts](../../packages/db/src/client.test.ts). + +- [ ] **DB-02 — Preserve disconnect and transaction safety.** `.end()` permanently + shuts down the pool and must not become an idle operation. Reconnection does + not authorize replaying an interrupted write. **Complete when:** existing + transaction isolation after disconnect and explicitly idempotent retry rules + still hold; ambiguous commits are not blindly replayed. See + [database retry behavior](../DATABASE.md#connection-loss-and-retries). + +- [ ] **DB-03 — Cover every client and socket.** Better Auth shares the main DB; + the optional plugin migration pool uses the same idle configuration. Utility, + backup, and dedicated advisory-lock connections are bounded work. The audited + postgres.js version has a 60-second TCP keepalive; keepalive is not SQL. + **Complete when:** all pools can reach zero when idle, reserved/active work is + protected, and no new persistent LISTEN or health-ping connection is introduced + as a substitute for polling. Sources: + [DB client](../../packages/db/src/client.ts), + [auth adapter](../../server/src/auth/better-auth.ts), + [backup library](../../packages/db/src/backup-lib.ts). + +- [ ] **DB-04 — Split process liveness from DB readiness.** Claimed-instance + [health](../../server/src/routes/health.ts) executes `SELECT 1`; auth and deep + checks may perform additional queries. Install cheap liveness before session + and actor resolution. **Complete when:** normal liveness produces no SQL, + explicit deep readiness still tests DB health, and signed claim/bootstrap + mutations retain their existing validation and tracking. + +- [ ] **DB-05 — Audit external probes and configured endpoints.** Inspect the + fleet gateway, uptime monitors, deployment checks, DB pool settings, and Neon + suspend configuration. The [ECS example](../../docker/ecs-task-definition.json) + probes every 30 seconds; it is not evidence of Railway fleet behavior. + **Complete when:** sleeping is an expected state, repeated checks do not wake + tenants or reconnect to Neon, and readiness errors remain distinguishable + from intentional sleep. Railway's built-in health checks are deployment-only: + [provider documentation](https://docs.railway.com/deployments/healthchecks). + +## 4. Server scheduling migration inventory + +Apply the triage above to each operation within these services. Remove polling +from event-driven normal paths and route only real deadlines through the internal +scheduler. Preserve durable delivery, startup recovery, and domain-level claims. +Longer polling intervals alone do not complete an item. + +- [ ] **JOB-01 — Chat reconciliation.** Current: every **1s**, with multiple + runtime/delivery/publication lanes and SQL even without connectors. Change: + disarm empty lanes and activate on committed events or deadlines. **Complete + when:** a tenant with no chat obligations issues no chat SQL and new inbound, + publication, and recovery work still runs. Sources: + [app.ts](../../server/src/app.ts), + [chat-channels.ts](../../server/src/services/chat-channels.ts). + +- [ ] **JOB-02 — Email reconciliation.** Current: every **1s**, querying active + endpoints even when none exist. Change: initialize resource/pending state and + update on endpoint changes, webhook receipt, sends, and retries. **Complete + when:** empty mail state produces no polling SQL and new work activates the + worker without restart. Source: + [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **JOB-03 — Browser-use cleanup.** Current: every **3s** with due-session + queries even when empty. Change: register actual session reconciliation/expiry + deadlines. **Complete when:** no sessions means no queries, while active leases, + expiry, and failed cleanup remain recoverable. Sources: + [app.ts](../../server/src/app.ts), + [browser-use.ts](../../server/src/services/browser-use.ts). + +- [ ] **JOB-04 — Feedback exports.** Current: every **5s**, querying the queue + even without a configured share client. Change: committed enqueue signals and + durable retries. **Complete when:** empty exports are quiet and queued exports + survive restart and temporary delivery failure. Sources: + [app.ts](../../server/src/app.ts), + [feedback.ts](../../server/src/services/feedback.ts). + +- [ ] **JOB-05 — Public MCP and Dot event dispatch.** Current: **2s** per started + dispatcher, including SQL-backed enablement, expiry, and delivery scans. + Change: cache capability state and schedule delivery/retry/expiry work. + **Complete when:** disabled/empty dispatchers are DB-free and pending events + retain subscription scope, expiry, and replay safety. Source: + [events.ts](../../server/src/services/public-mcp/events.ts). + +- [ ] **JOB-06 — Execution-control reconciliation.** Current: independent + **15s** recovery/delivery lanes, including when the main heartbeat is disabled. + Change: activate lanes from durable pending work and deadlines. **Complete + when:** empty lanes are quiet and abandoned finalizations, replacements, + continuations, status/disposition delivery, and login expiry still recover. + Sources: [index.ts](../../server/src/index.ts), + [execution-control-deadline.ts](../../server/src/services/execution-control-deadline.ts). + +- [ ] **JOB-07 — Plugin job scheduling.** Current: **30s** SQL even without jobs. + Change: register nearest `nextRunAt` and signal registration/update/removal. + **Complete when:** empty plugins/jobs produce no scheduler SQL and due jobs + retain atomic ownership. The plugin job coordinator itself is event-driven, + not another periodic loop. Sources: + [plugin-job-scheduler.ts](../../server/src/services/plugin-job-scheduler.ts), + [plugin-job-coordinator.ts](../../server/src/services/plugin-job-coordinator.ts). + +- [ ] **JOB-08 — Agent timer heartbeats and issue monitors.** Current: **30s** + agent/monitor scans; agent-level disabled heartbeats still require discovery. + Change: register each relevant deadline and aggregate locally. **Complete + when:** no timer work means no queries, and due work preserves pause, budget, + authorization, and atomic wake admission. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts), `tickTimers`. + +- [ ] **JOB-09 — Routine cron triggers.** Current: **30s** due-trigger query. + Change: use existing `nextRunAt` in the internal scheduler. **Complete when:** + timezone, catch-up/coalescing, concurrency, project pause, edits, and deletion + behave correctly; external waking can be added without relocating routines. + Source: [routines.ts](../../server/src/services/routines.ts), `tickScheduledTriggers`. + +- [ ] **JOB-10 — Decision retention and notifications.** Current: every **30s**, + builds the complete attention feed for each active company before archiving + and delivering notifications. Change: relevant mutation signals, actual + retention deadlines, and batched maintenance. **Complete when:** unchanged + idle companies do not rebuild feeds and required archive/notification behavior + remains timely. Source: [index.ts](../../server/src/index.ts), `runRetentionSweep`. + +- [ ] **JOB-11 — Decision expiry and recovery.** Current: **30s** scans for stuck + effects, continuations, and TTLs. Change: schedule their actual deadlines and + signal committed effects. **Complete when:** expiration remains authoritative + at action time and interrupted effects recover without perpetual empty scans. + Source: [decisions.ts](../../server/src/services/decisions.ts), `sweepExpired`. + +- [ ] **JOB-12 — External-object refresh.** Current: **30s** scans, including a + DB feature check when disabled. Change: invalidate cached capability on settings + changes; register `nextRefreshAt`; use webhook signals where supported. + **Complete when:** disabled/empty state is quiet and promised refresh freshness + has an owned deadline. Sources: + [external-objects.ts](../../server/src/services/external-objects.ts), + [instance-settings.ts](../../server/src/services/instance-settings.ts). + +- [ ] **JOB-13 — Status-card evaluation.** Current: **30s**, including a DB + feature check when disabled. Change: register `nextEvalAt` and relevant event + invalidations. **Complete when:** no capability/due cards means no queries, + and evaluation preserves existing assignment and cancellation behavior. + Source: [status-cards.ts](../../server/src/services/status-cards.ts). + +- [ ] **JOB-14 — GitHub events and connection continuity.** Current: **30s** + connection scans; provider calls depend on configured/due work. Change: + webhook-first activation plus explicit poll fallback, credential, and retry + deadlines. **Complete when:** empty state is quiet and configured connections + retain their delivery and continuity guarantees. Source: + [index.ts](../../server/src/index.ts), `scheduleGitHubConnectionEventPoll` and + `scheduleGitHubConnectionContinuitySweep`. + +- [ ] **JOB-15 — Merged pull-request confirmations.** Current: **30s** pending + confirmation scan, followed by GitHub checks for matching candidates. Change: + activate on merge events with bounded polling fallback only while candidates + exist. **Complete when:** confirmation resolution remains authorized and + restart-safe, with no empty scan. Source: + [issue-thread-interactions.ts](../../server/src/services/issue-thread-interactions.ts). + +- [ ] **JOB-16 — Terminal workspace cleanup.** Current: **30s** candidate scans. + Change: enqueue from terminal transitions and cooldown deadlines. **Complete + when:** empty state is quiet, and cleanup still checks current task state, + artifacts, deliveries, and ownership before deleting resources. Source: + [execution-workspaces.ts](../../server/src/services/execution-workspaces.ts). + +- [ ] **JOB-17 — Sandbox cleanup and orphan spools.** Current: **30s** scans even + when attempts have longer staleness/backoff. Change: durable cleanup queue with + next retry, immediate failed-release activation, and provider/fleet backstop. + **Complete when:** idle scans stop without leaking paid sandboxes or ignoring + malformed/unpersisted spool entries. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts), `sweepPendingCleanupLeases`. + +- [ ] **JOB-18 — Adapter login and setup-token reapers.** Current: **30s** + durable-session/lease scans, alongside local one-shot timers. Change: register + actual expiry and cleanup retry times. **Complete when:** abandoned logins and + interrupted teardown still release sandboxes across restart, without scans + when no obligations exist. Source: + [index.ts](../../server/src/index.ts), adapter and setup-token reaper wiring. + +- [ ] **JOB-19 — Custom-image setup expiry.** Current: **30s** active-session + expiry scans. Change: register `expiresAt` and cleanup attempts. **Complete + when:** no sessions means no SQL and expired provider resources are released. + Source: [environment-custom-images.ts](../../server/src/services/environment-custom-images.ts). + +- [ ] **JOB-20 — Tool connection health.** Current: **30s** DB inventory; + provider probes are conditional. Change: check on use/configuration change, + with explicit deadlines for any promised proactive health checks. **Complete + when:** quiet connections stop creating empty scans and failures remain + visible within the agreed freshness contract. Source: + [index.ts](../../server/src/index.ts), `tools.sweepConnectionHealth`. + +- [ ] **JOB-21 — Secret proposals and expired action tokens.** Current: **30s** + expiry/cleanup queries. Change: enforce expiry synchronously on use and + register required scrub/cleanup deadlines. **Complete when:** idle cleanup + does not extend authorization or secret-retention periods. Sources: + [secret-proposals.ts](../../server/src/services/secret-proposals.ts), + [index.ts](../../server/src/index.ts). + +- [ ] **JOB-22 — Chat completion delivery.** Current: **30s** outbox scan plus + an existing postcommit fast path. Change: extend that signal and register + retries only while pending. **Complete when:** committed completions survive + restart, duplicate activation, and provider failure without empty polling. + Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-23 — Connection continuation delivery.** Current: **30s** outbox + scans. Change: committed signals, retry deadlines, and startup reconciliation. + **Complete when:** newly available connections resume eligible work once, + preserve execution gates, and have no empty SQL loop. Source: + [index.ts](../../server/src/index.ts). + +- [ ] **JOB-24 — Tool action reviews and deliveries.** Current: **30s** + review/session/outbox scans. Change: register review expiry and delivery + deadlines; activate after committed decisions. **Complete when:** approval, + rejection, expiry, and delivery retain authorization and replay protection. + Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-25 — Question response delivery.** Current: **30s** outbox scans. + Change: committed-answer activation, durable retry deadlines, and startup + recovery. **Complete when:** responses preserve attempt-generation fencing + and recover from delivery failure without polling empty queues. Source: + [question-response-delivery.ts](../../server/src/services/question-response-delivery.ts). + +- [ ] **JOB-26 — Cost accounting and decision-model recovery.** Current: + **30s** reconciliation even when empty. Change: pending-receipt signals, + spool awareness, and durable retries. **Complete when:** no empty queries, + but unresolved receipts/reservations remain tracked and cannot be discarded + to permit sleep. Source: [heartbeat.ts](../../server/src/services/heartbeat.ts), + `reconcileCostAccounting`. + +- [ ] **JOB-27 — Run recovery, scheduled retries, and queued work.** Current: + **30s** orphan/retry/queue/stranded-assignment scans. Change: remain responsive + while work exists, register retry/control deadlines, and reconcile on startup + or wake. **Complete when:** quiet state has no scans and interrupted runs do + not duplicate provider work or lose continuation. Source: + [index.ts](../../server/src/index.ts), periodic heartbeat recovery chain. + +- [ ] **JOB-28 — Dependencies, task watchdogs, silent runs, and stale locks.** + Current: additional **30s** scans. Change: dependency/run transition signals, + active-run-only monitoring, actual timeout deadlines, and startup backstops. + **Complete when:** recovery remains scoped and fenced, and a completed task + still receives any owed watchdog review. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts). + +- [ ] **JOB-29 — Automatic database backups.** Current: enabled by default, + typically **hourly**, with separate dump connections. Change: register with + the internal scheduler; later assign durable waking or explicitly transfer + backup responsibility to managed infrastructure. **Complete when:** backup + and restore guarantees are demonstrated, active backup blocks sleep, and a + merely configured future backup no longer blocks indefinitely once covered + by an external alarm. Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-30 — Replace the main tick and its disabled branch.** Current: + default **30s** main tick; setting `HEARTBEAT_SCHEDULER_ENABLED=false` still + leaves accounting, external-object, cleanup, and GitHub work, plus independent + workers. Change: migrate all lanes before retiring aggregate polling. + **Complete when:** disabled scheduling semantics remain documented and no + legacy fallback quietly reintroduces recurring SQL. Sources: + [index.ts](../../server/src/index.ts), [config.ts](../../server/src/config.ts). + +## 5. Connectors and transport lifetime + +- [ ] **NET-01 — AgentMail socket ownership.** Current: outbound WebSocket and + **20s** DB lease renewal; its renewal bypasses the top-level background gate. + Change: prefer webhook mode for hosted stacks; otherwise retain an awake + receiver or use shared durable ingress. **Complete when:** explicit suspend + closes all associated timers and message reception/replay still works. + Source: [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **NET-02 — AgentMail catch-up.** Current: roughly **one-minute** provider + reconciliation even in webhook mode. Change: catch up on wake/connect and at + an owned recovery deadline. **Complete when:** checkpoints and deduplication + survive lost webhooks, with a defined maximum recovery delay. Source: + [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **NET-03 — Discord Gateway.** Current: persistent connection and **5s** DB + ownership renewal. Change: keep unsupported stacks awake initially; evaluate + a shared ingress worker owning the connection, lease, and durable delivery. + **Complete when:** idle tenant sleep does not silently stop timely reception + or weaken ownership fencing. Sources: + [chat-channels.ts](../../server/src/services/chat-channels.ts), + [chat-sdk-runtime.ts](../../server/src/services/chat-sdk-runtime.ts). + +- [ ] **NET-04 — Photon iMessage.** Current: several gRPC subscriptions, **5s** + ownership renewal, and **15s** catch-up. Change: explicit transport lifecycle + and, where required, shared durable receiving. **Complete when:** resume + preserves ordered checkpoints and detects replay/history gaps; stopping the + top-level chat loop also retires dependent work safely. Source: + [receiver.ts](../../server/src/services/photon/receiver.ts). + +- [ ] **NET-05 — Slack webhook ingress.** Current transport already uses webhook + mode. Change: qualify it through wake-aware ingress rather than introducing + a persistent receiver. **Complete when:** cold start meets provider timing + through durable acknowledgement/retry where necessary, with signature + verification, deduplication, and company scope preserved. Source: + [chat-sdk-runtime.ts](../../server/src/services/chat-sdk-runtime.ts). + +- [ ] **NET-06 — HTTP idle connections.** Current server keepalive is **185s** + to outlive common proxy defaults. Change: explicitly close idle sockets during + drain and qualify provider-specific connection policies. **Complete when:** + quiet sockets do not prevent intended sleep, and normal proxy reuse does not + gain intermittent 502/ECONNRESET failures. Source: + [index.ts](../../server/src/index.ts). + +- [ ] **NET-07 — Active execution transports and lease renewals.** Inventory + runner/PRP connections, HTTP/2 bridge pings, native/legacy controller leases, + terminal SSH keepalives, browser viewport ownership, and per-operation claims. + Retain their required cadence during work. **Complete when:** active operations + hold the instance awake and unused transports retire; no arbitrary heartbeat + disabling is used to manufacture sleep. Sources: + [native-session-executor.ts](../../server/src/services/native-runtime/native-session-executor.ts), + [legacy-controller-lease.ts](../../server/src/services/legacy-controller-lease.ts), + [HTTP/2 bridge](../../packages/adapter-utils/src/http2-bridge-server.ts), + [terminal WebSocket](../../server/src/realtime/environment-custom-image-terminal-ws.ts). + +## 6. Browser demand + +These client-side schedules do not run in the server scheduler. Consolidate +their activity policy so a parked browser does not create artificial tenant +demand. Preserve real-time behavior while a person is actively using the app. + +- [ ] **UI-01 — Live WebSocket and fallback polling.** Hidden tabs already close + the main socket; visible tabs receive **30s** pings and use **15s** refetch + fallback when disconnected. Change: distinguish intentional sleep from + outage and decide whether unattended visible tabs may become inactive. + **Complete when:** reconnection does not create a self-wake loop and returning + users reconcile missed state. Source: + [LiveUpdatesProvider.tsx](../../ui/src/context/LiveUpdatesProvider.tsx). + +- [ ] **UI-02 — Stable page queries.** Current issue views poll at **1–3s**, + browser/artifact views at **3–5s**, and agents/costs/reviews/settings at + **5–60s**. Change: extend event invalidation and active-work-only fallback; + reuse [polling.ts](../../ui/src/lib/polling.ts) and + [useSharedPolling.ts](../../ui/src/hooks/useSharedPolling.ts). + **Complete when:** idle views stop artificial demand without hiding updates; + multiple tabs do not duplicate avoidable work. Starting sources: + [IssueDetail.tsx](../../ui/src/pages/IssueDetail.tsx), + [Costs.tsx](../../ui/src/pages/Costs.tsx), + [useTaskBrowsers.ts](../../ui/src/hooks/useTaskBrowsers.ts). + +- [ ] **UI-03 — Cloud-access error recovery.** Current transient-error retries + run every **5s**, explicitly in background tabs; bootstrap has faster polling. + Change: visibility/online gating and wake-aware recovery, with bounded + bootstrap behavior. **Complete when:** a hidden tab that last received 502/503 + cannot keep waking its tenant indefinitely. Source: + [CloudAccessGate.tsx](../../ui/src/components/CloudAccessGate.tsx). + +- [ ] **UI-04 — Service-worker update checks.** Current update check is + **hourly**, regardless of visibility. Change: check on foreground activity or + serve shell/update assets from shared static hosting. **Complete when:** a + parked hidden tab does not periodically wake its tenant to fetch `/sw.js`. + Source: [service-worker-updates.ts](../../ui/src/lib/service-worker-updates.ts). + +- [ ] **UI-05 — Plugin SSE streams.** Current EventSource lacks visibility + gating and holds an HTTP response open. Change: explicit visibility/idle + lifetime with reconciliation or replay on reconnect. **Complete when:** + plugin state remains correct and abandoned streams do not block sleep. + Sources: [bridge.ts](../../ui/src/plugins/bridge.ts), + [plugin routes](../../server/src/routes/plugins.ts). + +- [ ] **UI-06 — Setup and enrollment polling.** OAuth/chat setup often polls at + **1.5–3s**, email setup at **10s**, commonly while pending. Change: keep these + responsive during the bounded workflow; stop on terminal state, close, or + inactivity with a deliberate resume path. **Complete when:** abandoned setup + creates neither permanent demand nor lost completion evidence. Sources: + [ConnectionSetupFlow.tsx](../../ui/src/features/connections/ConnectionSetupFlow.tsx), + [chat setup](../../ui/src/pages/apps/chat/ChatEndpointSetup.tsx), + [email setup](../../ui/src/pages/apps/chat/EmailEndpointSetup.tsx). + +## 7. Lower traffic housekeeping and plugins + +- [ ] **AUX-01 — First-party telemetry.** The **60s** flush returns without + sending when its queue is empty. Change: register flush/retry work only when + needed and drain or durably retain pending batches. **Complete when:** empty + telemetry is network-quiet and delivery semantics remain explicit. Preserve + generated-contract, README, and privacy-review requirements for event changes. + Sources: [telemetry.ts](../../server/src/telemetry.ts), + [telemetry client](../../packages/shared/src/telemetry/client.ts). + +- [ ] **AUX-02 — Optional OpenTelemetry.** No configured OTLP endpoint means a + no-op tracer; this is separate from first-party telemetry and the local run + log. Change: inspect exporter batching/connection behavior only when enabled; + avoid a perpetual stream of idle-maintenance spans. **Complete when:** the + endpoint gate remains intact and configured exporters do not defeat intended + sleep after their pending work settles. Source: + [instrumentation.ts](../../server/src/instrumentation.ts). + +- [ ] **AUX-03 — Plugin log buffer.** The **5s** flush avoids SQL when empty. + Change: schedule a bounded flush after the first buffered entry and include + pending writes in drain. **Complete when:** no empty timer is needed and log + persistence has a defined shutdown/sleep boundary. Source: + [plugin-host-services.ts](../../server/src/services/plugin-host-services.ts). + +- [ ] **AUX-04 — Import spools and in-memory status expiry.** Hourly import + cleanup already avoids SQL when the local spool is absent/empty. Runtime + status expiry is memory-only. Change: use actual deadlines or opportunistic + cleanup where timing is not contractual. **Complete when:** neither causes an + unnecessary external wake, while retained files and required cleanup remain + tracked. Sources: + [company-import-transfers.ts](../../server/src/services/company-import-transfers.ts), + [index.ts](../../server/src/index.ts). + +- [ ] **AUX-05 — Plugin worker lifecycle.** Enabled plugins can execute arbitrary + background work; host timers cannot prove they are quiet. Change: explicit + quiesce/resume, scheduled-job registration, ingress ownership, and lazy provider + worker startup where appropriate. **Complete when:** supported plugins prove + quiescence; unsupported plugins remain a named blocker. Source: + [plugin-worker-manager.ts](../../server/src/services/plugin-worker-manager.ts). + +- [ ] **AUX-06 — Verify low-priority and third-party sources at runtime.** + Announcements are request-driven with caching; no dedicated periodic server + application-update check was found. Plugin log retention has no production + caller in the audited revision. **Complete when:** packet traces confirm these + classifications and identify any SDK, gRPC, HTTP agent, monitoring, remote + client, or plugin traffic omitted by static inspection. Source: + [announcement-feed.ts](../../server/src/services/announcement-feed.ts). + +## 8. Sleep eligibility and alternative hosting + +- [ ] **SAFE-01 — Separate claimed inactivity from unclaimed warm standby.** + Existing warm standby bypasses SQL and recurring work only for empty, + unclaimed instances. Change: add claimed-instance behavior through explicit + scheduler/quiescence contracts. **Complete when:** an inactive claimed tenant + can become quiet without impersonating an unclaimed tenant or disrupting + signed claim handling. Source: + [cloud-warm-standby.ts](../../server/src/services/cloud-warm-standby.ts). + +- [ ] **SAFE-02 — Replace conservative blockers one capability at a time.** + Current safety blocks every enabled plugin, configured backups, API keys, + active routines/connections, nonterminal issues, and many retained records. + Change: distinguish current obligations from passive capability/history only + when a validated wake/recovery contract exists. **Complete when:** waiting for + a human or retaining completed history can permit sleep, while unknown state, + cleanup/accounting debt, and unhandled transports remain blockers. Sources: + [idle-sleep-safety.ts](../../server/src/services/idle-sleep-safety.ts), + [idle-local-work.ts](../../server/src/services/idle-local-work.ts). + +- [ ] **SAFE-03 — Validate cold start and stateful resume.** A snapshot preserves + memory, not the validity of external sockets, credentials, or leases. Change: + recheck deadlines and ownership on resume, recover stale connections, and + avoid replaying uncertain operations. **Complete when:** long sleeps, + expired sessions, and remote controller changes produce correct recovery + rather than duplicate execution or prolonged ingress rejection. + +- [ ] **HOST-01 — Evaluate colocated PostgreSQL separately.** Public authenticated + deployments currently reject an unset `DATABASE_URL` rather than falling back + to embedded PostgreSQL. Prototype explicit loopback PostgreSQL or deliberately + revise that deployment contract. **Complete when:** persistent storage, + backups/restores, upgrades, memory cost, and sleep/resume are qualified, with + agent execution isolated from the controller/database. This option does not + remove scheduler or wake requirements. Source: + [index.ts](../../server/src/index.ts), `assertCloudDatabaseContract`. + +## 9. Delivery order and acceptance + +1. Complete mechanism triage for the selected first slice: normal trigger, + producer coverage, durable dispatch, and remaining temporal obligations. +2. Ship DB-free health/probe handling and contained event-driven migrations. + Measure empty-worker SQL reduction while preserving delivery backstops until + their replacements are proven. These changes do not depend on a full scheduler. +3. Build the internal scheduler around true deadlines, starting with routines + and adding required retry/expiry/maintenance paths. Then add the external-waker + adapter and acknowledged alarm protocol before sleeping scheduled workloads. +4. Qualify connector transports, plugin lifecycle, browser demand, and provider + resume behavior. Expand eligibility only with evidence. + +- [ ] **TEST-01 — Measure an empty claimed tenant.** After startup settles, prove + zero app-originated SQL and no unintended outgoing packets, then observe actual + Neon and hosting sleep. Use provider/control-plane metrics so measurement + itself does not wake the tenant DB. Compare against the original baseline. + +- [ ] **TEST-02 — Prove first-use recovery.** After real sleep, perform an + authenticated read, write, and transaction. Verify persistent results, no + duplicate effects, and measured end-to-end latency including authentication + and database startup. + +- [ ] **TEST-03 — Prove scheduled work.** Exercise routines, a moved-earlier + deadline, cancellation, retry, expiry, and maintenance through restart/sleep. + Deliver duplicate/stale wakes and simulate external-waker failure. Verify + catch-up, claims, and awake fallback. + +- [ ] **TEST-04 — Prove event ingress.** Deliver a verified webhook while asleep + and while entering sleep. Exercise a connector with its supported receiver + strategy. Verify acknowledgements, replay, company scope, and persistence. + +- [ ] **TEST-05 — Prove active-work protection.** Attempt sleep during a run, + transaction, backup, finalization, cleanup attempt, and pending accounting + receipt. Verify the intended blocker/hold and safe release after settlement. + +- [ ] **TEST-06 — Prove browser inactivity behavior.** Test hidden tabs, + unattended visible tabs, reconnect/error states, plugin streams, and multiple + tabs. Verify returning users see reconciled state without an accidental wake + storm. + +- [ ] **TEST-07 — Prove crash, ownership, and rollout behavior.** Test a crash + around commit/alarm publication, stale drain owner, provider stop racing a + deployment, and long stateful suspension. Roll back migrated lanes without + duplicate workers. Measure fleet wake concurrency and actual compute savings + before broad enablement. + +Each implementation PR should run the smallest relevant checks first and the +repository-required verification for its handoff scope. This documentation-only +checklist does not claim any provider sleep/wake test has passed. diff --git a/docs/built-in-agents.md b/docs/built-in-agents.md index 90acac315f..2c563d1246 100644 --- a/docs/built-in-agents.md +++ b/docs/built-in-agents.md @@ -9,7 +9,7 @@ The first built-ins are `briefs` and `learning`. Operators can provision them fr The subsystem has four layers: - Registry: `server/src/services/built-in-agents.ts` defines the static `BuiltInAgentDefinition` list. -- Marker: `server/src/services/built-in-agent-metadata.ts` reads and writes `metadata.paperclipBuiltInAgent`. +- Marker: `server/src/lib/built-in-agent-metadata.ts` reads and writes `metadata.paperclipBuiltInAgent`. - Provisioning service: `builtInAgentService(db)` finds, creates, updates, resets, and requires built-ins per company. - Routes: `server/src/routes/built-in-agents.ts` exposes list, provision, and reset APIs. diff --git a/packages/adapters/claude-local/src/server/acp.auth.test.ts b/packages/adapters/claude-local/src/server/acp.auth.test.ts index 0e6b31f57b..0d1c96cd56 100644 --- a/packages/adapters/claude-local/src/server/acp.auth.test.ts +++ b/packages/adapters/claude-local/src/server/acp.auth.test.ts @@ -197,7 +197,7 @@ describe("probeClaudeAcpSandboxLogin", () => { // The parsed result is a success, so no login gate and no probe-unavailable // check appears. - expect(checks).toEqual([]); + expect(checks).toEqual([expect.objectContaining({ code: "claude_hello_probe_passed", level: "info" })]); }); it("keeps a non-auth failed probe with a token phrase on the probe-unavailable path", async () => { @@ -256,7 +256,7 @@ describe("probeClaudeAcpSandboxLogin", () => { expect(authRequired?.hint).toBe("Run `claude login` in this environment, then retry the probe."); }); - it("emits no checks when the sandbox probe reports a healthy login", async () => { + it("records a successful probe when the sandbox reports a healthy login", async () => { probeResult.value = { exitCode: 0, stdout: helloStdout, stderr: "", timedOut: false }; const checks = await probeClaudeAcpSandboxLogin({ @@ -264,7 +264,13 @@ describe("probeClaudeAcpSandboxLogin", () => { target: sandboxTarget, }); - expect(checks).toEqual([]); + expect(checks).toEqual([expect.objectContaining({ code: "claude_hello_probe_passed", level: "info" })]); + }); + + it("does not record success for an empty probe result", async () => { + probeResult.value = { exitCode: 0, stdout: "", stderr: "", timedOut: false }; + const checks = await probeClaudeAcpSandboxLogin({ config: { engine: "acp" }, target: sandboxTarget }); + expect(checks).toEqual([expect.objectContaining({ code: "claude_acp_login_probe_unavailable", level: "warn" })]); }); it("emits a distinct warn check, not a silent pass, when the probe cannot run", async () => { @@ -552,6 +558,8 @@ describe("Claude ACP hello probe on local and SSH targets", () => { expect(result.checks).toContainEqual(expect.objectContaining({ code: "claude_acp_anthropic_api_key_detected", level: "warn", })); + expect(result.status).toBe("warn"); + expect(result.checks).toContainEqual(expect.objectContaining({ code: "claude_hello_probe_passed", level: "info" })); // The host key value never enters a check. expect(JSON.stringify(result.checks)).not.toContain("sk-ant-host-key"); }); diff --git a/packages/adapters/claude-local/src/server/acp.ts b/packages/adapters/claude-local/src/server/acp.ts index 1ba468e42d..1a800e2c3a 100644 --- a/packages/adapters/claude-local/src/server/acp.ts +++ b/packages/adapters/claude-local/src/server/acp.ts @@ -721,7 +721,10 @@ export async function probeClaudeAcpSandboxLogin(input: { }); return [buildAcpLoginProbeUnavailableCheck("The Claude login probe did not complete.", targetIsSandbox)]; } - return []; + if (!/\bhello\b/i.test(parsedStream.summary)) { + return [buildAcpLoginProbeUnavailableCheck("The Claude login probe did not return hello.", targetIsSandbox)]; + } + return [{ code: "claude_hello_probe_passed", level: "info", message: "Claude login probe succeeded." }]; } export async function testClaudeAcpEnvironment( diff --git a/packages/adapters/openclaw-gateway/package.json b/packages/adapters/openclaw-gateway/package.json index 892f5e25ca..3f25ab08da 100644 --- a/packages/adapters/openclaw-gateway/package.json +++ b/packages/adapters/openclaw-gateway/package.json @@ -52,11 +52,11 @@ "dependencies": { "@paperclipai/adapter-utils": "workspace:*", "picocolors": "^1.1.1", - "ws": "^8.21.3" + "ws": "^8.22.0" }, "devDependencies": { "@types/node": "^24.0.0", - "@types/ws": "^8.18.1", + "@types/ws": "^8.18.2", "typescript": "^7.0.2" }, "engines": { diff --git a/packages/db/package.json b/packages/db/package.json index b942944cb4..6e9ece4a39 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -45,7 +45,7 @@ "generate": "pnpm run check:migrations && pnpm run clean && tsc -p tsconfig.json && drizzle-kit generate && pnpm run prune:snapshots", "prune:snapshots": "ls src/migrations/meta/*_snapshot.json | sort -r | tail -n +6 | xargs rm -f", "migrate": "pnpm run check:migrations && tsx src/migrate.ts", - "seed": "tsx src/seed.ts" + "seed": "pnpm --filter @paperclipai/server exec tsx scripts/seed.ts" }, "dependencies": { "@paperclipai/shared": "workspace:*", diff --git a/packages/db/src/backup-lib.test.ts b/packages/db/src/backup-lib.test.ts index de3166714c..c625208eac 100644 --- a/packages/db/src/backup-lib.test.ts +++ b/packages/db/src/backup-lib.test.ts @@ -7,6 +7,7 @@ import postgres from "postgres"; import { createBufferedTextFileWriter, runDatabaseBackup, runDatabaseRestore } from "./backup-lib.js"; import { ensurePostgresDatabase } from "./client.js"; import { + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./test-embedded-postgres.js"; @@ -105,7 +106,7 @@ describeEmbeddedPostgres("runDatabaseBackup", () => { if (previous === undefined) delete process.env.PAPERCLIP_PG_DUMP_PATH; else process.env.PAPERCLIP_PG_DUMP_PATH = previous; } - }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); it("preserves identity generation, sequence options and progress in JavaScript backups", async () => { const source = await createTempDatabase(); diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index c1b34d21eb..3f6061deaa 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -3,6 +3,7 @@ import { drizzle as drizzlePg } from "drizzle-orm/postgres-js"; import { readFile, readdir } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import postgres from "postgres"; +import { installDatabaseWorkSignals } from "./work-signals.js"; import * as schema from "./schema/index.js"; const MIGRATIONS_FOLDER = fileURLToPath(new URL("./migrations", import.meta.url)); @@ -268,6 +269,10 @@ export async function withDedicatedDbConnection(db: Db, action: (dedicated: D export function createDb(url: string, options?: DatabaseClientOptions) { const resolved = resolveDatabaseClientOptions(options ?? databaseClientOptionsFromEnv()); + return createDbClient(url, resolved); +} + +function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: object) { const sql = postgres(url, postgresJsOptions(resolved)); const key = hostPortKeyOrNull(url); if (key) registerClient(key, sql); @@ -275,10 +280,10 @@ export function createDb(url: string, options?: DatabaseClientOptions) { // postgres.js calls that error "write CONNECTION_CLOSED" too, so the // message cannot establish that replay is safe. Leave retries to callers // that know the complete operation is idempotent. - const db = drizzlePg(sql, { schema }); - dedicatedDbFactories.set(db, () => createDb(url, { + const db = installDatabaseWorkSignals(drizzlePg(sql, { schema }), owner); + dedicatedDbFactories.set(db, () => createDbClient(url, { ...resolved, maxConnections: 1, applicationName: "paperclip-workspace-finalization-lock", - })); + }, db)); return db; } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 378c9bd37b..73c6028d06 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -42,3 +42,5 @@ export { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecy export { issueRelations } from "./schema/issue_relations.js"; export { issueReferenceMentions } from "./schema/issue_reference_mentions.js"; export * from "./schema/index.js"; + +export { signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; diff --git a/packages/db/src/migrations/0320_daily_onslaught.sql b/packages/db/src/migrations/0320_daily_onslaught.sql new file mode 100644 index 0000000000..7e2fa9d804 --- /dev/null +++ b/packages/db/src/migrations/0320_daily_onslaught.sql @@ -0,0 +1,16 @@ +ALTER TABLE "agents" ADD COLUMN "lifecycle_state" text DEFAULT 'ready' NOT NULL;--> statement-breakpoint +ALTER TABLE "agents" ADD COLUMN "lifecycle_required_plugin_ids" jsonb;--> statement-breakpoint +ALTER TABLE "agents" ADD COLUMN "lifecycle_holds" jsonb DEFAULT '[]'::jsonb NOT NULL;--> statement-breakpoint +ALTER TABLE "agents" ADD COLUMN "lifecycle_version" integer DEFAULT 0 NOT NULL;--> statement-breakpoint +ALTER TABLE "agents" ADD COLUMN "lifecycle_error" text;--> statement-breakpoint +ALTER TABLE "agents" ADD COLUMN "lifecycle_operation" jsonb;--> statement-breakpoint +CREATE INDEX "agents_lifecycle_work_idx" ON "agents" USING btree ("lifecycle_state","updated_at","id"); +--> statement-breakpoint +UPDATE "agents" SET + "lifecycle_state" = CASE "status" + WHEN 'pending_approval' THEN 'pending_approval' + WHEN 'paused' THEN 'paused' + WHEN 'terminated' THEN 'terminated' + ELSE 'ready' END, + "lifecycle_holds" = CASE WHEN "status" = 'paused' + THEN jsonb_build_array(coalesce("pause_reason", 'manual')) ELSE '[]'::jsonb END; diff --git a/packages/db/src/migrations/meta/0315_snapshot.json b/packages/db/src/migrations/meta/0320_snapshot.json similarity index 97% rename from packages/db/src/migrations/meta/0315_snapshot.json rename to packages/db/src/migrations/meta/0320_snapshot.json index 6913129624..4282ad6c08 100644 --- a/packages/db/src/migrations/meta/0315_snapshot.json +++ b/packages/db/src/migrations/meta/0320_snapshot.json @@ -1,6 +1,6 @@ { - "id": "26691ae0-39a8-42ac-868d-e36ab7219073", - "prevId": "a1aefefe-8911-4583-9c52-7c658e46e9b4", + "id": "d6277736-9c05-4304-8d16-52dc0d2fbaef", + "prevId": "af0b2d33-956b-4b12-8d68-8b4e14c991b2", "version": "7", "dialect": "postgresql", "tables": { @@ -408,13 +408,19 @@ "name": "run_id", "type": "uuid", "primaryKey": false, - "notNull": true + "notNull": false + }, + "decision_invocation_id": { + "name": "decision_invocation_id", + "type": "uuid", + "primaryKey": false, + "notNull": false }, "agent_id": { "name": "agent_id", "type": "uuid", "primaryKey": false, - "notNull": true + "notNull": false }, "project_id": { "name": "project_id", @@ -456,6 +462,27 @@ } }, "indexes": { + "budget_reservations_decision_idx": { + "name": "budget_reservations_decision_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "decision_invocation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, "budget_reservations_run_idx": { "name": "budget_reservations_run_idx", "columns": [ @@ -551,6 +578,10 @@ "uniqueConstraints": {}, "policies": {}, "checkConstraints": { + "budget_reservations_source_check": { + "name": "budget_reservations_source_check", + "value": "(\"budget_reservations\".\"run_id\" is not null)::int + (\"budget_reservations\".\"decision_invocation_id\" is not null)::int = 1" + }, "budget_reservations_state_check": { "name": "budget_reservations_state_check", "value": "\"budget_reservations\".\"state\" in ('held', 'settled', 'released')" @@ -3883,6 +3914,45 @@ "notNull": true, "default": "'idle'" }, + "lifecycle_state": { + "name": "lifecycle_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'ready'" + }, + "lifecycle_required_plugin_ids": { + "name": "lifecycle_required_plugin_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "lifecycle_holds": { + "name": "lifecycle_holds", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "lifecycle_version": { + "name": "lifecycle_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "lifecycle_error": { + "name": "lifecycle_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lifecycle_operation": { + "name": "lifecycle_operation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "reports_to": { "name": "reports_to", "type": "uuid", @@ -3995,6 +4065,33 @@ } }, "indexes": { + "agents_lifecycle_work_idx": { + "name": "agents_lifecycle_work_idx", + "columns": [ + { + "expression": "lifecycle_state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "agents_company_status_idx": { "name": "agents_company_status_idx", "columns": [ @@ -11745,6 +11842,139 @@ }, "isRLSEnabled": false }, + "public.chat_slack_registrations": { + "name": "chat_slack_registrations", + "schema": "", + "columns": { + "endpoint_id": { + "name": "endpoint_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "manifest_hash": { + "name": "manifest_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "callback_uri": { + "name": "callback_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_ids": { + "name": "secret_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_user_id": { + "name": "bot_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "chat_slack_registrations_company_id_endpoint_id_chat_endpoints_company_id_id_fk": { + "name": "chat_slack_registrations_company_id_endpoint_id_chat_endpoints_company_id_id_fk", + "tableFrom": "chat_slack_registrations", + "tableTo": "chat_endpoints", + "columnsFrom": [ + "company_id", + "endpoint_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "chat_slack_registration_status_check": { + "name": "chat_slack_registration_status_check", + "value": "\"chat_slack_registrations\".\"status\" in ('creating', 'uncertain', 'failed', 'install', 'credentials_saved', 'configured', 'removed')" + } + }, + "isRLSEnabled": false + }, "public.chat_teams_file_transfers": { "name": "chat_teams_file_transfers", "schema": "", @@ -17284,7 +17514,20 @@ "name": "agent_id", "type": "uuid", "primaryKey": false, - "notNull": true + "notNull": false + }, + "usage_kind": { + "name": "usage_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'agent'" + }, + "responsible_user_id": { + "name": "responsible_user_id", + "type": "text", + "primaryKey": false, + "notNull": false }, "issue_id": { "name": "issue_id", @@ -17732,6 +17975,10 @@ "uniqueConstraints": {}, "policies": {}, "checkConstraints": { + "cost_events_usage_kind_check": { + "name": "cost_events_usage_kind_check", + "value": "\"cost_events\".\"usage_kind\" = 'decision' or (\"cost_events\".\"usage_kind\" = 'agent' and \"cost_events\".\"agent_id\" is not null)" + }, "cost_events_nonnegative_amounts": { "name": "cost_events_nonnegative_amounts", "value": "\"cost_events\".\"cost_cents\" >= 0 and \"cost_events\".\"input_tokens\" >= 0 and \"cost_events\".\"cached_input_tokens\" >= 0 and \"cost_events\".\"output_tokens\" >= 0" @@ -17739,6 +17986,346 @@ }, "isRLSEnabled": false }, + "public.company_decision_models": { + "name": "company_decision_models", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "allow_background": { + "name": "allow_background", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "company_decision_models_company_id_companies_id_fk": { + "name": "company_decision_models_company_id_companies_id_fk", + "tableFrom": "company_decision_models", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "company_decisions_configured": { + "name": "company_decisions_configured", + "value": "(\"company_decision_models\".\"connection_id\" is null) = (\"company_decision_models\".\"grant_id\" is null) and (not \"company_decision_models\".\"enabled\" or (\"company_decision_models\".\"connection_id\" is not null and \"company_decision_models\".\"provider\" is not null and \"company_decision_models\".\"model\" is not null))" + } + }, + "isRLSEnabled": false + }, + "public.decision_invocations": { + "name": "decision_invocations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "feature": { + "name": "feature", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_type": { + "name": "actor_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "responsible_user_id": { + "name": "responsible_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "identity_context_id": { + "name": "identity_context_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "question_types": { + "name": "question_types", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_request_id": { + "name": "provider_request_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cost_event_id": { + "name": "cost_event_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "decision_invocations_company_started_idx": { + "name": "decision_invocations_company_started_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "decision_invocations_pending_idx": { + "name": "decision_invocations_pending_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"decision_invocations\".\"status\" = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "decision_invocations_cost_idx": { + "name": "decision_invocations_cost_idx", + "columns": [ + { + "expression": "cost_event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "decision_invocations_company_id_companies_id_fk": { + "name": "decision_invocations_company_id_companies_id_fk", + "tableFrom": "decision_invocations", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "decision_invocations_cost_event_id_cost_events_id_fk": { + "name": "decision_invocations_cost_event_id_cost_events_id_fk", + "tableFrom": "decision_invocations", + "tableTo": "cost_events", + "columnsFrom": [ + "cost_event_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "decision_invocations_status_check": { + "name": "decision_invocations_status_check", + "value": "\"decision_invocations\".\"status\" in ('running', 'succeeded', 'failed', 'unknown')" + } + }, + "isRLSEnabled": false + }, "public.decision_archive_notification_outbox": { "name": "decision_archive_notification_outbox", "schema": "", @@ -21614,6 +22201,746 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.dot_agent_bindings": { + "name": "dot_agent_bindings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "operator_id": { + "name": "operator_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pairing'" + }, + "pairing_code_hash": { + "name": "pairing_code_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pairing_expires_at": { + "name": "pairing_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dot_url": { + "name": "dot_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "challenge_hash": { + "name": "challenge_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "challenge_expires_at": { + "name": "challenge_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "dot_bindings_active_agent_uq": { + "name": "dot_bindings_active_agent_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"dot_agent_bindings\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "dot_bindings_active_grant_uq": { + "name": "dot_bindings_active_grant_uq", + "columns": [ + { + "expression": "grant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"dot_agent_bindings\".\"revoked_at\" IS NULL AND \"dot_agent_bindings\".\"grant_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "dot_bindings_pairing_code_uq": { + "name": "dot_bindings_pairing_code_uq", + "columns": [ + { + "expression": "pairing_code_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "dot_agent_bindings_company_id_companies_id_fk": { + "name": "dot_agent_bindings_company_id_companies_id_fk", + "tableFrom": "dot_agent_bindings", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dot_agent_bindings_agent_id_agents_id_fk": { + "name": "dot_agent_bindings_agent_id_agents_id_fk", + "tableFrom": "dot_agent_bindings", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dot_agent_bindings_grant_id_mcp_oauth_grants_id_fk": { + "name": "dot_agent_bindings_grant_id_mcp_oauth_grants_id_fk", + "tableFrom": "dot_agent_bindings", + "tableTo": "mcp_oauth_grants", + "columnsFrom": [ + "grant_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dot_mailbox_items": { + "name": "dot_mailbox_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "binding_id": { + "name": "binding_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "binding_generation": { + "name": "binding_generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "assignment_id": { + "name": "assignment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_event_id": { + "name": "source_event_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "references": { + "name": "references", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "dot_mailbox_event_uq": { + "name": "dot_mailbox_event_uq", + "columns": [ + { + "expression": "binding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "binding_generation", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dot_mailbox_binding_cursor_idx": { + "name": "dot_mailbox_binding_cursor_idx", + "columns": [ + { + "expression": "binding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "binding_generation", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "dot_mailbox_items_company_id_companies_id_fk": { + "name": "dot_mailbox_items_company_id_companies_id_fk", + "tableFrom": "dot_mailbox_items", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dot_mailbox_items_binding_id_dot_agent_bindings_id_fk": { + "name": "dot_mailbox_items_binding_id_dot_agent_bindings_id_fk", + "tableFrom": "dot_mailbox_items", + "tableTo": "dot_agent_bindings", + "columnsFrom": [ + "binding_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dot_mailbox_items_assignment_id_dot_runner_assignments_id_fk": { + "name": "dot_mailbox_items_assignment_id_dot_runner_assignments_id_fk", + "tableFrom": "dot_mailbox_items", + "tableTo": "dot_runner_assignments", + "columnsFrom": [ + "assignment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dot_runner_assignments": { + "name": "dot_runner_assignments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "binding_id": { + "name": "binding_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "binding_generation": { + "name": "binding_generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "normalized_session_id": { + "name": "normalized_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "controller_generation": { + "name": "controller_generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "catalog_digest": { + "name": "catalog_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "projection": { + "name": "projection", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "accept_by": { + "name": "accept_by", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "accepted_at": { + "name": "accepted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "settled_at": { + "name": "settled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_activity_at": { + "name": "last_activity_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "dot_assignments_turn_uq": { + "name": "dot_assignments_turn_uq", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "turn_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dot_assignments_active_binding_uq": { + "name": "dot_assignments_active_binding_uq", + "columns": [ + { + "expression": "binding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"dot_runner_assignments\".\"status\" IN ('offered', 'accepted')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "dot_assignments_company_idx": { + "name": "dot_assignments_company_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "dot_runner_assignments_company_id_companies_id_fk": { + "name": "dot_runner_assignments_company_id_companies_id_fk", + "tableFrom": "dot_runner_assignments", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dot_runner_assignments_binding_id_dot_agent_bindings_id_fk": { + "name": "dot_runner_assignments_binding_id_dot_agent_bindings_id_fk", + "tableFrom": "dot_runner_assignments", + "tableTo": "dot_agent_bindings", + "columnsFrom": [ + "binding_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dot_runner_assignments_run_id_heartbeat_runs_id_fk": { + "name": "dot_runner_assignments_run_id_heartbeat_runs_id_fk", + "tableFrom": "dot_runner_assignments", + "tableTo": "heartbeat_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dot_runner_assignments_agent_id_agents_id_fk": { + "name": "dot_runner_assignments_agent_id_agents_id_fk", + "tableFrom": "dot_runner_assignments", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dot_runner_operations": { + "name": "dot_runner_operations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "assignment_id": { + "name": "assignment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "digest": { + "name": "digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "command": { + "name": "command", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'reserved'" + }, + "outcome": { + "name": "outcome", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "source_event_id": { + "name": "source_event_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "dot_operations_request_uq": { + "name": "dot_operations_request_uq", + "columns": [ + { + "expression": "assignment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "dot_runner_operations_company_id_companies_id_fk": { + "name": "dot_runner_operations_company_id_companies_id_fk", + "tableFrom": "dot_runner_operations", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dot_runner_operations_assignment_id_dot_runner_assignments_id_fk": { + "name": "dot_runner_operations_assignment_id_dot_runner_assignments_id_fk", + "tableFrom": "dot_runner_operations", + "tableTo": "dot_runner_assignments", + "columnsFrom": [ + "assignment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.email_endpoints": { "name": "email_endpoints", "schema": "", @@ -53799,7 +55126,13 @@ "name": "activity_id", "type": "uuid", "primaryKey": false, - "notNull": true + "notNull": false + }, + "mailbox_item_id": { + "name": "mailbox_item_id", + "type": "integer", + "primaryKey": false, + "notNull": false }, "event": { "name": "event", @@ -53835,6 +55168,27 @@ } }, "indexes": { + "mcp_event_deliveries_mailbox_uq": { + "name": "mcp_event_deliveries_mailbox_uq", + "columns": [ + { + "expression": "subscription_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "mailbox_item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, "mcp_event_deliveries_activity_uq": { "name": "mcp_event_deliveries_activity_uq", "columns": [ @@ -53890,7 +55244,12 @@ "compositePrimaryKeys": {}, "uniqueConstraints": {}, "policies": {}, - "checkConstraints": {}, + "checkConstraints": { + "mcp_delivery_source_check": { + "name": "mcp_delivery_source_check", + "value": "(\"mcp_event_deliveries\".\"activity_id\" IS NOT NULL AND \"mcp_event_deliveries\".\"mailbox_item_id\" IS NULL) OR (\"mcp_event_deliveries\".\"activity_id\" IS NULL AND \"mcp_event_deliveries\".\"mailbox_item_id\" IS NOT NULL)" + } + }, "isRLSEnabled": false }, "public.mcp_event_subscriptions": { @@ -53925,7 +55284,13 @@ "name": "task_id", "type": "uuid", "primaryKey": false, - "notNull": true + "notNull": false + }, + "binding_id": { + "name": "binding_id", + "type": "uuid", + "primaryKey": false, + "notNull": false }, "arguments": { "name": "arguments", @@ -54035,7 +55400,12 @@ "compositePrimaryKeys": {}, "uniqueConstraints": {}, "policies": {}, - "checkConstraints": {}, + "checkConstraints": { + "mcp_subscription_resource_check": { + "name": "mcp_subscription_resource_check", + "value": "(\"mcp_event_subscriptions\".\"task_id\" IS NOT NULL AND \"mcp_event_subscriptions\".\"binding_id\" IS NULL) OR (\"mcp_event_subscriptions\".\"task_id\" IS NULL AND \"mcp_event_subscriptions\".\"binding_id\" IS NOT NULL)" + } + }, "isRLSEnabled": false }, "public.mcp_file_tickets": { @@ -54591,6 +55961,19 @@ "primaryKey": false, "notNull": true }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'personal'" + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, "scopes": { "name": "scopes", "type": "jsonb", @@ -54673,6 +56056,19 @@ ], "onDelete": "cascade", "onUpdate": "no action" + }, + "mcp_oauth_grants_agent_id_agents_id_fk": { + "name": "mcp_oauth_grants_agent_id_agents_id_fk", + "tableFrom": "mcp_oauth_grants", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" } }, "compositePrimaryKeys": {}, @@ -54924,7 +56320,7 @@ "name": "expires_at", "type": "timestamp with time zone", "primaryKey": false, - "notNull": true + "notNull": false }, "created_at": { "name": "created_at", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index ce7a0e57d4..ea08e98553 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2227,6 +2227,13 @@ "when": 1791490887468, "tag": "0319_heavy_captain_midlands", "breakpoints": true + }, + { + "idx": 320, + "version": "7", + "when": 1791497222457, + "tag": "0320_daily_onslaught", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/agents.ts b/packages/db/src/schema/agents.ts index 88d0f50dc9..04e781f568 100644 --- a/packages/db/src/schema/agents.ts +++ b/packages/db/src/schema/agents.ts @@ -1,4 +1,4 @@ -import type { AgentAppearance } from "@paperclipai/shared"; +import type { AgentAppearance, AgentLifecycleState, AgentLifecycleOperation } from "@paperclipai/shared"; import { type AnyPgColumn, pgTable, @@ -25,6 +25,12 @@ export const agents = pgTable( icon: text("icon"), appearance: jsonb("appearance").$type(), status: text("status").notNull().default("idle"), + lifecycleState: text("lifecycle_state").$type().notNull().default("ready"), + lifecycleRequiredPluginIds: jsonb("lifecycle_required_plugin_ids").$type(), + lifecycleHolds: jsonb("lifecycle_holds").$type().notNull().default([]), + lifecycleVersion: integer("lifecycle_version").notNull().default(0), + lifecycleError: text("lifecycle_error"), + lifecycleOperation: jsonb("lifecycle_operation").$type(), reportsTo: uuid("reports_to").references((): AnyPgColumn => agents.id), capabilities: text("capabilities"), adapterType: text("adapter_type").notNull().default("process"), @@ -45,6 +51,7 @@ export const agents = pgTable( }, (table) => ({ companyIdUq: unique("agents_company_id_uq").on(table.companyId, table.id), + lifecycleWorkIdx: index("agents_lifecycle_work_idx").on(table.lifecycleState, table.updatedAt, table.id), companyStatusIdx: index("agents_company_status_idx").on(table.companyId, table.status), companyReportsToIdx: index("agents_company_reports_to_idx").on(table.companyId, table.reportsTo), companyDefaultEnvironmentIdx: index("agents_company_default_environment_idx").on(table.companyId, table.defaultEnvironmentId), diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts new file mode 100644 index 0000000000..9bc3f738b1 --- /dev/null +++ b/packages/db/src/work-signals.test.ts @@ -0,0 +1,165 @@ +import { sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { createDb, withDedicatedDbConnection } from "./client.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; +import { databaseWorkPending, installDatabaseWorkSignals, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("database work signals", () => { + let db: ReturnType; + let temporary: Awaited>; + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("work-signals-"); + db = createDb(temporary.connectionString); + await db.execute(sql`create table work_signal_test (id text primary key)`); + }, 90_000); + afterAll(async () => { await temporary?.cleanup(); }); + + it("defers nested savepoint notifications until the outer commit and preserves return values", async () => { + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "nested", event => events.push(event)); + try { + const result = await db.transaction(async tx => { + await tx.transaction(async nested => { + await signalDatabaseWork(nested, "nested"); + await nested.execute(sql`insert into work_signal_test values ('nested')`); + }); + expect(events).toEqual(["intent"]); + expect(databaseWorkPending(db, "nested")).toBe(true); + expect(await db.execute(sql`select * from work_signal_test where id = 'nested'`)).toHaveLength(0); + return 123; + }); + expect(result).toBe(123); + expect(events).toEqual(["intent", "settled"]); + expect(databaseWorkPending(db, "nested")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'nested'`)).toHaveLength(1); + } finally { unsubscribe(); } + }); + + it("reconciles a rollback without leaving a permanent pending flag", async () => { + await expect(db.transaction(async tx => { + await signalDatabaseWork(tx, "rollback"); + await tx.execute(sql`insert into work_signal_test values ('rollback')`); + throw new Error("abort"); + })).rejects.toThrow("abort"); + await reconcileDatabaseWork(db, "rollback"); + expect(databaseWorkPending(db, "rollback")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'rollback'`)).toHaveLength(0); + }); + + it("keeps outer ownership after a caught savepoint rollback", async () => { + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "savepoint", event => events.push(event)); + try { + await db.transaction(async tx => { + await expect(tx.transaction(async nested => { + await signalDatabaseWork(nested, "savepoint"); + await nested.execute(sql`insert into work_signal_test values ('savepoint')`); + throw new Error("rollback savepoint"); + })).rejects.toThrow("rollback savepoint"); + expect(databaseWorkPending(db, "savepoint")).toBe(true); + expect(events).toEqual(["intent"]); + await signalDatabaseWork(tx, "savepoint"); + await tx.execute(sql`insert into work_signal_test values ('savepoint-retry')`); + }); + expect(events).toEqual(["intent", "intent", "settled"]); + expect(databaseWorkPending(db, "savepoint")).toBe(false); + } finally { unsubscribe(); } + }); + + it.each(["commit", "abort"])("does not clear a late %s until PostgreSQL reports a settled transaction", async outcome => { + // Inject a lost response at the client boundary while a REAL transaction + // remains open on PostgreSQL. The status probe and row visibility are real. + const raw = drizzle(db.$client); + const transaction = raw.transaction.bind(raw); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + let wrote!: () => void; + const written = new Promise(resolve => { wrote = resolve; }); + let serverDone!: Promise; + raw.transaction = ((callback: any) => { + serverDone = transaction(async tx => { + const result = await callback(tx); + wrote(); + await gate; + if (outcome === "abort") throw new Error("server abort"); + return result; + }); + // Observe rejection immediately; tests await the same result below. + void serverDone.catch(() => {}); + return written.then(() => { throw new Error("lost transaction response"); }); + }) as typeof raw.transaction; + const owner = installDatabaseWorkSignals(raw); + const topic = `late-${outcome}`; + try { + await expect(owner.transaction(async tx => { + await signalDatabaseWork(tx, topic); + await tx.execute(sql`insert into work_signal_test values (${topic})`); + })).rejects.toThrow("lost transaction response"); + await reconcileDatabaseWork(owner, topic); + expect(databaseWorkPending(owner, topic)).toBe(true); + expect(await db.execute(sql`select * from work_signal_test where id = ${topic}`)).toHaveLength(0); + release(); await serverDone.catch(() => {}); + await reconcileDatabaseWork(owner, topic); + expect(databaseWorkPending(owner, topic)).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = ${topic}`)).toHaveLength(outcome === "commit" ? 1 : 0); + } finally { release(); await serverDone?.catch(() => {}); } + }); + + it("recovers after a real backend disconnect without replaying the queue write", async () => { + await expect(db.transaction(async tx => { + await signalDatabaseWork(tx, "disconnect"); + await tx.execute(sql`insert into work_signal_test values ('disconnect')`); + const [{ pid }] = await tx.execute(sql`select pg_backend_pid() as pid`); + await db.execute(sql`select pg_terminate_backend(${pid as number})`); + await tx.execute(sql`select 1`); + })).rejects.toThrow(); + await reconcileDatabaseWork(db, "disconnect"); + expect(databaseWorkPending(db, "disconnect")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'disconnect'`)).toHaveLength(0); + }); + + it("shares notifications with dedicated child pools but isolates unrelated roots and topics", async () => { + const listener = vi.fn(); + const other = createDb(temporary.connectionString); + const unsubscribe = subscribeDatabaseWork(db, "dedicated", listener); + try { + await other.transaction(tx => signalDatabaseWork(tx, "dedicated")); + await db.transaction(tx => signalDatabaseWork(tx, "other")); + expect(listener).not.toHaveBeenCalled(); + await withDedicatedDbConnection(db, dedicated => dedicated.transaction(tx => signalDatabaseWork(tx, "dedicated"))); + expect(listener.mock.calls.map(([event]) => event)).toEqual(["intent", "settled"]); + } finally { unsubscribe(); await other.$client.end(); } + }); + + it("never changes transaction success because a listener throws", async () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => {}); + const unsubscribe = subscribeDatabaseWork(db, "listener", () => { throw new Error("listener"); }); + try { + await expect(db.transaction(async tx => { await signalDatabaseWork(tx, "listener"); return 7; })).resolves.toBe(7); + expect(warn).toHaveBeenCalledTimes(2); + } finally { unsubscribe(); warn.mockRestore(); } + }); +}); + +describe("work signal query boundaries", () => { + it("adds no SQL to transactions which do not register queue work", async () => { + const execute = vi.fn(); + const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) }); + await expect(connection.transaction(async () => 19)).resolves.toBe(19); + expect(execute).not.toHaveBeenCalled(); + }); + it("fails before the write if XID acquisition fails, without retaining uncertainty", async () => { + const execute = vi.fn().mockRejectedValue(new Error("disconnected")); + const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) }); + const write = vi.fn(); + await expect(connection.transaction(async (tx: any) => { await signalDatabaseWork(tx, "failed"); write(); })).rejects.toThrow("disconnected"); + expect(write).not.toHaveBeenCalled(); + expect(databaseWorkPending(connection, "failed")).toBe(false); + }); + it("rejects registration on an autocommit root", async () => { + const connection = installDatabaseWorkSignals({ execute: vi.fn(), transaction: vi.fn() }); + await expect(signalDatabaseWork(connection, "bad")).rejects.toThrow("inside a transaction"); + }); +}); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts new file mode 100644 index 0000000000..219ea92446 --- /dev/null +++ b/packages/db/src/work-signals.ts @@ -0,0 +1,109 @@ +import { sql } from "drizzle-orm"; + +// Process-local hints about existing durable queues. No SQL triggers, tables, +// background timers, or driver query replay. Only opted-in transactions fetch +// an XID, so unrelated transactions keep their existing SQL behavior. +type Listener = (event: "intent" | "settled") => void; +type Transaction = { topics: Set; xid?: Promise; uncertain: boolean }; +type Scope = { listeners: Map>; transactions: Set; root: Connection }; +type Context = { scope: Scope; transaction?: Transaction }; +type Connection = { transaction: (...args: any[]) => Promise; execute: (...args: any[]) => Promise }; +const contexts = new WeakMap(); + +function publish(scope: Scope, topic: string, event: "intent" | "settled" = "settled") { + for (const listener of scope.listeners.get(topic) ?? []) { + try { listener(event); } + catch (error) { process.emitWarning(`Database work listener failed: ${String(error)}`); } + } +} + +/** Installed once by createDb; dedicated child pools share their owner's scope. */ +export function installDatabaseWorkSignals(db: T, owner?: object): T { + const scope = (owner && contexts.get(owner)?.scope) || { listeners: new Map(), transactions: new Set(), root: db }; + install(db, { scope }); + return db; +} + +function install(connection: Connection, context: Context) { + contexts.set(connection, context); + const transaction = connection.transaction; + connection.transaction = async function (callback, ...options) { + // Nested savepoints belong to the outermost transaction. Even a released + // savepoint must not wake a consumer before its parent commits. + const state = context.transaction ?? { topics: new Set(), uncertain: false }; + try { + const result = await transaction.call(this, async (tx: Connection) => { + install(tx, { scope: context.scope, transaction: state }); + return callback(tx); + }, ...options); + if (!context.transaction) context.scope.transactions.delete(state); + return result; + } catch (error) { + if (!context.transaction) { + // An XID is known before the first opted-in write can execute. If + // acquisition itself failed there cannot be a corresponding write. + if (state.xid) state.uncertain = true; + else context.scope.transactions.delete(state); + } + throw error; + } finally { + if (!context.transaction) for (const topic of state.topics) publish(context.scope, topic); + } + }; +} + +/** Await BEFORE the queue write, on its transaction. Nested transactions work. */ +export async function signalDatabaseWork(connection: object, topic: string): Promise { + const context = contexts.get(connection); + if (!context) return; // Lightweight service test doubles have no DB lifecycle. + const state = context.transaction; + if (!state) throw new Error("Database work must be registered inside a transaction"); + state.topics.add(topic); + context.scope.transactions.add(state); + publish(context.scope, topic, "intent"); + if (!state.xid) { + state.xid = (connection as Connection).execute(sql`select pg_current_xact_id()::text as xid`) + .then((rows: { xid: string }[]) => rows[0]!.xid); + } + // Keep a failed acquisition distinguishable from an uncertain queue write. + try { await state.xid; } + catch (error) { state.xid = undefined; throw error; } +} + +export function databaseWorkPending(owner: object, topic: string): boolean { + return [...(contexts.get(owner)?.scope.transactions ?? [])].some(state => state.topics.has(topic)); +} + +/** Resolve only failed transactions, on a fresh root-pool statement. Call + * BEFORE the queue scan so rows from a newly confirmed commit are visible. + * NULL means the XID aged out; it can no longer be in progress, so scanning + * the durable queue is sufficient even though its outcome is unavailable. + */ +export async function reconcileDatabaseWork(owner: object, topic: string): Promise { + const scope = contexts.get(owner)?.scope; + if (!scope) return; + for (const state of [...scope.transactions]) { + if (!state.uncertain || !state.topics.has(topic) || !state.xid) continue; + const xid = await state.xid; + const rows = await scope.root.execute(sql`select pg_xact_status(${xid}::xid8) as status`); + if (rows[0]?.status === "in progress") continue; + if (!rows.length || ![null, "committed", "aborted"].includes(rows[0].status)) { + throw new Error("Unexpected database transaction status"); + } + scope.transactions.delete(state); + for (const affected of state.topics) publish(scope, affected); + } +} + +/** Includes intent, outer settlement, and uncertainty-resolution changes. */ +export function subscribeDatabaseWork(owner: object, topic: string, listener: Listener): () => void { + const context = contexts.get(owner); + if (!context) throw new Error("Database work subscriptions require createDb"); + let listeners = context.scope.listeners.get(topic); + if (!listeners) context.scope.listeners.set(topic, listeners = new Set()); + listeners.add(listener); + if (databaseWorkPending(owner, topic)) { + try { listener("intent"); } catch (error) { process.emitWarning(`Database work listener failed: ${String(error)}`); } + } + return () => { listeners.delete(listener); }; +} diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index f3db235e71..ab37f30bb0 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -6513,17 +6513,23 @@ it("rotates PRP authority in place for a warm cross-run attachment", async () => it("reopens Pi after a completed turn within its cold admission budget during warm attachment", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-pi-warm-admission-")); + const providerNode = process.platform === "linux" ? join(root, "node") : process.execPath; + if (process.platform === "linux") { + await cp(process.execPath, providerNode, { dereference: true }); + await chmod(providerNode, 0o500); + expect((await stat(providerNode)).mode & 0o777).toBe(0o500); + } const cliRoot = join(root, "dist/cli"); - await mkdir(cliRoot, { recursive: true }); + await mkdir(cliRoot, { recursive: true, mode: 0o700 }); const sidecarPath = join(cliRoot, "acpx-runtime-sidecar.cjs"); const journal = join(root, "commands.ndjson"); const fixture = await readFile(fileURLToPath(new URL("./fixtures/fake-pi-warm-sidecar.cjs", import.meta.url)), "utf8"); - await writeFile(sidecarPath, fixture.replace("/* fixture-config */ null", JSON.stringify({ journal, resumeDelayMs: 32_000, commandDigest: QUALIFIED_ACPX_PROFILES.pi.commandDigest }))); + await writeFile(sidecarPath, fixture.replace("/* fixture-config */ null", JSON.stringify({ journal, resumeDelayMs: 32_000, commandDigest: QUALIFIED_ACPX_PROFILES.pi.commandDigest })), { mode: 0o600 }); const digest = (path: string) => `sha256:${createHash("sha256").update(readFileSync(path)).digest("hex")}`; const bundle = createCapabilityRunnerdCodexTransport({ provider: "acpx", acpxAgent: "pi", piThinkingLevel: "low", acpxPermissionMode: "deny-all", runnerBinary: defaultCapabilityRunnerdBinary(), stateDirectory: root, runnerFilesystemRoot: root, - providerNodeCommand: process.execPath, providerNodeCommandSha256: digest(process.execPath), + providerNodeCommand: providerNode, providerNodeCommandSha256: digest(providerNode), acpxSidecarPath: sidecarPath, acpxSidecarSha256: digest(sidecarPath), providerPackAuthorityDigest: `sha256:${"d".repeat(64)}`, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, diff --git a/packages/plugins/sdk/src/define-plugin.ts b/packages/plugins/sdk/src/define-plugin.ts index 12b763957c..43a2991229 100644 --- a/packages/plugins/sdk/src/define-plugin.ts +++ b/packages/plugins/sdk/src/define-plugin.ts @@ -1,3 +1,4 @@ +import type { AgentLifecycleRequest, AgentLifecycleResult } from "@paperclipai/shared"; import type { AiConnectionRouterRequest, AiConnectionRouterResult } from "@paperclipai/shared"; /** * `definePlugin` — the top-level helper for authoring a Paperclip plugin. @@ -358,6 +359,8 @@ export interface PluginDefinition { * Requires `external.objects.read`. */ /** Propose a member from host-authorized candidates. Requires ai.connections.route. */ + onAgentLifecycle?(params: AgentLifecycleRequest): Promise; + onRouteAiConnection?(params: AiConnectionRouterRequest): Promise; onResolveExternalObject?( diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index 5043091251..7c2c944b63 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -456,3 +456,5 @@ export { preserveEnvironmentSyncErrorDiagnostic, environmentSyncErrorData, readE export type { PluginEnvironmentSyncErrorDiagnostic } from "./environment-sync-error.js"; export type { AiConnectionPool, AiConnectionPoolConfig, AiConnectionPoolMember, AiConnectionRouterRequest, AiConnectionRouterResult, AiConnectionRouterSelection } from "@paperclipai/shared"; + +export type { AgentLifecycleRequest, AgentLifecycleResult } from "@paperclipai/shared"; diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index 7b2af37275..8f2ad9ce35 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -1,3 +1,4 @@ +import type { AgentLifecycleRequest, AgentLifecycleResult } from "@paperclipai/shared"; import type { AiConnectionRouterRequest, AiConnectionRouterResult } from "@paperclipai/shared"; /** * JSON-RPC 2.0 message types and protocol helpers for the host ↔ worker IPC @@ -1345,6 +1346,7 @@ export interface HostToWorkerMethods { /** Scoped plugin API route dispatch. */ handleApiRequest: [params: PluginApiRequestInput, result: PluginApiResponse]; /** @see PLUGIN_SPEC.md §13.8 */ + agentLifecycle: [params: AgentLifecycleRequest, result: AgentLifecycleResult]; getData: [params: GetDataParams, result: unknown]; /** @see PLUGIN_SPEC.md §13.9 */ performAction: [params: PerformActionParams, result: unknown]; @@ -1481,6 +1483,7 @@ export const HOST_TO_WORKER_OPTIONAL_METHODS: readonly HostToWorkerMethodName[] "runJob", "handleWebhook", "handleApiRequest", + "agentLifecycle", "getData", "performAction", "executeTool", diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index a374e31922..ef8de899f6 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -1,3 +1,4 @@ +import type { AgentLifecycleRequest, AgentLifecycleResult } from "@paperclipai/shared"; import type { AiConnectionRouterRequest, AiConnectionRouterResult } from "@paperclipai/shared"; import { environmentCreationCleanupErrorData } from "./environment-creation-cleanup.js"; import { environmentSyncErrorData, withEnvironmentSyncErrorCapture } from "./environment-sync-error.js"; @@ -1659,6 +1660,9 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost return handleExecuteTool(params as ExecuteToolParams); case "detectExternalObjects": return handleDetectExternalObjects(params as DetectExternalObjectsParams); + case "agentLifecycle": + if (!plugin.definition.onAgentLifecycle) throw methodNotImplemented("agentLifecycle"); + return plugin.definition.onAgentLifecycle(params as AgentLifecycleRequest); case "routeAiConnection": if (!plugin.definition.onRouteAiConnection) throw methodNotImplemented("routeAiConnection"); return plugin.definition.onRouteAiConnection(params as AiConnectionRouterRequest); @@ -1779,6 +1783,7 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost if (plugin.definition.onIdleDrain) supportedMethods.push("prepareIdleSleep", "releaseIdleSleep"); if (plugin.definition.onApiRequest) supportedMethods.push("handleApiRequest"); if (plugin.definition.onDetectExternalObjects) supportedMethods.push("detectExternalObjects"); + if (plugin.definition.onAgentLifecycle) supportedMethods.push("agentLifecycle"); if (plugin.definition.onRouteAiConnection) supportedMethods.push("routeAiConnection"); if (plugin.definition.onResolveExternalObject) supportedMethods.push("resolveExternalObject"); if (plugin.definition.onRefreshExternalObjects) supportedMethods.push("refreshExternalObjects"); diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index fed7872474..70840a8fd6 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -1340,6 +1340,7 @@ export const PLUGIN_CAPABILITIES = [ "approvals.read", "issue.documents.read", "agents.read", + "agents.lifecycle.manage", "goals.read", "goals.create", "goals.update", diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index a94a156b7f..9e7dfd5da9 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2858,3 +2858,7 @@ export { updatePrimaryAgentSchema, type UpdatePrimaryAgent, type PrimaryAgentPre export * from "./agent-avatar-upload.js"; export type { DotBinding, DotInvitation, DotConnection, DotPairing } from "./dot-invitations.js"; + +export { AGENT_LIFECYCLE_STATES, type AgentLifecycleState, type AgentLifecycleOperation } from "./types/agent-lifecycle.js"; + +export type { AgentLifecycleRequest, AgentLifecycleResult } from "./types/agent-lifecycle.js"; diff --git a/packages/shared/src/types/agent-lifecycle.ts b/packages/shared/src/types/agent-lifecycle.ts new file mode 100644 index 0000000000..a1b03e88e3 --- /dev/null +++ b/packages/shared/src/types/agent-lifecycle.ts @@ -0,0 +1,31 @@ +/** The lifecycle controls admission. Execution status describes current work. */ +export const AGENT_LIFECYCLE_STATES = [ + "pending_approval", "preparing", "verifying", "ready", "pausing", "paused", + "resuming", "terminating", "cleaning_up", "terminated", "rejected", +] as const; +export type AgentLifecycleState = typeof AGENT_LIFECYCLE_STATES[number]; + +export interface AgentLifecycleOperation { + id: string; + resumeState?: "preparing" | "verifying" | "ready"; + hostComplete: boolean; + completedPluginIds: string[]; + leaseOwner?: string; + leaseUntil?: string; + retryAt?: string; + attempts: number; + responsibleUserId?: string | null; +} + +export interface AgentLifecycleRequest { + companyId: string; + agentId: string; + operationId: string; + version: number; + phase: AgentLifecycleState; +} +export interface AgentLifecycleResult { + operationId: string; + version: number; + status: "complete" | "pending"; +} diff --git a/packages/shared/src/types/agent.ts b/packages/shared/src/types/agent.ts index 2908981ccd..2e0f121664 100644 --- a/packages/shared/src/types/agent.ts +++ b/packages/shared/src/types/agent.ts @@ -1,3 +1,4 @@ +import type { AgentLifecycleState } from "./agent-lifecycle.js"; import type { AgentAppearance } from "../agent-appearance.js"; import type { AiConnectionLoginIntent } from "../ai-connections.js"; import type { @@ -90,6 +91,9 @@ export interface Agent { appearance?: AgentAppearance | null; avatarUrl?: string; status: AgentStatus; + lifecycleState?: AgentLifecycleState; + lifecycleVersion?: number; + lifecycleError?: string | null; reportsTo: string | null; capabilities: string | null; adapterType: AgentAdapterType; diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index f149b2a842..ec590e8e14 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -1098,3 +1098,5 @@ export type { AgentInstructionErrorCode, AgentInstructionErrorDetails, AgentInst export type { AgentInstructionCandidate } from "./agent.js"; export * from "./skill-source.js"; + +export * from "./agent-lifecycle.js"; diff --git a/packages/shared/src/types/plugin.ts b/packages/shared/src/types/plugin.ts index 42643d3b00..8bf53cb713 100644 --- a/packages/shared/src/types/plugin.ts +++ b/packages/shared/src/types/plugin.ts @@ -688,6 +688,8 @@ export interface PaperclipPluginManifestV1 { /** Capabilities this plugin requires from the host. Enforced at runtime. */ capabilities: PluginCapability[]; /** Opt into the native pooled-connection catalog, setup, and management UI. */ + /** Required participant in agent preparation, stop, resume, and cleanup. */ + agentLifecycle?: true; aiConnectionRouter?: { name: string; description: string }; /** Entrypoint paths relative to the package root. */ entrypoints: { diff --git a/packages/shared/src/validators/agent.test.ts b/packages/shared/src/validators/agent.test.ts new file mode 100644 index 0000000000..d38767b0c1 --- /dev/null +++ b/packages/shared/src/validators/agent.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { updateAgentSchema } from "./agent.js"; + +describe("agent update schema", () => { + it.each(["paused", "idle", "terminated"])("accepts lifecycle status request %s", status => { + expect(updateAgentSchema.parse({ status })).toEqual({ status }); + }); + it.each(["active", "running", "error", "pending_approval"])("rejects direct status write %s", status => { + const result = updateAgentSchema.safeParse({ name: "Renamed", status }); + expect(result.success).toBe(false); + if (!result.success) expect(result.error.issues).toEqual(expect.arrayContaining([ + expect.objectContaining({ path: ["status"] }), + ])); + }); + it.each([{ spentMonthlyCents: 0 }, { name: "Renamed", spentMonthlyCents: 10 }])("rejects spend totals: %j", payload => { + const result = updateAgentSchema.safeParse(payload); + expect(result.success).toBe(false); + if (!result.success) expect(result.error.issues).toEqual(expect.arrayContaining([ + expect.objectContaining({ path: ["spentMonthlyCents"] }), + ])); + }); + it("accepts profile and budget limit changes", () => { + expect(updateAgentSchema.parse({ name: "Renamed", budgetMonthlyCents: 100 })).toEqual({ name: "Renamed", budgetMonthlyCents: 100 }); + }); +}); diff --git a/packages/shared/src/validators/agent.ts b/packages/shared/src/validators/agent.ts index 9f023c2b59..415bce05c1 100644 --- a/packages/shared/src/validators/agent.ts +++ b/packages/shared/src/validators/agent.ts @@ -4,7 +4,6 @@ import { z } from "zod"; import { AGENT_ICON_NAMES, AGENT_ROLES, - AGENT_STATUSES, INBOX_MINE_ISSUE_STATUS_FILTER, } from "../constants.js"; import { agentAdapterTypeSchema } from "../adapter-type.js"; @@ -155,8 +154,8 @@ export const updateAgentSchema = objectWithoutDefaults( .extend({ permissions: z.never().optional(), replaceAdapterConfig: z.boolean().optional(), - status: z.enum(AGENT_STATUSES).optional(), - spentMonthlyCents: z.number().int().nonnegative().optional(), + status: z.enum(["paused", "idle", "terminated"]).optional(), + spentMonthlyCents: z.never().optional(), }); export type UpdateAgent = z.infer; diff --git a/packages/shared/src/validators/plugin.test.ts b/packages/shared/src/validators/plugin.test.ts index 3c3b46cc9b..683272d130 100644 --- a/packages/shared/src/validators/plugin.test.ts +++ b/packages/shared/src/validators/plugin.test.ts @@ -33,6 +33,12 @@ describe("plugin capability constants", () => { }); describe("plugin manifest validators", () => { + it("requires authority for required agent lifecycle participants", () => { + const manifest = { id: "example.lifecycle", apiVersion: 1, version: "0.1.0", displayName: "Lifecycle", description: "Lifecycle tests", author: "Tests", categories: ["automation"], entrypoints: { worker: "worker.js" }, agentLifecycle: true }; + expect(pluginManifestV1Schema.safeParse({ ...manifest, capabilities: ["agents.read"] }).success).toBe(false); + expect(pluginManifestV1Schema.parse({ ...manifest, capabilities: ["agents.lifecycle.manage"] }).agentLifecycle).toBe(true); + }); + it("requires routing authority for native pooled connector declarations without a custom UI bundle", () => { const manifest = { id: "example.pool", apiVersion: 1, version: "0.1.0", displayName: "Pool", description: "Pool", author: "Tests", categories: ["connector"], entrypoints: { worker: "worker.js" }, aiConnectionRouter: { name: "AI connection pool", description: "Use saved connections" } }; expect(pluginManifestV1Schema.safeParse({ ...manifest, capabilities: ["ui.page.register"] }).success).toBe(false); diff --git a/packages/shared/src/validators/plugin.ts b/packages/shared/src/validators/plugin.ts index 0024977164..8f67466da5 100644 --- a/packages/shared/src/validators/plugin.ts +++ b/packages/shared/src/validators/plugin.ts @@ -782,6 +782,7 @@ export const pluginManifestV1Schema = z.object({ "minimumPaperclipVersion must follow semver (e.g. 1.0.0)", ).optional(), capabilities: z.array(z.enum(PLUGIN_CAPABILITIES)).min(1), + agentLifecycle: z.literal(true).optional(), aiConnectionRouter: z.object({ name: z.string().trim().min(1).max(100), description: z.string().trim().min(1).max(500) }).strict().optional(), entrypoints: z.object({ worker: z.string().min(1), @@ -806,6 +807,10 @@ export const pluginManifestV1Schema = z.object({ launchers: z.array(pluginLauncherDeclarationSchema).optional(), }).optional(), }).superRefine((manifest, ctx) => { + if (manifest.agentLifecycle && !manifest.capabilities.includes("agents.lifecycle.manage")) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Agent lifecycle participation requires agents.lifecycle.manage", path: ["capabilities"] }); + } + // ── Entrypoint ↔ UI slot consistency ────────────────────────────────── // Plugins that declare UI slots must also declare a UI entrypoint so the // host knows where to load the bundle from (PLUGIN_SPEC.md §10.1). diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 285de6537d..ba54e00a77 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,7 +43,7 @@ patchedDependencies: hash: 6st5fzvttegjv64llnfegwwpca path: patches/acpx@0.12.0.patch acpx@0.13.1: - hash: btis2ixbxfyq422xwpnza3ncle + hash: j4huaoi2baejrlq7vdcgzlx54m path: patches/acpx@0.13.1.patch embedded-postgres@18.1.0-beta.16: hash: 55uhvnotpqyiy37rn3pqpukhei @@ -347,15 +347,15 @@ importers: specifier: ^1.1.1 version: 1.1.1 ws: - specifier: ^8.21.3 - version: 8.21.3 + specifier: ^8.22.0 + version: 8.22.0 devDependencies: '@types/node': specifier: ^24.0.0 version: 24.13.3 '@types/ws': - specifier: ^8.18.1 - version: 8.18.1 + specifier: ^8.18.2 + version: 8.18.2 typescript: specifier: ^7.0.2 version: 7.0.2 @@ -508,7 +508,7 @@ importers: version: 0.160.0 acpx: specifier: 0.13.1 - version: 0.13.1(patch_hash=btis2ixbxfyq422xwpnza3ncle) + version: 0.13.1(patch_hash=j4huaoi2baejrlq7vdcgzlx54m) ajv: specifier: ^8.20.0 version: 8.20.0 @@ -982,7 +982,7 @@ importers: version: 8.18.0 acpx: specifier: 0.13.1 - version: 0.13.1(patch_hash=btis2ixbxfyq422xwpnza3ncle) + version: 0.13.1(patch_hash=j4huaoi2baejrlq7vdcgzlx54m) ai: specifier: 7.0.130 version: 7.0.130(zod@4.4.3) @@ -1065,8 +1065,8 @@ importers: specifier: 1.76.1 version: 1.76.1 ws: - specifier: ^8.21.3 - version: 8.21.3 + specifier: ^8.22.0 + version: 8.22.0 zod: specifier: ^4.4.3 version: 4.4.3 @@ -1109,8 +1109,8 @@ importers: specifier: ^7.2.1 version: 7.2.1 '@types/ws': - specifier: ^8.18.1 - version: 8.18.1 + specifier: ^8.18.2 + version: 8.18.2 cross-env: specifier: ^10.1.0 version: 10.1.0 @@ -5349,8 +5349,8 @@ packages: '@types/webxr@0.5.24': resolution: {integrity: sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==} - '@types/ws@8.18.1': - resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@types/ws@8.18.2': + resolution: {integrity: sha512-67MQl+fpWKVTT1NYdnmo3U4sc/xPo/zQBncVnI74qmQa0z/b+1g6iYqNmGCPbxO+zz2aklb08a0oHfegiVd0/w==} '@typescript/typescript-aix-ppc64@7.0.2': resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} @@ -8994,18 +8994,6 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - ws@8.21.3: - resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - ws@8.22.0: resolution: {integrity: sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==} engines: {node: '>=10.0.0'} @@ -10160,11 +10148,11 @@ snapshots: '@discordjs/rest': 2.6.3 '@discordjs/util': 1.2.0 '@sapphire/async-queue': 1.5.5 - '@types/ws': 8.18.1 + '@types/ws': 8.18.2 '@vladfrangu/async_event_emitter': 2.4.7 discord-api-types: 0.38.55 tslib: 2.8.1 - ws: 8.21.3 + ws: 8.22.0 transitivePeerDependencies: - bufferutil - utf-8-validate @@ -12723,9 +12711,9 @@ snapshots: '@slack/logger': 4.0.1 '@slack/web-api': 7.19.0 '@types/node': 24.13.3 - '@types/ws': 8.18.1 + '@types/ws': 8.18.2 eventemitter3: 5.0.4 - ws: 8.21.3 + ws: 8.22.0 transitivePeerDependencies: - bufferutil - debug @@ -13310,7 +13298,7 @@ snapshots: '@types/webxr@0.5.24': {} - '@types/ws@8.18.1': + '@types/ws@8.18.2': dependencies: '@types/node': 24.13.3 @@ -13503,7 +13491,7 @@ snapshots: - bare-buffer - react-native-b4a - acpx@0.13.1(patch_hash=btis2ixbxfyq422xwpnza3ncle): + acpx@0.13.1(patch_hash=j4huaoi2baejrlq7vdcgzlx54m): dependencies: '@agentclientprotocol/sdk': 1.4.0(zod@4.4.3) commander: 15.0.0 @@ -17297,8 +17285,6 @@ snapshots: wrappy@1.0.2: {} - ws@8.21.3: {} - ws@8.22.0: {} wsl-utils@0.1.0: diff --git a/scripts/check-agent-lifecycle-boundaries.mjs b/scripts/check-agent-lifecycle-boundaries.mjs new file mode 100644 index 0000000000..f3b85a8873 --- /dev/null +++ b/scripts/check-agent-lifecycle-boundaries.mjs @@ -0,0 +1,50 @@ +import { readdirSync, readFileSync } from "node:fs"; +import { resolve, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +const protectedFields = new Set(["pauseReason", "pausedAt", "lifecycleState", "lifecycleVersion", "lifecycleError", "lifecycleOperation", "lifecycleRequiredPluginIds", "lifecycleHolds", "adapterType", "adapterConfig", "runtimeConfig", "defaultEnvironmentId"]); +export function agentLifecycleWriteViolations(source, file = "") { + const aliases = [...source.matchAll(/\bagents(?:\s+as\s+(\w+))?\s*[,}]/g)].map(match => match[1] ?? "agents"); + const violations = []; + const text = source.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^[ \t]*\/\/.*$/gm, ""); + for (const alias of new Set(aliases)) { + for (const match of text.matchAll(new RegExp(`\\.(insert|delete|update)\\(\\s*${alias}\\s*\\)([^;]*)`, "g"))) { + const [, operation, chain] = match; + if (operation !== "update") { violations.push(operation === "delete" + ? "Use the agent lifecycle module to delete an agent" + : "Use the agent lifecycle module to insert an agent"); continue; } + for (const field of protectedFields) { + // The offline seed copy disables timers before any server can verify or run an agent. + if (field === "runtimeConfig" && file === "cli/src/commands/worktree.ts") continue; + if (new RegExp(`(? `import { agents as records } from "@paperclipai/db"; ${body}`; +test("finds inserts, deletes, and lifecycle writes through an import alias", () => { + for (const body of ["db.insert(records).values(input)", "db.delete(records)", "db.update(records).set({ lifecycleState: 'ready' })", "db.update(records).set({ pauseReason })"]) { + assert.equal(agentLifecycleWriteViolations(source(body)).length, 1); + } +}); +test("permits configuration writes and guards execution status writes", () => { + assert.deepEqual(agentLifecycleWriteViolations(source("db.update(records).set({ name })")), []); + assert.equal(agentLifecycleWriteViolations(source("db.update(records).set({ status: 'idle' })")).length, 1); + assert.deepEqual(agentLifecycleWriteViolations(source('db.update(records).set({ status: "idle" }).where(eq(records.lifecycleState, "ready"))')), []); +}); + +test("execution configuration writes must use lifecycle invalidation", () => { + for (const field of ["adapterType", "adapterConfig", "runtimeConfig", "defaultEnvironmentId"]) { + assert.equal(agentLifecycleWriteViolations(source(`db.update(records).set({ ${field}: value })`)).length, 1); + } +}); + +test("offline seed quarantine can disable timers but cannot change lifecycle state", () => { + const file = "cli/src/commands/worktree.ts"; + assert.deepEqual(agentLifecycleWriteViolations(source("db.update(records).set({ runtimeConfig })"), file), []); + assert.equal(agentLifecycleWriteViolations(source("db.update(records).set({ lifecycleState: 'ready' })"), file).length, 1); +}); + +test("rejects agent deletion outside the module even with a terminal state guard", () => { + assert.equal(agentLifecycleWriteViolations(source('db.delete(records).where(and(eq(records.id, id), inArray(records.lifecycleState, ["terminated", "rejected"])))')).length, 1); + assert.equal(agentLifecycleWriteViolations(source('db.delete(records).where(inArray(records.lifecycleState, ["ready", "terminated"]))')).length, 1); +}); diff --git a/scripts/check-module-boundaries.mjs b/scripts/check-module-boundaries.mjs index eed66df002..21f8f48d0e 100644 --- a/scripts/check-module-boundaries.mjs +++ b/scripts/check-module-boundaries.mjs @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { scanAgentLifecycleBoundaries } from "./check-agent-lifecycle-boundaries.mjs"; import { readdirSync, readFileSync } from "node:fs"; import { dirname, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; @@ -9,6 +10,10 @@ const defaultServerSrc = resolve(repoRoot, "server/src"); const defaultModulesRoot = resolve(defaultServerSrc, "modules"); const layerNames = new Set(["domain", "application", "adapters"]); const databasePackages = ["@paperclipai/db", "drizzle-orm", "embedded-postgres", "postgres"]; +const configurationTransactionCallers = new Set([ + "services/secret-proposals.ts", "services/connection-intents.ts", "services/dot-runner-broker.ts", + "services/public-mcp/oauth.ts", "services/agent-profile-avatar.ts", "services/agent-instruction-revisions.ts", "services/company-skills.ts", +]); function normalizedRelative(from, to) { return relative(from, to).split(sep).join("/"); @@ -99,6 +104,18 @@ export function scanModuleBoundaries({ const targetSegments = targetServerSegments(serverSrc, target); const targetLocation = target ? moduleLocation(modulesRoot, target) : null; + if (/^services\/agent-configuration-transaction\.(?:js|ts)$/.test(targetSegments.join("/")) && + !configurationTransactionCallers.has(normalizedRelative(serverSrc, sourceFile))) { + addViolation(violations, sourceLabel, sourceLocation?.layer ?? null, specifier, + "configuration transaction integration is limited to its explicit workflow callers"); + } + + if (sourceLocation?.moduleName === "agent-lifecycle" && + (targetSegments.includes("services") || targetSegments.includes("routes"))) { + addViolation(violations, sourceLabel, sourceLocation.layer, specifier, + "agent lifecycle must receive service integrations through its ports"); + } + if (sourceLocation?.layer === "domain") { if (isDatabasePackage(specifier)) { addViolation(violations, sourceLabel, "domain", specifier, "domain cannot import database packages"); @@ -135,7 +152,11 @@ export function scanModuleBoundaries({ if (targetLocation && sourceLocation?.moduleName !== targetLocation.moduleName) { const targetRelative = normalizedRelative(resolve(modulesRoot, targetLocation.moduleName), target); - if (targetRelative !== "index.js" && targetRelative !== "index.ts") { + const configurationIntegration = /^configuration\.(?:js|ts)$/.test(targetRelative) && + normalizedRelative(serverSrc, sourceFile) === "services/agent-configuration-transaction.ts"; + const companyDeletionIntegration = /^company-deletion\.(?:js|ts)$/.test(targetRelative) && + normalizedRelative(serverSrc, sourceFile) === "services/company-deletion.ts"; + if (targetRelative !== "index.js" && targetRelative !== "index.ts" && !companyDeletionIntegration && !configurationIntegration) { addViolation( violations, sourceLabel, @@ -158,13 +179,15 @@ export function formatViolation(violation) { function main() { const violations = scanModuleBoundaries(); + const lifecycleViolations = scanAgentLifecycleBoundaries(); + if (lifecycleViolations.length) { console.error(lifecycleViolations.join("\n")); process.exitCode = 1; } if (violations.length > 0) { console.error("Feature module boundary check failed:"); for (const violation of violations) console.error(`- ${formatViolation(violation)}`); process.exitCode = 1; return; } - console.log("Feature module boundary check passed."); + if (!lifecycleViolations.length) console.log("Feature module boundary check passed."); } if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) main(); diff --git a/scripts/check-module-boundaries.test.mjs b/scripts/check-module-boundaries.test.mjs index 671025cec3..94f7eeb452 100644 --- a/scripts/check-module-boundaries.test.mjs +++ b/scripts/check-module-boundaries.test.mjs @@ -1,3 +1,4 @@ +import "./check-agent-lifecycle-boundaries.test.mjs"; import assert from "node:assert/strict"; import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -88,3 +89,57 @@ test("scanModuleBoundaries rejects outward dependencies and module-internal impo test("the repository's feature modules satisfy their import boundaries", () => { assert.deepEqual(scanModuleBoundaries(), []); }); + +test("only the company deletion coordinator can import module deletion entry points", () => { + const serverSrc = mkdtempSync(join(tmpdir(), "paperclip-company-deletion-boundaries-")); + const modulesRoot = join(serverSrc, "modules"); + try { + mkdirSync(join(serverSrc, "services")); + mkdirSync(join(serverSrc, "routes")); + const integration = 'import { deletion } from "../modules/example/company-deletion.js";'; + writeFileSync(join(serverSrc, "services", "company-deletion.ts"), integration); + writeFileSync(join(serverSrc, "services", "other.ts"), integration); + writeFileSync(join(serverSrc, "routes", "companies.ts"), integration); + const violations = scanModuleBoundaries({ serverSrc, modulesRoot }); + assert.equal(violations.length, 2); + assert(violations.some(({ file }) => file.endsWith("services/other.ts"))); + assert(violations.some(({ file }) => file.endsWith("routes/companies.ts"))); + } finally { + rmSync(serverSrc, { recursive: true, force: true }); + } +}); + +test("configuration integration is limited to its composition and workflow callers", () => { + const serverSrc = mkdtempSync(join(tmpdir(), "paperclip-configuration-boundaries-")); + const modulesRoot = join(serverSrc, "modules"); + try { + mkdirSync(join(serverSrc, "services")); + mkdirSync(join(serverSrc, "routes")); + const integration = 'import { update } from "../modules/agent-lifecycle/configuration.js";'; + const workflow = 'import { update } from "./agent-configuration-transaction.js";'; + writeFileSync(join(serverSrc, "services", "agent-configuration-transaction.ts"), integration); + writeFileSync(join(serverSrc, "services", "secret-proposals.ts"), workflow); + writeFileSync(join(serverSrc, "services", "other.ts"), workflow); + writeFileSync(join(serverSrc, "routes", "agents.ts"), integration); + const violations = scanModuleBoundaries({ serverSrc, modulesRoot }); + assert.equal(violations.length, 2); + assert(violations.some(({ file }) => file.endsWith("services/other.ts"))); + assert(violations.some(({ file }) => file.endsWith("routes/agents.ts"))); + } finally { + rmSync(serverSrc, { recursive: true, force: true }); + } +}); + +test("agent lifecycle adapters and entry point cannot depend on services", () => { + const serverSrc = mkdtempSync(join(tmpdir(), "paperclip-lifecycle-boundaries-")); + const modulesRoot = join(serverSrc, "modules"); + try { + mkdirSync(join(modulesRoot, "agent-lifecycle", "adapters"), { recursive: true }); + writeFileSync(join(modulesRoot, "agent-lifecycle", "index.ts"), 'import "../../services/example.js";'); + writeFileSync(join(modulesRoot, "agent-lifecycle", "adapters", "records.ts"), 'import "../../../services/example.js";'); + assert.equal(scanModuleBoundaries({ serverSrc, modulesRoot }).filter(item => + item.reason === "agent lifecycle must receive service integrations through its ports").length, 2); + } finally { + rmSync(serverSrc, { recursive: true, force: true }); + } +}); diff --git a/server/package.json b/server/package.json index 81cd24e5f1..b0fa25129f 100644 --- a/server/package.json +++ b/server/package.json @@ -104,7 +104,7 @@ "smol-toml": "^1.4.2", "ssh2": "^1.17.0", "svix": "1.76.1", - "ws": "^8.21.3", + "ws": "^8.22.0", "zod": "^4.4.3" }, "bundleDependencies": [ @@ -127,7 +127,7 @@ "@types/node": "^24.0.0", "@types/sharp": "^0.32.0", "@types/supertest": "^7.2.1", - "@types/ws": "^8.18.1", + "@types/ws": "^8.18.2", "cross-env": "^10.1.0", "supertest": "^7.0.0", "tsx": "^4.23.15", diff --git a/packages/db/src/seed.ts b/server/scripts/seed.ts similarity index 85% rename from packages/db/src/seed.ts rename to server/scripts/seed.ts index ff6e4f63d3..452ad348e6 100644 --- a/packages/db/src/seed.ts +++ b/server/scripts/seed.ts @@ -1,5 +1,11 @@ -import { createDb } from "./client.js"; -import { companies, agents, goals, projects, issues } from "./schema/index.js"; +import { createDb } from "@paperclipai/db"; +import { createAgentLifecycle } from "../src/services/agent-lifecycle.js"; +import { + companies, + goals, + projects, + issues, +} from "@paperclipai/db"; const url = process.env.DATABASE_URL; if (!url) throw new Error("DATABASE_URL is required"); @@ -18,10 +24,7 @@ const [company] = await db }) .returning(); -const [ceo] = await db - .insert(agents) - .values({ - companyId: company!.id, +const ceo = await createAgentLifecycle(db).requestHire(company!.id, { name: "CEO Agent", role: "ceo", title: "Chief Executive Officer", @@ -29,13 +32,9 @@ const [ceo] = await db adapterType: "process", adapterConfig: { command: "echo", args: ["hello from ceo"] }, budgetMonthlyCents: 15000, - }) - .returning(); + }); -const [engineer] = await db - .insert(agents) - .values({ - companyId: company!.id, +const engineer = await createAgentLifecycle(db).requestHire(company!.id, { name: "Engineer Agent", role: "engineer", title: "Software Engineer", @@ -44,8 +43,7 @@ const [engineer] = await db adapterType: "process", adapterConfig: { command: "echo", args: ["hello from engineer"] }, budgetMonthlyCents: 10000, - }) - .returning(); + }); const [goal] = await db .insert(goals) diff --git a/server/scripts/smoke-native-task-monitor.ts b/server/scripts/smoke-native-task-monitor.ts index 76d327bd93..0231bdba09 100644 --- a/server/scripts/smoke-native-task-monitor.ts +++ b/server/scripts/smoke-native-task-monitor.ts @@ -1,4 +1,4 @@ -/** Opt-in live qualification: creates an isolated database/workspace and spends two Codex turns. +/** Opt-in live qualification: creates an isolated database/workspace and runs a harness test and two Codex turns. * PATH=::$PATH node --import tsx server/scripts/smoke-native-task-monitor.ts --run */ import assert from "node:assert/strict"; @@ -17,6 +17,9 @@ process.env.PAPERCLIP_INSTANCE_ID = "monitor-smoke"; process.env.PAPERCLIP_TELEMETRY_ENABLED = "false"; const { createDb, companies, agents, authUsers, companyMemberships, issues, heartbeatRuns, agentWakeupRequests, nativeRunResults, statusDecisions } = await import("@paperclipai/db"); const { startEmbeddedPostgresTestDatabase } = await import("../src/__tests__/helpers/embedded-postgres.js"); +const { createAgentLifecycle, startAgentLifecycle } = await import("../src/services/agent-lifecycle.js"); +const { createLifecycleDriver } = await import("../src/services/agent-lifecycle-driver.js"); +const { createPluginWorkerManager } = await import("../src/services/plugin-worker-manager.js"); const { heartbeatService } = await import("../src/services/heartbeat.js"); const { setupRunnerPrpWebSocketServer, runnerPrpWebSocketInternals } = await import("../src/realtime/runner-prp-ws.js"); const { closeIdleWarmNativeSessionsForRestart } = await import("../src/services/native-runtime/native-session-executor.js"); @@ -30,6 +33,7 @@ process.env.PAPERCLIP_API_URL = `http://127.0.0.1:${address.port}`; setupRunnerPrpWebSocketServer(server, { apiUrl: process.env.PAPERCLIP_API_URL }); const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); const heartbeat = heartbeatService(db); +const lifecycle = startAgentLifecycle(db, createLifecycleDriver(db, createPluginWorkerManager()), () => true); const readRuns = () => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)).orderBy(heartbeatRuns.createdAt); console.log(JSON.stringify({ root, companyId, agentId, issueId })); try { @@ -38,9 +42,18 @@ try { await db.insert(companies).values({ id: companyId, name: "Isolated monitor smoke", issuePrefix: "MON", defaultResponsibleUserId: "monitor-smoke", requireBoardApprovalForNewAgents: false }); await db.insert(authUsers).values({ id: "monitor-smoke", name: "Monitor smoke", email: "monitor-smoke@example.test", createdAt: new Date(), updatedAt: new Date() }); await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "monitor-smoke", status: "active", membershipRole: "owner" }); - await db.insert(agents).values({ id: agentId, companyId, name: "Monitor verifier", status: "active", adapterType: "paperclip_runner", + await createAgentLifecycle(db).requestHire(companyId, { id: agentId, name: "Monitor verifier", status: "active", adapterType: "paperclip_runner", adapterConfig: { provider: "codex", cwd, lifecycleMode: "warm", idleTimeoutMs: 300_000, timeoutSeconds: 180 }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } }); + const setupDeadline = Date.now() + 3 * 60_000; + while (true) { + const agent = await createAgentLifecycle(db).get(agentId); + assert.ok(agent, "the hired agent must exist"); + assert.equal(agent.lifecycleError, null, "agent setup failed"); + if (agent.lifecycleState === "ready") break; + assert.ok(Date.now() < setupDeadline, "agent setup timed out"); + await delay(250); + } await db.insert(issues).values({ id: issueId, companyId, title: "Verify a one-shot native task monitor", identifier: "MON-1", status: "in_progress", assigneeAgentId: agentId, description: "This is an isolated two-run integration test. First run: use set_task_monitor on this task with a fresh nextCheckAt about 45 seconds in the future (compute the current UTC time if needed), notes saying to verify issue_monitor_due, and idempotencyKey monitor-live-first. Confirm the receipt. Then call paperclip_finish with yielded and continuation.kind monitor, accurately retaining the second run as blocking remaining work. End immediately after acceptance. Do not sleep or poll. Second run: if the wake reason is issue_monitor_due, the requested check succeeded. Report that reason and finish done using the current completion contract. Do not schedule another monitor or request human review. No files or other deliverables are required." }); await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", reason: "monitor_smoke", @@ -90,6 +103,7 @@ try { await writeFile(join(root, "evidence.json"), JSON.stringify(evidence, null, 2)); console.log(JSON.stringify({ passed: true, evidence: join(root, "evidence.json"), warmReuse: evidence.warmReuse })); } finally { + await lifecycle.stop(); await writeFile(join(root, "run-records.json"), JSON.stringify(await readRuns(), null, 2)); await closeIdleWarmNativeSessionsForRestart(); runnerPrpWebSocketInternals.resetForTests(); diff --git a/server/scripts/verify-agent-commentary-live.ts b/server/scripts/verify-agent-commentary-live.ts index 78423b1849..d408ecf951 100644 --- a/server/scripts/verify-agent-commentary-live.ts +++ b/server/scripts/verify-agent-commentary-live.ts @@ -2,6 +2,7 @@ * Uses real Codex, disposable database/workspaces, and a private copy of login credentials. * Writes only content-free evidence to stdout; never runs as part of the unit suite. */ +import { createAgentLifecycle } from "../src/services/agent-lifecycle.js"; import assert from "node:assert/strict"; import { randomUUID } from "node:crypto"; import { chmod, copyFile, lstat, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; @@ -63,7 +64,7 @@ try { let result; if (mode === "legacy") { await server.db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null }).where(eq(heartbeatRuns.id, f.runId)); - await server.db.update(agents).set({ adapterType: "codex_local" }).where(eq(agents.id, f.agentId)); + await createAgentLifecycle(server.db).updateConfiguration(f.agentId, { adapterType: "codex_local" }); result = await executeLegacy({ runId: f.runId, agent: { id: f.agentId, companyId: f.companyId, name: "Feedback smoke", adapterType: "codex_local", adapterConfig: {} }, diff --git a/server/src/__tests__/adapter-auth-signal-routes.test.ts b/server/src/__tests__/adapter-auth-signal-routes.test.ts index 3a8fc01b97..8360e3e133 100644 --- a/server/src/__tests__/adapter-auth-signal-routes.test.ts +++ b/server/src/__tests__/adapter-auth-signal-routes.test.ts @@ -413,3 +413,9 @@ describe("adapter auth-signal route", () => { expect(mockResolveEnvironmentExecutionTarget).not.toHaveBeenCalled(); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + }) }; +}); diff --git a/server/src/__tests__/agent-adapter-validation-routes.test.ts b/server/src/__tests__/agent-adapter-validation-routes.test.ts index 1aa1217977..335381d98b 100644 --- a/server/src/__tests__/agent-adapter-validation-routes.test.ts +++ b/server/src/__tests__/agent-adapter-validation-routes.test.ts @@ -1199,3 +1199,10 @@ describe("agent routes adapter validation", () => { expect(mockAgentService.update).toHaveBeenCalledOnce(); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + requestHire: (...args: unknown[]) => mockAgentService.create(...args), + }) }; +}); diff --git a/server/src/__tests__/agent-commentary.integration.test.ts b/server/src/__tests__/agent-commentary.integration.test.ts index 3fce4e97a3..bc0655633d 100644 --- a/server/src/__tests__/agent-commentary.integration.test.ts +++ b/server/src/__tests__/agent-commentary.integration.test.ts @@ -212,7 +212,10 @@ describe("internal agent commentary through both transports", () => { await server.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId, status: "running", invocationSource: "on_demand" }); await submitAgentCommentary(server.db, { companyId: f.companyId, agentId, runId }, { ...input, kind: "complaint" }); expect((await rows({ ...f, runId }))[0].issueId).toBeNull(); - if (owner === "agent") await agentService(server.db).remove(agentId); + if (owner === "agent") { + await server.db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, agentId)); + await agentService(server.db).remove(agentId); + } else await companyService(server.db).remove(f.companyId); expect(await rows({ ...f, runId })).toHaveLength(0); }); diff --git a/server/src/__tests__/agent-cross-tenant-authz-routes.test.ts b/server/src/__tests__/agent-cross-tenant-authz-routes.test.ts index c7c1af4931..6884957481 100644 --- a/server/src/__tests__/agent-cross-tenant-authz-routes.test.ts +++ b/server/src/__tests__/agent-cross-tenant-authz-routes.test.ts @@ -755,3 +755,13 @@ describe("agent cross-tenant route authorization", () => { expect(mockLogActivity).not.toHaveBeenCalled(); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + requestHire: (...args: unknown[]) => mockAgentService.create(...args), + pauseAgent: (...args: unknown[]) => mockAgentService.pause(...args), + resumeAgent: (...args: unknown[]) => mockAgentService.resume(...args), + terminateAgent: (...args: unknown[]) => mockAgentService.terminate(...args), + }) }; +}); diff --git a/server/src/__tests__/agent-default-configure-grants.test.ts b/server/src/__tests__/agent-default-configure-grants.test.ts index 3094e12f70..b1bfa96d51 100644 --- a/server/src/__tests__/agent-default-configure-grants.test.ts +++ b/server/src/__tests__/agent-default-configure-grants.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { eq } from "drizzle-orm"; @@ -50,7 +51,7 @@ describeDatabase("new agent configuration defaults", () => { issuePrefix: `PD${companyId.slice(0, 6).toUpperCase()}`, }); const create = (name: string, permissions: Record = {}, metadata?: Record) => - agentService(db).create(companyId, { + createAgentLifecycle(db).requestHire(companyId, { name, role: "engineer", adapterType: "process", @@ -130,6 +131,7 @@ describeDatabase("new agent configuration defaults", () => { expect(await db.select().from(principalPermissionGrants) .where(eq(principalPermissionGrants.principalId, standard.id))).toEqual([]); + await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, peer.id)); await agentService(db).remove(peer.id); expect(await db.select().from(principalPermissionGrants) .where(eq(principalPermissionGrants.principalId, peer.id))).toEqual([]); @@ -190,7 +192,7 @@ describeDatabase("new agent configuration defaults", () => { name: "Pending permission default", issuePrefix: `PP${companyId.slice(0, 6).toUpperCase()}`, }); - const pending = await agentService(db).create(companyId, { + const pending = await createAgentLifecycle(db).requestHire(companyId, { name: "Pending hire", status: "pending_approval", role: "engineer", @@ -201,7 +203,7 @@ describeDatabase("new agent configuration defaults", () => { expect(await db.select().from(principalPermissionGrants) .where(eq(principalPermissionGrants.companyId, companyId))).toEqual([]); - const result = await agentService(db).activatePendingApproval(pending.id); + const result = await createAgentLifecycle(db).approveHire(pending.id); expect(result?.activated).toBe(true); expect((await db.select().from(principalPermissionGrants) .where(eq(principalPermissionGrants.companyId, companyId))) @@ -220,7 +222,7 @@ describeDatabase("new agent configuration defaults", () => { name: "Invited permission default", issuePrefix: `PI${companyId.slice(0, 6).toUpperCase()}`, }); - const invited = await agentService(db).create(companyId, { + const invited = await createAgentLifecycle(db).requestHire(companyId, { name: "Invited agent", role: "engineer", adapterType: "process", diff --git a/server/src/__tests__/agent-device-login-routes.test.ts b/server/src/__tests__/agent-device-login-routes.test.ts index 32488c6095..61b617d212 100644 --- a/server/src/__tests__/agent-device-login-routes.test.ts +++ b/server/src/__tests__/agent-device-login-routes.test.ts @@ -1709,3 +1709,10 @@ describe("adapter device-login routes", () => { expect(loggedText).not.toContain("provider-lease-"); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + requestHire: (...args: unknown[]) => mockAgentService.create(...args), + }) }; +}); diff --git a/server/src/__tests__/agent-harness-verification.test.ts b/server/src/__tests__/agent-harness-verification.test.ts new file mode 100644 index 0000000000..4728cdabd1 --- /dev/null +++ b/server/src/__tests__/agent-harness-verification.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import type { AdapterEnvironmentTestResult } from "@paperclipai/adapter-utils"; +import { ADAPTER_AUTH_MISSING_CHECK_CODE } from "@paperclipai/shared"; +import { assertHarnessTestPassed } from "../services/agent-harness-verification.js"; + +describe("saved harness verification", () => { + const result = (status: AdapterEnvironmentTestResult["status"], codes: string[]): AdapterEnvironmentTestResult => ({ + adapterType: "test", status, testedAt: new Date().toISOString(), + checks: codes.map(code => ({ code, level: "info", message: code })), + }); + it.each([ + ["fail", []], ["warn", ["claude_hello_probe_auth_required"]], + ["warn", ["gemini_hello_probe_timed_out"]], ["pass", ["claude_hello_probe_skipped_custom_command"]], + ["warn", ["claude_acp_anthropic_api_key_detected", "claude_acp_login_probe_unavailable"]], + ["warn", ["cli_version_probe_mismatch"]], ["pass", [ADAPTER_AUTH_MISSING_CHECK_CODE]], + ["warn", ["claude_hello_probe_passed", "claude_hello_probe_auth_required"]], + ] as const)("rejects %s with %j", (status, codes) => { + expect(() => assertHarnessTestPassed(result(status, [...codes]))).toThrow("The harness test failed"); + }); + it.each([ + ["pass", ["process_command_available"]], ["warn", ["claude_hello_probe_passed", "optional_environment_warning"]], + ["warn", ["claude_acp_anthropic_api_key_detected", "claude_hello_probe_passed"]], + ["pass", ["codex_hello_probe_succeeded"]], ["warn", ["ai_connection_api_key_reverified"]], + ] as const)("accepts %s with %j", (status, codes) => { + expect(() => assertHarnessTestPassed(result(status, [...codes]))).not.toThrow(); + }); +}); diff --git a/server/src/__tests__/agent-hire-ai-connections.test.ts b/server/src/__tests__/agent-hire-ai-connections.test.ts index e1be98a92b..86ca31b72f 100644 --- a/server/src/__tests__/agent-hire-ai-connections.test.ts +++ b/server/src/__tests__/agent-hire-ai-connections.test.ts @@ -22,6 +22,7 @@ import { aiConnectionRouterService } from "../services/ai-connection-router.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { toolAccessService } from "../services/tool-access.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; +import { configureAgentLifecycle } from "../services/agent-lifecycle.js"; const captureRunFailure = vi.hoisted(() => vi.fn()); vi.mock("../sentry.js", async (importOriginal) => ({ @@ -374,7 +375,7 @@ describe("agent-created hires use managed AI connections", () => { const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Other provider", role: "engineer", adapterType })); expect(agent.runtimeConfig.aiConnection).toMatchObject({ provider: otherProvider, mode: "responsible_user" }); await expect(prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: agent.id, responsibleUserId: f.userId, adapterType, binding: agent.runtimeConfig.aiConnection, config: agent.adapterConfig })).rejects.toMatchObject({ details: { code: "ai_connection_default_missing" } }); - expect(agent.status).toBe("idle"); + expect(agent).toMatchObject({ status: "paused", lifecycleState: "preparing" }); }); for (const endpoint of ["agent-hires", "agents"]) { @@ -486,11 +487,17 @@ describe("agent-created hires use managed AI connections", () => { }); describe("hired agents sharing a subscription", () => { + async function finishSetup(agentId: string) { + // These tests cover task credentials after setup; lifecycle tests cover verification. + const worker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" }); + try { await worker.process(agentId); } finally { await worker.stop(); } + } it("keeps a credential-lock timeout on automatic retry without blocking the task or starting a provider", async () => { const f = await fixture("openai", "subscription"); const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Waiting teammate", role: "engineer", adapterType: f.adapterType, reportsTo: f.agentId, adapterConfig: { engine: "cli" }, runtimeConfig: { heartbeat: { enabled: false } } })); // Only an operator may set host execution paths after the agent is hired. await db.update(agents).set({ adapterConfig: { ...agent.adapterConfig, cwd: home } }).where(eq(agents.id, agent.id)); + await finishSetup(agent.id); const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Wait for credential rotation", status: "todo", assigneeAgentId: agent.id, responsibleUserId: f.userId, createdByUserId: f.userId }).returning(); const execute = vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false, resultJson: {} })); registerServerAdapter({ ...getServerAdapter(f.adapterType), execute }); @@ -534,6 +541,7 @@ describe("hired agents sharing a subscription", () => { const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Concurrent teammate", role: "engineer", adapterType: f.adapterType, reportsTo: f.agentId, adapterConfig: { engine: "cli" }, runtimeConfig: { heartbeat: { enabled: false } } })); // Host working directories are configured by an operator, not an agent key. await db.update(agents).set({ adapterConfig: { ...agent.adapterConfig, cwd: home } }).where(eq(agents.id, agent.id)); + await finishSetup(agent.id); const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Subscription child task", status: "todo", assigneeAgentId: agent.id, responsibleUserId: f.userId, createdByUserId: f.userId }).returning(); const parentRuntime = await prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: f.agentId, responsibleUserId: f.userId, adapterType: f.adapterType, binding: f.binding, config: { cwd: home } }); const execute = vi.fn(async () => { diff --git a/server/src/__tests__/agent-identity.test.ts b/server/src/__tests__/agent-identity.test.ts index dcdbf1189b..d0e7dcbe24 100644 --- a/server/src/__tests__/agent-identity.test.ts +++ b/server/src/__tests__/agent-identity.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomBytes, randomUUID, verify, generateKeyPairSync } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -74,7 +75,7 @@ describe("agent cryptographic identity", () => { }); it("creates identities in the agent creation transaction and rolls both writes back on failure", async () => { - const created = await agentService(db).create(companyId, { name: "New identity", role: "engineer", status: "pending_approval" }); + const created = await createAgentLifecycle(db).requestHire(companyId, { name: "New identity", role: "engineer", status: "pending_approval" }); expect(await agentIdentityService(db).getPublicIdentity(companyId, created.id)).toMatchObject({ algorithm: "Ed25519" }); const id = randomUUID(); await expect(db.transaction(async tx => { @@ -85,7 +86,7 @@ describe("agent cryptographic identity", () => { expect(await db.select().from(agents).where(eq(agents.id, id))).toHaveLength(0); expect(await agentIdentityService(db).getPublicIdentity(companyId, id)).toBeNull(); const fail = vi.spyOn(localEncryptedProvider, "createSecret").mockRejectedValueOnce(new Error("encryption unavailable")); - await expect(agentService(db).create(companyId, { name: "Must roll back", role: "engineer", status: "pending_approval" })).rejects.toThrow("encryption unavailable"); + await expect(createAgentLifecycle(db).requestHire(companyId, { name: "Must roll back", role: "engineer", status: "pending_approval" })).rejects.toThrow("encryption unavailable"); fail.mockRestore(); expect(await db.select().from(agents).where(eq(agents.name, "Must roll back"))).toHaveLength(0); }); diff --git a/server/src/__tests__/agent-instructions-routes.test.ts b/server/src/__tests__/agent-instructions-routes.test.ts index a16333701a..4f7a65449b 100644 --- a/server/src/__tests__/agent-instructions-routes.test.ts +++ b/server/src/__tests__/agent-instructions-routes.test.ts @@ -844,3 +844,10 @@ describe("agent instructions bundle routes", () => { expect(res.body.adapterConfig.instructionsFilePath).toBeUndefined(); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + requestHire: (...args: unknown[]) => mockAgentService.create(...args), + }) }; +}); diff --git a/server/src/__tests__/agent-live-run-routes.test.ts b/server/src/__tests__/agent-live-run-routes.test.ts index 6210ac9947..60605adf32 100644 --- a/server/src/__tests__/agent-live-run-routes.test.ts +++ b/server/src/__tests__/agent-live-run-routes.test.ts @@ -1940,3 +1940,9 @@ describe("agent live run routes", () => { expect(mockWorkspaceDiffReprojection.persist).not.toHaveBeenCalled(); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + }) }; +}); diff --git a/server/src/__tests__/agent-permissions-routes.test.ts b/server/src/__tests__/agent-permissions-routes.test.ts index 72570a9e9d..0ad2e67909 100644 --- a/server/src/__tests__/agent-permissions-routes.test.ts +++ b/server/src/__tests__/agent-permissions-routes.test.ts @@ -73,8 +73,8 @@ const mockApprovalService = vi.hoisted(() => ({ create: vi.fn(), getById: vi.fn(), findOpenHireApprovalForAgent: vi.fn(), - approve: vi.fn(), - reject: vi.fn(), + approveHire: vi.fn(), + rejectHire: vi.fn(), })); const mockBudgetService = vi.hoisted(() => ({ @@ -317,8 +317,8 @@ describe("agent permission routes", () => { mockApprovalService.create.mockReset(); mockApprovalService.getById.mockReset(); mockApprovalService.findOpenHireApprovalForAgent.mockReset(); - mockApprovalService.approve.mockReset(); - mockApprovalService.reject.mockReset(); + mockApprovalService.approveHire.mockReset(); + mockApprovalService.rejectHire.mockReset(); mockBudgetService.upsertPolicy.mockReset(); mockHeartbeatService.listTaskSessions.mockReset(); mockHeartbeatService.resetRuntimeSession.mockReset(); @@ -679,6 +679,16 @@ describe("agent permission routes", () => { ]); }); + it.each([{ spentMonthlyCents: 0 }, { name: "Renamed", spentMonthlyCents: 0 }])( + "rejects accounting fields before applying an agent update: %j", async payload => { + const app = await createApp({ type: "board", userId: "board-user", source: "session", isInstanceAdmin: true, companyIds: [companyId] }); + const res = await requestApp(app, baseUrl => request(baseUrl).patch(`/api/agents/${agentId}`).send(payload)); + expect(res.status).toBe(400); + expect(res.body.details).toEqual(expect.arrayContaining([expect.objectContaining({ path: ["spentMonthlyCents"] })])); + expect(mockAgentService.update).not.toHaveBeenCalled(); + }, + ); + it("blocks agent updates for authenticated company members without agent admin permission", async () => { mockAccessService.canUser.mockResolvedValue(false); @@ -1251,7 +1261,7 @@ describe("agent permission routes", () => { expect.objectContaining({ status: "idle", }), - { createdByUserId: "agent-admin-user", claudeLogin: { storedSessionId: null, ownerUserId: "agent-admin-user", applyExistingWithoutClaim: false } }, + { createdByUserId: "agent-admin-user", responsibleUserId: "agent-admin-user", claudeLogin: { storedSessionId: null, ownerUserId: "agent-admin-user", applyExistingWithoutClaim: false } }, ); expect(mockAccessService.setPrincipalPermission).toHaveBeenCalledWith( companyId, @@ -1381,7 +1391,7 @@ describe("agent permission routes", () => { }, }, }), - { createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, + { createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, ); }); @@ -1417,7 +1427,7 @@ describe("agent permission routes", () => { model: DEFAULT_OPENCODE_LOCAL_MODEL, }), }), - { createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, + { createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, ); }); @@ -1455,7 +1465,7 @@ describe("agent permission routes", () => { model: "anthropic/claude-sonnet-4-5", }), }), - { createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, + { createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, ); }); @@ -1495,7 +1505,7 @@ describe("agent permission routes", () => { }, }), { - createdByUserId: "board-user", + createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", @@ -1516,9 +1526,10 @@ describe("agent permission routes", () => { status: "idle", }; mockAgentService.getById.mockResolvedValue(pendingAgent); - mockAgentService.activatePendingApproval.mockResolvedValue({ + mockApprovalService.approveHire.mockResolvedValue({ agent: approvedAgent, - activated: true, + applied: true, + approval: null, }); const app = await createApp({ @@ -1534,8 +1545,7 @@ describe("agent permission routes", () => { .send({})); expect(res.status).toBe(200); - expect(mockAgentService.activatePendingApproval).toHaveBeenCalledWith(agentId); - expect(mockApprovalService.approve).not.toHaveBeenCalled(); + expect(mockApprovalService.approveHire).toHaveBeenCalledWith(agentId, "board-user"); expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ companyId, actorType: "user", @@ -1556,19 +1566,9 @@ describe("agent permission routes", () => { ...baseAgent, status: "idle", }; - // First getById (getAccessibleAgent) sees the pending agent; the second - // (after the approval resolves) sees the activated agent. - mockAgentService.getById - .mockResolvedValueOnce(pendingAgent) - .mockResolvedValue(approvedAgent); - mockApprovalService.findOpenHireApprovalForAgent.mockResolvedValue({ - id: "approval-1", - companyId, - type: "hire_agent", - status: "pending", - payload: { agentId }, - }); - mockApprovalService.approve.mockResolvedValue({ + mockAgentService.getById.mockResolvedValue(pendingAgent); + mockApprovalService.approveHire.mockResolvedValue({ + agent: approvedAgent, approval: { id: "approval-1", status: "approved" }, applied: true, }); @@ -1587,7 +1587,7 @@ describe("agent permission routes", () => { expect(res.status).toBe(200); // The shared approval flow handles activation; we must not double-activate. - expect(mockApprovalService.approve).toHaveBeenCalledWith("approval-1", "board-user"); + expect(mockApprovalService.approveHire).toHaveBeenCalledWith(agentId, "board-user"); expect(mockAgentService.activatePendingApproval).not.toHaveBeenCalled(); expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: "agent.approved", @@ -1604,19 +1604,9 @@ describe("agent permission routes", () => { ...baseAgent, status: "terminated", }; - // getAccessibleAgent sees the pending agent; after the rejection resolves - // (which terminates internally) the route re-reads the terminated agent. - mockAgentService.getById - .mockResolvedValueOnce(pendingAgent) - .mockResolvedValue(terminatedAgent); - mockApprovalService.findOpenHireApprovalForAgent.mockResolvedValue({ - id: "approval-1", - companyId, - type: "hire_agent", - status: "pending", - payload: { agentId }, - }); - mockApprovalService.reject.mockResolvedValue({ + mockAgentService.getById.mockResolvedValue(pendingAgent); + mockApprovalService.rejectHire.mockResolvedValue({ + agent: terminatedAgent, approval: { id: "approval-1", status: "rejected" }, applied: true, }); @@ -1639,8 +1629,8 @@ describe("agent permission routes", () => { .send({})); expect(res.status).toBe(200); - expect(mockApprovalService.reject).toHaveBeenCalledWith("approval-1", "board-user"); - // reject() terminates the agent internally; the route must not terminate again. + expect(mockApprovalService.rejectHire).toHaveBeenCalledWith(agentId, "board-user"); + // The hire decision rejects the pending agent; the route must not terminate again. expect(mockAgentService.terminate).not.toHaveBeenCalled(); }); @@ -1670,7 +1660,7 @@ describe("agent permission routes", () => { expect(res.status).toBe(200); expect(mockAgentService.terminate).toHaveBeenCalledWith(agentId); expect(mockApprovalService.findOpenHireApprovalForAgent).not.toHaveBeenCalled(); - expect(mockApprovalService.reject).not.toHaveBeenCalled(); + expect(mockApprovalService.rejectHire).not.toHaveBeenCalled(); }); it("rejects direct approval for agents that are not pending approval", async () => { @@ -1687,7 +1677,7 @@ describe("agent permission routes", () => { .send({})); expect(res.status).toBe(409); - expect(mockAgentService.activatePendingApproval).not.toHaveBeenCalled(); + expect(mockApprovalService.approveHire).not.toHaveBeenCalled(); expect(mockLogActivity).not.toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: "agent.approved", })); @@ -1726,7 +1716,7 @@ describe("agent permission routes", () => { expect.objectContaining({ defaultEnvironmentId: environmentId, }), - { createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, + { createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, ); }); @@ -1812,7 +1802,7 @@ describe("agent permission routes", () => { adapterType: adapterCase.adapterType, defaultEnvironmentId: environmentId, }), - { createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, + { createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } }, ); }); } @@ -2291,3 +2281,12 @@ describe("agent permission routes", () => { expect(mockHeartbeatService.cancelRun).not.toHaveBeenCalled(); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + requestHire: (...args: unknown[]) => mockAgentService.create(...args), + approveHire: (...args: unknown[]) => mockAgentService.activatePendingApproval(...args), + terminateAgent: (...args: unknown[]) => mockAgentService.terminate(...args), + }) }; +}); diff --git a/server/src/__tests__/agent-permissions-service.test.ts b/server/src/__tests__/agent-permissions-service.test.ts index 6f8e37e300..e45ecb944a 100644 --- a/server/src/__tests__/agent-permissions-service.test.ts +++ b/server/src/__tests__/agent-permissions-service.test.ts @@ -8,7 +8,7 @@ import { defaultAgentPermissions, normalizeAgentPermissions, permissionsImplyLowTrust, -} from "../services/agent-permissions.js"; +} from "../lib/agent-permissions.js"; describe("agent permissions service", () => { it("grants agent-creation authority to new agents by default", () => { diff --git a/server/src/__tests__/agent-persona-persistence.test.ts b/server/src/__tests__/agent-persona-persistence.test.ts index fb39304ddb..3b1ca4e4c2 100644 --- a/server/src/__tests__/agent-persona-persistence.test.ts +++ b/server/src/__tests__/agent-persona-persistence.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { readFile } from "node:fs/promises"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -20,15 +21,17 @@ describe("persisted agent personas", () => { afterAll(async () => { await database?.cleanup(); }); it("randomizes once, accepts a saved draft and preserves identity across edits and revision rollback", async () => { const service = agentService(db); - const random = await service.create(companyId, { name: "Random", role: "engineer", adapterType: "process" }); + const serviceLifecycle = createAgentLifecycle(db); + const random = await serviceLifecycle.requestHire(companyId, { name: "Random", role: "engineer", adapterType: "process" }); expect(agentAppearanceSchema.safeParse(random.appearance).success).toBe(true); const appearance = appearanceForPalette("arctic-blue"); - const original = await service.create(companyId, { name: "Draft", role: "engineer", adapterType: "process", appearance }); + const original = await serviceLifecycle.requestHire(companyId, { name: "Draft", role: "engineer", adapterType: "process", appearance }); expect(original.appearance).toEqual(appearance); const updated = await service.update(original.id, { name: "Renamed" }, { recordRevision: { source: "test" } }); expect(updated?.appearance).toEqual(appearance); const [revision] = await service.listConfigRevisions(original.id); - await service.update(original.id, { name: "Changed again", status: "paused" }); + await service.update(original.id, { name: "Changed again" }); + await serviceLifecycle.pauseAgent(original.id); const restored = await service.rollbackConfigRevision(original.id, revision.id, {}); expect(restored?.appearance).toEqual(appearance); expect((await agentService(db).getById(original.id))?.avatarUrl).toContain("/arctic-blue/rest.png?size=512"); diff --git a/server/src/__tests__/agent-skills-routes.test.ts b/server/src/__tests__/agent-skills-routes.test.ts index ae267e9fd4..9f0848f2d5 100644 --- a/server/src/__tests__/agent-skills-routes.test.ts +++ b/server/src/__tests__/agent-skills-routes.test.ts @@ -1005,7 +1005,7 @@ describe("agent skill routes", () => { }), }), }), - { createdByUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", applyExistingWithoutClaim: false } }, + { createdByUserId: "local-board", responsibleUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", applyExistingWithoutClaim: false } }, ); expect(mockTrackAgentCreated).toHaveBeenCalledWith( expect.anything(), @@ -1055,7 +1055,7 @@ describe("agent skill routes", () => { expect.objectContaining({ role: "security", }), - { createdByUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", applyExistingWithoutClaim: false } }, + { createdByUserId: "local-board", responsibleUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", applyExistingWithoutClaim: false } }, ); expect(mockTrackAgentCreated).toHaveBeenCalledWith( expect.anything(), @@ -1491,7 +1491,7 @@ describe("agent skill routes", () => { }), }), { - createdByUserId: "local-board", + createdByUserId: "local-board", responsibleUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", @@ -1575,3 +1575,10 @@ describe("agent skill routes", () => { expect(mockAgentInstructionsService.materializeManagedBundle).not.toHaveBeenCalled(); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + requestHire: (...args: unknown[]) => mockAgentService.create(...args), + }) }; +}); diff --git a/server/src/__tests__/agent-test-environment-routes.test.ts b/server/src/__tests__/agent-test-environment-routes.test.ts index 2c2499e4f3..d78375724a 100644 --- a/server/src/__tests__/agent-test-environment-routes.test.ts +++ b/server/src/__tests__/agent-test-environment-routes.test.ts @@ -116,10 +116,7 @@ vi.mock("../services/ai-connection-runtime.js", async (importOriginal) => ({ }, })); const mockValidateAiApiKey = vi.hoisted(() => vi.fn(async () => undefined)); -vi.mock("../routes/ai-connections.js", async (importOriginal) => ({ - ...(await importOriginal()), - validateAiApiKey: mockValidateAiApiKey, -})); +vi.mock("../services/ai-api-key-test.js", () => ({ validateAiApiKey: mockValidateAiApiKey })); const mockMarkAuthenticationFailed = vi.hoisted(() => vi.fn(async () => undefined)); vi.mock("../services/ai-connections.js", async (importOriginal) => ({ ...(await importOriginal()), @@ -1030,3 +1027,9 @@ describe("agent test-environment route", () => { }); }); }); + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + }) }; +}); diff --git a/server/src/__tests__/agents-adapter-config-user-secret.test.ts b/server/src/__tests__/agents-adapter-config-user-secret.test.ts index 57a0f8dc06..e91be3a11a 100644 --- a/server/src/__tests__/agents-adapter-config-user-secret.test.ts +++ b/server/src/__tests__/agents-adapter-config-user-secret.test.ts @@ -416,3 +416,9 @@ describeEmbeddedPostgres("agents adapter-config user-secret resolution routes", function beforeEachActor(actor: TestActor) { currentActor = actor; } + +vi.mock("../services/agent-lifecycle.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAgentLifecycle: () => ({ + }) }; +}); diff --git a/server/src/__tests__/agents-claude-oauth-binding.test.ts b/server/src/__tests__/agents-claude-oauth-binding.test.ts index 1a133380da..4d645e6388 100644 --- a/server/src/__tests__/agents-claude-oauth-binding.test.ts +++ b/server/src/__tests__/agents-claude-oauth-binding.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { mkdirSync, rmSync } from "node:fs"; import os from "node:os"; @@ -362,7 +363,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); const sessionId = await seedStoredClaim(scope, Date.now() + 60_000); - const created = await agentService(db).create(scope.companyId, createInput(scope), { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }); @@ -376,7 +377,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { // A replay of the same claim inserts no second agent. await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -386,7 +387,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { it("rejects a missing claim and inserts no binding", async () => { const scope = await seedScope(); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: undefined, ownerUserId: scope.ownerUserId }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -397,7 +398,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); const sessionId = await seedStoredClaim(scope, Date.now() + 60_000); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: "intruder" }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -409,7 +410,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); const sessionId = await seedStoredClaim(scope, Date.now() - 1_000); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -420,7 +421,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); const sessionId = await seedStoredClaim(scope, Date.now() + 60_000, "awaiting_code"); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -432,8 +433,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { // Missing claim. const missingScope = await seedScope(); messages.push( - await agentService(db) - .create(missingScope.companyId, createInput(missingScope), { + await createAgentLifecycle(db).requestHire(missingScope.companyId, createInput(missingScope), { claudeLogin: { storedSessionId: undefined, ownerUserId: missingScope.ownerUserId }, }) .then(() => "no-error") @@ -443,8 +443,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const foreignScope = await seedScope(); const foreignSession = await seedStoredClaim(foreignScope, Date.now() + 60_000); messages.push( - await agentService(db) - .create(foreignScope.companyId, createInput(foreignScope), { + await createAgentLifecycle(db).requestHire(foreignScope.companyId, createInput(foreignScope), { claudeLogin: { storedSessionId: foreignSession, ownerUserId: "intruder" }, }) .then(() => "no-error") @@ -454,8 +453,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const expiredScope = await seedScope(); const expiredSession = await seedStoredClaim(expiredScope, Date.now() - 1_000); messages.push( - await agentService(db) - .create(expiredScope.companyId, createInput(expiredScope), { + await createAgentLifecycle(db).requestHire(expiredScope.companyId, createInput(expiredScope), { claudeLogin: { storedSessionId: expiredSession, ownerUserId: expiredScope.ownerUserId }, }) .then(() => "no-error") @@ -465,8 +463,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const nonStoredScope = await seedScope(); const nonStoredSession = await seedStoredClaim(nonStoredScope, Date.now() + 60_000, "submitting"); messages.push( - await agentService(db) - .create(nonStoredScope.companyId, createInput(nonStoredScope), { + await createAgentLifecycle(db).requestHire(nonStoredScope.companyId, createInput(nonStoredScope), { claudeLogin: { storedSessionId: nonStoredSession, ownerUserId: nonStoredScope.ownerUserId }, }) .then(() => "no-error") @@ -475,12 +472,11 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { // Already consumed. const consumedScope = await seedScope(); const consumedSession = await seedStoredClaim(consumedScope, Date.now() + 60_000); - await agentService(db).create(consumedScope.companyId, createInput(consumedScope), { + await createAgentLifecycle(db).requestHire(consumedScope.companyId, createInput(consumedScope), { claudeLogin: { storedSessionId: consumedSession, ownerUserId: consumedScope.ownerUserId }, }); messages.push( - await agentService(db) - .create(consumedScope.companyId, createInput(consumedScope), { + await createAgentLifecycle(db).requestHire(consumedScope.companyId, createInput(consumedScope), { claudeLogin: { storedSessionId: consumedSession, ownerUserId: consumedScope.ownerUserId }, }) .then(() => "no-error") @@ -495,10 +491,10 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const sessionId = await seedStoredClaim(scope, Date.now() + 60_000); const results = await Promise.allSettled([ - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }), - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }), ]); @@ -536,8 +532,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { }); await locked; - const createPromise = agentService(db) - .create(scope.companyId, createInput(scope), { + const createPromise = createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }) .then(() => "created") @@ -555,7 +550,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { it("removes the fixed binding on a normal update", async () => { const scope = await seedScope(); const sessionId = await seedStoredClaim(scope, Date.now() + 60_000); - const created = await agentService(db).create(scope.companyId, createInput(scope), { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }); @@ -568,7 +563,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { it("re-points the binding to a plain value on a normal update", async () => { const scope = await seedScope(); const sessionId = await seedStoredClaim(scope, Date.now() + 60_000); - const created = await agentService(db).create(scope.companyId, createInput(scope), { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }); @@ -583,7 +578,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { it("removes the binding by a move to another adapter type on a normal update", async () => { const scope = await seedScope(); const sessionId = await seedStoredClaim(scope, Date.now() + 60_000); - const created = await agentService(db).create(scope.companyId, createInput(scope), { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }); @@ -603,7 +598,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { it("keeps an existing binding when the update changes another field", async () => { const scope = await seedScope(); const sessionId = await seedStoredClaim(scope, Date.now() + 60_000); - const created = await agentService(db).create(scope.companyId, createInput(scope), { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId }, }); @@ -616,7 +611,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { it("rejects a newly introduced binding on update, because it carries no claim", async () => { const scope = await seedScope(); - const created = await agentService(db).create(scope.companyId, { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, { name: "Api Key Agent", role: "engineer", status: "idle", @@ -645,7 +640,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); await seedStoredOwnerValue(scope); - const created = await agentService(db).create(scope.companyId, createInput(scope), { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { ownerUserId: scope.ownerUserId, applyExistingWithoutClaim: true }, }); @@ -666,9 +661,9 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); await seedStoredOwnerValue(scope); - const created = await agentService( + const created = await createAgentLifecycle( db, - ).create(scope.companyId, { ...createInput(scope), status: "pending_approval" }, { + ).requestHire(scope.companyId, { ...createInput(scope), status: "pending_approval" }, { claudeLogin: { ownerUserId: scope.ownerUserId, applyExistingWithoutClaim: true }, }); @@ -680,7 +675,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { it("binds the fixed reference on an update from a stored owner value with no claim", async () => { const scope = await seedScope(); await seedStoredOwnerValue(scope); - const created = await agentService(db).create(scope.companyId, { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, { name: "Claude Agent", role: "engineer", status: "idle", @@ -706,7 +701,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { ownerUserId: scope.ownerUserId, applyExistingWithoutClaim: true }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -723,7 +718,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { // The route sets no owner for an agent actor. The gate rejects the no-claim // bind, so an agent actor never binds the fixed reference. await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { ownerUserId: null, applyExistingWithoutClaim: true }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -739,7 +734,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { foreignScope.ownerUserId = ownerScope.ownerUserId; await expect( - agentService(db).create(foreignScope.companyId, createInput(foreignScope), { + createAgentLifecycle(db).requestHire(foreignScope.companyId, createInput(foreignScope), { claudeLogin: { ownerUserId: foreignScope.ownerUserId, applyExistingWithoutClaim: true }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -753,7 +748,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { // The apply-existing flag is set, but the config carries no fixed binding. The // gate mints the fixed reference only when the client presents the exact fixed // binding, so the flag alone binds nothing and creates no fixed definition. - const created = await agentService(db).create( + const created = await createAgentLifecycle(db).requestHire( scope.companyId, { name: "Plain Agent", @@ -781,7 +776,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { // A client adds a secret id and a foreign owner to the binding. The persist // normalization keeps only the fixed reference fields, so the extra selectors // never reach the stored binding. The runtime derives the owner from context. - const created = await agentService(db).create( + const created = await createAgentLifecycle(db).requestHire( scope.companyId, createInput(scope, { CLAUDE_CODE_OAUTH_TOKEN: { @@ -804,7 +799,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { it("resolves the bound reference to the stored value without leaking it in a log", async () => { const scope = await seedScope(); await seedStoredOwnerValue(scope, "sk-secret-resolve"); - const created = await agentService(db).create(scope.companyId, createInput(scope), { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { ownerUserId: scope.ownerUserId, applyExistingWithoutClaim: true }, }); @@ -858,7 +853,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); const parent = await seedParentAgent(scope); - const created = await agentService(db).create(scope.companyId, createInput(scope), { + const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { inheritedFromAgentId: parent.id }, }); @@ -871,7 +866,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); const parent = await seedParentAgent(scope, { version: 5 }); - const created = await agentService(db).create( + const created = await createAgentLifecycle(db).requestHire( scope.companyId, createInput(scope, { CLAUDE_CODE_OAUTH_TOKEN: { ...FIXED_BINDING, version: 5 } }), { claudeLogin: { inheritedFromAgentId: parent.id } }, @@ -890,7 +885,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const parent = await seedParentAgent(scope, { version: 5 }); await expect( - agentService(db).create( + createAgentLifecycle(db).requestHire( scope.companyId, createInput(scope, { CLAUDE_CODE_OAUTH_TOKEN: { ...FIXED_BINDING, version: 2 } }), { claudeLogin: { inheritedFromAgentId: parent.id } }, @@ -906,7 +901,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const parent = await seedParentAgent(scope, { holdsFixedBinding: false }); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { inheritedFromAgentId: parent.id }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -921,7 +916,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const parent = await seedParentAgent(foreignScope); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { inheritedFromAgentId: parent.id }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -934,7 +929,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const scope = await seedScope(); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { inheritedFromAgentId: randomUUID() }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -950,7 +945,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { const parent = await seedParentAgent(scope, { adapterType: "codex_local" }); await expect( - agentService(db).create(scope.companyId, createInput(scope), { + createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), { claudeLogin: { inheritedFromAgentId: parent.id }, }), ).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED }); @@ -988,8 +983,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => { // The child's copied reference still names version 5, the version the // route read before the rotation started. The create call must wait for // the parent row lock, so it can only proceed once the rotation commits. - const createPromise = agentService(db) - .create( + const createPromise = createAgentLifecycle(db).requestHire( scope.companyId, createInput(scope, { CLAUDE_CODE_OAUTH_TOKEN: { ...FIXED_BINDING, version: 5 } }), { claudeLogin: { inheritedFromAgentId: parent.id } }, diff --git a/server/src/__tests__/agents-pending-approval-config.test.ts b/server/src/__tests__/agents-pending-approval-config.test.ts index 7c02a10da1..28d27e6e66 100644 --- a/server/src/__tests__/agents-pending-approval-config.test.ts +++ b/server/src/__tests__/agents-pending-approval-config.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { eq } from "drizzle-orm"; @@ -66,8 +67,9 @@ describeEmbeddedPostgres("pending approval agent config integrity", () => { it("freezes generic pending hire config and reapplies the approval snapshot on activation", async () => { const companyId = await seedCompany(); const agentSvc = agentService(db); + const agentSvcLifecycle = createAgentLifecycle(db); const approvalSvc = approvalService(db); - const pending = await agentSvc.create(companyId, { + const pending = await agentSvcLifecycle.requestHire(companyId, { name: "Pending Coder", role: "engineer", title: "Software Engineer", @@ -145,7 +147,8 @@ describeEmbeddedPostgres("pending approval agent config integrity", () => { await approvalSvc.approve(approval.id, "board-user", "Approved generic hire"); await expect(agentSvc.getById(pending.id)).resolves.toMatchObject({ - status: "idle", + status: "paused", + lifecycleState: "preparing", appearance: pending.appearance, name: "Pending Coder", role: "engineer", diff --git a/server/src/__tests__/agents-service-secret-bindings.test.ts b/server/src/__tests__/agents-service-secret-bindings.test.ts index a155bb72ab..23f24ebaaa 100644 --- a/server/src/__tests__/agents-service-secret-bindings.test.ts +++ b/server/src/__tests__/agents-service-secret-bindings.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { mkdirSync, rmSync } from "node:fs"; import os from "node:os"; @@ -86,7 +87,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { value: "sk-ant-123", }); - const created = await agentService(db).create(companyId, { + const created = await createAgentLifecycle(db).requestHire(companyId, { name: "Claude Novita", role: "engineer", status: "pending_approval", @@ -137,7 +138,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { value: "slack-test-token", }); - const created = await agentService(db).create(companyId, { + const created = await createAgentLifecycle(db).requestHire(companyId, { name: "Slack Briefing", role: "briefing", adapterType: "codex_local", @@ -185,7 +186,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { }); await expect( - agentService(db).create(companyId, { + createAgentLifecycle(db).requestHire(companyId, { name: "Unlisted Static Lease", role: "engineer", adapterType: "codex_local", @@ -220,7 +221,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { const companyId = await seedCompany(); const literalApiKey = `hermes-key-${randomUUID()}`; - const created = await agentService(db).create(companyId, { + const created = await createAgentLifecycle(db).requestHire(companyId, { name: "Hermes Gateway", role: "engineer", status: "idle", @@ -289,7 +290,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { value: "next-value", }); - const created = await agentService(db).create(companyId, { + const created = await createAgentLifecycle(db).requestHire(companyId, { name: "Binding Swapper", role: "engineer", adapterType: "codex_local", @@ -359,13 +360,14 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { .where(eq(companySecretBindings.targetId, agentId)); expect(beforeBindings).toHaveLength(0); - const approved = await agentService(db).activatePendingApproval(agentId); + const approved = await createAgentLifecycle(db).approveHire(agentId); expect(approved).toMatchObject({ activated: true, agent: { id: agentId, - status: "idle", + status: "paused", + lifecycleState: "preparing", }, }); @@ -390,7 +392,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { const missingSecretId = randomUUID(); await expect( - agentService(db).create(companyId, { + createAgentLifecycle(db).requestHire(companyId, { name: "Broken Create", role: "engineer", adapterType: "claude_local", @@ -420,7 +422,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { provider: "local_encrypted", value: "valid-value", }); - const created = await agentService(db).create(companyId, { + const created = await createAgentLifecycle(db).requestHire(companyId, { name: "Transactional Update", role: "engineer", adapterType: "codex_local", @@ -483,7 +485,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => { permissions: {}, }); - await expect(agentService(db).activatePendingApproval(agentId)).rejects.toBeTruthy(); + await expect(createAgentLifecycle(db).approveHire(agentId)).rejects.toBeTruthy(); const reloaded = await agentService(db).getById(agentId); expect(reloaded?.status).toBe("pending_approval"); diff --git a/server/src/__tests__/approvals-service.test.ts b/server/src/__tests__/approvals-service.test.ts index 47aedf9169..1a4cbfe22d 100644 --- a/server/src/__tests__/approvals-service.test.ts +++ b/server/src/__tests__/approvals-service.test.ts @@ -2,21 +2,23 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { approvalService } from "../services/approvals.ts"; import { companies } from "@paperclipai/db"; -vi.mock("../services/budgets.js", () => ({ +vi.mock("../services/budgets.js", async importOriginal => ({ + ...await importOriginal(), budgetService: () => ({ deliverPendingEnforcement: vi.fn(async () => {}) }), budgetServiceInTransaction: () => ({ upsertPolicy: vi.fn(async () => {}) }), })); const mockAgentService = vi.hoisted(() => ({ activatePendingApproval: vi.fn(), + getById: vi.fn(async () => ({ id: "agent-1" })), create: vi.fn(), terminate: vi.fn(), })); const mockNotifyHireApproved = vi.hoisted(() => vi.fn()); -vi.mock("../services/agents.js", () => ({ - agentService: vi.fn(() => mockAgentService), +vi.mock("../modules/agent-lifecycle/adapters/records.js", () => ({ + agentRecords: vi.fn(() => ({ ...mockAgentService, rejectPendingHire: mockAgentService.terminate })), })); vi.mock("../services/hire-hook.js", () => ({ @@ -114,7 +116,7 @@ describe("approvalService resolution idempotency", () => { const result = await svc.approve("approval-1", "board", "ship it"); expect(result.applied).toBe(true); - expect(mockAgentService.activatePendingApproval).toHaveBeenCalledWith("agent-1", approved.payload); + expect(mockAgentService.activatePendingApproval).toHaveBeenCalledWith("agent-1", approved.payload, approved.requestedByUserId); expect(mockNotifyHireApproved).toHaveBeenCalledTimes(1); }); @@ -161,7 +163,7 @@ describe("approvalService resolution idempotency", () => { expect.objectContaining({ adapterConfig: approved.payload.adapterConfig, }), - { createdByUserId: expectedCreator }, + { createdByUserId: expectedCreator, responsibleUserId: requestedByUserId }, ); }); }); diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index d9d43e4c19..26934470fb 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; @@ -795,8 +796,8 @@ describeEmbeddedPostgres("attention service", () => { }, ]); - await agentService(db).pause(pausedReviewerId); - await agentService(db).terminate(terminatedReviewerId); + await createAgentLifecycle(db).pauseAgent(pausedReviewerId); + await createAgentLifecycle(db).terminateAgent(terminatedReviewerId); const feed = await attentionService(db).list(companyId, { userId: "board-user" }); const interactionTitles = feed.items @@ -886,7 +887,7 @@ describeEmbeddedPostgres("attention service", () => { effectiveResolverPolicy: "human_only", payload: { version: 1, questions: [] }, }); - await agentService(db).pause(reviewerId); + await createAgentLifecycle(db).pauseAgent(reviewerId); const feed = await attentionService(db).list(companyId, { userId: "board-user" }); const otherUserFeed = await attentionService(db).list(companyId, { userId: "other-user" }); diff --git a/server/src/__tests__/built-in-agents.test.ts b/server/src/__tests__/built-in-agents.test.ts index 7a90b5b5ec..920a1c9537 100644 --- a/server/src/__tests__/built-in-agents.test.ts +++ b/server/src/__tests__/built-in-agents.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js"; import fs from "node:fs/promises"; import { fileHash } from "../services/agent-file-store.js"; import { stockHash } from "../services/managed-resource-drift.js"; @@ -9,6 +10,9 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest import { and, eq, sql } from "drizzle-orm"; import { activityLog, + heartbeatRuns, + agentWakeupRequests, + routineRuns, agentConfigRevisions, agents, approvals, @@ -35,14 +39,14 @@ import { agentInstructionsService } from "../services/agent-instructions.ts"; import { agentService } from "../services/agents.ts"; import { approvalService } from "../services/approvals.ts"; import { - builtInAgentService, + builtInAgentService as createBuiltInAgentService, deriveBuiltInAgentStatus, listBuiltInAgentDefinitions, readBuiltInTextWithFallback, reconcileBuiltInAgentsOnStartup, validateBuiltInAgentDefinitions, } from "../services/built-in-agents.ts"; -import { readBuiltInAgentMarker, withBuiltInAgentMarker } from "../services/built-in-agent-metadata.ts"; +import { readBuiltInAgentMarker, withBuiltInAgentMarker } from "../lib/built-in-agent-metadata.ts"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); @@ -119,6 +123,22 @@ describe("built-in agent asset loading", () => { describeEmbeddedPostgres("built-in agents", () => { let db!: ReturnType; + let lifecycleWorker: ReturnType; + function builtInAgentService(database: typeof db) { + const service = createBuiltInAgentService(database); + return { ...service, + async ensure(...args: Parameters) { + const result = await service.ensure(...args); + if (result.agentId) await lifecycleWorker.process(result.agentId); + return service.get(args[0], args[1]); + }, + async get(...args: Parameters) { + const result = await service.get(...args); + if (result.agentId) await lifecycleWorker.process(result.agentId); + return service.get(...args); + }, + }; + } let tempDb: Awaited> | null = null; const instructionOperator = { type: "board", userId: "local-board", source: "local_implicit" } as const; async function writeInstructionEntry(agent: { id: string; companyId: string }, entryFile: string, content: string) { @@ -134,9 +154,11 @@ describeEmbeddedPostgres("built-in agents", () => { beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-built-in-agents-"); db = createDb(tempDb.connectionString); + lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" }); }, 20_000); afterEach(async () => { + await lifecycleWorker.sweep(); await db.delete(routineTriggers); await db.delete(routines); await db.delete(issueThreadInteractions); @@ -158,6 +180,7 @@ describeEmbeddedPostgres("built-in agents", () => { }); afterAll(async () => { + await lifecycleWorker.stop(); await tempDb?.cleanup(); }); @@ -498,7 +521,7 @@ describeEmbeddedPostgres("built-in agents", () => { }); expect(ready.status).toBe("ready"); - await agentService(db).pause(ready.agentId!, "manual"); + await createAgentLifecycle(db).pauseAgent(ready.agentId!, "manual"); await expect(builtIns.get(companyId, "learning")).resolves.toMatchObject({ status: "paused", agentId: ready.agentId, @@ -540,7 +563,7 @@ describeEmbeddedPostgres("built-in agents", () => { warning: null, }); - await agentService(db).pause(ready.agentId!, "maintenance"); + await createAgentLifecycle(db).pauseAgent(ready.agentId!, "maintenance"); await expect(builtIns.requireBuiltInAgent(companyId, "briefs")).resolves.toMatchObject({ agent: { id: ready.agentId }, warning: { @@ -585,7 +608,7 @@ describeEmbeddedPostgres("built-in agents", () => { it("reconciles an enabled Reflection Coach bundle with skill sync and a disabled routine", async () => { const companyId = await seedCompany({ requireApproval: false }); - const root = await agentService(db).create(companyId, { + const root = await createAgentLifecycle(db).requestHire(companyId, { name: "CEO", role: "ceo", status: "idle", @@ -677,7 +700,7 @@ describeEmbeddedPostgres("built-in agents", () => { it("recreates missing managed resource bindings idempotently during concurrent reconcile", async () => { const companyId = await seedCompany({ requireApproval: false }); - await agentService(db).create(companyId, { + await createAgentLifecycle(db).requestHire(companyId, { name: "CEO", role: "ceo", status: "idle", @@ -715,7 +738,7 @@ describeEmbeddedPostgres("built-in agents", () => { it("preserves new-agent approval gates during on-demand Reflection Coach provisioning", async () => { const companyId = await seedCompany({ requireApproval: true }); - const root = await agentService(db).create(companyId, { + const root = await createAgentLifecycle(db).requestHire(companyId, { name: "CEO", role: "ceo", status: "idle", @@ -959,7 +982,7 @@ describeEmbeddedPostgres("built-in agents", () => { it("prevents direct marker add, remove, or mutation", async () => { const companyId = await seedCompany(); const builtIn = await builtInAgentService(db).ensure(companyId, "briefs"); - const normal = await agentService(db).create(companyId, { + const normal = await createAgentLifecycle(db).requestHire(companyId, { name: "Normal", role: "engineer", status: "idle", @@ -969,7 +992,7 @@ describeEmbeddedPostgres("built-in agents", () => { permissions: {}, }); - await expect(agentService(db).create(companyId, { + await expect(createAgentLifecycle(db).requestHire(companyId, { name: "Spoof", role: "engineer", status: "idle", @@ -1222,7 +1245,7 @@ describeEmbeddedPostgres("built-in agents", () => { it("automatically materializes the Reflection Coach bundle without enabling background work", async () => { const companyId = await seedCompany(); - const root = await agentService(db).create(companyId, { + const root = await createAgentLifecycle(db).requestHire(companyId, { name: "CEO", role: "ceo", status: "idle", @@ -1297,7 +1320,7 @@ describeEmbeddedPostgres("built-in agents", () => { it("materializes the Summarizer bundle paused on Claude Haiku with a disabled routine", async () => { const companyId = await seedCompany(); - const root = await agentService(db).create(companyId, { + const root = await createAgentLifecycle(db).requestHire(companyId, { name: "CEO", role: "ceo", status: "idle", @@ -1409,9 +1432,35 @@ describeEmbeddedPostgres("built-in agents", () => { expect(resetFile.content).not.toContain("Operator edit."); }); + it("keeps a built-in routine task while the agent prepares", async () => { + const companyId = await seedCompany({ requireApproval: false }); + const builtIns = createBuiltInAgentService(db); + const created = await builtIns.ensure(companyId, "reflection-coach"); + await lifecycleWorker.stop(); + lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "pending" }); + try { + const result = await builtIns.runRoutine(companyId, "reflection-coach", "recent-agent-reflection", { userId: "responsible-user" }); + expect(result.status).toBe("issue_created"); + const tasks = await db.select().from(issues).where(eq(issues.companyId, companyId)); + expect(tasks).toHaveLength(1); + expect(tasks[0]!.assigneeAgentId).toBe(created.agentId); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))) + .toEqual([expect.objectContaining({ status: "queued", agentId: created.agentId })]); + // A second request during setup must not try to resume an already preparing agent. + await expect(builtIns.runRoutine(companyId, "reflection-coach", "recent-agent-reflection", { userId: "responsible-user" })) + .resolves.toHaveProperty("id"); + } finally { + await lifecycleWorker.stop(); + await db.delete(routineRuns).where(eq(routineRuns.companyId, companyId)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" }); + } + }); + it("controls the Reflection Coach routine schedule without enabling it by default", async () => { const companyId = await seedCompany(); - await agentService(db).create(companyId, { + await createAgentLifecycle(db).requestHire(companyId, { name: "CEO", role: "ceo", status: "idle", @@ -1434,7 +1483,8 @@ describeEmbeddedPostgres("built-in agents", () => { "recent-agent-reflection", { userId: "board-user" }, ); - expect(enabled.status).toBe("needs_setup"); + await lifecycleWorker.process(enabled.agentId!); + expect((await builtIns.get(companyId, "reflection-coach")).status).toBe("needs_setup"); expect(enabled.resources.find((resource) => resource.resourceKind === "routine")).toMatchObject({ stockStatus: "stock_current", scheduleEnabled: true, @@ -1462,7 +1512,7 @@ describeEmbeddedPostgres("built-in agents", () => { it("surfaces pending Reflection Coach proposal interactions on the routine resource", async () => { const companyId = await seedCompany(); - await agentService(db).create(companyId, { + await createAgentLifecycle(db).requestHire(companyId, { name: "CEO", role: "ceo", status: "idle", @@ -1515,7 +1565,8 @@ describeEmbeddedPostgres("built-in agents", () => { it("gates Reflection Coach proposal mutations until an accepted follow-up apply step", async () => { const companyId = await seedCompany(); const agentsSvc = agentService(db); - await agentsSvc.create(companyId, { + const agentsSvcLifecycle = createAgentLifecycle(db); + await agentsSvcLifecycle.requestHire(companyId, { name: "CEO", role: "ceo", status: "idle", @@ -1524,7 +1575,7 @@ describeEmbeddedPostgres("built-in agents", () => { runtimeConfig: {}, permissions: {}, }); - const target = await agentsSvc.create(companyId, { + const target = await agentsSvcLifecycle.requestHire(companyId, { name: "Target Coder", role: "engineer", status: "idle", diff --git a/server/src/__tests__/chat-completion-delivery.test.ts b/server/src/__tests__/chat-completion-delivery.test.ts index bb8e7f076d..694f511d0e 100644 --- a/server/src/__tests__/chat-completion-delivery.test.ts +++ b/server/src/__tests__/chat-completion-delivery.test.ts @@ -1,3 +1,5 @@ +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { subscribeDeliveryWork, DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, eq } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -7,7 +9,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; import { getExecutionBlocker } from "../services/execution-blocker.js"; import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js"; -import { issueService } from "../services/issues.js"; +import { executeIssuePostCommitActions, type IssuePostCommitAction, issueService } from "../services/issues.js"; import { buildLowTrustSourceTrust } from "../services/source-trust.js"; import { documentService } from "../services/documents.js"; import { instanceSettingsService } from "../services/instance-settings.js"; @@ -51,6 +53,57 @@ const support = await getEmbeddedPostgresTestSupport(); }; return { companyId, agentId, sourceId, runId, task, create, finish, rows, due, wakeup, service, run }; } + it("dispatches a committed completion without activity publication and recovers after restart", async () => { + const f = await seed(); + const run = vi.fn(() => f.service.sweepPending()); + const makeWorker = () => { + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + const worker = coordinator.register(DELIVERY_QUEUES.chatCompletion, { retryMs: 60_000, run, hasPending: f.service.hasPending }); + return { ...worker, stop: coordinator.stop }; + }; + const worker = makeWorker(); + try { + await worker.ready; + expect(await f.service.hasPending()).toBe(false); + expect(run).toHaveBeenCalledTimes(1); + await f.finish(); // Service mutation alone: no route activity callback. + await vi.waitFor(() => expect(f.wakeup).toHaveBeenCalledTimes(1)); + expect(await f.service.hasPending()).toBe(true); // queued receipt still owed + } finally { await worker.stop(); } + // Admit the first reporting turn; later completions need a new turn. + await f.run(); + // A second task commits while no consumer is registered. + const next = await f.create(); + await f.finish(next.id); + const restarted = makeWorker(); + try { + await restarted.ready; + expect(f.wakeup).toHaveBeenCalledTimes(2); + } finally { await restarted.stop(); } + }); + it("automatically notifies after a caller-owned transaction commits", async () => { + const f = await seed(); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatCompletion, notified); + try { + const actions: IssuePostCommitAction[] = []; + await db.transaction(async tx => { + await issueService(db).update(f.task.id, { status: "done" }, tx, [], actions); + expect(notified).not.toHaveBeenCalled(); + }); + expect(notified).toHaveBeenCalledTimes(1); + expect(actions).toEqual([]); + await executeIssuePostCommitActions(db, actions); + expect(notified).toHaveBeenCalledTimes(1); + const rolledBack: IssuePostCommitAction[] = []; + await expect(db.transaction(async tx => { + await issueService(db).update(f.task.id, { status: "todo" }, tx, [], rolledBack); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + expect(notified).toHaveBeenCalledTimes(1); + expect((await f.rows())[0]?.status).toBe("pending"); + } finally { unsubscribe(); } + }); it("records authenticated origins and atomically creates one event per Done transition", async () => { const f = await seed(); expect(await db.select().from(handoffs).where(eq(handoffs.taskId, f.task.id))).toMatchObject([{ conversationId: f.sourceId, sessionGeneration: 0 }]); diff --git a/server/src/__tests__/cleanup-removal-service.test.ts b/server/src/__tests__/cleanup-removal-service.test.ts index 68bf1ce2c1..6c7761e2b5 100644 --- a/server/src/__tests__/cleanup-removal-service.test.ts +++ b/server/src/__tests__/cleanup-removal-service.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -41,6 +42,7 @@ describeEmbeddedPostgres("cleanup removal services", () => { beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-cleanup-removal-"); db = createDb(tempDb.connectionString); + configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" }); }, 20_000); afterEach(async () => { @@ -146,6 +148,7 @@ describeEmbeddedPostgres("cleanup removal services", () => { createdByRunId: runId, }); + await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, agentId)); const removed = await agentService(db).remove(agentId); expect(removed?.id).toBe(agentId); diff --git a/server/src/__tests__/companies-service.test.ts b/server/src/__tests__/companies-service.test.ts index aa79a5b93a..1b7e6fe8b0 100644 --- a/server/src/__tests__/companies-service.test.ts +++ b/server/src/__tests__/companies-service.test.ts @@ -33,9 +33,10 @@ vi.mock("../services/cloud-lifecycle-sync.js", () => ({ notifyCloudOfPrimaryCompanyLifecycleChange: notifyCloudSpy, })); +import { configureAgentLifecycle } from "../services/agent-lifecycle.js"; import { companyService } from "../services/companies.js"; import { deriveIssuePrefixBase } from "../services/issue-prefix.js"; -import { readBuiltInAgentMarker } from "../services/built-in-agent-metadata.js"; +import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService, reconcileBuiltInAgentsOnStartup } from "../services/built-in-agents.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); @@ -49,6 +50,16 @@ if (!embeddedPostgresSupport.supported) { describeEmbeddedPostgres("companyService", () => { let db!: ReturnType; + let lifecycleWorker: ReturnType; + beforeEach(() => { lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" }); }); + async function insertAgents(input: typeof agents.$inferInsert | Array) { + const rows = Array.isArray(input) ? input : [input]; + await db.insert(agents).values(rows.map(row => ({ ...row, + lifecycleState: ["paused", "pending_approval", "terminated"].includes(row.status ?? "") ? row.status! : "ready", + lifecycleHolds: row.status === "paused" ? [row.pauseReason ?? "manual"] : [], + }))); + } + let tempDb: Awaited> | null = null; beforeAll(async () => { @@ -57,6 +68,7 @@ describeEmbeddedPostgres("companyService", () => { }, 20_000); afterEach(async () => { + await lifecycleWorker.stop(); await db.delete(routineTriggers); await db.delete(routines); await db.delete(builtInManagedResources); @@ -165,7 +177,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values([ + await insertAgents([ { id: runningAgentId, companyId, @@ -335,7 +347,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values([ + await insertAgents([ { id: archivedPausedAgentId, companyId, @@ -382,6 +394,7 @@ describeEmbeddedPostgres("companyService", () => { ); expect(reactivated?.status).toBe("active"); + await lifecycleWorker.process(archivedPausedAgentId); const reactivateActivity = await db .select({ @@ -441,7 +454,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values([ + await insertAgents([ { id: runningAgentId, companyId, @@ -551,7 +564,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values([ + await insertAgents([ { id: archivedPausedAgentId, companyId, @@ -587,6 +600,7 @@ describeEmbeddedPostgres("companyService", () => { ); expect(reactivated?.status).toBe("active"); + await lifecycleWorker.process(archivedPausedAgentId); const rows = await db .select({ id: agents.id, status: agents.status, pauseReason: agents.pauseReason }) @@ -618,7 +632,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values({ + await insertAgents({ id: terminatedAgentId, companyId, name: "Terminated Agent", @@ -661,7 +675,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values({ + await insertAgents({ id: manualPausedAgentId, companyId, name: "Manual Paused Agent", @@ -713,7 +727,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values({ + await insertAgents({ id: agentId, companyId, name: "Idle Agent", @@ -788,7 +802,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values({ + await insertAgents({ id: agentId, companyId, name: "Idle Agent", @@ -863,7 +877,7 @@ describeEmbeddedPostgres("companyService", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(agents).values({ + await insertAgents({ id: idleAgentId, companyId, name: "Idle Agent", diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index 461da04f26..b5c265a2a9 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -122,6 +122,13 @@ vi.mock("../services/agents.js", () => ({ agentService: () => agentSvc, })); +vi.mock("../services/agent-lifecycle.js", () => ({ + createAgentLifecycle: () => ({ + requestHire: (...args: unknown[]) => agentSvc.create(...args), + pauseAgent: vi.fn(), + }), +})); + vi.mock("../services/access.js", () => ({ accessService: () => accessSvc, })); diff --git a/server/src/__tests__/connection-intents-service.test.ts b/server/src/__tests__/connection-intents-service.test.ts index c6c318c372..94613ec1f3 100644 --- a/server/src/__tests__/connection-intents-service.test.ts +++ b/server/src/__tests__/connection-intents-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; @@ -773,7 +775,11 @@ describeEmbeddedPostgres("connectionIntentService", () => { it("recovers a resolution after a failed dispatch and acknowledges an already queued wake exactly once", async () => { const service = connectionIntentService(db); const pending = await service.request(claims, "airtable"); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.connection, notified); await service.decline(pending.interactionId!, claims.responsible_user_id); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); const wakeup = vi.fn().mockRejectedValueOnce(new Error("simulated crash before durable enqueue")); const deliveries = connectionIntentDeliveryService(db, { wakeup } as never); await expect(deliveries.deliver(pending.interactionId!)).rejects.toThrow("simulated crash"); diff --git a/server/src/__tests__/decision-models.test.ts b/server/src/__tests__/decision-models.test.ts index aa3bfbdb4b..b106157c81 100644 --- a/server/src/__tests__/decision-models.test.ts +++ b/server/src/__tests__/decision-models.test.ts @@ -189,12 +189,16 @@ describe("company decision service", () => { const context: DecisionContext = { companyId: f.companyId, feature: settingsDecisionTest, actor: { type: "agent", agentId: f.agentId, runId, companyId: f.companyId, source: "agent_jwt" } }; const remove = () => agentService(db).remove(f.agentId); + let inFlightDeletionError: unknown; f.provider.mockImplementationOnce(async () => { - await expect(remove()).rejects.toMatchObject({ status: 409, details: { code: "agent_decision_accounting_pending" } }); + // Exercise the accounting guard after lifecycle termination. + await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, f.agentId)); + inFlightDeletionError = await remove().catch(error => error); expect(await f.service.decide(f.context, DECISION_TEST_REQUEST)).toMatchObject({ status: "unavailable", reason: "budget_blocked" }); return { errorCode: "timeout", receipt: decisionReceipt("openai", null) }; }); expect((await f.service.decide(context, DECISION_TEST_REQUEST)).status).toBe("failed"); + expect(inFlightDeletionError).toMatchObject({ status: 409, details: { code: "agent_decision_accounting_pending" } }); await expect(remove()).rejects.toMatchObject({ status: 409, details: { code: "agent_decision_accounting_pending" } }); expect((await db.select().from(budgetReservations).where(eq(budgetReservations.companyId, f.companyId)))[0]).toMatchObject({ state: "held", amountCents: "1.0000000" }); }); diff --git a/server/src/__tests__/delivery-work-coordinator.test.ts b/server/src/__tests__/delivery-work-coordinator.test.ts new file mode 100644 index 0000000000..1be5395f9d --- /dev/null +++ b/server/src/__tests__/delivery-work-coordinator.test.ts @@ -0,0 +1,192 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyDeliveryWork, DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const coordinators: ReturnType[] = []; +afterEach(async () => { for (const c of coordinators.splice(0)) await c.stop(); vi.useRealTimers(); }); +function setup() { + vi.useFakeTimers(); + let outcome = "committed"; + let rejectCommit = false; + const execute = vi.fn(async () => [{ xid: "42", status: outcome }]); + const owner = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise) { + const result = await callback({ execute, transaction: owner.transaction }); + if (rejectCommit) throw new Error("lost commit reply"); + return result; + } }); + const canRun = vi.fn(() => true), canReconcile = vi.fn(() => false), onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile, onError }); + coordinators.push(coordinator); + const enqueue = (queue = DELIVERY_QUEUES.feedback as typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]) => + owner.transaction(tx => notifyDeliveryWork(tx, queue)); + return { owner, canRun, canReconcile, onError, coordinator, enqueue, execute, + failCommit: () => { rejectCommit = true; outcome = "in progress"; }, settle: (status = "committed") => { outcome = status; } }; +} +function task() { return { retryMs: 5000, run: vi.fn(async () => {}), hasPending: vi.fn(async () => false) }; } + +describe("delivery work coordinator", () => { + it("leaves five empty queues with no deadlines, timers, or periodic queries", async () => { + const s = setup(); + const tasks = Object.values(DELIVERY_QUEUES).map(queue => { + const t = task(); return { queue, t, worker: s.coordinator.register(queue, t) }; + }); + await Promise.all(tasks.map(t => t.worker.ready)); + expect(vi.getTimerCount()).toBe(0); + expect(s.coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(3_600_000); + for (const { queue, t } of tasks) { + expect(t.run).toHaveBeenCalledTimes(1); + await s.enqueue(queue); await s.enqueue(queue); + } + await vi.advanceTimersByTimeAsync(1); + for (const { t } of tasks) expect(t.run).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not postpone an already committed wake when another producer starts writing", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + await s.enqueue(); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const writing = s.owner.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback); + await gate; + }); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); + release(); await writing; + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(3); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("fences unresolved writes until the server settles, then scans and releases the idle hold", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.failCommit(); + await expect(s.enqueue()).rejects.toThrow("lost commit reply"); + expect(idleWorkSnapshot().active).toBe(1); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); // Empty scan cannot clear unresolved write. + expect(idleWorkSnapshot().active).toBe(1); + expect(s.coordinator.nextWakeAt()).not.toBeNull(); + s.settle(); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run.mock.calls.length).toBeGreaterThanOrEqual(3); + expect(idleWorkSnapshot().active).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); + it("retains unresolved intent across coordinator replacement", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.failCommit(); await expect(s.enqueue()).rejects.toThrow(); + await s.coordinator.stop(); + expect(idleWorkSnapshot().active).toBe(0); + const replacement = createDeliveryWorkCoordinator({ owner: s.owner, canRun: () => true, onError: vi.fn() }); + coordinators.push(replacement); + await replacement.register(DELIVERY_QUEUES.feedback, t).ready; + expect(idleWorkSnapshot().active).toBe(1); + s.settle(); await vi.advanceTimersByTimeAsync(5001); + expect(idleWorkSnapshot().active).toBe(0); + expect(replacement.nextWakeAt()).toBeNull(); + }); + it("retries pending deliveries and transient failures, then goes quiet", async () => { + const s = setup(), t = task(); + t.run.mockRejectedValueOnce(new Error("database disconnected")); + t.hasPending.mockResolvedValueOnce(true); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + expect(s.onError).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(10_000); + expect(t.run).toHaveBeenCalledTimes(3); + expect(s.coordinator.nextWakeAt()).toBeNull(); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not overlap a sweep or lose a commit during its final pending check", async () => { + const s = setup(), t = task(); + let release!: (pending: boolean) => void; + t.hasPending.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + s.coordinator.register(DELIVERY_QUEUES.connection, t); + await vi.advanceTimersByTimeAsync(1); + await s.enqueue(DELIVERY_QUEUES.connection); + await vi.advanceTimersByTimeAsync(1000); + expect(t.run).toHaveBeenCalledTimes(1); + release(false); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); + }); + it("reconciles a rollback during idle drain and releases the empty-queue hold without dispatching", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.canRun.mockReturnValue(false); + s.canReconcile.mockReturnValue(true); + s.failCommit(); await expect(s.enqueue()).rejects.toThrow("lost commit reply"); + await vi.advanceTimersByTimeAsync(1); + expect(idleWorkSnapshot().active).toBe(1); + expect(t.run).toHaveBeenCalledTimes(1); + s.settle("aborted"); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(1); + expect(t.hasPending).toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBe(0); + expect(s.coordinator.nextWakeAt()).toBeNull(); + }); + it("keeps committed pending work recoverable during drain without starting deliveries", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.canRun.mockReturnValue(false); + s.canReconcile.mockReturnValue(true); + t.hasPending.mockResolvedValue(true); + await s.enqueue(); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(1); + expect(s.coordinator.nextWakeAt()).not.toBeNull(); + s.canRun.mockReturnValue(true); + t.hasPending.mockResolvedValue(false); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(2); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not query during standby and resumes when admission opens", async () => { + const s = setup(), t = task(); s.canRun.mockReturnValue(false); + await s.coordinator.register(DELIVERY_QUEUES.toolAction, t).ready; + await s.enqueue(DELIVERY_QUEUES.toolAction); + await vi.advanceTimersByTimeAsync(120_000); + expect(t.run).not.toHaveBeenCalled(); + expect(t.hasPending).not.toHaveBeenCalled(); + s.canRun.mockReturnValue(true); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(1); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("awaits in-flight work on shutdown and releases the subscription for restart", async () => { + const s = setup(), t = task(); + let release!: () => void; + t.run.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + s.coordinator.register(DELIVERY_QUEUES.chatCompletion, t); + await vi.advanceTimersByTimeAsync(1); + const stopped = vi.fn(); + const stopping = s.coordinator.stop().then(stopped); + await s.enqueue(DELIVERY_QUEUES.chatCompletion); + expect(stopped).not.toHaveBeenCalled(); + release(); await stopping; + expect(vi.getTimerCount()).toBe(0); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("cancels an abortable delivery before awaiting shutdown", async () => { + const s = setup(); + const hasPending = vi.fn(async () => false); + const run = vi.fn((signal: AbortSignal) => new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + })); + s.coordinator.register(DELIVERY_QUEUES.feedback, { retryMs: 5000, run, hasPending }); + await vi.advanceTimersByTimeAsync(1); + await s.coordinator.stop(); + expect(run.mock.calls[0]?.[0].aborted).toBe(true); + expect(hasPending).not.toHaveBeenCalled(); + expect(s.onError).not.toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/server/src/__tests__/dot-invitations.test.ts b/server/src/__tests__/dot-invitations.test.ts index 0a31da0f2f..84fd5d102c 100644 --- a/server/src/__tests__/dot-invitations.test.ts +++ b/server/src/__tests__/dot-invitations.test.ts @@ -1,3 +1,8 @@ +import { logActivity } from "../services/activity-log.js"; +vi.mock("../services/activity-log.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, logActivity: vi.fn(actual.logActivity) }; +}); import { randomBytes, randomUUID } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -38,7 +43,8 @@ it("atomically resumes one pending invite across retries and never persists a pa const service = dotInvitationService(db); const [a, b] = await Promise.all([service.create(f.companyId, f.userId), service.create(f.companyId, f.userId)]); expect(a.agent.id).toBe(b.agent.id); - expect(a.agent.status).toBe("idle"); + expect(a.agent.status).toBe("paused"); + expect((await db.select().from(agents).where(eq(agents.id, a.agent.id)))[0]!.lifecycleState).toBe("preparing"); expect(await service.resume(f.companyId, f.userId)).toMatchObject({ agent: { id: a.agent.id } }); expect(await service.resume(f.companyId, randomUUID())).toBeNull(); expect(await db.select().from(agents).where(eq(agents.companyId, f.companyId))).toHaveLength(1); @@ -47,6 +53,26 @@ it("atomically resumes one pending invite across retries and never persists a pa expect(JSON.stringify(logs)).not.toContain(code.pairingCode); expect(JSON.stringify(await service.resume(f.companyId, f.userId))).not.toContain(code.pairingCode); }); +it("reuses a committed hire when invitation completion must retry", async () => { + const f = await fixture(true); + const actual = await vi.importActual("../services/activity-log.js"); + vi.mocked(logActivity).mockImplementation(async (...args) => { + if (args[1].action === "agent.hire_created") { + vi.mocked(logActivity).mockImplementation(actual.logActivity); + throw new Error("Invitation audit unavailable"); + } + return actual.logActivity(...args); + }); + const service = dotInvitationService(db); + await expect(service.create(f.companyId, f.userId)).rejects.toThrow("Invitation audit unavailable"); + const before = await db.select().from(agents).where(eq(agents.companyId, f.companyId)); + expect(before).toHaveLength(1); + expect(await service.resume(f.companyId, f.userId)).toBeNull(); + const retried = await service.create(f.companyId, f.userId); + expect(retried.agent.id).toBe(before[0]!.id); + expect(await db.select().from(approvals).where(eq(approvals.companyId, f.companyId))).toHaveLength(1); + expect(await db.select().from(agents).where(eq(agents.companyId, f.companyId))).toHaveLength(1); +}); it("honors hiring approval before issuing a pairing capability", async () => { const f = await fixture(true); const invite = await dotInvitationService(db).create(f.companyId, f.userId); @@ -91,7 +117,7 @@ it("keeps invite routes company-scoped and denies viewers and agent actors", asy expect(result.status, JSON.stringify(result.body)).toBe(200); expect(result.body.agent.id).toBeTruthy(); expect((await request(app).get(path)).body.agent.id).toBe(result.body.agent.id); - expect(await db.select().from(agents).where(and(eq(agents.companyId, f.companyId), eq(agents.status, "idle")))).toHaveLength(1); + expect(await db.select().from(agents).where(and(eq(agents.companyId, f.companyId), eq(agents.lifecycleState, "preparing")))).toHaveLength(1); }); it("does not trust editable agent metadata as an invitation ownership receipt", async () => { diff --git a/server/src/__tests__/dot-oauth-onboarding.test.ts b/server/src/__tests__/dot-oauth-onboarding.test.ts index 2f88ebeda2..53e24b5606 100644 --- a/server/src/__tests__/dot-oauth-onboarding.test.ts +++ b/server/src/__tests__/dot-oauth-onboarding.test.ts @@ -4,7 +4,7 @@ import express from "express"; import request from "supertest"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { createDb, authUsers, companies, companyMemberships, agents, dotAgentBindings, mcpOauthGrants, mcpOauthRequests, mcpOauthTokens } from "@paperclipai/db"; +import { createDb, authUsers, companies, companyMemberships, agents, dotAgentBindings, dotMailboxItems, mcpEventSubscriptions, mcpOauthGrants, mcpOauthRequests, mcpOauthTokens } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createPublicMcpOAuth, publicMcpConfig, hashMcpSecret, DEVICE_GRANT } from "../services/public-mcp/oauth.js"; import { instanceSettingsService } from "../services/instance-settings.js"; @@ -22,12 +22,12 @@ describe("Dot onboarding with an operator-issued pairing capability", () => { }, 120000); beforeEach(async () => { await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true, enableOpenAiDot: true, enableNativeRunner: false }); }); afterAll(async () => { await temp?.cleanup(); }); - async function fixture(authorizationOrigin?: string) { + async function fixture(authorizationOrigin?: string, lifecycleState = "ready") { const userId = randomUUID(); await db.insert(authUsers).values({ id: userId, name: "Operator", email: userId + "@example.test", createdAt: new Date(), updatedAt: new Date() }); const [company] = await db.insert(companies).values({ name: "Dot onboarding", issuePrefix: "DO" + randomBytes(3).toString("hex") }).returning(); await db.insert(companyMemberships).values({ companyId: company!.id, principalType: "user", principalId: userId, membershipRole: "owner", status: "active" }); - const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Dot", adapterType: "paperclip_runner", adapterConfig: { provider: "openai_dot" }, status: "active" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Dot", adapterType: "paperclip_runner", adapterConfig: { provider: "openai_dot" }, lifecycleState, status: lifecycleState === "ready" ? "active" : "paused" }).returning(); const oauth = createPublicMcpOAuth(db, { ...config, ...(authorizationOrigin ? { authorizationOrigin } : {}) }); const client = await oauth.register({ client_name: "Dot", redirect_uris: [callback], grant_types: ["authorization_code", "refresh_token", DEVICE_GRANT] }, randomUUID()); const verifier = randomBytes(32).toString("base64url"); @@ -43,6 +43,49 @@ describe("Dot onboarding with an operator-issued pairing capability", () => { return f.oauth.token({ grant_type: "authorization_code", client_id: f.client.client_id, redirect_uri: callback, resource: config.resource, code: new URL(consent.redirectUrl).searchParams.get("code"), code_verifier: f.verifier }); } + + it("invalidates setup when a broker pairing is applied or revoked", async () => { + const f = await fixture(undefined, "preparing"); + const consent = await f.oauth.consent(f.id, { type: "board", source: "session", userId: f.userId }, + { decision: "approve", companyId: f.company.id, allowWrites: false }); + const tokens = await f.oauth.token({ grant_type: "authorization_code", client_id: f.client.client_id, + redirect_uri: callback, resource: config.resource, code: new URL(consent.redirectUrl).searchParams.get("code"), code_verifier: f.verifier }); + const principal = await f.oauth.authenticate(tokens.access_token); + const broker = dotRunnerBroker(db); + await broker.pair(principal, f.pairing.pairingCode); + const [paired] = await db.select().from(agents).where(eq(agents.id, f.agent.id)); + expect(paired.adapterConfig.dotBindingId).toBe(f.pairing.bindingId); + expect(paired.lifecycleVersion).toBe(f.agent.lifecycleVersion + 1); + await expect(broker.pair(principal, f.pairing.pairingCode)).rejects.toThrow("Pairing code expired or was consumed"); + + await broker.revoke(f.company.id, f.agent.id, f.userId); + const [revoked] = await db.select().from(agents).where(eq(agents.id, f.agent.id)); + expect(revoked.adapterConfig).not.toHaveProperty("dotBindingId"); + expect(revoked.lifecycleVersion).toBe(paired.lifecycleVersion + 1); + await expect(f.oauth.authenticate(tokens.access_token)).rejects.toThrow(); + }); + it.each(["preparing", "verifying"])("allows Dot setup during %s without allowing task access", async (state) => { + const f = await fixture(undefined, state); + const tokens = await connect(f); + const principal = await f.oauth.authenticate(tokens.access_token); + const broker = dotRunnerBroker(db); + await db.insert(mcpEventSubscriptions).values({ id: randomUUID(), companyId: f.company.id, + grantId: principal.grant.id, name: "paperclip.dot.mailbox_updated", bindingId: f.pairing.bindingId, + arguments: {}, deliveryMaterial: {}, verifiedAt: new Date(), expiresAt: new Date(Date.now() + 60_000) }); + await db.insert(dotMailboxItems).values({ companyId: f.company.id, bindingId: f.pairing.bindingId, + bindingGeneration: 1, kind: "assignment", sourceEventId: randomUUID(), references: { task: "hidden" } }); + await broker.challenge(f.company.id, f.agent.id); + const inbox = await broker.mailbox(principal); + expect(inbox.items.map(item => item.kind)).toEqual(["readiness_challenge"]); + expect(inbox.nextCursor).toBe(0); + await expect(broker.confirmChallenge(principal, String(inbox.items[0]!.references.challenge))).resolves.toEqual({ status: "ready" }); + expect(await broker.capabilities(principal)).toMatchObject({ ready: false }); + await expect(broker.tasks(principal)).rejects.toThrow("authority"); + await expect(broker.snapshot(f.company.id, f.agent.id, f.pairing.bindingId)).rejects.toThrow("authority"); + await db.update(agents).set({ lifecycleState: "paused" }).where(eq(agents.id, f.agent.id)); + expect((await broker.mailbox(principal)).items).toEqual([]); + await expect(broker.confirmChallenge(principal, "already-used")).rejects.toThrow("authority"); + }); it("refreshes a Dot connection after years of inactivity, still rotates and revokes on replay", async () => { const f = await fixture(); const tokens = await connect(f); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index b683491a82..0d3a2a76dd 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -12,6 +12,7 @@ import { stopSshEnvLabFixture, } from "@paperclipai/adapter-utils/ssh"; import { + activityLog, costEvents, agents, builtInManagedResources, @@ -219,6 +220,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { await stopSshEnvLabFixture(path.join(root, "state.json")).catch(() => undefined); await rm(root, { recursive: true, force: true }).catch(() => undefined); } + await db.delete(activityLog); await db.delete(environmentLeases); await db.delete(issues); await db.delete(costEvents); @@ -5780,6 +5782,21 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }); }); + it("destroys a failed retained lease when the agent stops", async () => { + const { lease, workerManager, runtimeWithPlugin, runId } = await seedStaleLifecycleReusableLease("retain_on_failure"); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + await runtimeWithPlugin.releaseRunLeases(runId, "failed"); + await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "failed" }); + vi.mocked(workerManager.getWorker).mockReturnValue({ supportedMethods: ["environmentDestroyLease"] } as never); + vi.mocked(workerManager.call).mockResolvedValue({ providerLeaseId: lease.providerLeaseId, state: "destroyed" }); + const { heartbeatService } = await import("../services/heartbeat.js"); + const stopped = await heartbeatService(db, { pluginWorkerManager: workerManager }) + .stopInvocationsForAgents([lease.metadata!.agentId as string], "Agent lifecycle stop requested"); + expect(stopped).toBe(true); + expect(workerManager.call).toHaveBeenCalledWith(expect.any(String), "environmentDestroyLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId }), expect.any(Number)); + await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success" }); + }); + it("fails closed on expiry destruction when the worker no longer advertises the destroy lifecycle method", async () => { const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease(); diff --git a/server/src/__tests__/feedback-service.test.ts b/server/src/__tests__/feedback-service.test.ts index ed3c7f2189..08a4444203 100644 --- a/server/src/__tests__/feedback-service.test.ts +++ b/server/src/__tests__/feedback-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; @@ -800,6 +802,8 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { sessionId, }); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.feedback, notified); await flushingSvc.saveIssueVote({ issueId, targetType: "issue_comment", @@ -808,7 +812,12 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { authorUserId: "user-1", allowSharing: true, }); - await flushingSvc.flushPendingFeedbackTraces(); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(true); + // The route and worker may request a flush together after the commit. + await Promise.all([flushingSvc.flushPendingFeedbackTraces(), flushingSvc.flushPendingFeedbackTraces()]); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(false); expect(uploadTraceBundle).toHaveBeenCalledTimes(1); const bundle = uploadTraceBundle.mock.calls[0]?.[0] as Record | undefined; @@ -1152,6 +1161,30 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { expect(uploadTraceBundle).toHaveBeenCalledTimes(1); }); + it("leaves aborted and unstarted exports available for recovery after shutdown", async () => { + const first = await seedIssueWithAgentComment(); + const second = await seedIssueWithAgentComment(); + const shutdown = new AbortController(); + const uploadTraceBundle = vi.fn().mockResolvedValue({ objectKey: "saved" }) + .mockImplementationOnce(async (_bundle, signal: AbortSignal) => { + shutdown.abort(new Error("shutdown")); + signal.throwIfAborted(); + }); + const flushingSvc = feedbackService(db, { shareClient: { uploadTraceBundle } }); + for (const item of [first, second]) { + await flushingSvc.saveIssueVote({ issueId: item.issueId, targetType: "issue_comment", + targetId: item.commentId, vote: "up", authorUserId: "user-1", allowSharing: true }); + } + expect(await flushingSvc.flushPendingFeedbackTraces({ signal: shutdown.signal })) + .toMatchObject({ attempted: 1, sent: 0, failed: 1 }); + const rows = await db.select().from(feedbackExports); + expect(rows.filter(row => row.status === "failed")).toHaveLength(1); + expect(rows.filter(row => row.status === "pending")).toHaveLength(1); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(true); + expect(await flushingSvc.flushPendingFeedbackTraces()).toMatchObject({ attempted: 2, sent: 2, failed: 0 }); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(false); + }); + it("marks pending shared traces as failed when no feedback export backend is configured", async () => { const { companyId, issueId, commentId } = await seedIssueWithAgentComment(); diff --git a/server/src/__tests__/feedback-share-client.test.ts b/server/src/__tests__/feedback-share-client.test.ts index fe71026283..d47d01ede4 100644 --- a/server/src/__tests__/feedback-share-client.test.ts +++ b/server/src/__tests__/feedback-share-client.test.ts @@ -1,6 +1,6 @@ import { gunzipSync } from "node:zlib"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { createFeedbackTraceShareClientFromConfig } from "../services/feedback-share-client.js"; +import { FEEDBACK_UPLOAD_TIMEOUT_MS, createFeedbackTraceShareClientFromConfig } from "../services/feedback-share-client.js"; describe("feedback trace share client", () => { beforeEach(() => { @@ -12,6 +12,7 @@ describe("feedback trace share client", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); it("defaults to telemetry.paperclip.ing when no backend url is configured", async () => { @@ -98,4 +99,43 @@ describe("feedback trace share client", () => { expect(parsed.objectKey.endsWith("/export-1.json")).toBe(true); expect(parsed.bundle.envelope).toEqual({ hello: "world" }); }); + it.each(["headers", "body"])("cancels a stalled upload at its deadline while waiting for %s", async phase => { + const deadline = new AbortController(); + const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(deadline.signal); + let started!: () => void; + const waiting = new Promise(resolve => { started = resolve; }); + vi.mocked(fetch).mockImplementation(async (_url, options) => { + const block = () => new Promise((_resolve, reject) => { + options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true }); + started(); + }); + if (phase === "headers") return block(); + return { ok: true, json: block } as unknown as Response; + }); + const client = createFeedbackTraceShareClientFromConfig({ feedbackExportBackendUrl: undefined, feedbackExportBackendToken: undefined }); + const bundle = { companyId: "company", traceId: "trace" } as Parameters[0]; + const uploading = client.uploadTraceBundle(bundle); + const rejected = expect(uploading).rejects.toThrow("upload deadline"); + await waiting; + expect(timeout).toHaveBeenCalledWith(FEEDBACK_UPLOAD_TIMEOUT_MS); + deadline.abort(new Error("upload deadline")); + await rejected; + }); + + it("passes shutdown cancellation through to a pending upload", async () => { + const shutdown = new AbortController(); + let started!: () => void; + const waiting = new Promise(resolve => { started = resolve; }); + vi.mocked(fetch).mockImplementation((_url, options) => new Promise((_resolve, reject) => { + options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true }); + started(); + })); + const client = createFeedbackTraceShareClientFromConfig({ feedbackExportBackendUrl: undefined, feedbackExportBackendToken: undefined }); + const bundle = { companyId: "company", traceId: "trace" } as Parameters[0]; + const rejected = expect(client.uploadTraceBundle(bundle, shutdown.signal)).rejects.toThrow("shutdown"); + await waiting; + shutdown.abort(new Error("shutdown")); + await rejected; + }); + }); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index f02983143e..a4bde74c6c 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -9046,7 +9046,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(mockTerminateLocalService).toHaveBeenCalledWith( expect.objectContaining({ pid: 81_501, processGroupId: 81_502 }), - { forceAfterMs: 3000 }, + { forceAfterMs: 3000, signal: "SIGINT" }, ); expect(runningProcesses.has(runId)).toBe(false); }); diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index 4711ad72c2..a8c7882c71 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -216,6 +216,26 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { return { companyId, agentId }; } + it.each(["preparing", "verifying", "resuming"] as const)("keeps assigned work queued through %s, including recovery with timers disabled", async lifecycleState => { + const { companyId, agentId } = await seedCompanyAndAgent({ heartbeatConfig: { enabled: false } }); + await db.update(agents).set({ status: "paused", lifecycleState }).where(eq(agents.id, agentId)); + const [issue] = await db.insert(issues).values({ companyId, title: "Work accepted during setup", status: "todo", assigneeAgentId: agentId }).returning(); + const run = await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned", + requestedByActorType: "system", payload: { issueId: issue.id } }); + expect(run?.status).toBe("queued"); + await heartbeat.resumeQueuedRuns(); + expect((await heartbeat.getRun(run!.id))?.status).toBe("queued"); + expect(countExecuteCallsForRun(run!.id)).toBe(0); + expect((await heartbeat.reconcileStrandedAssignedIssues()).escalated).toBe(0); + expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0]).toMatchObject({ status: "todo", assigneeAgentId: agentId }); + await db.update(agents).set({ status: "idle", lifecycleState: "ready" }).where(eq(agents.id, agentId)); + const recovered = heartbeatService(db, { runtimeEnv: { ...process.env, PAPERCLIP_IN_WORKTREE: "false" } }); + await recovered.resumeQueuedRuns(); + expect(await waitForCondition(async () => countExecuteCallsForRun(run!.id) === 1)).toBe(true); + await recovered.waitForRunExecutionDrain(run!.id); + expect(countExecuteCallsForRun(run!.id)).toBe(1); + }); + async function seedQueuedRun(input: { companyId: string; agentId: string; diff --git a/server/src/__tests__/invite-create-route.test.ts b/server/src/__tests__/invite-create-route.test.ts index 27372692df..e6241153fc 100644 --- a/server/src/__tests__/invite-create-route.test.ts +++ b/server/src/__tests__/invite-create-route.test.ts @@ -7,6 +7,10 @@ const canUserMock = vi.fn(); const insertMock = vi.fn(); function registerModuleMocks() { + vi.doMock("../services/agent-lifecycle.js", () => ({ + createAgentLifecycle: () => ({}), + AgentLifecycleConflict: class extends Error {}, + })); vi.doMock("../services/index.js", () => ({ accessService: () => ({ isInstanceAdmin: vi.fn(), diff --git a/server/src/__tests__/invite-join-grants.test.ts b/server/src/__tests__/invite-join-grants.test.ts index 63c614e66d..c514e5d918 100644 --- a/server/src/__tests__/invite-join-grants.test.ts +++ b/server/src/__tests__/invite-join-grants.test.ts @@ -8,7 +8,7 @@ import { normalizeHumanRole, resolveHumanInviteRole, } from "../services/company-member-roles.js"; -import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope } from "../services/agent-permissions.js"; +import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope } from "../lib/agent-permissions.js"; const invitedAgentId = "agent-1"; const defaultAgentGrants = NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({ diff --git a/server/src/__tests__/issue-feedback-routes.test.ts b/server/src/__tests__/issue-feedback-routes.test.ts index e59fe6d1e9..643956ae22 100644 --- a/server/src/__tests__/issue-feedback-routes.test.ts +++ b/server/src/__tests__/issue-feedback-routes.test.ts @@ -8,6 +8,7 @@ const mockFeedbackService = vi.hoisted(() => ({ listIssueVotesForUser: vi.fn(), listFeedbackTraces: vi.fn(), saveIssueVote: vi.fn(), + flushPendingFeedbackTraces: vi.fn(() => new Promise(() => {})), })); const mockIssueService = vi.hoisted(() => ({ @@ -18,9 +19,6 @@ const mockIssueService = vi.hoisted(() => ({ findMentionedAgents: vi.fn(), })); -const mockFeedbackExportService = vi.hoisted(() => ({ - flushPendingFeedbackTraces: vi.fn(async () => ({ attempted: 1, sent: 1, failed: 0 })), -})); const mockAccessService = vi.hoisted(() => ({ canUser: vi.fn(), hasPermission: vi.fn(), @@ -138,7 +136,7 @@ async function createApp(actor: Record) { (req as any).actor = actor; next(); }); - app.use("/api", issueRoutes({} as any, {} as any, { feedbackExportService: mockFeedbackExportService })); + app.use("/api", issueRoutes({} as any, {} as any)); const routeErrors: string[] = []; app.locals.routeErrors = routeErrors; app.use((error: unknown, _req: express.Request, _res: express.Response, next: express.NextFunction) => { @@ -152,11 +150,6 @@ async function createApp(actor: Record) { describe("issue feedback trace routes", () => { beforeEach(() => { vi.clearAllMocks(); - mockFeedbackExportService.flushPendingFeedbackTraces.mockResolvedValue({ - attempted: 1, - sent: 1, - failed: 0, - }); mockHeartbeatService.wakeup.mockResolvedValue(undefined); mockHeartbeatService.reportRunActivity.mockResolvedValue(undefined); mockHeartbeatService.getRun.mockResolvedValue(null); @@ -174,7 +167,7 @@ describe("issue feedback trace routes", () => { mockLogActivity.mockResolvedValue(undefined); }); - it("flushes a newly shared feedback trace immediately after saving the vote", async () => { + it("returns the saved vote without waiting for feedback uploads", async () => { const targetId = "11111111-1111-4111-8111-111111111111"; mockIssueService.getById.mockResolvedValue({ id: "issue-1", @@ -211,11 +204,9 @@ describe("issue feedback trace routes", () => { }); expect([200, 201]).toContain(res.status); - expect(mockFeedbackExportService.flushPendingFeedbackTraces).toHaveBeenCalledWith({ - companyId: "company-1", - traceId: "trace-1", - limit: 1, - }); + expect(mockFeedbackService.saveIssueVote).toHaveBeenCalledTimes(1); + expect(mockFeedbackService.flushPendingFeedbackTraces).not.toHaveBeenCalled(); + }); it("rejects non-board callers before fetching a feedback trace", async () => { diff --git a/server/src/__tests__/issue-thread-interactions-service.test.ts b/server/src/__tests__/issue-thread-interactions-service.test.ts index 6e909fc8b3..c6c0b07254 100644 --- a/server/src/__tests__/issue-thread-interactions-service.test.ts +++ b/server/src/__tests__/issue-thread-interactions-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -231,11 +233,15 @@ describeEmbeddedPostgres("issueThreadInteractionService", () => { expect(created.payload.questions.map((question) => question.id)).toEqual(["repo", "scope", "hosting"]); expect(await interactionsSvc.create(issue, input, { userId: "local-board" })).toEqual(created); await expect(interactionsSvc.answerQuestions(issue, created.id, { answers: [{ questionId: "repo", optionIds: [], otherText: "https://example.com/repo" }] }, { userId: "local-board" })).rejects.toThrow("requires an answer"); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.question, notified); const answered = await interactionsSvc.answerQuestions(issue, created.id, { answers: [ { questionId: "repo", optionIds: [], otherText: "https://example.com/repo" }, { questionId: "scope", optionIds: ["all"] }, { questionId: "hosting", optionIds: ["existing", "new"] }, ] }, { userId: "local-board" }); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); expect(answered.status).toBe("answered"); }); @@ -1025,6 +1031,7 @@ describeEmbeddedPostgres("issueThreadInteractionService", () => { { agentId: creatorAgentId }, ); + await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, addresseeAgentId)); await agentService(db).remove(addresseeAgentId); const cancelled = await interactionsSvc.getById(created.id); diff --git a/server/src/__tests__/onboarding-seed-route.test.ts b/server/src/__tests__/onboarding-seed-route.test.ts index 766d8c9668..a7d75aa3c4 100644 --- a/server/src/__tests__/onboarding-seed-route.test.ts +++ b/server/src/__tests__/onboarding-seed-route.test.ts @@ -208,7 +208,9 @@ describeEmbeddedPostgres("POST /api/companies/:companyId/onboarding-seed", () => expect(companyGoals[0]?.title).toBe("Make robotics boring"); expect(companyGoals[0]?.description).toBe("Boring enough that hospitals buy it."); // No agent and no task were sent, so none were invented. - expect(await ctx.db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0); + const committedAgents = await ctx.db.select().from(agents).where(eq(agents.companyId, companyId)); + expect(committedAgents).toHaveLength(0); + expect(await ctx.db.select().from(projects).where(eq(projects.companyId, companyId))).toHaveLength(0); expect(await ctx.db.select().from(issues).where(eq(issues.companyId, companyId))).toHaveLength(0); expect(response.body.agentId).toBeNull(); expect(response.body.issueId).toBeNull(); @@ -245,12 +247,7 @@ describeEmbeddedPostgres("POST /api/companies/:companyId/onboarding-seed", () => expect(entries).toHaveLength(1); }); - it("rolls the whole seed back when the audit entry cannot be written", async () => { - // The audit entry shares the seed's transaction, so a failure to write it - // must leave nothing behind. The alternative — commit the seed and lose the - // entry — is unrecoverable: Cloud stops retrying on a 2xx, and a later - // replay reports `changed: false` and never logs, so the entry would be - // permanently absent. + it("reuses the committed hire after the content transaction fails", async () => { const { companyId, app } = await seedCompany(); // Injected at the module boundary, not on `ctx.db`: the audit write goes @@ -261,18 +258,20 @@ describeEmbeddedPostgres("POST /api/companies/:companyId/onboarding-seed", () => const failed = await post(app, companyId, SEED); expect(failed.status).toBeGreaterThanOrEqual(500); - // Nothing committed: no seed record, so Cloud has no acknowledged revision - // and keeps retrying, and no orphaned agent from the rolled-back attempt. + // The failed content transaction must not acknowledge the revision. expect( await ctx.db .select() .from(companyOnboardingSeeds) .where(eq(companyOnboardingSeeds.companyId, companyId)), ).toHaveLength(0); - expect(await ctx.db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0); + const committedAgents = await ctx.db.select().from(agents).where(eq(agents.companyId, companyId)); + expect(committedAgents).toHaveLength(1); + expect(await ctx.db.select().from(projects).where(eq(projects.companyId, companyId))).toHaveLength(0); - // And the retry recovers completely — seed applied, entry present. await post(app, companyId, SEED).expect(200); + expect((await ctx.db.select().from(agents).where(eq(agents.companyId, companyId))).map(agent => agent.id)) + .toEqual(committedAgents.map(agent => agent.id)); expect( await ctx.db .select() @@ -299,7 +298,9 @@ describeEmbeddedPostgres("POST /api/companies/:companyId/onboarding-seed", () => const response = await post(strangerApp, companyId, SEED); expect(response.status).toBe(403); - expect(await ctx.db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0); + const committedAgents = await ctx.db.select().from(agents).where(eq(agents.companyId, companyId)); + expect(committedAgents).toHaveLength(0); + expect(await ctx.db.select().from(projects).where(eq(projects.companyId, companyId))).toHaveLength(0); expect(await ctx.db.select().from(companyOnboardingSeeds)).toHaveLength(0); }); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index cd8cdded1b..ed79f9537f 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -279,6 +279,15 @@ function loadSpecRoutes() { } describe("openapi routes", () => { + it("documents only writable agent update fields and lifecycle status requests", () => { + const document = buildOpenApiSpec() as any; + const properties = document.paths["/api/agents/{id}"].patch.requestBody.content["application/json"].schema.properties; + expect(properties).not.toHaveProperty("spentMonthlyCents"); + expect(properties.status.enum).toEqual(["paused", "idle", "terminated"]); + expect(properties).toHaveProperty("name"); + expect(properties).toHaveProperty("budgetMonthlyCents"); + }); + it("documents manager-only task privacy hints without protected scope identity", () => { const spec = buildOpenApiSpec() as any; const operation = spec.paths["/api/issues/{id}/privacy-constraints"].get; diff --git a/server/src/__tests__/plugin-idle-rpc.test.ts b/server/src/__tests__/plugin-idle-rpc.test.ts index c1e626c1b8..a60365612e 100644 --- a/server/src/__tests__/plugin-idle-rpc.test.ts +++ b/server/src/__tests__/plugin-idle-rpc.test.ts @@ -21,6 +21,18 @@ function fixture(definition: PluginDefinition, rpcTimeoutMs = 30_000) { } describe("plugin idle RPC", () => { + it("advertises and dispatches the agent lifecycle method", async () => { + const request = { companyId: "company", agentId: "agent", operationId: "operation", version: 7, phase: "preparing" }; + const f = fixture({ async setup() {}, async onAgentLifecycle(input) { + expect(input).toEqual(request); + return { operationId: input.operationId, version: input.version, status: "pending" }; + } }); + try { + expect((await f.initialize()).result.supportedMethods).toContain("agentLifecycle"); + expect((await f.call("agentLifecycle", request)).result).toEqual({ operationId: "operation", version: 7, status: "pending" }); + } finally { f.close(); } + }); + const hold = () => ({ ownerId: "idle-owner", expiresAt: Date.now() + 30_000 }); it("seals an idle worker, fences stale releases, and restores normal calls", async () => { diff --git a/server/src/__tests__/plugin-managed-agents.test.ts b/server/src/__tests__/plugin-managed-agents.test.ts index 95abe9db66..a1661fef5e 100644 --- a/server/src/__tests__/plugin-managed-agents.test.ts +++ b/server/src/__tests__/plugin-managed-agents.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import os from "node:os"; @@ -256,11 +257,15 @@ describeEmbeddedPostgres("plugin-managed agents", () => { it("keeps an explicit resume durable across managed-agent reconcile", async () => { const { companyId, services } = await seedCompanyAndPlugin({ manifest: pausedManifest() }); const created = await services.agents.managedReconcile({ companyId, agentKey: "wiki-maintainer" }); - await agentService(db).resume(created.agentId!); + await createAgentLifecycle(db).resumeAgent(created.agentId!); const reconciled = await services.agents.managedReconcile({ companyId, agentKey: "wiki-maintainer" }); - expect(reconciled.agent).toMatchObject({ + expect(reconciled.agent).toMatchObject({ lifecycleState: "resuming", pauseReason: null }); + const worker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" }); + await worker.process(created.agentId!); + await worker.stop(); + expect((await services.agents.managedReconcile({ companyId, agentKey: "wiki-maintainer" })).agent).toMatchObject({ status: "idle", pauseReason: null, pausedAt: null, diff --git a/server/src/__tests__/primary-agent.test.ts b/server/src/__tests__/primary-agent.test.ts index 1f4f6f6a51..a8adb69a2d 100644 --- a/server/src/__tests__/primary-agent.test.ts +++ b/server/src/__tests__/primary-agent.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { readFile } from "node:fs/promises"; import express from "express"; @@ -42,7 +43,7 @@ if (!support.supported) console.warn(`Primary agent database tests unavailable: await db.insert(companyMemberships).values(["user-a", "user-b", "explicit"].map(principalId => ({ companyId: id, principalType: "user", principalId, status: "active", membershipRole: "admin" }))); return id; } - const create = (companyId: string, name: string, userId?: string) => agentService(db).create(companyId, { + const create = (companyId: string, name: string, userId?: string) => createAgentLifecycle(db).requestHire(companyId, { name, adapterType: "process", status: "idle", }, { createdByUserId: userId }); const get = (companyId: string, userId = "user-a") => primaryAgentService(db).get(companyId, userId); @@ -123,9 +124,9 @@ if (!support.supported) console.warn(`Primary agent database tests unavailable: it("retains paused/error primaries, clears on leave, rejoins when chosen, and preserves stars", async () => { const c = await company(); const a = await create(c, "Maia", "user-a"); - await agentService(db).pause(a.id); + await createAgentLifecycle(db).pauseAgent(a.id); expect((await get(c)).primaryAgentId).toBe(a.id); - await agentService(db).update(a.id, { status: "error" }); + await db.update(agents).set({ status: "error", lifecycleState: "ready", lifecycleHolds: [] }).where(eq(agents.id, a.id)); expect((await get(c)).primaryAgentId).toBe(a.id); const memberships = resourceMembershipService(db); await memberships.updateAgent({ companyId: c, userId: "user-a", agentId: a.id, state: "left", actor: actor(c) }); @@ -153,23 +154,24 @@ if (!support.supported) console.warn(`Primary agent database tests unavailable: it("clears terminated/deleted primaries and rejects terminated targets and explicit removal", async () => { const c = await company(); const a = await create(c, "Maia", "user-a"); - await agentService(db).terminate(a.id); + await createAgentLifecycle(db).terminateAgent(a.id); expect(await get(c)).toMatchObject({ primaryAgentId: null, initialized: true }); await request(app(actor(c))).put(url(c)).send({ primaryAgentId: a.id }).expect(422); await request(app(actor(c))).put(url(c)).send({ primaryAgentId: null }).expect(400); const b = await create(c, "Alex", "user-a"); expect((await get(c)).primaryAgentId).toBeNull(); await request(app(actor(c))).put(url(c)).send({ primaryAgentId: b.id }).expect(200); + await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, b.id)); await agentService(db).remove(b.id); expect(await get(c)).toMatchObject({ primaryAgentId: null, initialized: true }); }); - it("rolls creation and initialization back together", async () => { + it("rejects creation inside a caller transaction", async () => { const c = await company(); await expect(db.transaction(async tx => { - await agentService(tx as unknown as typeof db).create(c, { name: "Rolled back", adapterType: "process" }, { createdByUserId: "user-a" }); + await createAgentLifecycle(tx as unknown as typeof db).requestHire(c, { name: "Rolled back", adapterType: "process" }, { createdByUserId: "user-a" }); throw new Error("rollback"); - })).rejects.toThrow("rollback"); + })).rejects.toThrow("root database"); expect(await get(c)).toMatchObject({ primaryAgentId: null, initialized: false }); expect(await db.select().from(agents).where(eq(agents.companyId, c))).toHaveLength(0); }); @@ -177,7 +179,7 @@ if (!support.supported) console.warn(`Primary agent database tests unavailable: it("backfills the earliest human event, leaves gone/terminated originals empty, and preserves explicit preferences", async () => { const c = await company(); const first = await create(c, "First"), later = await create(c, "Later"); const terminated = await create(c, "Terminated"); - await agentService(db).terminate(terminated.id); + await createAgentLifecycle(db).terminateAgent(terminated.id); const events = [ ["human", first.id, "user", 1], ["human", later.id, "user", 2], ["gone", randomUUID(), "user", 1], ["gone", later.id, "user", 2], diff --git a/server/src/__tests__/resource-lifecycle-events.test.ts b/server/src/__tests__/resource-lifecycle-events.test.ts index 95aae83d32..dd54856dfd 100644 --- a/server/src/__tests__/resource-lifecycle-events.test.ts +++ b/server/src/__tests__/resource-lifecycle-events.test.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js"; import { randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; import { eq, sql } from "drizzle-orm"; @@ -22,23 +23,29 @@ describePostgres("Resource lifecycle events", () => { let database: Awaited>; let db: Db; let companyId: string; + let worker: ReturnType; beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-resource-events-"); db = createDb(database.connectionString); }, 90_000); - afterAll(async () => { await database?.cleanup(); }); + afterAll(async () => { await worker?.stop(); await database?.cleanup(); }); beforeEach(async () => { vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined); vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + await worker?.stop(); + worker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" }); companyId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Lifecycle fixture", issuePrefix: `L${companyId.replaceAll("-", "").slice(0, 6)}` }); }); afterEach(() => { vi.unstubAllEnvs(); }); const events = () => db.select().from(resourceLifecycleEvents).where(eq(resourceLifecycleEvents.companyId, companyId)); - const createAgent = (status: "idle" | "pending_approval" | "terminated" = "idle", database: Db = db) => - agentService(database).create(companyId, { name: "Lifecycle agent", adapterType: "process", adapterConfig: {}, status }); + const createAgent = async (status: "idle" | "pending_approval" | "terminated" = "idle", database: Db = db) => { + const agent = await createAgentLifecycle(database).requestHire(companyId, { name: "Lifecycle agent", adapterType: "process", adapterConfig: {}, status }); + await worker.process(agent.id); + return (await agentService(db).getById(agent.id))!; + }; const createPlugin = async () => { const [plugin] = await db.insert(plugins).values({ pluginKey: randomUUID(), packageName: "lifecycle-fixture", version: "1.0.0", status: "ready", manifestJson: {} as never }).returning(); return plugin; @@ -56,7 +63,7 @@ describePostgres("Resource lifecycle events", () => { const approval = await approvalService(db).create(companyId, { type: "hire_agent", status: "pending", payload: { agentId: agent.id } }); await approvalService(db).approve(approval.id, "fixture-board"); await approvalService(db).approve(approval.id, "fixture-board"); - await agentService(db).activatePendingApproval(agent.id); + await createAgentLifecycle(db).approveHire(agent.id); expect(await events()).toEqual([expect.objectContaining({ resourceType: "agent", resourceId: agent.id })]); }); @@ -64,6 +71,7 @@ describePostgres("Resource lifecycle events", () => { const approval = await approvalService(db).create(companyId, { type: "hire_agent", status: "pending", payload: { name: "Legacy hire", adapterType: "process" } }); await approvalService(db).approve(approval.id, "fixture-board"); const [intent] = await events(); + if (intent) await worker.process(intent.resourceId); expect(intent?.resourceType).toBe("agent"); expect(await agentService(db).getById(intent!.resourceId)).toMatchObject({ status: "idle", companyId }); }); @@ -111,25 +119,32 @@ describePostgres("Resource lifecycle events", () => { it("records every pause/resume cycle and termination in resource order, without duplicate hooks", async () => { const agent = await createAgent(); const service = agentService(db); + const serviceLifecycle = createAgentLifecycle(db); await db.insert(agentApiKeys).values({ agentId: agent.id, companyId, name: "Lifecycle key", keyHash: "fixture-hash" }); - await Promise.all(Array.from({ length: 4 }, () => service.pause(agent.id))); - await Promise.all(Array.from({ length: 4 }, () => service.resume(agent.id))); - await service.update(agent.id, { status: "paused" }); - await service.update(agent.id, { status: "idle" }); - await Promise.all(Array.from({ length: 4 }, () => service.terminate(agent.id))); + await Promise.all(Array.from({ length: 4 }, () => serviceLifecycle.pauseAgent(agent.id))); + await worker.process(agent.id); + await Promise.all(Array.from({ length: 4 }, () => serviceLifecycle.resumeAgent(agent.id))); + await worker.process(agent.id); + await serviceLifecycle.pauseAgent(agent.id); + await worker.process(agent.id); + await worker.process(agent.id); + await serviceLifecycle.resumeAgent(agent.id); + await worker.process(agent.id); + await worker.process(agent.id); + await Promise.all(Array.from({ length: 4 }, () => serviceLifecycle.terminateAgent(agent.id))); const rows = (await events()).sort((a, b) => a.id - b.id); expect(rows.map(row => row.action)).toEqual(["create", "pause", "resume", "pause", "resume", "terminate"]); expect(rows.every(row => row.resourceId === agent.id)).toBe(true); const [key] = await db.select().from(agentApiKeys).where(eq(agentApiKeys.agentId, agent.id)); expect(key.revokedAt).not.toBeNull(); - await expect(service.pause(agent.id)).rejects.toMatchObject({ status: 409 }); - await expect(service.resume(agent.id)).rejects.toMatchObject({ status: 409 }); + await expect(serviceLifecycle.pauseAgent(agent.id)).rejects.toMatchObject({ name: "AgentLifecycleConflict" }); + await expect(serviceLifecycle.resumeAgent(agent.id)).rejects.toMatchObject({ name: "AgentLifecycleConflict" }); }); it("does not bypass pending hire approval with pause or resume", async () => { const agent = await createAgent("pending_approval"); - await expect(agentService(db).pause(agent.id)).rejects.toMatchObject({ status: 409 }); - await expect(agentService(db).resume(agent.id)).rejects.toMatchObject({ status: 409 }); + await expect(createAgentLifecycle(db).pauseAgent(agent.id)).rejects.toMatchObject({ name: "AgentLifecycleConflict" }); + await expect(createAgentLifecycle(db).resumeAgent(agent.id)).rejects.toMatchObject({ name: "AgentLifecycleConflict" }); expect(await events()).toEqual([]); }); @@ -142,6 +157,7 @@ describePostgres("Resource lifecycle events", () => { await service.evaluateCostEvent(event); expect(await agentService(db).getById(agent.id)).toMatchObject({ status: "paused", pauseReason: "budget" }); await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agent.id, amount: 200 }, "fixture-board"); + await worker.process(agent.id); expect(await agentService(db).getById(agent.id)).toMatchObject({ status: "idle", pauseReason: null }); expect((await events()).sort((a, b) => a.id - b.id).map(row => row.action)).toEqual(["create", "pause", "resume"]); }); @@ -149,14 +165,15 @@ describePostgres("Resource lifecycle events", () => { it("rolls back pause, resume, termination, and key revocation if a hook write fails", async () => { const agent = await createAgent(); const paused = await createAgent(); - await agentService(db).pause(paused.id); + await createAgentLifecycle(db).pauseAgent(paused.id); + await worker.process(paused.id); await db.insert(agentApiKeys).values({ agentId: agent.id, companyId, name: "Retained key", keyHash: "fixture-retained-hash" }); const before = await events(); await db.execute(sql`ALTER TABLE resource_lifecycle_events ADD CONSTRAINT fixture_reject_hook CHECK (false) NOT VALID`); try { - await expect(agentService(db).pause(agent.id)).rejects.toThrow(); - await expect(agentService(db).resume(paused.id)).rejects.toThrow(); - await expect(agentService(db).terminate(agent.id)).rejects.toThrow(); + await expect(createAgentLifecycle(db).pauseAgent(agent.id)).rejects.toThrow(); + await expect(createAgentLifecycle(db).resumeAgent(paused.id)).rejects.toThrow(); + await expect(createAgentLifecycle(db).terminateAgent(agent.id)).rejects.toThrow(); expect(await agentService(db).getById(agent.id)).toMatchObject({ status: "idle" }); expect(await agentService(db).getById(paused.id)).toMatchObject({ status: "paused" }); const [key] = await db.select().from(agentApiKeys).where(eq(agentApiKeys.agentId, agent.id)); @@ -172,7 +189,8 @@ describePostgres("Resource lifecycle events", () => { const projectId = randomUUID(); await expect(db.transaction(async tx => { const txDb = tx as unknown as Db; - await agentService(txDb).create(companyId, { id: agentId, name: "Rolled back agent" }); + await tx.insert(agents).values({ id: agentId, companyId, name: "Rolled back agent" }); + await recordResourceCreationEvent(txDb, companyId, "agent", agentId); await projectService(txDb).createWithRepositories(companyId, { id: projectId, name: "Rolled back project" }, []); throw new Error("rollback fixture"); })).rejects.toThrow("rollback fixture"); @@ -387,8 +405,10 @@ describePostgres("Resource lifecycle events", () => { const first = await createPlugin(); const second = await createPlugin(); const agent = await createAgent(); - await agentService(db).pause(agent.id); - await agentService(db).resume(agent.id); + await createAgentLifecycle(db).pauseAgent(agent.id); + await worker.process(agent.id); + await createAgentLifecycle(db).resumeAgent(agent.id); + await worker.process(agent.id); const journal = (await events()).sort((a, b) => a.id - b.id); const inbox = pluginLifecycleInbox(db, first.id); const [event] = await inbox.list(companyId); @@ -426,7 +446,8 @@ describePostgres("Resource lifecycle events", () => { const plugin = await createPlugin(); const inbox = pluginLifecycleInbox(db, plugin.id); await db.transaction(async tx => { - const low = await agentService(tx as unknown as Db).create(companyId, { name: "Delayed commit" }); + const [low] = await tx.insert(agents).values({ companyId, name: "Delayed commit" }).returning(); + await recordResourceCreationEvent(tx as unknown as Db, companyId, "agent", low.id); const high = await createAgent(); const [visible] = await inbox.list(companyId); expect(visible.resourceId).toBe(high.id); diff --git a/server/src/__tests__/secret-proposals-routes.test.ts b/server/src/__tests__/secret-proposals-routes.test.ts index 124c8fcabc..3354345e4f 100644 --- a/server/src/__tests__/secret-proposals-routes.test.ts +++ b/server/src/__tests__/secret-proposals-routes.test.ts @@ -335,7 +335,7 @@ describeEmbeddedPostgres("secret proposal routes", () => { ]); }); - it("requires agent JWT and atomically cascade-approves a secret plus binding with dual audits", async () => { + it("requires agent JWT and approves a secret before its binding with dual audits", async () => { const fixture = await seedRun(); const denied = await request(createAgentApp(fixture, "agent_key")) .post("/api/agents/me/secret-proposals") @@ -475,6 +475,46 @@ describeEmbeddedPostgres("secret proposal routes", () => { ]); }); + it("keeps an approved secret after binding fails and reuses it on retry", async () => { + const fixture = await seedRun(); + const agentApp = createAgentApp(fixture); + const boardApp = createBoardApp(fixture); + const secret = await request(agentApp).post("/api/agents/me/secret-proposals").send({ + kind: "secret", name: "dev/retry/token", value: "retry-fixture", justification: "Needed by task", + }); + const binding = await request(agentApp).post("/api/agents/me/secret-proposals").send({ + kind: "binding", secretProposalId: secret.body.id, configPath: "env.RETRY_TOKEN", justification: "Use saved secret", + }); + expect(secret.status).toBe(201); + expect(binding.status).toBe(201); + await db.update(agents).set({ adapterConfig: { env: { RETRY_TOKEN: "existing-value" } } }) + .where(eq(agents.id, fixture.agentId)); + const approve = () => request(boardApp) + .post(`/api/companies/${fixture.companyId}/secret-proposals/${binding.body.id}/approve`).send({ cascade: true }); + + expect((await approve()).status).toBe(409); + const [saved] = await db.select().from(companySecrets); + expect(saved).toMatchObject({ name: "dev/retry/token", status: "active" }); + expect(await db.select().from(companySecretProposals)).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: secret.body.id, status: "approved", createdSecretId: saved.id, valueCiphertext: null }), + expect.objectContaining({ id: binding.body.id, status: "pending" }), + ])); + expect(await db.select().from(companySecretBindings)).toHaveLength(0); + expect((await db.select().from(agents))[0].adapterConfig).toEqual({ env: { RETRY_TOKEN: "existing-value" } }); + expect((await db.select().from(issueThreadInteractions))[0].status).toBe("pending"); + + await db.update(agents).set({ adapterConfig: {} }).where(eq(agents.id, fixture.agentId)); + expect((await approve()).status).toBe(200); + expect(await db.select().from(companySecrets)).toHaveLength(1); + expect(await db.select().from(companySecretVersions)).toHaveLength(1); + expect(await db.select().from(companySecretBindings)).toEqual([ + expect.objectContaining({ secretId: saved.id, targetId: fixture.agentId, configPath: "env.RETRY_TOKEN" }), + ]); + const history = await db.select().from(activityLog); + expect(history.filter(row => row.action === "secret.created")).toHaveLength(1); + expect(history.filter(row => row.action === "secret.proposal.approved")).toHaveLength(2); + }); + it("approves a binding without cascade after its secret proposal was approved separately", async () => { const fixture = await seedRun(); const agentApp = createAgentApp(fixture); @@ -1446,7 +1486,7 @@ describeEmbeddedPostgres("secret proposal routes", () => { ])); }); - it("holds the org graph stable until binding approval commits", async () => { + it("rechecks the org graph after secret approval commits", async () => { const fixture = await seedRun(); const targetAgentId = randomUUID(); await db.insert(agents).values({ @@ -1532,8 +1572,9 @@ describeEmbeddedPostgres("secret proposal routes", () => { expect(reorgFinished).toBe(false); const approval = await approvalPromise; - expect(approval.status).toBe(200); await reorgPromise; + expect(approval.status).toBe(409); + expect(approval.body.error).toContain("chain-of-command policy"); } finally { await db.execute(sql.raw(` DROP TRIGGER IF EXISTS paperclip_test_pause_secret_approval ON company_secrets; @@ -1541,9 +1582,12 @@ describeEmbeddedPostgres("secret proposal routes", () => { `)); } - expect(await db.select().from(companySecretBindings)).toEqual([ - expect.objectContaining({ targetId: targetAgentId, configPath: "access.CONCURRENT_REORG_TOKEN" }), - ]); + expect(await db.select().from(companySecretBindings)).toHaveLength(0); + expect(await db.select().from(companySecrets)).toHaveLength(1); + expect(await db.select().from(companySecretProposals)).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: secretProposal.body.id, status: "approved", valueCiphertext: null }), + expect.objectContaining({ id: bindingProposal.body.id, status: "pending" }), + ])); expect(await db.select({ reportsTo: agents.reportsTo }).from(agents).where(eq(agents.id, targetAgentId))) .toEqual([{ reportsTo: null }]); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 449d8d5b5d..5eaa40e414 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -44,11 +44,17 @@ const { const completionSweepMock = vi.fn(async () => undefined); const createAppMock = vi.fn(async () => Object.assign((_: unknown, __: unknown) => {}, { locals: { + deliveryWork: { + register: (_queue: string, task: { run: () => Promise }) => ({ + ready: task.run().catch(() => undefined), wake: vi.fn(), + }), + stop: vi.fn(async () => {}), + }, toolGateway: { sweepActionReviews: vi.fn(async () => ({ scanned: 0 })), cleanupExpiredSessions: vi.fn(async () => ({ deletedCount: 0 })), }, - toolActionDeliveries: { sweepPending: vi.fn(async () => ({ scanned: 0, delivered: 0 })) }, + toolActionDeliveries: { hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, delivered: 0 })) }, }, }) as never); const createBetterAuthInstanceMock = vi.fn(() => ({})); @@ -133,6 +139,7 @@ const { }; const routineServiceFactoryMock = vi.fn(() => routineServiceMock); const feedbackExportServiceMock = { + hasPendingFeedbackTraces: vi.fn(async () => false), flushPendingFeedbackTraces: vi.fn(async () => ({ attempted: 0, sent: 0, failed: 0 })), }; const feedbackServiceFactoryMock = vi.fn(() => feedbackExportServiceMock); @@ -363,16 +370,18 @@ vi.mock("../services/index.js", () => ({ })), })); -vi.mock("../services/chat-completion-delivery.js", () => ({ chatCompletionDeliveryService: () => ({ sweepPending: completionSweepMock }) })); +vi.mock("../services/chat-completion-delivery.js", () => ({ chatCompletionDeliveryService: () => ({ hasPending: vi.fn(async () => false), sweepPending: completionSweepMock }) })); vi.mock("../services/connection-intent-delivery.js", () => ({ connectionIntentDeliveryService: vi.fn(() => ({ + hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, failed: 0 })), })), })); vi.mock("../services/question-response-delivery.js", () => ({ questionResponseDeliveryService: vi.fn(() => ({ + hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, steered: 0, @@ -675,6 +684,22 @@ describe("startServer feedback export wiring", () => { expect(completionSweepMock).toHaveBeenCalled(); }); + it("keeps activity-driven completion scans scoped to the changed task", async () => { + await startServer(); + completionSweepMock.mockClear(); + const { publishLiveEvent } = await import("../services/live-events.js"); + publishLiveEvent({ companyId: "company-one", type: "activity.logged", payload: { + action: "issue.updated", entityId: "task-one", + } }); + expect(completionSweepMock).toHaveBeenCalled(); + for (const args of completionSweepMock.mock.calls) { + expect(args).toEqual([{ companyId: "company-one", taskId: "task-one" }]); + } + completionSweepMock.mockClear(); + publishLiveEvent({ companyId: "company-two", type: "activity.logged", payload: { action: "issue.updated" } }); + expect(completionSweepMock).not.toHaveBeenCalled(); + }); + it("never invokes the retired review detector at startup or on periodic recovery", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, diff --git a/server/src/__tests__/status-cards.test.ts b/server/src/__tests__/status-cards.test.ts index f0aef9e6da..e143666631 100644 --- a/server/src/__tests__/status-cards.test.ts +++ b/server/src/__tests__/status-cards.test.ts @@ -27,7 +27,7 @@ import { } from "@paperclipai/shared"; import { errorHandler } from "../middleware/index.js"; import { statusCardRoutes } from "../routes/status-cards.js"; -import { withBuiltInAgentMarker } from "../services/built-in-agent-metadata.js"; +import { withBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import type { IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js"; import { issueService } from "../services/issues.js"; diff --git a/server/src/__tests__/summary-slots.test.ts b/server/src/__tests__/summary-slots.test.ts index ac9fba3bff..77f32cc082 100644 --- a/server/src/__tests__/summary-slots.test.ts +++ b/server/src/__tests__/summary-slots.test.ts @@ -20,7 +20,7 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { summarySlotService } from "../services/summary-slots.ts"; -import { withBuiltInAgentMarker } from "../services/built-in-agent-metadata.ts"; +import { withBuiltInAgentMarker } from "../lib/built-in-agent-metadata.ts"; import { issueService } from "../services/issues.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); diff --git a/server/src/__tests__/tool-gateway-service.test.ts b/server/src/__tests__/tool-gateway-service.test.ts index 3e9b984672..0db14e6f47 100644 --- a/server/src/__tests__/tool-gateway-service.test.ts +++ b/server/src/__tests__/tool-gateway-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -219,6 +221,8 @@ describeEmbeddedPostgres("tool gateway service", () => { runId: run.id, }); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.toolAction, notified); await expect(gateway.executeTool({ sessionToken: session.token, tool: "mcp-remote-fixture:update_note", @@ -228,6 +232,9 @@ describeEmbeddedPostgres("tool gateway service", () => { details: { instructions: expect.stringContaining("A human approval card was posted on task") }, }); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); + await expect(gateway.executeTool({ sessionToken: session.token, tool: "mcp-remote-fixture:update_note", diff --git a/server/src/__tests__/work-scheduler.test.ts b/server/src/__tests__/work-scheduler.test.ts new file mode 100644 index 0000000000..dcfde0220a --- /dev/null +++ b/server/src/__tests__/work-scheduler.test.ts @@ -0,0 +1,47 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createWorkScheduler } from "../services/work-scheduler.js"; +afterEach(() => vi.useRealTimers()); +describe("work scheduler", () => { + it("owns one timer, replaces named deadlines, and exposes the earliest wake", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const s = createWorkScheduler(); + const first = vi.fn(), second = vi.fn(); + s.schedule("first", 100, first); + s.schedule("second", 50, second); + s.schedule("first", 20, first); + expect(vi.getTimerCount()).toBe(1); + expect(s.nextWakeAt()).toBe(20); + await vi.advanceTimersByTimeAsync(20); + expect(first).toHaveBeenCalledTimes(1); + expect(second).not.toHaveBeenCalled(); + expect(s.nextWakeAt()).toBe(50); + s.cancel("second"); + expect(s.nextWakeAt()).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + it("honors cancellation during dispatch and isolates a failing callback", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => {}); + const s = createWorkScheduler(), cancelled = vi.fn(), later = vi.fn(); + s.schedule("first", 10, () => { s.cancel("cancelled"); throw new Error("failure"); }); + s.schedule("cancelled", 10, cancelled); + s.schedule("later", 20, later); + await vi.advanceTimersByTimeAsync(20); + expect(cancelled).not.toHaveBeenCalled(); + expect(later).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); s.stop(); + }); + it("does not fire distant deadlines early and cannot restart after stop", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const s = createWorkScheduler(), run = vi.fn(); + s.schedule("far", 3_000_000_000, run); + await vi.advanceTimersByTimeAsync(2_147_483_647); + expect(run).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(852_516_353); + expect(run).toHaveBeenCalledTimes(1); + s.stop(); s.schedule("late", Date.now(), run); + expect(vi.getTimerCount()).toBe(0); + expect(() => createWorkScheduler().schedule("bad", NaN, run)).toThrow("finite"); + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index bda8217716..fdf1d385de 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1,3 +1,7 @@ +import { createDeliveryWorkCoordinator } from "./services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js"; +import { createLifecycleDriver } from "./services/agent-lifecycle-driver.js"; +import { startAgentLifecycle } from "./services/agent-lifecycle.js"; import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./middleware/idle-admission.js"; import { isIdleTaskDrainActive, trackIdleWork } from "./services/task-admission.js"; import { customerSuccessRoutes } from "./routes/customer-success.js"; @@ -483,11 +487,13 @@ export async function createApp( serverPort: number; storageService: StorageService; feedbackExportService?: { + hasPendingFeedbackTraces(): Promise; flushPendingFeedbackTraces(input?: { companyId?: string; traceId?: string; limit?: number; now?: Date; + signal?: AbortSignal; }): Promise; }; databaseBackupService?: InstanceDatabaseBackupService; @@ -618,6 +624,8 @@ export async function createApp( const hostServicesDisposers = new Map void>(); const workerManager = opts.pluginWorkerManager ?? createPluginWorkerManager(); + let lifecyclePluginsReady = false; + const agentLifecycle = startAgentLifecycle(db, createLifecycleDriver(db, workerManager), () => lifecyclePluginsReady && !isWarmStandby() && !isIdleTaskDrainActive()); const connectionIntentHeartbeat = heartbeatService(db, { pluginWorkerManager: workerManager, }); @@ -877,7 +885,6 @@ export async function createApp( // route prefixes, so this dependency does not change issue-route precedence. api.use(issueRoutes(db, opts.storageService, { chatRunRetries: chatChannels, - feedbackExportService: opts.feedbackExportService, pluginWorkerManager: workerManager, approveToolActionRequest: (input) => toolGateway.approveActionRequest(input), declineToolActionRequest: (input) => toolGateway.declineActionRequest(input), @@ -1190,40 +1197,19 @@ export async function createApp( jobCoordinator.start(); scheduler.start(); - let feedbackExportShuttingDown = false; - let feedbackExportTimer: ReturnType | null = null; - const disableFeedbackExportFlushes = () => { - feedbackExportShuttingDown = true; - if (feedbackExportTimer) { - clearInterval(feedbackExportTimer); - feedbackExportTimer = null; - } - }; - const flushPendingFeedbackExports = async () => { - if (feedbackExportShuttingDown || isWarmStandby() || isIdleTaskDrainActive()) return; - try { - await opts.feedbackExportService?.flushPendingFeedbackTraces(); - } catch (err) { - if (isDatabaseConnectionUnavailableError(err)) { - disableFeedbackExportFlushes(); - logger.warn( - { err }, - "Disabling pending feedback export flushes because the database is unavailable", - ); - return; - } - logger.error({ err }, "Failed to flush pending feedback exports"); - } - }; - - feedbackExportTimer = opts.feedbackExportService - ? setInterval(() => { - void flushPendingFeedbackExports(); - }, FEEDBACK_EXPORT_FLUSH_INTERVAL_MS) - : null; - feedbackExportTimer?.unref?.(); + const deliveryWork = createDeliveryWorkCoordinator({ + owner: db, + canRun: () => !isWarmStandby() && !isIdleTaskDrainActive(), + canReconcile: () => !isWarmStandby(), + onError: (err, queue) => logger.error({ err, queue }, "Delivery reconciliation failed"), + }); + app.locals.deliveryWork = deliveryWork; if (opts.feedbackExportService) { - void flushPendingFeedbackExports(); + deliveryWork.register(DELIVERY_QUEUES.feedback, { + retryMs: FEEDBACK_EXPORT_FLUSH_INTERVAL_MS, + run: (signal) => opts.feedbackExportService!.flushPendingFeedbackTraces({ signal }), + hasPending: () => opts.feedbackExportService!.hasPendingFeedbackTraces(), + }); } emailChannels.start(); const flushChatPublications = async () => { @@ -1376,6 +1362,7 @@ export async function createApp( logger.error({ err }, "Failed to load ready plugins on startup"); }); app.locals.bundledPluginsStartup = trackIdleWork(bundledPluginsStartup); + void bundledPluginsStartup.then(() => { lifecyclePluginsReady = true; return agentLifecycle.sweep(); }).catch(() => logger.warn("Agent lifecycle recovery failed; retrying.")); // The shutdown hook runs at most once. It caches the in-flight promise, so a // second caller (for example the `exit` handler) awaits the same completion // instead of starting a second teardown. @@ -1386,10 +1373,11 @@ export async function createApp( // The scheduler tick queries the database. Stop it here, inside the // awaited teardown, so no tick runs after the caller ends the pool. scheduler.stop(); + await agentLifecycle.stop(); await publicMcpEvents?.stop(); await dotMcpEvents?.stop(); jobCoordinator.stop(); - disableFeedbackExportFlushes(); + await deliveryWork.stop(); unsubscribeChatPublicationSignals(); chatReconciliation.stop(); if (chatPublicationTimer) { diff --git a/server/src/http/server.test.ts b/server/src/http/server.test.ts new file mode 100644 index 0000000000..6798ab9a47 --- /dev/null +++ b/server/src/http/server.test.ts @@ -0,0 +1,52 @@ +import { once } from "node:events"; +import { request } from "node:http"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DurablePrpControlPlane } from "../vendor/paperclip-runner/index.js"; +import { registerRunnerPrpAuthority, runnerPrpWebSocketInternals } from "../realtime/runner-prp-ws.js"; +import { createPaperclipHttpServer } from "./server.js"; + +describe("Paperclip HTTP server runner transport", () => { + afterEach(() => runnerPrpWebSocketInternals.resetForTests()); + + it("registers native runner authority without a separate startup call", async () => { + const server = createPaperclipHttpServer((_req, res) => res.end("ok"), { + apiUrl: "http://127.0.0.1:3210", + }); + const handleUpgrade = vi.fn(); + const runId = "00000000-0000-4000-8000-000000000777"; + const authority = await registerRunnerPrpAuthority({ + companyId: "company-1", runId, + authority: { handleUpgrade } as unknown as DurablePrpControlPlane, + }); + expect(authority.connectUrl).toBe(`ws://127.0.0.1:3210/api/runner/v1/connect/${runId}`); + expect(server.listenerCount("upgrade")).toBe(1); + await authority.release(); + server.close(); + }); + + it("serves HTTP and rejects an unregistered runner over the real listener", async () => { + const server = createPaperclipHttpServer((_req, res) => res.end("ready"), { + apiUrl: "http://127.0.0.1:3210", + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Listener not bound"); + const origin = `http://127.0.0.1:${address.port}`; + try { + expect(await (await fetch(origin)).text()).toBe("ready"); + const status = await new Promise((resolve, reject) => { + const req = request(`${origin}/api/runner/v1/connect/00000000-0000-4000-8000-000000000778`, { + headers: { connection: "Upgrade", upgrade: "websocket" }, + }, res => { res.resume(); resolve(res.statusCode); }); + req.on("error", reject); + req.setTimeout(1000, () => req.destroy(new Error("Runner upgrade was not handled"))); + req.end(); + }); + expect(status).toBe(404); + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } + }); +}); diff --git a/server/src/http/server.ts b/server/src/http/server.ts new file mode 100644 index 0000000000..f3097ee455 --- /dev/null +++ b/server/src/http/server.ts @@ -0,0 +1,14 @@ +import { createServer, type RequestListener, type ServerOptions } from "node:http"; +import { setupRunnerPrpWebSocketServer } from "../realtime/runner-prp-ws.js"; + +/** Create the HTTP listener used by both production and custom launchers. */ +export function createPaperclipHttpServer( + listener: RequestListener, + options: { apiUrl: string; httpOptions?: ServerOptions }, +) { + const server = createServer(options.httpOptions ?? {}, listener); + // Native execution uses the listener even when a launcher has no browser + // realtime connection. Register it before any run can be admitted. + setupRunnerPrpWebSocketServer(server, { apiUrl: options.apiUrl }); + return server; +} diff --git a/server/src/index.ts b/server/src/index.ts index ce72014df0..6bef07ad23 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,9 +1,11 @@ +import { subscribeAllCompanyLiveEvents } from "./services/live-events.js"; +import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js"; +import type { createDeliveryWorkCoordinator } from "./services/delivery-work-coordinator.js"; import { isIdleTaskDrainActive, beginIdleTrackedWork, trackIdleWork } from "./services/task-admission.js"; import { markIdleStartupComplete } from "./services/idle-local-work.js"; import { clearIdleBackupWakeRequired, idleBackupWakeMarker, markIdleBackupWakeRequired, verifyIdleDatabaseBackup } from "./services/idle-database-backup.js"; import { cloudWarmStandbyServerOptions } from "./middleware/cloud-warm-standby.js"; import { createCloudWarmStandby } from "./services/cloud-warm-standby.js"; -import { subscribeAllCompanyLiveEvents } from "./services/live-events.js"; import { chatCompletionDeliveryService } from "./services/chat-completion-delivery.js"; /// // Kicks off the OTel bootstrap as early as possible (no-op unless @@ -22,7 +24,8 @@ import { reconcileAbandonedExecutionControl } from "./services/execution-control import { EXECUTION_RECONCILIATION_INTERVAL_MS } from "./services/execution-control-deadline.js"; import { connectionIntentDeliveryService } from "./services/connection-intent-delivery.js"; import { existsSync, readFileSync, rmSync } from "node:fs"; -import { createServer, type RequestListener } from "node:http"; +import type { Server, RequestListener } from "node:http"; +import { createPaperclipHttpServer } from "./http/server.js"; import { resolve } from "node:path"; import { createInterface } from "node:readline/promises"; import { stdin, stdout } from "node:process"; @@ -64,7 +67,6 @@ import { import { getOperatorSettingDefaults } from "./services/setting-defaults.js"; import { setupEnvironmentCustomImageTerminalWebSocketServer } from "./realtime/environment-custom-image-terminal-ws.js"; import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js"; -import { setupRunnerPrpWebSocketServer } from "./realtime/runner-prp-ws.js"; import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./middleware/auth.js"; import { feedbackService, @@ -168,7 +170,7 @@ type EmbeddedPostgresCtor = new (opts: { export interface StartedServer { - server: ReturnType; + server: Server; host: string; listenPort: number; apiUrl: string; @@ -943,14 +945,6 @@ async function startServerWithDatabaseTeardown( decisionServiceOptions, managedPluginAutoInstall, }); - // Upgrade admission runs before every WebSocket listener, outside Express. - const server = createServer(cloudWarmStandbyServerOptions(() => isWarmStandby() || isIdleTaskDrainActive()), app as unknown as RequestListener); - - // Increase keep-alive timeouts to safely outlive default idle timeouts - // of common reverse proxies and load balancers (like AWS ALB, Nginx, or Traefik). - // This prevents intermittent 502/ECONNRESET errors caused by Node's 5s default. - server.keepAliveTimeout = 185000; - server.headersTimeout = 186000; if (listenPort !== requestedListenPort) { logger.warn(`Requested port is busy; using next free port (requestedPort=${requestedListenPort}, selectedPort=${listenPort})`); @@ -964,6 +958,14 @@ async function startServerWithDatabaseTeardown( port: listenPort, }); const configuredApiUrl = process.env.PAPERCLIP_API_URL?.trim() || runtimeApiUrl; + // Upgrade admission runs before every WebSocket listener, outside Express. + const server = createPaperclipHttpServer(app as unknown as RequestListener, { + apiUrl: configuredApiUrl, + httpOptions: cloudWarmStandbyServerOptions(() => isWarmStandby() || isIdleTaskDrainActive()), + }); + // Outlive reverse-proxy idle timeouts to avoid intermittent resets. + server.keepAliveTimeout = 185000; + server.headersTimeout = 186000; const runtimeApiCandidates = buildRuntimeApiCandidateUrls({ preferredApiUrl: configuredApiUrl, authPublicBaseUrl: config.authPublicBaseUrl ?? null, @@ -979,7 +981,6 @@ async function startServerWithDatabaseTeardown( let startupListenerBound = false; try { - setupRunnerPrpWebSocketServer(server, { apiUrl: configuredApiUrl }); setupEnvironmentCustomImageTerminalWebSocketServer(server, db as any, { pluginWorkerManager, }); @@ -1253,20 +1254,31 @@ async function startServerWithDatabaseTeardown( const ENVIRONMENT_LEASE_CLEANUP_SWEEP_BACKOFF_MS = 5 * 60 * 1000; const environmentLeaseCleanupHeartbeat = accountingHeartbeat; const chatCompletionDeliveries = chatCompletionDeliveryService(db as any, environmentLeaseCleanupHeartbeat); - // Activity publication happens after the status transaction commits. This is - // a best-effort fast path; the durable outbox and sweeps remain authoritative. - const unsubscribeChatCompletions = subscribeAllCompanyLiveEvents(event => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped || event.type !== "activity.logged" || - event.payload.action !== "issue.updated" || typeof event.payload.entityId !== "string") return; - trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending({ companyId: event.companyId, taskId: event.payload.entityId }) - .catch(err => logger.error({ err }, "post-commit chat completion delivery failed"))); - }); - server.on("close", unsubscribeChatCompletions); const connectionDeliveries = connectionIntentDeliveryService(db as any, environmentLeaseCleanupHeartbeat); const questionResponseDeliveries = questionResponseDeliveryService(db as any, { heartbeat: environmentLeaseCleanupHeartbeat, resolveNativeQuestion: (interaction) => deliverNativeQuestionResponse(db as any, interaction), }); + const deliveryWork = app.locals.deliveryWork as ReturnType; + const deliveryWorkers = ([ + [DELIVERY_QUEUES.chatCompletion, chatCompletionDeliveries], + [DELIVERY_QUEUES.connection, connectionDeliveries], + [DELIVERY_QUEUES.question, questionResponseDeliveries], + [DELIVERY_QUEUES.toolAction, app.locals.toolActionDeliveries as ReturnType], + ] as const).map(([queue, service]) => deliveryWork.register(queue, { + retryMs: config.heartbeatSchedulerIntervalMs, + run: () => service.sweepPending(), + hasPending: () => service.hasPending(), + })); + // Preserve the existing task-scoped activity fast path. Claims in the + // delivery service arbitrate with the coordinator's startup/recovery sweep. + const unsubscribeChatCompletions = subscribeAllCompanyLiveEvents(event => { + if (isIdleTaskDrainActive() || isWarmStandby() || heartbeatSchedulerStopped || event.type !== "activity.logged" || + event.payload.action !== "issue.updated" || typeof event.payload.entityId !== "string") return; + trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending({ companyId: event.companyId, taskId: event.payload.entityId }) + .catch(err => logger.error({ err }, "post-commit chat completion delivery failed"))); + }); + server.on("close", unsubscribeChatCompletions); const runEnvironmentLeaseCleanupSweep = (backoffMs: number) => environmentLeaseCleanupHeartbeat .sweepPendingCleanupLeases({ backoffMs }) @@ -1317,18 +1329,9 @@ async function startServerWithDatabaseTeardown( })); }; - await chatCompletionDeliveries.sweepPending().catch((err) => logger.error({ err }, "startup chat completion delivery recovery failed")); - await connectionDeliveries.sweepPending(); await app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "startup tool review recovery failed")); await app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "startup gateway token cleanup failed")); - await app.locals.toolActionDeliveries.sweepPending().catch((err: unknown) => logger.error({ err }, "startup tool review delivery sweep failed")); - await questionResponseDeliveries.sweepPending().then((result) => { - if (result.scanned > 0) { - logger.info(result, "startup question-response delivery sweep completed"); - } - }).catch((err) => { - logger.error({ err }, "startup question-response delivery sweep failed"); - }); + await Promise.all(deliveryWorkers.map(worker => worker.ready)); scheduleGitHubConnectionEventPoll(); scheduleGitHubConnectionContinuitySweep(); @@ -1782,23 +1785,10 @@ async function startServerWithDatabaseTeardown( logger.error({ err }, "periodic secret proposal expiry sweep failed"); })); - trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending().catch((err) => logger.error({ err }, "chat completion delivery failed"))); - trackHeartbeatSchedulerWork(accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed"))); - trackHeartbeatSchedulerWork(connectionDeliveries.sweepPending().catch((err) => logger.error({ err }, "connection continuation delivery failed"))); trackHeartbeatSchedulerWork(app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "tool review recovery failed"))); trackHeartbeatSchedulerWork(app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "gateway token cleanup failed"))); - trackHeartbeatSchedulerWork(app.locals.toolActionDeliveries.sweepPending().catch((err: unknown) => logger.error({ err }, "tool review delivery sweep failed"))); - trackHeartbeatSchedulerWork(questionResponseDeliveries.sweepPending() - .then((result) => { - if (result.scanned > 0) { - logger.info(result, "periodic question-response delivery sweep completed"); - } - }) - .catch((err) => { - logger.error({ err }, "periodic question-response delivery sweep failed"); - })); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) { @@ -1975,6 +1965,7 @@ async function startServerWithDatabaseTeardown( await systemdNotify(["--stopping", `--status=Stopping after ${signal}`]); heartbeatSchedulerStopped = true; unsubscribeChatCompletions(); + await deliveryWork.stop(); clearInterval(executionControlInterval); if (heartbeatSchedulerInterval) { clearInterval(heartbeatSchedulerInterval); diff --git a/server/src/services/agent-permissions.ts b/server/src/lib/agent-permissions.ts similarity index 100% rename from server/src/services/agent-permissions.ts rename to server/src/lib/agent-permissions.ts diff --git a/server/src/lib/agent-records.ts b/server/src/lib/agent-records.ts new file mode 100644 index 0000000000..da1702f595 --- /dev/null +++ b/server/src/lib/agent-records.ts @@ -0,0 +1,473 @@ +import { createHash, randomBytes } from "node:crypto"; +import { and, eq, gte, inArray, lt, sql } from "drizzle-orm"; +import { agents, costEvents, type Db } from "@paperclipai/db"; +import { + agentAppearanceSchema, resolveAgentAppearance, agentAvatarUrl, + AGENT_DEFAULT_MAX_CONCURRENT_RUNS, agentRuntimeConfigSchema, getAgentWorkEligibility, + normalizeAgentUrlKey, type AgentEligibilityAgent, +} from "@paperclipai/shared"; +import { conflict, notFound, unprocessable } from "../errors.js"; +import { REDACTED_EVENT_VALUE, sanitizeRecord } from "../redaction.js"; +import { normalizeAgentPermissions } from "./agent-permissions.js"; +import { builtInAgentMarkersEqual, readBuiltInAgentMarker } from "./built-in-agent-metadata.js"; + +export function hashToken(token: string) { + return createHash("sha256").update(token).digest("hex"); +} + +export function createToken() { + return `pcp_${randomBytes(24).toString("hex")}`; +} + +const CONFIG_REVISION_FIELDS = [ + "name", + "role", + "title", + "icon", + "appearance", + "reportsTo", + "capabilities", + "adapterType", + "adapterConfig", + "runtimeConfig", + "defaultEnvironmentId", + "budgetMonthlyCents", + "metadata", +] as const; + +type ConfigRevisionField = (typeof CONFIG_REVISION_FIELDS)[number]; +type AgentConfigSnapshot = Pick; + +export interface RevisionMetadata { + createdByAgentId?: string | null; + createdByUserId?: string | null; + source?: string; + rolledBackFromRevisionId?: string | null; +} + +/** + * The Claude login context for an agent write. The route derives the owner user + * from the authenticated actor, not from the request body, and forwards the + * non-secret `storedSessionId` claim from a completed Claude login session. A + * controlled internal override permits a migration or an administrator repair to + * bind or unbind the fixed OAuth token without a claim. + * + * The `applyExistingWithoutClaim` field is the user-actor apply-existing path. + * The route sets it only for an authenticated user actor and derives the owner + * from that actor. The path binds the fixed reference to the owner stored value + * with no login round trip. It is distinct from `allowInternalBindingOverride`, + * which does no ownership check. + * + * The `inheritedFromAgentId` field is the hire-inheritance path. The route + * sets it only for an authenticated agent actor whose hire request inherited + * the fixed reference from that named parent. The service re-reads the parent + * agent inside the write transaction and binds the fixed reference only when + * the parent exists, is in the same company, is a `claude_local` agent, and + * already holds the exact fixed binding. + */ +export interface ClaudeLoginContext { + storedSessionId?: string | null; + ownerUserId?: string | null; + allowInternalBindingOverride?: boolean; + applyExistingWithoutClaim?: boolean; + inheritedFromAgentId?: string | null; +} + +export interface UpdateAgentOptions { + recordRevision?: RevisionMetadata; + allowBuiltInAgentMetadata?: boolean; + allowPendingApprovalConfigUpdate?: boolean; + claudeLogin?: ClaudeLoginContext; +} + +export type CreateAgentData = Omit; +export type AgentConfigurationRecord = typeof agents.$inferSelect; +export const AGENT_CONFIGURATION_FIELDS = [...CONFIG_REVISION_FIELDS, "permissions"] as const; +export type AgentConfigurationPatch = Partial>; +export type AgentHireRecord = Pick; + +export interface CreateAgentOptions { + responsibleUserId?: string | null; + createdByUserId?: string | null; + aiConnectionInstall?: { connectionId: string; memberConnectionIds?: string[]; createdByUserId: string | null }; + allowBuiltInAgentMetadata?: boolean; + claudeLogin?: ClaudeLoginContext; +} + +interface AgentShortnameRow { + id: string; + name: string; + status: string; +} + +interface AgentShortnameCollisionOptions { + excludeAgentId?: string | null; +} + +export function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function jsonEqual(left: unknown, right: unknown): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +export function buildConfigSnapshot( + row: Pick, +): AgentConfigSnapshot { + const adapterConfig = + typeof row.adapterConfig === "object" && row.adapterConfig !== null && !Array.isArray(row.adapterConfig) + ? sanitizeRecord(row.adapterConfig as Record) + : {}; + const runtimeConfig = + typeof row.runtimeConfig === "object" && row.runtimeConfig !== null && !Array.isArray(row.runtimeConfig) + ? sanitizeRecord(row.runtimeConfig as Record) + : {}; + const metadata = + typeof row.metadata === "object" && row.metadata !== null && !Array.isArray(row.metadata) + ? sanitizeRecord(row.metadata as Record) + : row.metadata ?? null; + return { + name: row.name, + role: row.role, + title: row.title, + icon: row.icon, + appearance: row.appearance, + reportsTo: row.reportsTo, + capabilities: row.capabilities, + adapterType: row.adapterType, + adapterConfig, + runtimeConfig, + defaultEnvironmentId: row.defaultEnvironmentId, + budgetMonthlyCents: row.budgetMonthlyCents, + metadata, + }; +} + +export function containsRedactedMarker(value: unknown): boolean { + if (value === REDACTED_EVENT_VALUE) return true; + if (Array.isArray(value)) return value.some((item) => containsRedactedMarker(item)); + if (typeof value !== "object" || value === null) return false; + return Object.values(value as Record).some((entry) => containsRedactedMarker(entry)); +} + +export function hasConfigPatchFields(data: Partial) { + return CONFIG_REVISION_FIELDS.some((field) => Object.prototype.hasOwnProperty.call(data, field)); +} + +export function changedPendingApprovalConfigFields( + existing: Omit, + data: Partial, +) { + return CONFIG_REVISION_FIELDS.filter((field) => + Object.prototype.hasOwnProperty.call(data, field) && !jsonEqual(data[field], existing[field]), + ); +} + +export function configPatchFromApprovalPayload(payload: Record) { + const patch: Partial = {}; + if (typeof payload.name === "string") patch.name = payload.name; + if (typeof payload.role === "string") patch.role = payload.role; + if (payload.appearance != null) patch.appearance = agentAppearanceSchema.parse(payload.appearance); + if (Object.prototype.hasOwnProperty.call(payload, "title")) { + patch.title = typeof payload.title === "string" ? payload.title : null; + } + if (Object.prototype.hasOwnProperty.call(payload, "icon")) { + patch.icon = typeof payload.icon === "string" ? payload.icon : null; + } + if (Object.prototype.hasOwnProperty.call(payload, "reportsTo")) { + patch.reportsTo = typeof payload.reportsTo === "string" ? payload.reportsTo : null; + } + if (Object.prototype.hasOwnProperty.call(payload, "capabilities")) { + patch.capabilities = typeof payload.capabilities === "string" ? payload.capabilities : null; + } + if (typeof payload.adapterType === "string") patch.adapterType = payload.adapterType; + if (isPlainRecord(payload.adapterConfig)) patch.adapterConfig = payload.adapterConfig; + if (isPlainRecord(payload.runtimeConfig)) patch.runtimeConfig = payload.runtimeConfig; + if (Object.prototype.hasOwnProperty.call(payload, "defaultEnvironmentId")) { + patch.defaultEnvironmentId = + typeof payload.defaultEnvironmentId === "string" ? payload.defaultEnvironmentId : null; + } + if (typeof payload.budgetMonthlyCents === "number") { + patch.budgetMonthlyCents = payload.budgetMonthlyCents; + } + if (Object.prototype.hasOwnProperty.call(payload, "metadata")) { + patch.metadata = isPlainRecord(payload.metadata) ? payload.metadata : null; + } + if (isPlainRecord(payload.permissions)) { + patch.permissions = payload.permissions; + } + return patch; +} + +function parseFiniteNumberLike(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value !== "string") return null; + const parsed = Number(value.trim()); + return Number.isFinite(parsed) ? parsed : null; +} + +export function normalizeRuntimeConfigForNewAgent(runtimeConfig: unknown): Record { + const normalizedRuntimeConfig = isPlainRecord(runtimeConfig) ? { ...runtimeConfig } : {}; + const heartbeat = isPlainRecord(normalizedRuntimeConfig.heartbeat) + ? { ...normalizedRuntimeConfig.heartbeat } + : {}; + if (parseFiniteNumberLike(heartbeat.maxConcurrentRuns) == null) { + heartbeat.maxConcurrentRuns = AGENT_DEFAULT_MAX_CONCURRENT_RUNS; + } + normalizedRuntimeConfig.heartbeat = heartbeat; + return normalizedRuntimeConfig; +} + +export function diffConfigSnapshot( + before: AgentConfigSnapshot, + after: AgentConfigSnapshot, +): string[] { + return CONFIG_REVISION_FIELDS.filter((field) => !jsonEqual(before[field], after[field])); +} + +export function configPatchFromSnapshot(snapshot: unknown): Partial { + if (!isPlainRecord(snapshot)) throw unprocessable("Invalid revision snapshot"); + + if (typeof snapshot.name !== "string" || snapshot.name.length === 0) { + throw unprocessable("Invalid revision snapshot: name"); + } + if (typeof snapshot.role !== "string" || snapshot.role.length === 0) { + throw unprocessable("Invalid revision snapshot: role"); + } + if (typeof snapshot.adapterType !== "string" || snapshot.adapterType.length === 0) { + throw unprocessable("Invalid revision snapshot: adapterType"); + } + if (typeof snapshot.budgetMonthlyCents !== "number" || !Number.isFinite(snapshot.budgetMonthlyCents)) { + throw unprocessable("Invalid revision snapshot: budgetMonthlyCents"); + } + const runtimeConfig = agentRuntimeConfigSchema.safeParse( + isPlainRecord(snapshot.runtimeConfig) ? snapshot.runtimeConfig : {}, + ); + if (!runtimeConfig.success) { + throw unprocessable("Invalid revision snapshot: runtimeConfig"); + } + + return { + name: snapshot.name, + role: snapshot.role, + title: typeof snapshot.title === "string" || snapshot.title === null ? snapshot.title : null, + reportsTo: + typeof snapshot.reportsTo === "string" || snapshot.reportsTo === null ? snapshot.reportsTo : null, + capabilities: + typeof snapshot.capabilities === "string" || snapshot.capabilities === null + ? snapshot.capabilities + : null, + adapterType: snapshot.adapterType, + adapterConfig: isPlainRecord(snapshot.adapterConfig) ? snapshot.adapterConfig : {}, + runtimeConfig: runtimeConfig.data, + defaultEnvironmentId: + typeof snapshot.defaultEnvironmentId === "string" || snapshot.defaultEnvironmentId === null + ? snapshot.defaultEnvironmentId + : null, + budgetMonthlyCents: Math.max(0, Math.floor(snapshot.budgetMonthlyCents)), + metadata: isPlainRecord(snapshot.metadata) || snapshot.metadata === null ? snapshot.metadata : null, + }; +} + +export function hasAgentShortnameCollision( + candidateName: string, + existingAgents: AgentShortnameRow[], + options?: AgentShortnameCollisionOptions, +): boolean { + const candidateShortname = normalizeAgentUrlKey(candidateName); + if (!candidateShortname) return false; + + return existingAgents.some((agent) => { + if (agent.status === "terminated") return false; + if (options?.excludeAgentId && agent.id === options.excludeAgentId) return false; + return normalizeAgentUrlKey(agent.name) === candidateShortname; + }); +} + +export function deduplicateAgentName( + candidateName: string, + existingAgents: AgentShortnameRow[], +): string { + if (!hasAgentShortnameCollision(candidateName, existingAgents)) { + return candidateName; + } + for (let i = 2; i <= 100; i++) { + const suffixed = `${candidateName} ${i}`; + if (!hasAgentShortnameCollision(suffixed, existingAgents)) { + return suffixed; + } + } + return `${candidateName} ${Date.now()}`; +} + +export function agentRecordQueries(db: Db) { + function currentUtcMonthWindow(now = new Date()) { + const year = now.getUTCFullYear(); + const month = now.getUTCMonth(); + return { + start: new Date(Date.UTC(year, month, 1, 0, 0, 0, 0)), + end: new Date(Date.UTC(year, month + 1, 1, 0, 0, 0, 0)), + }; + } + + function withUrlKey(row: T) { + return { + ...row, + urlKey: normalizeAgentUrlKey(row.name) ?? row.id, + }; + } + + function normalizeAgentBaseRow(row: typeof agents.$inferSelect) { + const { lifecycleOperation, lifecycleRequiredPluginIds, ...publicRow } = row; + return withUrlKey({ + ...publicRow, + permissions: normalizeAgentPermissions(row.permissions), + }); + } + + function toEligibilityAgent(row: Pick): AgentEligibilityAgent { + return { + id: row.id, + companyId: row.companyId, + name: row.name, + status: row.status, + reportsTo: row.reportsTo, + }; + } + + function normalizeAgentRows(rows: (typeof agents.$inferSelect)[], allCompanyRows = rows) { + const eligibilityAgents = allCompanyRows.map(toEligibilityAgent); + return rows.map((row) => { + const base = normalizeAgentBaseRow(row); + const appearance = resolveAgentAppearance(row.appearance, row.id); + return { + ...base, + appearance, + avatarUrl: agentAvatarUrl(appearance), + orgChainHealth: getAgentWorkEligibility({ + agent: toEligibilityAgent(row), + agents: eligibilityAgents, + }).orgChainHealth, + }; + }); + } + + function normalizeAgentRow(row: typeof agents.$inferSelect, allCompanyRows?: (typeof agents.$inferSelect)[]) { + return normalizeAgentRows([row], allCompanyRows)[0]!; + } + + async function listCompanyAgentRows(companyId: string) { + return db.select().from(agents).where(eq(agents.companyId, companyId)); + } + + async function getMonthlySpendByAgentIds(companyId: string, agentIds: string[]) { + if (agentIds.length === 0) return new Map(); + const { start, end } = currentUtcMonthWindow(); + const rows = await db + .select({ + agentId: costEvents.agentId, + spentMonthlyCents: sql`coalesce(sum(${costEvents.costCents}), 0)::double precision`, + }) + .from(costEvents) + .where( + and( + eq(costEvents.companyId, companyId), + inArray(costEvents.agentId, agentIds), + gte(costEvents.occurredAt, start), + lt(costEvents.occurredAt, end), + ), + ) + .groupBy(costEvents.agentId); + return new Map(rows.map((row) => [row.agentId, Number(row.spentMonthlyCents ?? 0)])); + } + + async function hydrateAgentSpend(rows: T[]) { + const agentIds = rows.map((row) => row.id); + const companyId = rows[0]?.companyId; + if (!companyId || agentIds.length === 0) return rows; + const spendByAgentId = await getMonthlySpendByAgentIds(companyId, agentIds); + return rows.map((row) => ({ + ...row, + spentMonthlyCents: spendByAgentId.get(row.id) ?? 0, + })); + } + + async function getById(id: string) { + const row = await db + .select() + .from(agents) + .where(eq(agents.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const [companyRows, hydrated] = await Promise.all([ + listCompanyAgentRows(row.companyId), + hydrateAgentSpend([row]).then((rows) => rows[0]!), + ]); + return normalizeAgentRow(hydrated, companyRows); + } + + async function ensureManager(companyId: string, managerId: string) { + const manager = await getById(managerId); + if (!manager) throw notFound("Manager not found"); + if (manager.companyId !== companyId) { + throw unprocessable("Manager must belong to same company"); + } + return manager; + } + + async function assertNoCycle(agentId: string, reportsTo: string | null | undefined) { + if (!reportsTo) return; + if (reportsTo === agentId) throw unprocessable("Agent cannot report to itself"); + + let cursor: string | null = reportsTo; + while (cursor) { + if (cursor === agentId) throw unprocessable("Reporting relationship would create cycle"); + const next = await getById(cursor); + cursor = next?.reportsTo ?? null; + } + } + + async function assertCompanyShortnameAvailable( + companyId: string, + candidateName: string, + options?: AgentShortnameCollisionOptions, + ) { + const candidateShortname = normalizeAgentUrlKey(candidateName); + if (!candidateShortname) return; + + const existingAgents = await db + .select({ + id: agents.id, + name: agents.name, + status: agents.status, + }) + .from(agents) + .where(eq(agents.companyId, companyId)); + + const hasCollision = hasAgentShortnameCollision(candidateName, existingAgents, options); + if (hasCollision) { + throw conflict( + `Agent shortname '${candidateShortname}' is already in use in this company`, + ); + } + } + + function assertBuiltInAgentMetadataMutationAllowed( + beforeMetadata: unknown, + afterMetadata: unknown, + options?: { allowBuiltInAgentMetadata?: boolean }, + ) { + if (options?.allowBuiltInAgentMetadata) return; + const beforeMarker = readBuiltInAgentMarker(beforeMetadata); + const afterMarker = readBuiltInAgentMarker(afterMetadata); + if (builtInAgentMarkersEqual(beforeMarker, afterMarker)) return; + throw conflict("Built-in agent marker is managed by Paperclip and cannot be edited directly", { + code: "built_in_agent_marker_readonly", + key: beforeMarker?.key ?? afterMarker?.key ?? null, + }); + } + + return { normalizeAgentRows, normalizeAgentRow, listCompanyAgentRows, hydrateAgentSpend, + getById, ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed }; +} diff --git a/server/src/lib/approval-records.ts b/server/src/lib/approval-records.ts new file mode 100644 index 0000000000..957dfac03e --- /dev/null +++ b/server/src/lib/approval-records.ts @@ -0,0 +1,68 @@ +import { and, eq, inArray } from "drizzle-orm"; +import { approvals, type Db } from "@paperclipai/db"; +import { notFound, unprocessable } from "../errors.js"; + +export const resolvableApprovalStatuses = ["pending", "revision_requested"]; + +export function approvalResolutionRecords(db: Db) { + const canResolveStatuses = new Set(resolvableApprovalStatuses); + const resolvableStatuses = resolvableApprovalStatuses; + type ApprovalRecord = typeof approvals.$inferSelect; + type ResolutionResult = { approval: ApprovalRecord; applied: boolean }; + async function getExistingApproval(id: string, database: Db = db) { + const existing = await database + .select() + .from(approvals) + .where(eq(approvals.id, id)) + .then((rows) => rows[0] ?? null); + if (!existing) throw notFound("Approval not found"); + return existing; + } + + async function resolveApproval( + id: string, + targetStatus: "approved" | "rejected", + decidedByUserId: string, + decisionNote: string | null | undefined, + database: Db = db, + ): Promise { + const existing = await getExistingApproval(id, database); + if (!canResolveStatuses.has(existing.status)) { + if (existing.status === targetStatus) { + return { approval: existing, applied: false }; + } + throw unprocessable( + `Only pending or revision requested approvals can be ${targetStatus === "approved" ? "approved" : "rejected"}`, + ); + } + + const now = new Date(); + const updated = await database + .update(approvals) + .set({ + status: targetStatus, + decidedByUserId, + decisionNote: decisionNote ?? null, + decidedAt: now, + updatedAt: now, + }) + .where(and(eq(approvals.id, id), inArray(approvals.status, resolvableStatuses))) + .returning() + .then((rows) => rows[0] ?? null); + + if (updated) { + return { approval: updated, applied: true }; + } + + const latest = await getExistingApproval(id, database); + if (latest.status === targetStatus) { + return { approval: latest, applied: false }; + } + + throw unprocessable( + `Only pending or revision requested approvals can be ${targetStatus === "approved" ? "approved" : "rejected"}`, + ); + } + + return { getExistingApproval, resolveApproval }; +} diff --git a/server/src/services/built-in-agent-metadata.ts b/server/src/lib/built-in-agent-metadata.ts similarity index 100% rename from server/src/services/built-in-agent-metadata.ts rename to server/src/lib/built-in-agent-metadata.ts diff --git a/server/src/lib/company-deletion.ts b/server/src/lib/company-deletion.ts new file mode 100644 index 0000000000..c6caae4045 --- /dev/null +++ b/server/src/lib/company-deletion.ts @@ -0,0 +1,9 @@ +import type { Db } from "@paperclipai/db"; + +type CompanyDeletionTransaction = Parameters[0]>[0]; + +export interface CompanyDeletionParticipant { + // The caller holds the company lock and removes dependent records first. + // Use this transaction only. Throw on failure. Do not call external systems. + deleteCompanyData(tx: CompanyDeletionTransaction, companyId: string): Promise; +} diff --git a/server/src/middleware/error-handler.ts b/server/src/middleware/error-handler.ts index 23103185c7..f138c06e4d 100644 --- a/server/src/middleware/error-handler.ts +++ b/server/src/middleware/error-handler.ts @@ -1,3 +1,4 @@ +import { AgentLifecycleConflict } from "../modules/agent-lifecycle/index.js"; import type { Request, Response, NextFunction } from "express"; import type { Db } from "@paperclipai/db"; import { ZodError } from "zod"; @@ -129,6 +130,7 @@ export function errorHandler( res: Response, _next: NextFunction, ) { + if (err instanceof AgentLifecycleConflict) { res.status(409).json({ error: err.message }); return; } if (err instanceof HttpError) { const details = err.details && diff --git a/server/src/modules/README.md b/server/src/modules/README.md index 05ff555034..6c3bb456c4 100644 --- a/server/src/modules/README.md +++ b/server/src/modules/README.md @@ -22,10 +22,37 @@ adapters → application → domain Postgres queries, transactions, and process control. An adapter file may import `drizzle-orm`, `@paperclipai/db`, and Node.js I/O modules. -A module exposes one entry point, `index.ts`, which composes the adapters +A module exposes its commands through `index.ts`, which composes the adapters and the use cases behind a factory function. Code outside the module imports -only that entry point, never a file inside `domain/`, `application/`, or +that entry point, never a file inside `domain/`, `application/`, or `adapters/` directly. `pnpm check:module-boundaries` enforces these rules for production source -files. It also rejects imports that bypass another module's `index.ts`. +files. It also checks access to the company deletion entry point below. + +## Company deletion + +The existing company deletion service owns the database transaction and deletion +order. A module can expose a separate `company-deletion.ts` entry point. +Only `services/company-deletion.ts` can import this entry point. +The entry point exports an implementation of `CompanyDeletionParticipant` from +`lib/company-deletion.ts`. Do not export it through the normal module index. + +The service must complete external cleanup before it starts the transaction. +It must lock the company before it deletes dependent records or calls a module. +Creation commands must use the same company lock. +The service removes dependent records before it calls the module. +The module checks its deletion conditions and deletes the records it owns. +Use the supplied transaction. Do not start another transaction or call external +systems. Throw an error if deletion cannot proceed. +An error must roll back all database changes in the deletion transaction. + +Call each module explicitly in dependency order. No module registry is required. + +The agent lifecycle module also has a restricted `configuration.ts` entry point. +It saves configuration and invalidates verification in the caller's transaction. +The [lifecycle guide](../../../doc/AGENT-LIFECYCLE.md) lists its permitted callers. +Ordinary configuration updates use the root lifecycle command. + +The [agent lifecycle module](../../../doc/AGENT-LIFECYCLE.md) owns agent creation, +pause, resume, and termination. Use its commands for these changes. diff --git a/server/src/modules/active-run-watchdog/adapters/postgres.ts b/server/src/modules/active-run-watchdog/adapters/postgres.ts index ac46b6c095..356ef375c2 100644 --- a/server/src/modules/active-run-watchdog/adapters/postgres.ts +++ b/server/src/modules/active-run-watchdog/adapters/postgres.ts @@ -456,6 +456,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch eq(agents.id, input.run.agentId), eq(agents.companyId, companyId), notInArray(agents.status, ["paused", "terminated"]), + eq(agents.lifecycleState, "ready"), )); return updatedRun; diff --git a/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts b/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts new file mode 100644 index 0000000000..4f7a8db721 --- /dev/null +++ b/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { createLifecycleWorker } from "../application/worker.js"; +import type { LifecycleAgent, LifecycleStore, LifecycleDriver } from "../application/ports.js"; +import { startAgentLifecycle } from "../index.js"; +import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js"; +import { logger } from "../../../middleware/logger.js"; + +const agent: LifecycleAgent = { id: "agent", companyId: "company", lifecycleState: "verifying", lifecycleVersion: 3, + lifecycleRequiredPluginIds: null, lifecycleError: null, + lifecycleOperation: { id: "operation", hostComplete: false, completedPluginIds: [], attempts: 1 } }; +function fixture() { + const store = { get: vi.fn(), claim: vi.fn().mockResolvedValue(agent), renew: vi.fn().mockResolvedValue(true), + setRequiredPlugins: vi.fn().mockResolvedValue(true), completeHost: vi.fn().mockResolvedValue(true), + completePlugin: vi.fn().mockResolvedValue(true), defer: vi.fn().mockResolvedValue(true), + change: vi.fn().mockResolvedValue({ ...agent, lifecycleState: "ready" }), pending: vi.fn().mockResolvedValue([agent.id]) } satisfies LifecycleStore; + const driver = { requiredPluginIds: vi.fn().mockResolvedValue(["plugin"]), runHost: vi.fn().mockResolvedValue("complete"), + runPlugin: vi.fn().mockResolvedValue("complete") } satisfies LifecycleDriver; + const report = vi.fn(); + return { store, driver, report, worker: createLifecycleWorker(store, driver, report) }; +} +afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); }); + +describe("lifecycle failure diagnostics", () => { + it.each(["host", "plugin", "claim", "renew", "scan", "defer"] as const)("reports %s failures with the available context", async stage => { + const { store, driver, report, worker } = fixture(); + const error = new Error("private provider detail"); + if (stage === "host") driver.runHost.mockRejectedValue(error); + else if (stage === "plugin") driver.runPlugin.mockRejectedValue(error); + else if (stage === "scan") store.pending.mockRejectedValue(error); + else if (stage === "defer") { driver.runHost.mockResolvedValue("pending"); store.defer.mockRejectedValue(error); } + else store[stage].mockRejectedValue(error); + const work = stage === "scan" ? worker.sweep() : worker.process(agent.id); + if (["claim", "scan", "defer"].includes(stage)) await expect(work).rejects.toThrow(error); + else await work; + expect(report).toHaveBeenCalledWith(expect.objectContaining({ stage }), error); + if (!["claim", "scan"].includes(stage)) expect(report).toHaveBeenCalledWith(expect.objectContaining({ + agentId: agent.id, companyId: agent.companyId, phase: "verifying", operationId: "operation", version: 3, + ...(stage === "plugin" ? { pluginId: "plugin" } : {}), + }), error); + await worker.stop(); + }); + + it("reports periodic policy scan failures", async () => { + vi.useFakeTimers(); + const error = new Error("private database detail"); + const db = { select: () => { throw error; } } as unknown as Db; + const effects = { ...createAgentLifecycleEffects(), reportFailure: vi.fn() }; + const work = startAgentLifecycle(db, effects, fixture().driver, () => true); + try { + await vi.advanceTimersByTimeAsync(5_000); + expect(effects.reportFailure).toHaveBeenCalledWith({ stage: "policy_scan" }, error); + } finally { await work.stop(); } + }); + + it.each([ + [{ code: "required_plugin_unavailable" }, "required_plugin_unavailable"], + [{ code: "harness_test_failed" }, "harness_test_failed"], + [{ cause: { code: "40P01", message: "private query" } }, "40P01"], + [{ code: "private credential value" }, "unclassified"], + ])("logs known codes without provider error contents", (details, failureCode) => { + const log = vi.spyOn(logger, "warn").mockImplementation(() => {}); + const context = { stage: "host" as const, agentId: "agent", phase: "verifying" as const, operationId: "operation", version: 3 }; + createAgentLifecycleEffects().reportFailure(context, Object.assign(new Error("private provider detail"), details)); + expect(log).toHaveBeenCalledExactlyOnceWith({ ...context, failureCode }, "Agent lifecycle operation failed"); + expect(JSON.stringify(log.mock.calls)).not.toContain("private"); + }); +}); diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts new file mode 100644 index 0000000000..b2110f065f --- /dev/null +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -0,0 +1,673 @@ +import { createAgentLifecycle as createLifecycleCommands } from "../index.js"; +import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js"; +import { deleteCompany } from "../../../services/company-deletion.js"; +import { agentLifecycleCompanyDeletion } from "../company-deletion.js"; +import { requireServerAdapter } from "../../../adapters/index.js"; +import { agentExecutionsHaveStopped } from "../../../services/agent-execution-stop.js"; +import { remoteTerminationReceipt } from "../../../services/remote-execution-termination.js"; +import { approvalService } from "../../../services/approvals.js"; +import { updateAgentConfigurationInTransaction } from "../../../services/agent-configuration-transaction.js"; +import { agentService as agentConfiguration } from "../../../services/agents.js"; +import { readFileSync } from "node:fs"; +import express from "express"; +import request from "supertest"; +import { agentRoutes } from "../../../routes/agents.js"; +import { errorHandler } from "../../../middleware/error-handler.js"; +import { createLifecycleDriver } from "../../../services/agent-lifecycle-driver.js"; +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { agents, approvals, projects, issues, heartbeatRuns, nativeRunFinalizations, budgetPolicies, environmentLeases, agentApiKeys, activityLog, agentConfigRevisions, userCompanyPreferences, companies, companyMemberships, principalPermissionGrants, plugins, pluginCompanySettings, createDb, type Db } from "@paperclipai/db"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { createAgentLifecycle, configureAgentLifecycle } from "../../../services/agent-lifecycle.js"; +import { createLifecycleStore as createStore } from "../adapters/postgres.js"; +import { transition } from "../domain/policy.js"; +import type { LifecycleAgent } from "../application/ports.js"; +import { budgetService } from "../../../services/budgets.js"; + +const createLifecycleStore = (db: Db) => createStore(db, createAgentLifecycleEffects()); + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("agent lifecycle commands", () => { + let database: Awaited>; + let db: Db; + let companyId: string; + const workers: ReturnType[] = []; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); }, 90_000); + afterAll(async () => { for (const worker of workers) await worker.stop(); await database?.cleanup(); }); + beforeEach(async () => { + for (const work of workers) await work.stop(); + companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Lifecycle test", issuePrefix: `L${companyId.slice(0, 7)}` }); + }); + function worker(runHost: (agent: LifecycleAgent) => Promise<"complete" | "pending"> = async () => "complete") { + const value = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost }); + workers.push(value); return value; + } + async function hire(status = "idle") { return createAgentLifecycle(db).requestHire(companyId, { name: "Agent", status, adapterType: "process" }); } + async function current(id: string) { return (await db.select().from(agents).where(eq(agents.id, id)))[0]!; } + + it("maps existing states without starting setup or clearing pause reasons", async () => { + const rows = await db.insert(agents).values(["idle", "running", "error", "paused", "pending_approval", "terminated"].map(status => ({ + companyId, name: status, status, pauseReason: status === "paused" ? "budget" : null, + }))).returning(); + const migration = readFileSync(new URL("../../../../../packages/db/src/migrations/0320_daily_onslaught.sql", import.meta.url), "utf8"); + await db.execute(sql.raw(migration.slice(migration.indexOf('UPDATE "agents"')))); + for (const row of rows) { + const mapped = await current(row.id); + expect(mapped.lifecycleState).toBe(["paused", "pending_approval", "terminated"].includes(row.status) ? row.status : "ready"); + expect(mapped.lifecycleOperation).toBeNull(); + expect(mapped.lifecycleRequiredPluginIds).toBeNull(); + expect(mapped.lifecycleHolds).toEqual(row.status === "paused" ? ["budget"] : []); + } + }); + + it("commits the record before preparation and verifies before admission", async () => { + const observed: string[] = []; + const work = worker(async agent => { + expect((await current(agent.id)).lifecycleState).toBe(agent.lifecycleState); + observed.push(agent.lifecycleState); return "complete"; + }); + const agent = await hire(); + expect(agent).toMatchObject({ lifecycleState: "preparing", status: "paused" }); + await work.process(agent.id); + expect(observed).toEqual(["preparing", "verifying"]); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle" }); + expect(agent).not.toHaveProperty("lifecycleOperation"); + }); + + it("rolls back a hire and its grants when hire initialization fails", async () => { + const effects = createAgentLifecycleEffects(); + const lifecycle = createLifecycleCommands(db, { ...effects, + initializeHire: async (tx, agent, options) => { + await effects.initializeHire(tx, agent, options); + throw new Error("Hire initialization failed"); + }, + }); + await expect(lifecycle.requestHire(companyId, { name: "Rollback hire", adapterType: "process" })) + .rejects.toThrow("Hire initialization failed"); + expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0); + expect(await db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.companyId, companyId))).toHaveLength(0); + }); + + it("runs the saved harness test and keeps a failed configuration out of ready", async () => { + const driver = createLifecycleDriver(db, {} as never); + const work = worker(driver.runHost); + const agent = await hire(); + await work.process(agent.id); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", status: "paused" }); + await agentConfiguration(db).update(agent.id, { adapterConfig: { command: process.execPath } }); + await work.process(agent.id); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle", lifecycleError: null }); + }); + + it.each(["claude_local", "gemini_local"])("keeps %s authentication warnings out of ready", async adapterType => { + const adapter = requireServerAdapter(adapterType); + const probe = vi.spyOn(adapter, "testEnvironment").mockResolvedValue({ adapterType, status: "warn", + testedAt: new Date().toISOString(), checks: [{ code: `${adapterType.split("_")[0]}_hello_probe_auth_required`, level: "warn", message: "Login required" }] }); + try { + const agent = await createAgentLifecycle(db).requestHire(companyId, { name: "Login test", adapterType }); + const work = worker(createLifecycleDriver(db, {} as never).runHost); + await work.process(agent.id); + expect(probe).toHaveBeenCalled(); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: "The lifecycle step failed. Retry the operation." }); + } finally { probe.mockRestore(); } + }); + + it("recovers an expired native owner only after the cancelled run and controller stop", async () => { + const agent = await hire(); + const [issue] = await db.insert(issues).values({ companyId, title: "Cancelled native run" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, nativeIssueId: issue.id, + status: "cancelled", invocationSource: "on_demand", runtimeMode: "native", + startedAt: new Date(), finishedAt: new Date(), processPid: process.pid }).returning(); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId: run.id, + phase: "executing", leaseOwner: "stopped-server", leaseExpiresAt: new Date(0), controllerPid: process.pid }); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false); + await db.update(nativeRunFinalizations).set({ controllerPid: null }).where(eq(nativeRunFinalizations.runId, run.id)); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false); + await db.update(heartbeatRuns).set({ processPid: 2_000_000_000 }).where(eq(heartbeatRuns.id, run.id)); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, run.id)); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, run.id)); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(true); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0]).toMatchObject({ leaseOwner: null, leaseExpiresAt: null }); + }); + + it("does not advance termination while a cancelled process still runs", async () => { + const agent = await hire(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, + status: "cancelled", invocationSource: "on_demand", runtimeMode: "legacy", + startedAt: new Date(), finishedAt: new Date(), processPid: process.pid }).returning(); + const driver = createLifecycleDriver(db, {} as never); + const work = worker(driver.runHost); + await createAgentLifecycle(db).terminateAgent(agent.id); + await work.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("terminating"); + await db.update(heartbeatRuns).set({ processPid: null, + resultJson: { executionCancellation: { state: "acknowledged" } } }).where(eq(heartbeatRuns.id, run.id)); + await createAgentLifecycle(db).retry(agent.id); + await work.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("terminated"); + }); + + it("does not cancel a reporting agent when its manager pauses", async () => { + const parent = await hire(); + const [child] = await db.insert(agents).values({ companyId, name: "Report", reportsTo: parent.id, + status: "idle", lifecycleState: "ready" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: child.id, + status: "queued", invocationSource: "on_demand" }).returning(); + const work = worker(createLifecycleDriver(db, {} as never).runHost); + await createAgentLifecycle(db).pauseAgent(parent.id); + await work.process(parent.id); + expect((await current(parent.id)).lifecycleState).toBe("paused"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]!.status).toBe("queued"); + }); + + it("retains company data until every agent finishes termination", async () => { + const agent = await hire(); + const [project] = await db.insert(projects).values({ companyId, name: "Keep until cleanup", leadAgentId: agent.id }).returning(); + await db.insert(issues).values({ companyId, projectId: project.id, title: "Keep this task", assigneeAgentId: agent.id }); + const pending = worker(async () => "pending"); + await expect(deleteCompany(db, companyId)).rejects.toThrow("Complete termination"); + expect(await db.select().from(projects).where(eq(projects.id, project.id))).toHaveLength(1); + expect(await db.select().from(issues).where(eq(issues.companyId, companyId))).toHaveLength(1); + await pending.stop(); + const complete = worker(); + await createAgentLifecycle(db).retry(agent.id); + await complete.process(agent.id); + await deleteCompany(db, companyId); + expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0); + expect(await db.select().from(companies).where(eq(companies.id, companyId))).toHaveLength(0); + }); + + it("requires a transaction for the company deletion integration", async () => { + await expect(agentLifecycleCompanyDeletion.deleteCompanyData(db as never, companyId)) + .rejects.toThrow("Company deletion requires a database transaction"); + }); + + it("rejects individual deletion before cleanup and rolls back failed cleanup", async () => { + const agent = await hire(); + const effects = createAgentLifecycleEffects(); + const cleanup = vi.fn(effects.deleteDependencies); + const lifecycle = createLifecycleCommands(db, { ...effects, deleteDependencies: cleanup }); + await expect(lifecycle.purgeAgent(agent.id)).rejects.toThrow("Complete termination"); + expect(cleanup).not.toHaveBeenCalled(); + + await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, agent.id)); + const [key] = await db.insert(agentApiKeys).values({ companyId, agentId: agent.id, name: "Keep on rollback", keyHash: randomUUID() }).returning(); + cleanup.mockImplementation(async (tx, companyId, id) => { + await effects.deleteDependencies(tx, companyId, id); + throw new Error("Dependency cleanup failed"); + }); + await expect(lifecycle.purgeAgent(agent.id)).rejects.toThrow("Dependency cleanup failed"); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "terminated" }); + expect(await db.select().from(agentApiKeys).where(eq(agentApiKeys.id, key.id))).toHaveLength(1); + }); + + it("deletes terminated and rejected agents only in the requested company", async () => { + await db.insert(agents).values((["terminated", "rejected"] as const).map(lifecycleState => ({ + companyId, name: lifecycleState, status: "terminated", lifecycleState, + }))); + const [otherCompany] = await db.insert(companies).values({ name: "Keep this company", issuePrefix: `K${companyId.slice(0, 7)}` }).returning(); + const [otherAgent] = await db.insert(agents).values({ companyId: otherCompany.id, name: "Keep this agent", + status: "terminated", lifecycleState: "terminated" }).returning(); + + await deleteCompany(db, companyId); + + expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0); + expect(await current(otherAgent.id)).toMatchObject({ companyId: otherCompany.id }); + expect(await db.select().from(companies).where(eq(companies.id, otherCompany.id))).toHaveLength(1); + }); + + it("rolls back agent and company data deletion when a later operation fails", async () => { + const agent = await hire("terminated"); + const [project] = await db.insert(projects).values({ companyId, name: "Keep on rollback" }).returning(); + const removeAgents = agentLifecycleCompanyDeletion.deleteCompanyData; + const failure = vi.spyOn(agentLifecycleCompanyDeletion, "deleteCompanyData").mockImplementation(async (tx, id) => { + await removeAgents(tx, id); + expect(await tx.select().from(agents).where(eq(agents.companyId, id))).toHaveLength(0); + throw new Error("Later deletion failed"); + }); + try { + await expect(deleteCompany(db, companyId)).rejects.toThrow("Later deletion failed"); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "terminated" }); + expect(await db.select().from(projects).where(eq(projects.id, project.id))).toHaveLength(1); + expect(await db.select().from(companies).where(eq(companies.id, companyId))).toHaveLength(1); + } finally { failure.mockRestore(); } + }); + + it("blocks a concurrent hire until company deletion commits", async () => { + let reportLock!: (pid: number) => void; + let rejectLock!: (error: unknown) => void; + const locked = new Promise((resolve, reject) => { reportLock = resolve; rejectLock = reject; }); + let release!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const removeAgents = agentLifecycleCompanyDeletion.deleteCompanyData; + const pause = vi.spyOn(agentLifecycleCompanyDeletion, "deleteCompanyData").mockImplementation(async (tx, id) => { + const [backend] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + reportLock(backend!.pid); + await released; + await removeAgents(tx, id); + }); + const deletion = deleteCompany(db, companyId); + void deletion.catch(rejectLock); + let hiring: Promise | undefined; + try { + const pid = await locked; + hiring = hire().then(() => "created", error => error.message); + await vi.waitFor(async () => { + const blocked = await db.execute(sql`select 1 from pg_stat_activity + where datname = current_database() and ${pid} = any(pg_blocking_pids(pid))`); + expect(blocked).toHaveLength(1); + }); + release(); + await deletion; + expect(await hiring).toBe("Company not found"); + expect(await db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0); + } finally { + release(); + await Promise.allSettled([deletion, hiring]); + pause.mockRestore(); + } + }); + + it("requires cleanup proof after a remote run is already cancelled", async () => { + const agent = await hire(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, + status: "cancelled", invocationSource: "on_demand", runtimeMode: "native", + startedAt: new Date(), finishedAt: new Date() }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId, heartbeatRunId: run.id, + provider: "test", providerLeaseId: "test-lease", status: "pending_cleanup", + cleanupStatus: "failed", releasedAt: new Date() }).returning(); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success" }) + .where(eq(environmentLeases.id, lease.id)); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false); + await db.update(environmentLeases).set({ metadata: { + remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }), + } }).where(eq(environmentLeases.id, lease.id)); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(true); + }); + + it("does not prepare a proposed hire until approval", async () => { + const observed: string[] = []; + const work = worker(async agent => { observed.push(agent.lifecycleState); return "complete"; }); + const agent = await hire("pending_approval"); + await work.process(agent.id); + expect(observed).toEqual([]); + const approval = await approvalService(db).create(companyId, { type: "hire_agent", payload: { agentId: agent.id }, status: "pending" }); + await approvalService(db).approve(approval.id, "board"); + await vi.waitFor(async () => expect((await current(agent.id)).lifecycleState).toBe("ready")); + expect(observed).toEqual(["preparing", "verifying"]); + }); + + it.each(["approved", "rejected"] as const)("uses the same %s hire decision for an agent and its approval", async decision => { + const agent = await hire("pending_approval"); + const approval = await approvalService(db).create(companyId, { type: "hire_agent", + requestedByUserId: "requester", payload: { agentId: agent.id }, status: "pending" }); + const lifecycle = createAgentLifecycle(db); + const decide = decision === "approved" ? lifecycle.approveHire : lifecycle.rejectHire; + const first = await decide(agent.id, "board"); + expect(first).toMatchObject({ applied: true, approval: { id: approval.id, status: decision }, + agent: { lifecycleState: decision === "approved" ? "preparing" : "rejected" } }); + const version = (await current(agent.id)).lifecycleVersion; + expect(await decide({ approvalId: approval.id }, "board")).toMatchObject({ applied: false, hireApprovedAgentId: null }); + expect((await current(agent.id)).lifecycleVersion).toBe(version); + }); + + it("rolls back the approval and agent together when activation fails", async () => { + const agent = await hire("pending_approval"); + const approval = await approvalService(db).create(companyId, { type: "hire_agent", + payload: { agentId: agent.id }, status: "pending" }); + const lifecycle = createLifecycleCommands(db, { ...createAgentLifecycleEffects(), + recordCreation: async () => { throw new Error("Creation event failed"); } }); + await expect(lifecycle.approveHire(agent.id, "board")).rejects.toThrow("Creation event failed"); + expect((await current(agent.id)).lifecycleState).toBe("pending_approval"); + expect((await db.select().from(approvals).where(eq(approvals.id, approval.id)))[0].status).toBe("pending"); + }); + + it("keeps the hire credential owner and recovers it for migrated approvals", async () => { + for (const savedOwner of [null, "original-owner"]) { + const [agent] = await db.insert(agents).values({ companyId, name: "Migrated hire", status: "pending_approval", + lifecycleState: "pending_approval", lifecycleOperation: savedOwner ? { + id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, responsibleUserId: savedOwner, + } : null }).returning(); + const approval = await approvalService(db).create(companyId, { type: "hire_agent", + requestedByUserId: "requesting-member", payload: { agentId: agent.id }, status: "pending" }); + await approvalService(db).approve(approval.id, "approving-member"); + expect((await current(agent.id)).lifecycleOperation?.responsibleUserId).toBe(savedOwner ?? "requesting-member"); + } + }); + + it("rejects transaction injection and generic state writes", async () => { + const agent = await hire("pending_approval"); + await expect(db.transaction(async tx => createAgentLifecycle(tx as unknown as Db))).rejects.toThrow("root database"); + await expect(agentConfiguration(db).update(agent.id, { status: "idle" } as never)).rejects.toThrow("lifecycle command"); + await expect(createAgentLifecycle(db).requestHire(companyId, { name: "Bypass", lifecycleRequiredPluginIds: [] } as never)).rejects.toThrow("Lifecycle fields"); + }); + + it("clears a rejected hire from the user's primary selection", async () => { + const agent = await createAgentLifecycle(db).requestHire(companyId, { name: "Proposed", status: "pending_approval" }, { createdByUserId: "board" }); + const preference = () => db.select().from(userCompanyPreferences).where(eq(userCompanyPreferences.companyId, companyId)); + expect((await preference())[0]!.primaryAgentId).toBe(agent.id); + const approval = await approvalService(db).create(companyId, { type: "hire_agent", payload: { agentId: agent.id }, status: "pending" }); + await approvalService(db).reject(approval.id, "board"); + expect((await preference())[0]!.primaryAgentId).toBeNull(); + expect((await current(agent.id)).lifecycleState).toBe("rejected"); + }); + + it("rolls back configuration and its revision when the requested transition fails", async () => { + const lifecycle = createAgentLifecycle(db); + const agent = await hire("terminated"); + await expect(lifecycle.updateAndTransition(agent.id, "resume", { name: "Must not persist" }, + { recordRevision: { createdByUserId: "board", source: "patch" } })).rejects.toThrow("Cannot resume"); + expect((await current(agent.id)).name).toBe("Agent"); + expect(await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id))).toEqual([]); + const active = await hire(); + expect(await lifecycle.updateAndTransition(active.id, "pause", { name: "Paused agent" })).toMatchObject({ name: "Paused agent", lifecycleState: "pausing" }); + }); + + it("preserves a disabled positive budget at approval and applies explicit configuration changes", async () => { + const agent = await hire("pending_approval"); + const budgets = budgetService(db); + await budgets.upsertPolicy(companyId, { scopeType: "agent", scopeId: agent.id, amount: 1000, isActive: false }, null); + const approval = await approvalService(db).create(companyId, { type: "hire_agent", + payload: { agentId: agent.id, budgetMonthlyCents: 1000 }, status: "pending" }); + const policy = async () => (await db.select().from(budgetPolicies).where(eq(budgetPolicies.scopeId, agent.id)))[0]!; + await approvalService(db).approve(approval.id, "board"); + expect(await policy()).toMatchObject({ amount: 1000, isActive: false }); + await agentConfiguration(db).update(agent.id, { budgetMonthlyCents: 2000 }); + expect(await policy()).toMatchObject({ amount: 2000, isActive: true }); + await agentConfiguration(db).update(agent.id, { budgetMonthlyCents: 0 }); + expect(await policy()).toMatchObject({ amount: 0, isActive: false }); + }); + + it.each(["updateConfiguration", "updateAndTransition"] as const)("limits %s budget reconciliation to the edited agent", async command => { + const lifecycle = createAgentLifecycle(db); + const one = await hire(); + const two = await lifecycle.requestHire(companyId, { name: "Other budget target" }); + await db.update(companies).set({ status: "paused" }).where(eq(companies.id, companyId)); + if (command === "updateConfiguration") await lifecycle.updateConfiguration(one.id, { budgetMonthlyCents: 1000 }); + else await lifecycle.updateAndTransition(one.id, "pause", { budgetMonthlyCents: 1000 }); + expect((await current(one.id)).lifecycleHolds).toContain("company_paused"); + expect((await current(two.id)).lifecycleHolds).toEqual([]); + }); + + it("limits invocation policy checks to the requested agent", async () => { + const one = await hire(); + const two = await createAgentLifecycle(db).requestHire(companyId, { name: "Another agent" }); + await db.update(companies).set({ status: "paused" }).where(eq(companies.id, companyId)); + await budgetService(db).getInvocationBlock(companyId, one.id); + expect((await current(one.id)).lifecycleHolds).toContain("company_paused"); + expect((await current(two.id)).lifecycleHolds).toEqual([]); + }); + + it("keeps a manual hold when company policy changes", async () => { + const work = worker(); const lifecycle = createAgentLifecycle(db); const agent = await hire(); await work.process(agent.id); + await lifecycle.pauseAgent(agent.id); await work.process(agent.id); + await db.update(companies).set({ status: "archived" }).where(eq(companies.id, companyId)); + await createAgentLifecycle(db).reconcilePolicyHolds(companyId); await work.process(agent.id); + await db.update(companies).set({ status: "active" }).where(eq(companies.id, companyId)); + await createAgentLifecycle(db).reconcilePolicyHolds(companyId); await work.process(agent.id); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "paused", lifecycleHolds: ["manual"] }); + await lifecycle.resumeAgent(agent.id); await work.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("ready"); + }); + + it("defers automatic policy changes while a task holds the agent lock", async () => { + const agent = await hire("paused"); + await db.transaction(async tx => { + await tx.select().from(agents).where(eq(agents.id, agent.id)).for("update"); + expect(await createLifecycleStore(db).change(agent.id, "reconcile")).toBeNull(); + }); + expect(await createLifecycleStore(db).change(agent.id, "reconcile")).not.toBeNull(); + }); + + it("retains the failed phase and the same operation for retry", async () => { + let fail = true; + const work = worker(async agent => { if (agent.lifecycleState === "verifying" && fail) throw new Error("private detail"); return "complete"; }); + const agent = await hire(); await work.process(agent.id); + const failed = await current(agent.id); + expect(failed.lifecycleState).toBe("verifying"); + expect(failed.lifecycleError).not.toContain("private detail"); + fail = false; + await createAgentLifecycle(db).retry(agent.id); await work.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("ready"); + }); + + it("fences a late completion when termination replaces preparation", async () => { + const agent = await hire("paused"); + const store = createLifecycleStore(db); + await createAgentLifecycle(db).resumeAgent(agent.id); + const claimed = await store.claim(agent.id, "old", new Date()); + expect(claimed).not.toBeNull(); + await store.setRequiredPlugins(claimed!, "old", []); + await createAgentLifecycle(db).terminateAgent(agent.id); + expect(await store.completeHost(claimed!, "old")).toBe(false); + expect(await store.change(agent.id, "complete", { owner: "old", version: claimed!.lifecycleVersion })).toBeNull(); + expect((await current(agent.id)).lifecycleState).toBe("terminating"); + }); + + it("revokes keys at termination and waits for cleanup before deletion", async () => { + let pending = true; + const phases: string[] = []; + const work = worker(async agent => { phases.push(agent.lifecycleState); return agent.lifecycleState === "cleaning_up" && pending ? "pending" : "complete"; }); + const lifecycle = createAgentLifecycle(db); const agent = await hire(); await work.process(agent.id); + await db.insert(agentApiKeys).values({ agentId: agent.id, companyId, name: "Test", keyHash: randomUUID() }); + await lifecycle.terminateAgent(agent.id); await work.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("cleaning_up"); + expect((await db.select().from(agentApiKeys).where(eq(agentApiKeys.agentId, agent.id)))[0].revokedAt).not.toBeNull(); + await expect(agentConfiguration(db).remove(agent.id)).rejects.toThrow("Complete termination"); + pending = false; await lifecycle.retry(agent.id); await work.process(agent.id); + expect(phases).toContain("terminating"); + expect((await current(agent.id)).lifecycleState).toBe("terminated"); + await agentConfiguration(db).remove(agent.id); + expect(await current(agent.id)).toBeUndefined(); + }); + + it("retains required plugin IDs and checks the operation and company on each reply", async () => { + const agent = await hire("paused"); + const pluginId = randomUUID(); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "lifecycle-test", version: "1.0.0", status: "ready", + manifestJson: { agentLifecycle: true, capabilities: ["agents.lifecycle.manage"] } as never }); + const call = vi.fn(async (_id, _method, input) => ({ operationId: input.operationId, version: input.version, status: "complete" })); + const driver = createLifecycleDriver(db, { call } as never); + const snapshot = (await createLifecycleStore(db).get(agent.id))!; + expect(await driver.requiredPluginIds(snapshot)).toContain(pluginId); + await db.insert(pluginCompanySettings).values({ companyId, pluginId, enabled: false }); + expect(await driver.requiredPluginIds(snapshot)).not.toContain(pluginId); + expect(await driver.requiredPluginIds({ ...snapshot, lifecycleRequiredPluginIds: [pluginId] })).toContain(pluginId); + await expect(driver.runPlugin(snapshot, pluginId)).rejects.toThrow("unavailable"); + await db.update(pluginCompanySettings).set({ enabled: true }).where(eq(pluginCompanySettings.pluginId, pluginId)); + expect(await driver.runPlugin(snapshot, pluginId)).toBe("complete"); + expect(call).toHaveBeenCalledWith(pluginId, "agentLifecycle", { companyId, agentId: agent.id, + operationId: snapshot.lifecycleOperation!.id, version: snapshot.lifecycleVersion, phase: "paused" }, 30_000); + call.mockResolvedValueOnce({ operationId: "stale", version: snapshot.lifecycleVersion, status: "complete" }); + await expect(driver.runPlugin(snapshot, pluginId)).rejects.toThrow("Invalid lifecycle result"); + }); + + it("keeps host completion and required plugin results across a worker restart", async () => { + const pluginIds = [randomUUID(), randomUUID()]; + const runHost = vi.fn(async () => "complete" as const); + const firstPlugin = vi.fn(async (_agent: LifecycleAgent, pluginId: string) => + pluginId === pluginIds[0] ? "complete" as const : "pending" as const); + const first = configureAgentLifecycle(db, { requiredPluginIds: async () => pluginIds, runHost, runPlugin: firstPlugin }); + workers.push(first); + const agent = await hire(); + await first.process(agent.id); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "preparing", lifecycleRequiredPluginIds: pluginIds, + lifecycleOperation: { hostComplete: true, completedPluginIds: [pluginIds[0]] } }); + await first.stop(); + + const discover = vi.fn(async () => []); + const resumedPlugin = vi.fn(async (_agent: LifecycleAgent, _pluginId: string) => "complete" as const); + const resumed = configureAgentLifecycle(db, { requiredPluginIds: discover, runHost, runPlugin: resumedPlugin }); + workers.push(resumed); + await createAgentLifecycle(db).retry(agent.id); + await resumed.process(agent.id); + expect((await current(agent.id)).lifecycleState).toBe("ready"); + expect(discover).not.toHaveBeenCalled(); + expect(runHost).toHaveBeenCalledTimes(2); + expect(resumedPlugin.mock.calls.map(call => call[1])).toEqual([pluginIds[1], ...pluginIds]); + }); + + it("requires host and plugin completion and freezes the selected plugin IDs", async () => { + const agent = await hire(); + const store = createLifecycleStore(db); + const claimed = (await store.claim(agent.id, "test", new Date()))!; + const pluginId = randomUUID(); + expect(await store.setRequiredPlugins(claimed, "test", [pluginId])).toBe(true); + expect(await store.setRequiredPlugins(claimed, "test", [])).toBe(false); + expect(await store.completePlugin(claimed, "test", randomUUID())).toBe(false); + const complete = () => store.change(agent.id, "complete", { owner: "test", version: claimed.lifecycleVersion }); + expect(await complete()).toBeNull(); + expect(await store.completePlugin(claimed, "test", pluginId)).toBe(true); + expect(await complete()).toBeNull(); + expect(await store.completeHost(claimed, "test")).toBe(true); + expect(await complete()).toMatchObject({ lifecycleState: "verifying" }); + expect(await store.completePlugin(claimed, "test", pluginId)).toBe(false); + }); + + it("fences verification when the saved configuration changes", async () => { + const lifecycle = createAgentLifecycle(db); const store = createLifecycleStore(db); + const agent = await hire(); + const claimed = (await store.claim(agent.id, "configuration-test", new Date()))!; + await store.setRequiredPlugins(claimed, "configuration-test", []); + await agentConfiguration(db).update(agent.id, { adapterConfig: { command: "echo" } }); + expect(await store.completeHost(claimed, "configuration-test")).toBe(false); + expect((await lifecycle.get(agent.id))!.lifecycleVersion).toBeGreaterThan(claimed.lifecycleVersion); + }); + + it("invalidates only changed execution configuration during setup", async () => { + const lifecycle = createAgentLifecycle(db); + const agent = await hire(); + const before = await current(agent.id); + await lifecycle.updateConfiguration(agent.id, { name: "Profile only" }); + await lifecycle.updateConfiguration(agent.id, { adapterConfig: before.adapterConfig }); + expect((await current(agent.id)).lifecycleVersion).toBe(before.lifecycleVersion); + await lifecycle.updateConfiguration(agent.id, { runtimeConfig: { heartbeat: { enabled: false } } }); + expect((await current(agent.id)).lifecycleVersion).toBe(before.lifecycleVersion + 1); + const work = worker(); + await work.process(agent.id); + const ready = await current(agent.id); + expect(ready.lifecycleState).toBe("ready"); + await lifecycle.updateConfiguration(agent.id, { adapterConfig: { command: "echo" } }); + expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", lifecycleVersion: ready.lifecycleVersion }); + }); + + it("rejects identity, accounting, and unknown fields through every configuration entry point", async () => { + const agent = await hire(); + const lifecycle = createAgentLifecycle(db); + const before = await current(agent.id); + for (const patch of [{ id: randomUUID() }, { companyId: randomUUID() }, { spentMonthlyCents: 0 }, + { spendMonthUtc: "2099-01" }, { lastHeartbeatAt: new Date() }, { errorReason: null }, + { createdAt: new Date() }, { updatedAt: new Date() }, { lifecycleState: "ready" }, { futureColumn: true }]) { + await expect(lifecycle.updateConfiguration(agent.id, patch as never)).rejects.toThrow("Unsupported configuration field"); + await expect(lifecycle.updateAndTransition(agent.id, "pause", patch as never)).rejects.toThrow("Unsupported configuration field"); + await expect(db.transaction(tx => updateAgentConfigurationInTransaction(tx as unknown as Db, agent.id, patch as never))) + .rejects.toThrow("Unsupported configuration field"); + } + expect(await current(agent.id)).toEqual(before); + const effects = createAgentLifecycleEffects(); + const malformed = createLifecycleCommands(db, { ...effects, prepareConfiguration: async () => ({ id: randomUUID() }) as never }); + await expect(malformed.updateConfiguration(agent.id, { name: "Changed" })).rejects.toThrow("Unsupported configuration field"); + expect(await current(agent.id)).toEqual(before); + }); + + it("records only the changed configuration fields for a legacy appearance", async () => { + const [agent] = await db.insert(agents).values({ companyId, name: "Legacy", appearance: null }).returning(); + await createAgentLifecycle(db).updateConfiguration(agent.id, { name: "Renamed" }, + { recordRevision: { source: "test", createdByUserId: "board" } }); + const revisions = await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id)); + expect(revisions).toHaveLength(1); + expect(revisions[0].changedKeys).toEqual(["name"]); + expect(revisions[0].beforeConfig.appearance).toEqual(revisions[0].afterConfig.appearance); + }); + + it("accepts the unchanged resolved appearance of a pending hire", async () => { + const [agent] = await db.insert(agents).values({ companyId, name: "Pending legacy", status: "pending_approval", appearance: null }).returning(); + const resolved = await agentConfiguration(db).getById(agent.id); + await expect(createAgentLifecycle(db).updateConfiguration(agent.id, { appearance: resolved!.appearance })) + .resolves.toMatchObject({ id: agent.id }); + await expect(createAgentLifecycle(db).updateConfiguration(agent.id, { name: "Changed" })) + .rejects.toThrow("Pending approval agent configuration cannot be changed"); + }); + + it("rolls back a root configuration command when dependent records fail", async () => { + const agent = await hire(); + const before = await current(agent.id); + const effects = createAgentLifecycleEffects(); + const lifecycle = createLifecycleCommands(db, { ...effects, completeConfiguration: async (...args) => { + await effects.completeConfiguration(...args); + throw new Error("Configuration completion failed"); + } }); + await expect(lifecycle.updateConfiguration(agent.id, { adapterConfig: { command: "echo" } }, + { recordRevision: { source: "test", createdByUserId: "board" } })).rejects.toThrow("Configuration completion failed"); + expect(await current(agent.id)).toMatchObject({ adapterConfig: before.adapterConfig, + lifecycleVersion: before.lifecycleVersion, lifecycleOperation: before.lifecycleOperation }); + expect(await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agent.id))).toHaveLength(0); + }); + + it("rolls back configuration and verification invalidation in the binding transaction", async () => { + const agent = await hire(); + const before = await current(agent.id); + await expect(updateAgentConfigurationInTransaction(db, agent.id, { name: "Uncommitted" })) + .rejects.toThrow("requires a database transaction"); + expect(() => updateAgentConfigurationInTransaction(db, agent.id, { budgetMonthlyCents: 100 } as never)) + .toThrow("Budget changes require a root lifecycle command"); + await expect(db.transaction(async tx => agentConfiguration(tx as unknown as Db).update(agent.id, { name: "Uncommitted" }))) + .rejects.toThrow("root database"); + await expect(db.transaction(async tx => { + await updateAgentConfigurationInTransaction(tx as unknown as Db, agent.id, { adapterConfig: { command: "echo" } }); + expect((await tx.select().from(agents).where(eq(agents.id, agent.id)))[0].lifecycleVersion).toBe(before.lifecycleVersion + 1); + throw new Error("Binding change failed"); + })).rejects.toThrow("Binding change failed"); + expect(await current(agent.id)).toMatchObject({ adapterConfig: before.adapterConfig, + lifecycleVersion: before.lifecycleVersion, lifecycleOperation: before.lifecycleOperation }); + }); + + it("permits a board retry and rejects an agent or a different company", async () => { + const agent = await hire(); + function app(actor: Express.Request["actor"]) { + const app = express(); app.use(express.json()); + app.use((req, _res, next) => { req.actor = actor; next(); }); + app.use("/api", agentRoutes(db)); app.use(errorHandler); return app; + } + const board: Express.Request["actor"] = { type: "board", source: "session", userId: "board", companyIds: [companyId], + memberships: [{ companyId, membershipRole: "admin", status: "active" }] }; + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "admin" }); + await db.insert(principalPermissionGrants).values(["agents:create", "agents:configure"].map(permissionKey => ({ companyId, principalType: "user", principalId: "board", permissionKey }))); + const url = `/api/agents/${agent.id}/lifecycle/retry`; + const response = await request(app(board)).post(url).send({}); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body.lifecycleState).toBe("preparing"); + expect(response.body).not.toHaveProperty("lifecycleOperation"); + expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, agent.id), eq(activityLog.action, "agent.lifecycle_retried")))) + .toEqual([expect.objectContaining({ actorType: "user", actorId: "board" })]); + await request(app({ type: "agent", source: "agent_key", companyId, agentId: agent.id })).post(url).send({}).expect(403); + const denied = await request(app({ ...board, companyIds: [], memberships: [] })).post(url).send({}); + expect([403, 404]).toContain(denied.status); + }); + + it("recovers an expired lease and never accepts an expired completion", async () => { + const agent = await hire("paused"); const store = createLifecycleStore(db); + await createAgentLifecycle(db).resumeAgent(agent.id); + const claim = await store.claim(agent.id, "first", new Date(Date.now() - 180_000)); + expect(claim).not.toBeNull(); + await store.setRequiredPlugins(claim!, "first", []); + await store.completeHost(claim!, "first"); + expect(await store.change(agent.id, "complete", { owner: "first", version: claim!.lifecycleVersion })).toBeNull(); + expect(await store.claim(agent.id, "second", new Date())).not.toBeNull(); + }); +}); + +describe("lifecycle transition rules", () => { + it("does not bypass authorization, verification, or cleanup", () => { + expect(() => transition("pending_approval", "resume")).toThrow(); + expect(transition("preparing", "complete")).toBe("verifying"); + expect(transition("terminating", "complete")).toBe("cleaning_up"); + expect(transition("cleaning_up", "complete")).toBe("terminated"); + expect(() => transition("terminated", "resume")).toThrow(); + expect(transition("terminating", "terminate")).toBe("terminating"); + }); +}); diff --git a/server/src/modules/agent-lifecycle/__tests__/policy-sweep.test.ts b/server/src/modules/agent-lifecycle/__tests__/policy-sweep.test.ts new file mode 100644 index 0000000000..0358e9e4a7 --- /dev/null +++ b/server/src/modules/agent-lifecycle/__tests__/policy-sweep.test.ts @@ -0,0 +1,63 @@ +import { agents, companies, createDb, type Db } from "@paperclipai/db"; +import { asc, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { startAgentLifecycle } from "../index.js"; +import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("lifecycle policy sweep", () => { + let database: Awaited>; + let db: Db; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-lifecycle-policy-"); + db = createDb(database.connectionString); + }, 90_000); + afterAll(async () => { await database?.cleanup(); }); + + it("caps each policy page, skips ineligible agents, and returns to the first page", async () => { + const [company] = await db.insert(companies).values({ name: "Policy pages", issuePrefix: "POL" }).returning(); + const eligible = await db.insert(agents).values(Array.from({ length: 101 }, (_, index) => ({ + id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + companyId: company.id, name: `Agent ${index}`, status: "idle", lifecycleState: "ready" as const, + lifecycleHolds: ["company_paused"], pauseReason: "company_paused", + }))).returning(); + const excluded = await db.insert(agents).values((["pending_approval", "terminating", "cleaning_up", "terminated", "rejected"] as const).map(lifecycleState => ({ + companyId: company.id, name: lifecycleState, status: "terminated", lifecycleState, + lifecycleHolds: ["company_paused"], pauseReason: "company_paused", + }))).returning(); + const effects = createAgentLifecycleEffects(); + const transaction = vi.spyOn(effects, "transaction"); + const now = Date.now(); + const clock = vi.spyOn(Date, "now").mockReturnValue(now); + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => ({ unref() {} }) as ReturnType); + const clearTimer = vi.spyOn(globalThis, "clearInterval").mockImplementation(() => {}); + const work = startAgentLifecycle(db, effects, { + requiredPluginIds: async () => [], runHost: async () => "pending", runPlugin: async () => "pending", + }, () => true); + const rows = () => db.select().from(agents).where(eq(agents.companyId, company.id)).orderBy(asc(agents.id)); + try { + await work.sweep(); + expect(transaction).toHaveBeenCalledTimes(100); + expect((await rows()).filter(row => row.lifecycleHolds.length === 0)).toHaveLength(100); + await work.sweep(); + expect(transaction).toHaveBeenCalledTimes(100); + clock.mockReturnValue(now + 60_001); + await work.sweep(); + expect(transaction).toHaveBeenCalledTimes(101); + expect((await rows()).filter(row => row.lifecycleHolds.length > 0).map(row => row.id).sort()) + .toEqual(excluded.map(row => row.id).sort()); + await db.update(agents).set({ lifecycleHolds: ["company_paused"] }).where(eq(agents.id, eligible[0].id)); + clock.mockReturnValue(now + 120_002); + await work.sweep(); + expect(transaction).toHaveBeenCalledTimes(201); + expect((await rows())[0].lifecycleHolds).toEqual([]); + } finally { + await work.stop(); + transaction.mockRestore(); + clock.mockRestore(); + timer.mockRestore(); + clearTimer.mockRestore(); + } + }); +}); diff --git a/server/src/modules/agent-lifecycle/adapters/approvals.ts b/server/src/modules/agent-lifecycle/adapters/approvals.ts new file mode 100644 index 0000000000..e1c21fe195 --- /dev/null +++ b/server/src/modules/agent-lifecycle/adapters/approvals.ts @@ -0,0 +1,96 @@ +import { agentAppearanceSchema } from "@paperclipai/shared"; +import { approvals, type Db } from "@paperclipai/db"; +import { and, desc, eq, inArray, sql } from "drizzle-orm"; +import { unprocessable } from "../../../errors.js"; +import { approvalResolutionRecords, resolvableApprovalStatuses } from "../../../lib/approval-records.js"; +import { agentRecords } from "./records.js"; +import { createLifecycleStore } from "./postgres.js"; +import type { LifecycleEffects } from "./effects.js"; + +export type HireDecisionTarget = string | { approvalId: string }; +type HireDecisionResult = { + approval: typeof approvals.$inferSelect | null; + agent: Awaited["getById"]>>; + applied: boolean; + activated: boolean; + hireApprovedAgentId: string | null; +}; + +export function hireApprovalService(db: Db, effects: LifecycleEffects) { + const { getExistingApproval, resolveApproval } = approvalResolutionRecords(db); + async function decide(target: HireDecisionTarget, status: "approved" | "rejected", userId: string, note?: string | null): Promise { + const initial = typeof target === "string" + ? await agentRecords(db, effects).getById(target) + : await getExistingApproval(target.approvalId); + if (!initial) return null; + if (typeof target !== "string" && (!("type" in initial) || initial.type !== "hire_agent")) throw unprocessable("Expected a hire approval"); + const companyId = initial.companyId; + // Hire decisions acquire the company lock before changing an approval or agent. + return effects.transaction(db, companyId, async (tx, publications) => { + const records = agentRecords(tx, effects); + let approvalId = typeof target === "string" ? null : target.approvalId; + if (typeof target === "string") { + const [open] = await tx.select({ id: approvals.id }).from(approvals).where(and( + eq(approvals.companyId, companyId), eq(approvals.type, "hire_agent"), + inArray(approvals.status, resolvableApprovalStatuses), + sql`${approvals.payload}->>'agentId' = ${target}`, + )).orderBy(desc(approvals.createdAt)).limit(1); + approvalId = open?.id ?? null; + } + let approval: typeof approvals.$inferSelect | null = null; + let applied = true; + if (approvalId) { + const result = await resolveApproval(approvalId, status, userId, note, tx); + approval = result.approval; + applied = result.applied; + } + const payload = approval?.payload ?? {}; + let agentId = typeof target === "string" ? target : typeof payload.agentId === "string" ? payload.agentId : null; + if (applied && status === "approved") { + if (agentId) { + const activated = await records.activatePendingApproval(agentId, approval ? payload : null, approval ? approval.requestedByUserId : userId); + if (!approval) applied = activated?.activated ?? false; + } else { + const created = await records.create(companyId, { + name: String(payload.name ?? "New Agent"), + appearance: payload.appearance == null ? undefined : agentAppearanceSchema.parse(payload.appearance), + role: String(payload.role ?? "general"), + title: typeof payload.title === "string" ? payload.title : null, + reportsTo: typeof payload.reportsTo === "string" ? payload.reportsTo : null, + capabilities: typeof payload.capabilities === "string" ? payload.capabilities : null, + adapterType: String(payload.adapterType ?? "process"), + adapterConfig: + typeof payload.adapterConfig === "object" && payload.adapterConfig !== null + ? (payload.adapterConfig as Record) + : {}, + budgetMonthlyCents: + typeof payload.budgetMonthlyCents === "number" ? payload.budgetMonthlyCents : 0, + metadata: + typeof payload.metadata === "object" && payload.metadata !== null + ? (payload.metadata as Record) + : null, + status: "idle", + spentMonthlyCents: 0, + permissions: undefined, + lastHeartbeatAt: null, + }, { createdByUserId: approval!.requestedByAgentId ? null : approval!.requestedByUserId, responsibleUserId: approval!.requestedByUserId }); + agentId = created.id; + } + if (approval && agentId && typeof payload.budgetMonthlyCents === "number" && payload.budgetMonthlyCents > 0) { + await effects.setAgentBudget(tx, publications, companyId, agentId, payload.budgetMonthlyCents, userId); + } + } else if (applied && agentId) { + if (approval) await records.rejectPendingHire(agentId); + else { + const before = await records.getById(agentId); + await createLifecycleStore(tx, effects).change(agentId, "reject"); + applied = before?.lifecycleState !== "rejected"; + } + } + return { approval, agent: agentId ? await records.getById(agentId) : null, applied, + activated: status === "approved" && applied, + hireApprovedAgentId: status === "approved" && applied ? agentId : null }; + }); + } + return { decide }; +} diff --git a/server/src/modules/agent-lifecycle/adapters/company-deletion.ts b/server/src/modules/agent-lifecycle/adapters/company-deletion.ts new file mode 100644 index 0000000000..5d27552b39 --- /dev/null +++ b/server/src/modules/agent-lifecycle/adapters/company-deletion.ts @@ -0,0 +1,19 @@ +import { agents } from "@paperclipai/db"; +import { and, eq, inArray } from "drizzle-orm"; +import type { CompanyDeletionParticipant } from "../../../lib/company-deletion.js"; +import { AgentLifecycleConflict, assertAgentPurgeAllowed } from "../domain/policy.js"; + +export const agentLifecycleCompanyDeletion: CompanyDeletionParticipant = { + async deleteCompanyData(tx, companyId) { + if (!("nestedIndex" in tx)) { + throw new AgentLifecycleConflict("Company deletion requires a database transaction"); + } + const rows = await tx.select({ lifecycleState: agents.lifecycleState }).from(agents) + .where(eq(agents.companyId, companyId)).for("update"); + for (const agent of rows) assertAgentPurgeAllowed(agent); + await tx.delete(agents).where(and( + eq(agents.companyId, companyId), + inArray(agents.lifecycleState, ["terminated", "rejected"]), + )); + }, +}; diff --git a/server/src/modules/agent-lifecycle/adapters/configuration.ts b/server/src/modules/agent-lifecycle/adapters/configuration.ts new file mode 100644 index 0000000000..70a8e44d5d --- /dev/null +++ b/server/src/modules/agent-lifecycle/adapters/configuration.ts @@ -0,0 +1,32 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { agents, type Db } from "@paperclipai/db"; +import { agentRecordQueries, AGENT_CONFIGURATION_FIELDS, jsonEqual, type AgentConfigurationPatch, type UpdateAgentOptions } from "../../../lib/agent-records.js"; +import type { ActivityPublication } from "../../../types/activity-publication.js"; +import { AgentLifecycleConflict } from "../domain/policy.js"; +import type { LifecycleEffects } from "./effects.js"; + +export async function updateAgentConfiguration(tx: Db, effects: LifecycleEffects, id: string, data: AgentConfigurationPatch, options?: UpdateAgentOptions, publications: ActivityPublication[] = []) { + if (!("nestedIndex" in tx)) throw new AgentLifecycleConflict("Configuration integration requires a database transaction"); + assertConfigurationFields(data); + const [current] = await tx.select().from(agents).where(eq(agents.id, id)).for("update"); + if (!current) return null; + const patch = await effects.prepareConfiguration(tx, current, data, options); + assertConfigurationFields(patch); + const changed = ["adapterType", "adapterConfig", "runtimeConfig", "defaultEnvironmentId"].some(key => + Object.prototype.hasOwnProperty.call(patch, key) && !jsonEqual(patch[key as keyof typeof patch], current[key as keyof typeof current])); + const invalidate = changed && ["preparing", "verifying", "resuming"].includes(current.lifecycleState); + const [updated] = await tx.update(agents).set({ ...patch, updatedAt: new Date(), ...(invalidate ? { + lifecycleVersion: current.lifecycleVersion + 1, lifecycleError: null, + lifecycleOperation: { ...current.lifecycleOperation!, id: randomUUID(), hostComplete: false, + completedPluginIds: [], leaseOwner: undefined, leaseUntil: undefined, retryAt: undefined }, + } : {}) }).where(eq(agents.id, id)).returning(); + await effects.completeConfiguration(tx, current, updated, patch, options, publications); + return agentRecordQueries(tx).getById(id); +} + +function assertConfigurationFields(data: AgentConfigurationPatch) { + if (Object.keys(data).some(field => !AGENT_CONFIGURATION_FIELDS.some(allowed => allowed === field))) { + throw new AgentLifecycleConflict("Unsupported configuration field; use the command that owns this field, such as a lifecycle command"); + } +} diff --git a/server/src/modules/agent-lifecycle/adapters/deletion.ts b/server/src/modules/agent-lifecycle/adapters/deletion.ts new file mode 100644 index 0000000000..e198181a4d --- /dev/null +++ b/server/src/modules/agent-lifecycle/adapters/deletion.ts @@ -0,0 +1,27 @@ +import { agents, type Db } from "@paperclipai/db"; +import { and, eq, inArray } from "drizzle-orm"; +import { conflict } from "../../../errors.js"; +import { agentRecordQueries } from "../../../lib/agent-records.js"; +import { readBuiltInAgentMarker } from "../../../lib/built-in-agent-metadata.js"; +import { assertAgentPurgeAllowed } from "../domain/policy.js"; +import type { LifecycleEffects } from "./effects.js"; + +export async function purgeAgent(db: Db, effects: LifecycleEffects, id: string) { + const { getById, normalizeAgentRow } = agentRecordQueries(db); + const existing = await getById(id); + if (!existing) return null; + return effects.transaction(db, existing.companyId, async tx => { + const [agent] = await tx.select().from(agents).where(eq(agents.id, id)).for("update"); + if (!agent) return null; + const marker = readBuiltInAgentMarker(agent.metadata); + if (marker) throw conflict("Built-in agents cannot be deleted; pause them instead", { + code: "built_in_agent_undeletable", key: marker.key, featureKeys: marker.featureKeys, + }); + assertAgentPurgeAllowed(agent); + await effects.deleteDependencies(tx, agent.companyId, id); + const [deleted] = await tx.delete(agents).where(and( + eq(agents.id, id), inArray(agents.lifecycleState, ["terminated", "rejected"]), + )).returning(); + return deleted ? normalizeAgentRow(deleted) : null; + }); +} diff --git a/server/src/modules/agent-lifecycle/adapters/effects.ts b/server/src/modules/agent-lifecycle/adapters/effects.ts new file mode 100644 index 0000000000..3df05e49b2 --- /dev/null +++ b/server/src/modules/agent-lifecycle/adapters/effects.ts @@ -0,0 +1,24 @@ +import type { LifecycleFailure } from "../application/ports.js"; +import type { Db } from "@paperclipai/db"; +import type { budgetPolicies } from "@paperclipai/db"; +import type { CreateAgentData, CreateAgentOptions, AgentConfigurationPatch, AgentConfigurationRecord, AgentHireRecord, UpdateAgentOptions } from "../../../lib/agent-records.js"; +import type { ActivityPublication } from "../../../types/activity-publication.js"; + +/** Services supply integrations; lifecycle owns the transaction and state writes. */ +export interface LifecycleEffects { + reportFailure(context: LifecycleFailure, error: unknown): void; + deleteDependencies(db: Db, companyId: string, agentId: string): Promise; + prepareConfiguration(db: Db, current: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise; + completeConfiguration(db: Db, before: AgentConfigurationRecord, after: AgentConfigurationRecord, patch: AgentConfigurationPatch, options: UpdateAgentOptions | undefined, publications: ActivityPublication[]): Promise; + transaction(db: Db, companyId: string, work: (tx: Db, publications: ActivityPublication[]) => Promise): Promise; + policyBlocks(db: Db, policy: typeof budgetPolicies.$inferSelect): Promise; + setAgentBudget(db: Db, publications: ActivityPublication[], companyId: string, agentId: string, amount: number, userId: string | null, isActive?: boolean): Promise; + enforceBudget(db: Db, companyId: string, agentId: string): Promise; + recordCreation(db: Db, companyId: string, agentId: string): Promise; + recordStatus(db: Db, companyId: string, agentId: string, before: string, after: string): Promise; + clearPrimary(db: Db, companyId: string, agentId: string): Promise; + prepareHire(db: Db, companyId: string, data: CreateAgentData, options?: CreateAgentOptions): Promise; + initializeHire(db: Db, agent: AgentHireRecord, options?: CreateAgentOptions): Promise; + prepareHireApproval(db: Db, agent: AgentHireRecord, payload?: Record | null): Promise; + completeHireApproval(db: Db, agent: AgentHireRecord, previous: AgentHireRecord): Promise; +} diff --git a/server/src/modules/agent-lifecycle/adapters/postgres.ts b/server/src/modules/agent-lifecycle/adapters/postgres.ts new file mode 100644 index 0000000000..a5cde41e74 --- /dev/null +++ b/server/src/modules/agent-lifecycle/adapters/postgres.ts @@ -0,0 +1,151 @@ +import { randomUUID } from "node:crypto"; +import { and, asc, eq, inArray, sql } from "drizzle-orm"; +import { agents, agentApiKeys, companies, budgetPolicies, type Db } from "@paperclipai/db"; +import type { AgentLifecycleOperation } from "@paperclipai/shared"; +import { AgentLifecycleConflict, compatibilityStatus, transition } from "../domain/policy.js"; +import type { LifecycleEffects } from "./effects.js"; +import type { LifecycleStore } from "../application/ports.js"; + +export function assertRootDatabase(db: Db) { + if ("nestedIndex" in db) throw new AgentLifecycleConflict("Lifecycle commands require the root database connection"); +} + +export function createLifecycleStore(db: Db, effects: LifecycleEffects): LifecycleStore { + const get = (id: string) => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0] ?? null); + return { + get, + async change(id, command, options = {}) { + const existing = await get(id); + if (!existing) return null; + return effects.transaction(db, existing.companyId, async tx => { + const [agent] = await tx.select().from(agents).where(eq(agents.id, id)).for("update", command === "reconcile" ? { skipLocked: true } : undefined); + if (!agent) return null; + if (options.version !== undefined && options.version !== agent.lifecycleVersion) return null; + if (command === "complete" && (!options.owner || !agent.lifecycleOperation?.leaseUntil || agent.lifecycleOperation?.leaseOwner !== options.owner || + Date.parse(agent.lifecycleOperation?.leaseUntil ?? "") <= Date.now() || + !agent.lifecycleOperation.hostComplete || agent.lifecycleRequiredPluginIds === null || + agent.lifecycleRequiredPluginIds.some(id => !agent.lifecycleOperation!.completedPluginIds.includes(id)))) return null; + if (command === "retry") { + transition(agent.lifecycleState, command); + if (agent.lifecycleOperation?.leaseUntil && Date.parse(agent.lifecycleOperation.leaseUntil) > Date.now()) return agent; + const [updated] = await tx.update(agents).set({ lifecycleError: null, + lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt' - 'leaseOwner' - 'leaseUntil'`, updatedAt: new Date() }).where(eq(agents.id, id)).returning(); + return updated; + } + if (command === "resume" && !["ready", "paused", "pausing", "resuming"].includes(agent.lifecycleState)) transition(agent.lifecycleState, command); + let holds = agent.lifecycleHolds; + const reason = options.reason ?? "manual"; + if (command === "reconcile") { + if (["pending_approval", "terminating", "cleaning_up", "terminated", "rejected"].includes(agent.lifecycleState)) return agent; + const [company] = await tx.select().from(companies).where(eq(companies.id, agent.companyId)); + const policies = await tx.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, agent.companyId), + eq(budgetPolicies.scopeType, "agent"), eq(budgetPolicies.scopeId, id))); + let budgetBlocked = false; + for (const policy of policies) if (await effects.policyBlocks(tx, policy)) budgetBlocked = true; + holds = holds.filter(hold => hold !== "budget" && hold !== "company_archived" && hold !== "company_paused"); + if (budgetBlocked) holds.push("budget"); + if (company.status === "archived") holds.push("company_archived"); + if (company.status === "paused") holds.push("company_paused"); + } + if (command === "pause") holds = [...new Set([...holds, reason])]; + if (command === "resume") holds = holds.filter(hold => reason === "user" ? ["budget", "company_archived", "company_paused"].includes(hold) : hold !== reason); + let next = command === "reconcile" + ? holds.length && !["pausing", "paused"].includes(agent.lifecycleState) ? "pausing" as const + : !holds.length && agent.lifecycleState === "paused" ? "resuming" as const : agent.lifecycleState + : command === "resume" && (holds.length || agent.lifecycleState === "pausing") ? agent.lifecycleState + : transition(agent.lifecycleState, command, agent.lifecycleOperation?.resumeState); + if (command === "complete" && next === "ready" && holds.length) next = "pausing"; + if (command === "complete" && next === "paused" && !holds.length) next = "resuming"; + if (next === agent.lifecycleState) { + if (JSON.stringify(holds) !== JSON.stringify(agent.lifecycleHolds) || agent.pauseReason !== (holds[0] ?? null)) { + await tx.update(agents).set({ lifecycleHolds: holds, pauseReason: holds[0] ?? null }).where(eq(agents.id, id)); + } + return agent; + } + const operation: AgentLifecycleOperation = { + responsibleUserId: agent.lifecycleOperation?.responsibleUserId, + id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, + resumeState: next === "pausing" + ? agent.lifecycleState === "preparing" || agent.lifecycleState === "verifying" ? agent.lifecycleState : agent.lifecycleOperation?.resumeState ?? "ready" + : agent.lifecycleOperation?.resumeState, + }; + if (next === "ready") operation.resumeState = "ready"; + const status = compatibilityStatus(next, agent.status); + const [updated] = await tx.update(agents).set({ + lifecycleHolds: holds, lifecycleState: next, lifecycleVersion: agent.lifecycleVersion + 1, lifecycleError: null, + lifecycleOperation: operation, status, + pauseReason: holds[0] ?? null, + pausedAt: next === "pausing" ? new Date() : next === "ready" ? null : agent.pausedAt, + updatedAt: new Date(), + }).where(eq(agents.id, id)).returning(); + if (next === "terminating" || next === "rejected") { + await tx.update(agentApiKeys).set({ revokedAt: new Date() }).where(eq(agentApiKeys.agentId, id)); + await effects.clearPrimary(tx, agent.companyId, id); + } + if (command === "approve") await effects.recordCreation(tx, agent.companyId, id); + if (next === "pausing" || next === "resuming" || next === "terminating" || next === "rejected") { + await effects.recordStatus(tx, agent.companyId, id, + next === "resuming" ? "paused" : "idle", + next === "resuming" ? "idle" : next === "pausing" ? "paused" : "terminated"); + } + return updated; + }); + }, + async claim(id, owner, now) { + const [agent] = await db.update(agents).set({ + lifecycleOperation: sql`coalesce(${agents.lifecycleOperation}, '{"id":"", "hostComplete":false, "completedPluginIds":[], "attempts":0}'::jsonb) + || jsonb_build_object('leaseOwner', ${owner}::text, 'leaseUntil', ${(new Date(now.getTime() + 120_000)).toISOString()}::text, + 'attempts', coalesce((${agents.lifecycleOperation}->>'attempts')::integer, 0) + 1)`, + }).where(and(eq(agents.id, id), inArray(agents.lifecycleState, ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"]), + sql`coalesce((${agents.lifecycleOperation}->>'leaseUntil')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`, + sql`coalesce((${agents.lifecycleOperation}->>'retryAt')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`, + )).returning(); + return agent ?? null; + }, + async renew(agent, owner, now) { + const rows = await db.update(agents).set({ lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{leaseUntil}', ${JSON.stringify(new Date(now.getTime() + 120_000).toISOString())}::jsonb)` }) + .where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion), sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id }); + return rows.length > 0; + }, + async setRequiredPlugins(agent, owner, pluginIds) { + const rows = await db.update(agents).set({ lifecycleRequiredPluginIds: pluginIds }) + .where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion), + sql`${agents.lifecycleRequiredPluginIds} is null`, + sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id }); + return rows.length > 0; + }, + async completeHost(agent, owner) { + const rows = await db.update(agents).set({ + lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{hostComplete}', 'true'::jsonb)`, + }).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion), + sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id }); + return rows.length > 0; + }, + async completePlugin(agent, owner, pluginId) { + const rows = await db.update(agents).set({ + lifecycleOperation: sql`jsonb_set(${agents.lifecycleOperation}, '{completedPluginIds}', + coalesce(${agents.lifecycleOperation}->'completedPluginIds', '[]'::jsonb) || ${JSON.stringify([pluginId])}::jsonb)`, + }).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion), + sql`${agents.lifecycleRequiredPluginIds} @> ${JSON.stringify([pluginId])}::jsonb`, + sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id }); + return rows.length > 0; + }, + async defer(agent, owner, error, now) { + const rows = await db.update(agents).set({ + lifecycleError: error, + lifecycleOperation: sql`(${agents.lifecycleOperation} - 'leaseOwner' - 'leaseUntil') + || jsonb_build_object('retryAt', ${new Date(now.getTime() + (error ? 60_000 : 2_000)).toISOString()}::text)`, + updatedAt: now, + }).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion), + sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id }); + return rows.length > 0; + }, + async pending(limit, now) { + return db.select({ id: agents.id }).from(agents).where(and( + inArray(agents.lifecycleState, ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"]), + sql`coalesce((${agents.lifecycleOperation}->>'leaseUntil')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`, + sql`coalesce((${agents.lifecycleOperation}->>'retryAt')::timestamptz, '-infinity') <= ${now.toISOString()}::timestamptz`, + )).orderBy(asc(agents.updatedAt), asc(agents.id)).limit(limit).then(rows => rows.map(row => row.id)); + }, + }; +} diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts new file mode 100644 index 0000000000..bf2ec0cd47 --- /dev/null +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -0,0 +1,118 @@ +import type { LifecycleEffects } from "./effects.js"; +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { agents, agentApiKeys, type Db } from "@paperclipai/db"; +import { conflict, notFound } from "../../../errors.js"; +import { agentRecordQueries, type CreateAgentOptions, type CreateAgentData } from "../../../lib/agent-records.js"; + +export function agentRecords(db: Db, effects: LifecycleEffects) { + const { getById } = agentRecordQueries(db); + + return { + getById, + create: async (companyId: string, data: CreateAgentData, options?: CreateAgentOptions) => { + if (Object.keys(data).some(key => key.startsWith("lifecycle"))) throw conflict("Lifecycle fields belong to the lifecycle module"); + return effects.transaction(db, companyId, async (txDb) => { + const tx = txDb; + const prepared = await effects.prepareHire(txDb, companyId, data, options); + const created = await tx + .insert(agents) + .values({ + ...prepared, + lifecycleState: data.status === "pending_approval" ? "pending_approval" : data.status === "terminated" ? "terminated" : data.status === "paused" ? "paused" : "preparing", + lifecycleVersion: 1, + lifecycleHolds: data.status === "paused" ? [data.pauseReason ?? "manual"] : [], + pauseReason: data.status === "paused" ? data.pauseReason ?? "manual" : null, + pausedAt: data.status === "paused" ? new Date() : null, + lifecycleOperation: { id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, resumeState: "preparing", responsibleUserId: options?.responsibleUserId ?? options?.createdByUserId }, + status: data.status === "pending_approval" ? "pending_approval" : data.status === "terminated" ? "terminated" : "paused", + companyId, + }) + .returning() + .then((rows) => rows[0]); + await effects.initializeHire(txDb, created, options); + if (created.status !== "pending_approval" && created.status !== "terminated") { + await effects.recordCreation(txDb, companyId, created.id); + } + const normalizedCreated = await agentRecordQueries(txDb).getById(created.id); + if (!normalizedCreated) { + throw notFound("Agent not found"); + } + return normalizedCreated; + }); + }, + + clearError: async (id: string) => { + const existing = await getById(id); + if (!existing) return null; + if (existing.status === "terminated") throw conflict("Cannot clear error on terminated agent"); + if (existing.status === "pending_approval") { + throw conflict("Pending approval agents cannot have errors cleared"); + } + if (existing.status !== "error") { + throw conflict("Only agents in error status can have their error cleared"); + } + + const updated = await db + .update(agents) + .set({ + status: "idle", + pauseReason: null, + pausedAt: null, + errorReason: null, + updatedAt: new Date(), + }) + .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) + .returning() + .then((rows) => rows[0] ?? null); + + if (!updated) { + throw conflict("Only agents in error status can have their error cleared"); + } + return getById(updated.id); + }, + + rejectPendingHire: async (id: string) => { + const rows = await db.update(agents).set({ status: "terminated", lifecycleState: "rejected", + lifecycleVersion: sql`${agents.lifecycleVersion} + 1`, lifecycleOperation: null, + updatedAt: new Date() }).where(and(eq(agents.id, id), eq(agents.lifecycleState, "pending_approval"))).returning(); + for (const row of rows) { + await effects.clearPrimary(db, row.companyId, id); + await db.update(agentApiKeys).set({ revokedAt: new Date() }).where(eq(agentApiKeys.agentId, id)); + await effects.recordStatus(db, row.companyId, id, "pending_approval", "terminated"); + } + return rows; + }, + + activatePendingApproval: async (id: string, approvedPayload?: Record | null, requestedByUserId?: string | null) => { + const activatedAgent = await db.transaction(async (tx) => { + const txDb = tx as unknown as Db; + const existing = await agentRecordQueries(txDb).getById(id); + if (!existing || existing.status !== "pending_approval") return null; + const patch = await effects.prepareHireApproval(txDb, existing, approvedPayload); + const updated = await tx + .update(agents) + .set({ ...patch, status: "paused", lifecycleState: "preparing", lifecycleVersion: existing.lifecycleVersion + 1, lifecycleError: null, lifecycleOperation: sql`jsonb_build_object('id', ${randomUUID()}::text, 'hostComplete', false, 'completedPluginIds', '[]'::jsonb, 'attempts', 0, 'responsibleUserId', coalesce(nullif(${agents.lifecycleOperation}->'responsibleUserId', 'null'::jsonb), to_jsonb(${requestedByUserId ?? null}::text)))`, updatedAt: new Date() }) + .where(and(eq(agents.id, id), eq(agents.status, "pending_approval"))) + .returning() + .then((rows) => rows[0] ?? null); + if (!updated) return null; + await effects.completeHireApproval(txDb, updated, existing); + await effects.recordCreation(txDb, existing.companyId, updated.id); + const agent = await agentRecordQueries(txDb).getById(updated.id); + if (!agent) { + throw notFound("Agent not found"); + } + return agent; + }); + + if (activatedAgent) { + return { agent: activatedAgent, activated: true }; + } + + const existing = await getById(id); + return existing ? { agent: existing, activated: false } : null; + }, + + }; +} diff --git a/server/src/modules/agent-lifecycle/application/ports.ts b/server/src/modules/agent-lifecycle/application/ports.ts new file mode 100644 index 0000000000..1845ad8b7d --- /dev/null +++ b/server/src/modules/agent-lifecycle/application/ports.ts @@ -0,0 +1,41 @@ +import type { AgentLifecycleOperation, AgentLifecycleState } from "@paperclipai/shared"; +import type { LifecycleCommand } from "../domain/policy.js"; + +export interface LifecycleAgent { + id: string; + companyId: string; + lifecycleState: AgentLifecycleState; + lifecycleVersion: number; + lifecycleRequiredPluginIds: string[] | null; + lifecycleError: string | null; + lifecycleOperation: AgentLifecycleOperation | null; +} + +export interface LifecycleStore { + get(id: string): Promise; + change(id: string, command: LifecycleCommand, options?: { version?: number; owner?: string; reason?: string }): Promise; + claim(id: string, owner: string, now: Date): Promise; + renew(agent: LifecycleAgent, owner: string, now: Date): Promise; + setRequiredPlugins(agent: LifecycleAgent, owner: string, pluginIds: string[]): Promise; + completeHost(agent: LifecycleAgent, owner: string): Promise; + completePlugin(agent: LifecycleAgent, owner: string, pluginId: string): Promise; + defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date): Promise; + pending(limit: number, now: Date): Promise; +} + +export interface LifecycleDriver { + requiredPluginIds(agent: LifecycleAgent): Promise; + runHost(agent: LifecycleAgent): Promise<"complete" | "pending">; + runPlugin(agent: LifecycleAgent, pluginId: string): Promise<"complete" | "pending">; +} + +export interface LifecycleFailure { + stage: "claim" | "renew" | "select_plugins" | "save_plugins" | "host" | "plugin" + | "complete_host" | "complete_plugin" | "transition" | "defer" | "scan" | "policy_scan"; + agentId?: string; + companyId?: string; + phase?: AgentLifecycleState; + operationId?: string; + version?: number; + pluginId?: string; +} diff --git a/server/src/modules/agent-lifecycle/application/worker.ts b/server/src/modules/agent-lifecycle/application/worker.ts new file mode 100644 index 0000000000..ad1c143dd9 --- /dev/null +++ b/server/src/modules/agent-lifecycle/application/worker.ts @@ -0,0 +1,96 @@ +import { randomUUID } from "node:crypto"; +import type { LifecycleDriver, LifecycleStore, LifecycleFailure } from "./ports.js"; +import { isLifecycleWorkPending } from "../domain/policy.js"; + +/** Database leases protect each attempt across server processes. */ +export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDriver, reportFailure: (context: LifecycleFailure, error: unknown) => void, canRun = () => true) { + let stopped = false; + const active = new Map>(); + let running: Promise | undefined; + + async function attempt(context: LifecycleFailure, work: () => Promise): Promise { + try { return await work(); } + catch (error) { reportFailure(context, error); throw error; } + } + + async function process(id: string) { + phase: for (let step = 0; !stopped && canRun() && step < 8; step++) { + const owner = randomUUID(); + const agent = await attempt({ stage: "claim", agentId: id }, () => store.claim(id, owner, new Date())); + if (!agent || !isLifecycleWorkPending(agent.lifecycleState)) return; + const context = { agentId: agent.id, companyId: agent.companyId, phase: agent.lifecycleState, + operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion }; + const runStep = (stage: LifecycleFailure["stage"], work: () => Promise, pluginId?: string) => + attempt({ ...context, stage, ...(pluginId ? { pluginId } : {}) }, work); + const renewal = setInterval(() => { + void runStep("renew", () => store.renew(agent, owner, new Date())).catch(() => {}); + }, 30_000); + renewal.unref?.(); + try { + const operation = agent.lifecycleOperation!; + const pluginIds = agent.lifecycleRequiredPluginIds ?? await runStep("select_plugins", () => driver.requiredPluginIds(agent)); + if (agent.lifecycleRequiredPluginIds === null && !await runStep("save_plugins", () => store.setRequiredPlugins(agent, owner, pluginIds))) continue; + if (!canRun()) { + await runStep("defer", () => store.defer(agent, owner, null, new Date())); + return; + } + if (!operation.hostComplete) { + if (!await runStep("renew", () => store.renew(agent, owner, new Date()))) continue; + if (await runStep("host", () => driver.runHost(agent)) === "pending") { + if (!await runStep("defer", () => store.defer(agent, owner, null, new Date()))) continue; + return; + } + if (!await runStep("complete_host", () => store.completeHost(agent, owner))) continue; + } + for (const pluginId of pluginIds) { + if (operation.completedPluginIds.includes(pluginId)) continue; + if (!canRun()) { + await runStep("defer", () => store.defer(agent, owner, null, new Date())); + return; + } + if (!await runStep("renew", () => store.renew(agent, owner, new Date()))) continue phase; + if (await runStep("plugin", () => driver.runPlugin(agent, pluginId), pluginId) === "pending") { + if (!await runStep("defer", () => store.defer(agent, owner, null, new Date()))) continue phase; + return; + } + if (!await runStep("complete_plugin", () => store.completePlugin(agent, owner, pluginId), pluginId)) continue phase; + } + const changed = await runStep("transition", () => store.change(id, "complete", { version: agent.lifecycleVersion, owner })); + if (changed && !isLifecycleWorkPending(changed.lifecycleState)) return; + } catch { + if (!await runStep("defer", () => store.defer(agent, owner, "The lifecycle step failed. Retry the operation.", new Date()))) continue; + return; + } finally { + clearInterval(renewal); + } + } + } + + async function sweep() { + const pending = await attempt({ stage: "scan" }, () => store.pending(100, new Date())); + const entries = pending.values(); + await Promise.all(Array.from({ length: Math.min(4, pending.length) }, async () => { + for (const id of entries) await start(id); + })); + } + + function start(id: string): Promise { + if (stopped || !canRun()) return Promise.resolve(); + const existing = active.get(id); + if (existing) return existing; + const work = process(id).finally(() => { active.delete(id); }); + active.set(id, work); + return work; + } + + return { + process: start, + async stop() { + stopped = true; + await Promise.allSettled([...active.values()]); + }, + sweep() { + return running ??= sweep().finally(() => { running = undefined; }); + }, + }; +} diff --git a/server/src/modules/agent-lifecycle/company-deletion.ts b/server/src/modules/agent-lifecycle/company-deletion.ts new file mode 100644 index 0000000000..6ec8a8bad6 --- /dev/null +++ b/server/src/modules/agent-lifecycle/company-deletion.ts @@ -0,0 +1 @@ +export { agentLifecycleCompanyDeletion } from "./adapters/company-deletion.js"; diff --git a/server/src/modules/agent-lifecycle/configuration.ts b/server/src/modules/agent-lifecycle/configuration.ts new file mode 100644 index 0000000000..c82463ec52 --- /dev/null +++ b/server/src/modules/agent-lifecycle/configuration.ts @@ -0,0 +1 @@ +export { updateAgentConfiguration } from "./adapters/configuration.js"; diff --git a/server/src/modules/agent-lifecycle/domain/policy.ts b/server/src/modules/agent-lifecycle/domain/policy.ts new file mode 100644 index 0000000000..520193ee6f --- /dev/null +++ b/server/src/modules/agent-lifecycle/domain/policy.ts @@ -0,0 +1,63 @@ +import type { AgentLifecycleState } from "@paperclipai/shared"; + +export type LifecycleCommand = "reconcile" | "approve" | "reject" | "pause" | "resume" | "terminate" | "complete" | "retry"; +export class AgentLifecycleConflict extends Error { + constructor(message: string) { super(message); this.name = "AgentLifecycleConflict"; } +} + +export function canConfigureAgentConnection(agent: { lifecycleState: string; status: string }): boolean { + return agent.lifecycleState === "preparing" || agent.lifecycleState === "verifying" + || (agent.lifecycleState === "ready" && !["paused", "terminated", "pending_approval"].includes(agent.status)); +} + +export function isAgentAwaitingSetup(agent: { lifecycleState: string; lifecycleHolds: readonly string[] }): boolean { + return ["preparing", "verifying", "resuming"].includes(agent.lifecycleState) && agent.lifecycleHolds.length === 0; +} + +export function transition(state: AgentLifecycleState, command: LifecycleCommand, + resumeState: "preparing" | "verifying" | "ready" = "ready"): AgentLifecycleState { + if (command === "retry") { + if (["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"].includes(state)) return state; + } else if (command === "terminate") { + if (["terminated", "rejected", "terminating", "cleaning_up"].includes(state)) return state; + return "terminating"; + } else if (command === "approve") { + if (state === "pending_approval") return "preparing"; + } else if (command === "reject") { + if (state === "pending_approval") return "rejected"; + if (state === "rejected") return state; + } else if (command === "pause") { + if (state === "paused" || state === "pausing") return state; + if (["ready", "preparing", "verifying", "resuming"].includes(state)) return "pausing"; + } else if (command === "resume") { + if (state === "ready" || state === "resuming") return state; + if (state === "paused") return "resuming"; + } else if (command === "complete") { + switch (state) { + case "preparing": return "verifying"; + case "verifying": return "ready"; + case "pausing": return "paused"; + case "resuming": return resumeState; + case "terminating": return "cleaning_up"; + case "cleaning_up": return "terminated"; + } + } + throw new AgentLifecycleConflict(`Cannot ${command} an agent in state ${state}`); +} + +export function compatibilityStatus(state: AgentLifecycleState, executionStatus = "idle"): string { + if (state === "ready") return ["active", "idle", "running", "error"].includes(executionStatus) ? executionStatus : "idle"; + if (state === "pending_approval") return "pending_approval"; + if (["terminating", "cleaning_up", "terminated", "rejected"].includes(state)) return "terminated"; + return "paused"; +} + +export function isLifecycleWorkPending(state: AgentLifecycleState): boolean { + return ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"].includes(state); +} + +export function assertAgentPurgeAllowed(agent: { lifecycleState: string }) { + if (!["terminated", "rejected"].includes(agent.lifecycleState)) { + throw new AgentLifecycleConflict("Complete termination before deleting the agent"); + } +} diff --git a/server/src/modules/agent-lifecycle/index.ts b/server/src/modules/agent-lifecycle/index.ts new file mode 100644 index 0000000000..ba8abce8c3 --- /dev/null +++ b/server/src/modules/agent-lifecycle/index.ts @@ -0,0 +1,144 @@ +import { hireApprovalService as approvalRecords, type HireDecisionTarget } from "./adapters/approvals.js"; +import { agents } from "@paperclipai/db"; +import { and, asc, eq, gt, inArray } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import type { LifecycleEffects } from "./adapters/effects.js"; +export type { LifecycleEffects } from "./adapters/effects.js"; +import { updateAgentConfiguration } from "./adapters/configuration.js"; +import type { AgentConfigurationPatch, UpdateAgentOptions } from "../../lib/agent-records.js"; +import { purgeAgent } from "./adapters/deletion.js"; +import { agentRecords } from "./adapters/records.js"; +import { assertRootDatabase, createLifecycleStore } from "./adapters/postgres.js"; +import { createLifecycleWorker } from "./application/worker.js"; +import type { LifecycleDriver } from "./application/ports.js"; + +export { AgentLifecycleConflict, canConfigureAgentConnection, isAgentAwaitingSetup } from "./domain/policy.js"; + +export type { HireDecisionTarget } from "./adapters/approvals.js"; +export type { LifecycleDriver, LifecycleAgent, LifecycleFailure } from "./application/ports.js"; + +const policyStates = ["preparing", "verifying", "ready", "pausing", "paused", "resuming"] as const; +const policyPageSize = 100; + +const workers = new WeakMap>(); +export function configureAgentLifecycle(db: Db, effects: LifecycleEffects, driver: LifecycleDriver, canRun = () => true) { + assertRootDatabase(db); + const worker = createLifecycleWorker(createLifecycleStore(db, effects), driver, effects.reportFailure, canRun); + workers.set(db, worker); + return worker; +} +export function scheduleAgentLifecycle(db: Db, id: string) { + // The periodic sweep retries work if the process stops before this call. + return workers.get(db)?.process(id).catch(() => {}); +} + +export function createAgentLifecycle(db: Db, effects: LifecycleEffects) { + assertRootDatabase(db); + const records = agentRecords(db, effects); + const store = createLifecycleStore(db, effects); + async function change(id: string, command: "pause" | "resume" | "terminate" | "retry", reason?: string) { + const result = await store.change(id, command, { reason }); + if (result) scheduleAgentLifecycle(db, id); + return result ? records.getById(id) : null; + } + async function decideHire(target: HireDecisionTarget, decision: "approved" | "rejected", userId: string, note?: string | null) { + const result = await approvalRecords(db, effects).decide(target, decision, userId, note); + if (result?.hireApprovedAgentId) scheduleAgentLifecycle(db, result.hireApprovedAgentId); + return result; + } + return { + async get(id: string) { + const agent = await store.get(id); + return agent ? { + id: agent.id, companyId: agent.companyId, lifecycleState: agent.lifecycleState, + lifecycleVersion: agent.lifecycleVersion, lifecycleError: agent.lifecycleError, + } : null; + }, + async requestHire(...args: Parameters) { + const agent = await records.create(...args); + scheduleAgentLifecycle(db, agent.id); + return agent; + }, + approveHire: (target: HireDecisionTarget, userId = "board", note?: string | null) => decideHire(target, "approved", userId, note), + rejectHire: (target: HireDecisionTarget, userId = "board", note?: string | null) => decideHire(target, "rejected", userId, note), + async updateConfiguration(id: string, data: AgentConfigurationPatch, options?: UpdateAgentOptions) { + const existing = await records.getById(id); + if (!existing) return null; + const agent = await effects.transaction(db, existing.companyId, (tx, publications) => + updateAgentConfiguration(tx, effects, id, data, options, publications)); + if (data.budgetMonthlyCents !== undefined) await effects.enforceBudget(db, existing.companyId, id); + scheduleAgentLifecycle(db, id); + return agent; + }, + async updateAndTransition(id: string, command: "pause" | "resume" | "terminate", data: AgentConfigurationPatch, options?: UpdateAgentOptions) { + const existing = await records.getById(id); + if (!existing) return null; + const agent = await effects.transaction(db, existing.companyId, async (tx, publications) => { + await updateAgentConfiguration(tx, effects, id, data, options, publications); + await createLifecycleStore(tx, effects).change(id, command, { reason: command === "resume" ? "user" : "manual" }); + return agentRecords(tx, effects).getById(id); + }); + if (data.budgetMonthlyCents !== undefined) await effects.enforceBudget(db, existing.companyId, id); + scheduleAgentLifecycle(db, id); + return agent; + }, + async reconcilePolicyHolds(companyId: string, agentId?: string | null) { + if (agentId === null) return; + const rows = await db.select({ id: agents.id }).from(agents).where(and( + eq(agents.companyId, companyId), agentId ? eq(agents.id, agentId) : undefined, + inArray(agents.lifecycleState, policyStates), + )); + for (const row of rows) { + await store.change(row.id, "reconcile"); + scheduleAgentLifecycle(db, row.id); + } + }, + pauseAgent: (id: string, reason = "manual") => change(id, "pause", reason), + resumeAgent: (id: string, reason = "user") => change(id, "resume", reason), + terminateAgent: (id: string) => change(id, "terminate"), + retry: (id: string) => change(id, "retry"), + clearError: records.clearError, + purgeAgent: (id: string) => purgeAgent(db, effects, id), + }; +} + +export function startAgentLifecycle(db: Db, effects: LifecycleEffects, driver: LifecycleDriver, canRun: () => boolean) { + const worker = configureAgentLifecycle(db, effects, driver, canRun); + let running: Promise | undefined; + let stopped = false; + let nextPolicySweep = 0; + let policyCursor: string | undefined; + const store = createLifecycleStore(db, effects); + function sweep() { + if (stopped || !canRun()) return Promise.resolve(); + return running ??= (async () => { + if (Date.now() >= nextPolicySweep) { + try { + const rows = await db.select({ id: agents.id }).from(agents).where(and( + inArray(agents.lifecycleState, policyStates), policyCursor ? gt(agents.id, policyCursor) : undefined, + )).orderBy(asc(agents.id)).limit(policyPageSize); + for (const row of rows) { + await store.change(row.id, "reconcile"); + scheduleAgentLifecycle(db, row.id); + policyCursor = row.id; + } + if (rows.length < policyPageSize) policyCursor = undefined; + } + catch (error) { effects.reportFailure({ stage: "policy_scan" }, error); throw error; } + nextPolicySweep = Date.now() + 60_000; + } + await worker.sweep(); + })().finally(() => { running = undefined; }); + } + const timer = setInterval(() => { void sweep().catch(() => {}); }, 5_000); + timer.unref?.(); + return { + sweep, + async stop() { + stopped = true; + clearInterval(timer); + workers.delete(db); + try { await running; } finally { await worker.stop(); } + }, + }; +} diff --git a/server/src/routes/access.ts b/server/src/routes/access.ts index 460cd3c3dd..f22b25db30 100644 --- a/server/src/routes/access.ts +++ b/server/src/routes/access.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { createHash, generateKeyPairSync, @@ -2640,6 +2641,7 @@ export function accessRoutes( const access = accessService(db); const boardAuth = boardAuthService(db); const agents = agentService(db); + const agentsLifecycle = createAgentLifecycle(db); const routeInviteResolutionNetwork = opts.inviteResolutionNetwork ? { ...defaultInviteResolutionNetwork, ...opts.inviteResolutionNetwork } : inviteResolutionNetwork; @@ -4268,7 +4270,7 @@ export function accessRoutes( })) ); - const created = await agents.create(companyId, { + const created = await agentsLifecycle.requestHire(companyId, { name: agentName, role: "general", title: null, diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 3c68a3d49d..0484bce3ca 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -1,5 +1,7 @@ import { cancellationRequestId } from "../services/native-runtime/native-cancellation-request.js"; -import { aiRoutingHarness } from "@paperclipai/shared"; +import { agentEnvironmentTestService } from "../services/agent-environment-test.js"; +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; + import { agentIdentityService } from "../services/agent-identity.js"; import { aiConnectionRouterService, poolMemberRuntimeConfig } from "../services/ai-connection-router.js"; import { connectionIntentService } from "../services/connection-intents.js"; @@ -11,12 +13,17 @@ import { agentFileStore, agentFileTokenFromHash } from "../services/agent-file-s import { pipeline } from "node:stream/promises"; import { resolveAgentAppearance, agentAvatarUrl } from "@paperclipai/shared"; import { listOpenRouterModels } from "../services/openrouter-models.js"; -import { prepareManagedAiRuntime, withManagedAiProbe, assertManagedAiProjectAuth, stripAiAuthBindings } from "../services/ai-connection-runtime.js"; -import { ADAPTER_AUTH_MISSING_CHECK_CODE, AI_CONNECTION_CAPABILITIES, aiConnectionBindingSchema, type AiConnectionBinding, aiRuntimeConnectionBindingSchema, type AiRuntimeConnectionBinding } from "@paperclipai/shared"; +import { withManagedAiProbe, stripAiAuthBindings } from "../services/ai-connection-runtime.js"; +import { + ADAPTER_AUTH_MISSING_CHECK_CODE, + aiConnectionBindingSchema, + aiRuntimeConnectionBindingSchema, + type AiRuntimeConnectionBinding, +} from "@paperclipai/shared"; import { toolConnections } from "@paperclipai/db"; import { aiConnectionService } from "../services/ai-connections.js"; import { defaultAiConnectionForHire } from "../services/agent-ai-connection-default.js"; -import { assertAiConnectionCreateAccess, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest, validateAiApiKey } from "./ai-connections.js"; +import { assertAiConnectionCreateAccess, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest } from "./ai-connections.js"; import { isAiConnectionCompatible } from "@paperclipai/shared"; import { applyConnectorSkills, resolveConnectorAssignments, annotateConnectorSkills, isConnectorSkill } from "../services/connector-runtime.js"; import { getExecutionBlocker } from "../services/execution-blocker.js"; @@ -114,7 +121,7 @@ import { } from "./workspace-command-authz.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { environmentService } from "../services/environments.js"; -import { resolveEnvironmentExecutionTarget } from "../services/environment-execution-target.js"; + import { environmentRuntimeService } from "../services/environment-runtime.js"; import { resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; @@ -561,10 +568,12 @@ export function agentRoutes( const router = Router(); const svc = agentService(db, { cancelWorkForScope: scope => heartbeat.cancelBudgetScopeWork(scope) }); + const svcLifecycle = createAgentLifecycle(db, { cancelWorkForScope: scope => heartbeat.cancelBudgetScopeWork(scope) }); const access = accessService(db); const approvalsSvc = approvalService(db); const budgets = budgetService(db); const environmentsSvc = environmentService(db); + const { resolveAdapterTestEnvironmentId, resolveAdapterTestExecutionContext, assertAdapterTestEnvironmentForCompany, testManagedEnvironment } = agentEnvironmentTestService(db, options.pluginWorkerManager); const environmentRuntime = environmentRuntimeService(db, { pluginWorkerManager: options.pluginWorkerManager, }); @@ -1221,415 +1230,6 @@ export function agentRoutes( return rows.filter((_, index) => decisions[index]?.allowed); } - // A null agent override inherits the instance default, just like dispatch. - // Resolve this before secrets or probes so a default remote environment can - // never accidentally validate the account on the control-plane host. - async function resolveAdapterTestEnvironmentId(companyId: string, environmentId: string | null | undefined) { - if (environmentId) return environmentId; - const settings = await instanceSettings.get(); - if (settings.defaultEnvironmentId) return settings.defaultEnvironmentId; - if ((await instanceSettings.getExperimental()).enableManagedSandboxOnly === true) { - const managed = await environmentsSvc.findManagedSandboxEnvironment(companyId); - if (!managed) { - throw unprocessable("The managed sandbox is unavailable. Restore Paperclip Computer and retry.", { - code: "managed_sandbox_unavailable", - }); - } - return managed.id; - } - return null; - } - - /** - * Resolve the execution target the adapter should run its test probes against. - * - * - No environmentId / local environment → returns a local target so the - * adapter probes the Paperclip host (legacy behavior). - * - SSH environment → builds an SSH execution target from the environment - * config so the adapter probes the remote box. No lease is required: - * the SSH spec is fully derived from the saved environment config. - * - Sandbox / plugin environments → acquires an ad-hoc lease, realizes the - * workspace, and resolves a sandbox execution target wired to the runtime - * so the adapter probe runs inside the sandbox the same way a heartbeat - * would. The returned `release` callback rolls the lease back when the - * route is done. - * - * The caller MUST always invoke `release()` (typically in a `finally` block). - */ - async function resolveAdapterTestExecutionContext(input: { - companyId: string; - adapterType: string; - environmentId: string | null; - }): Promise<{ - executionTarget: AdapterExecutionTarget | null; - environmentName: string | null; - fallbackChecks: AdapterEnvironmentCheck[]; - sandboxIdentityCheck?: AdapterEnvironmentCheck | null; - release: (status?: "released" | "failed") => Promise; - }> { - const noopRelease = async () => {}; - - if (!input.environmentId) { - return { - executionTarget: null, - environmentName: null, - fallbackChecks: [], - release: noopRelease, - }; - } - - const requestedEnvironment = await environmentsSvc.getById(input.environmentId); - if (!requestedEnvironment) { - return { - executionTarget: null, - environmentName: null, - fallbackChecks: [ - { - code: "environment_not_found", - level: "warn", - message: "Selected environment was not found. The test did not run.", - }, - ], - release: noopRelease, - }; - } - - // Managed-sandbox-only policy: redirect a Test that would run on the local - // host onto the platform-managed sandbox, the same as a real run does - // (resolveExecutionWorkspaceEnvironmentId in heartbeat). Without this - // redirect the Test probes the local host while the run executes in the - // managed sandbox, so a passing Test validates the wrong execution target. - // With no active managed sandbox the Test fails closed — never local. - let environment = requestedEnvironment; - if (requestedEnvironment.driver === "local") { - const managedSandboxOnly = - (await instanceSettings.getExperimental()).enableManagedSandboxOnly === true; - if (managedSandboxOnly) { - const managedSandboxEnvironment = await environmentsSvc.findManagedSandboxEnvironment( - input.companyId, - ); - if (!managedSandboxEnvironment) { - return { - executionTarget: null, - environmentName: requestedEnvironment.name, - fallbackChecks: [ - { - code: "managed_sandbox_unavailable", - level: "error", - message: - "This instance runs agents only in its platform-managed sandbox, but no active managed sandbox environment exists. The test did not run.", - hint: "Restore the managed sandbox environment, then test again.", - }, - ], - release: noopRelease, - }; - } - environment = managedSandboxEnvironment; - } - } - - if (environment.driver === "local") { - return { - executionTarget: null, - environmentName: environment.name, - fallbackChecks: [], - release: noopRelease, - }; - } - - if (environment.driver === "ssh") { - try { - const target = await resolveEnvironmentExecutionTarget({ - db, - companyId: input.companyId, - adapterType: input.adapterType, - environment: { - id: environment.id, - driver: environment.driver, - config: environment.config ?? null, - }, - leaseMetadata: null, - }); - if (target) { - return { - executionTarget: target, - environmentName: environment.name, - fallbackChecks: [], - release: noopRelease, - }; - } - return { - executionTarget: null, - environmentName: environment.name, - fallbackChecks: [ - { - code: "environment_target_unavailable", - level: "warn", - message: - `Could not resolve an execution target for environment "${environment.name}". The test did not run.`, - }, - ], - release: noopRelease, - }; - } catch (err) { - return { - executionTarget: null, - environmentName: environment.name, - fallbackChecks: [ - { - code: "environment_target_failed", - level: "warn", - message: - `Could not connect to environment "${environment.name}" to run the test.`, - detail: err instanceof Error ? err.message : String(err), - }, - ], - release: noopRelease, - }; - } - } - - // sandbox / plugin / other remote drivers: spin up an ad-hoc lease, realize - // the workspace inside the box, and run the same probe SSH uses against - // a sandbox execution target wired to the environment runtime. - // - // We pass `heartbeatRunId: null` because there's no heartbeat run for an - // operator-initiated `Test` invocation — the leases table FKs heartbeat - // run id to heartbeat_runs.id, and we don't want to manufacture a fake - // run row. Cleanup goes through the driver's `releaseRunLease` directly - // (by lease record), since the batch helper queries by heartbeatRunId. - // - // Sandbox tests boot a fresh throwaway sandbox (never resume a retained - // agent lease) and archive it on release instead of deleting it, so the - // operator can inspect the exact sandbox from the provider dashboard while - // provider-side expiry reaps it later. - const testEnvironment = environment.driver === "sandbox" - ? { - ...environment, - config: { - ...(environment.config ?? {}), - reuseLease: false, - archiveOnRelease: true, - }, - } - : environment; - let leaseRecord: Awaited>; - try { - leaseRecord = await environmentRuntime.acquireRunLease({ - companyId: input.companyId, - environment: testEnvironment, - issueId: null, - heartbeatRunId: null, - persistedExecutionWorkspace: null, - // Re-check the company binding atomically at lease time. The route - // guard already rejected a foreign environment, but the binding could - // change between the guard check and the lease acquire. This closes - // that check-to-lease race so a foreign sandbox never gets a lease. - assertCompanyBinding: true, - // Apply the active custom-image template so the Test boots with the - // operator's captured sandbox customizations and prepared image state, - // matching what real agent runs use. Without this the test would - // silently fall back to the base image. - applyCustomImageTemplate: true, - }); - } catch (err) { - return { - executionTarget: null, - environmentName: environment.name, - fallbackChecks: [ - { - code: "environment_lease_acquire_failed", - level: "error", - message: `Could not acquire a lease for environment "${environment.name}".`, - detail: err instanceof Error ? err.message : String(err), - hint: "Check the environment's provider credentials and quota.", - }, - ], - release: noopRelease, - }; - } - - const driver = environmentRuntime.getDriver(environment.driver); - const releaseLease = async (status: "released" | "failed" = "released") => { - try { - if (driver) { - await driver.releaseRunLease({ - environment: testEnvironment, - lease: leaseRecord.lease, - status, - }); - } else { - await environmentsSvc.releaseLease(leaseRecord.lease.id, status); - } - } catch (err) { - // Cleanup failures must not mask the test result. - // eslint-disable-next-line no-console - console.warn( - `[adapter-test] Failed to release lease ${leaseRecord.lease.id}: ${err instanceof Error ? err.message : String(err)}`, - ); - } - }; - - let realizedCwd: string | null = null; - try { - const realized = await environmentRuntime.realizeWorkspace({ - environment: testEnvironment, - lease: leaseRecord.lease, - // No host workspace to copy for a Test invocation; sandbox/plugin - // realize implementations use the lease metadata's remoteCwd to - // create the working directory inside the box. - workspace: {}, - }); - realizedCwd = - typeof realized.cwd === "string" && realized.cwd.trim().length > 0 - ? realized.cwd.trim() - : null; - } catch (err) { - await releaseLease("failed"); - return { - executionTarget: null, - environmentName: environment.name, - fallbackChecks: [ - { - code: "environment_workspace_realize_failed", - level: "error", - message: `Could not realize a workspace inside "${environment.name}".`, - detail: err instanceof Error ? err.message : String(err), - }, - ], - release: noopRelease, - }; - } - - let target: AdapterExecutionTarget | null; - try { - // Prefer the cwd the realize step returned; fall back to lease metadata. - const leaseMetadataForTarget: Record | null = - realizedCwd - ? { ...(leaseRecord.lease.metadata ?? {}), remoteCwd: realizedCwd } - : (leaseRecord.lease.metadata as Record | null) ?? null; - - target = await resolveEnvironmentExecutionTarget({ - db, - companyId: input.companyId, - adapterType: input.adapterType, - environment: { - id: testEnvironment.id, - driver: testEnvironment.driver, - config: testEnvironment.config ?? null, - }, - leaseId: leaseRecord.lease.id, - leaseMetadata: leaseMetadataForTarget, - lease: leaseRecord.lease, - environmentRuntime, - }); - } catch (err) { - await releaseLease("failed"); - return { - executionTarget: null, - environmentName: environment.name, - fallbackChecks: [ - { - code: "environment_target_failed", - level: "error", - message: `Could not resolve an execution target for "${environment.name}".`, - detail: err instanceof Error ? err.message : String(err), - }, - ], - release: noopRelease, - }; - } - - if (!target) { - await releaseLease("failed"); - return { - executionTarget: null, - environmentName: environment.name, - fallbackChecks: [ - { - code: "environment_target_unsupported", - level: "warn", - message: - `Adapter "${input.adapterType}" is not allowed in "${environment.name}" environments.`, - }, - ], - release: noopRelease, - }; - } - - return { - executionTarget: target, - environmentName: environment.name, - fallbackChecks: [], - sandboxIdentityCheck: buildSandboxIdentityCheck({ - environmentName: environment.name, - lease: leaseRecord.lease, - }), - release: releaseLease, - }; - } - - function readMetadataString(metadata: Record, keys: string[]): string | null { - for (const key of keys) { - const value = metadata[key]; - if (typeof value === "string" && value.trim().length > 0) return value.trim(); - } - return null; - } - - function buildSandboxIdentityCheck(input: { - environmentName: string; - lease: { - id: string; - provider?: string | null; - providerLeaseId?: string | null; - metadata?: Record | null; - }; - }): AdapterEnvironmentCheck { - const metadata = input.lease.metadata ?? {}; - const provider = input.lease.provider ?? readMetadataString(metadata, ["provider"]); - const sandboxId = readMetadataString(metadata, ["sandboxId", "sandboxID", "sandbox_id", "id"]); - const sandboxName = readMetadataString(metadata, ["sandboxName", "sandbox_name", "name"]); - const snapshotRef = readMetadataString(metadata, [ - "snapshot", - "snapshotId", - "snapshotID", - "snapshotRef", - "snapshot_ref", - "templateRef", - "template_ref", - "templateId", - "templateID", - "image", - "imageId", - "imageID", - "imageRef", - "image_ref", - ]); - const templateKind = readMetadataString(metadata, [ - "templateKind", - "template_kind", - "templateRefKind", - "template_ref_kind", - ]); - const detailParts = [ - `paperclipLeaseId=${input.lease.id}`, - input.lease.providerLeaseId ? `providerLeaseId=${input.lease.providerLeaseId}` : null, - provider ? `provider=${provider}` : null, - sandboxId ? `sandboxId=${sandboxId}` : null, - sandboxName ? `sandboxName=${sandboxName}` : null, - snapshotRef ? `${templateKind ? `${templateKind}Ref` : "snapshotOrTemplateRef"}=${snapshotRef}` : null, - ].filter((part): part is string => Boolean(part)); - - return { - code: "sandbox_test_identity", - level: "info", - message: `Environment test identity for "${input.environmentName}".`, - detail: detailParts.join("; "), - hint: "Use these provider-neutral IDs when comparing model-test output with provider logs or refreshed environment snapshots.", - }; - } - async function getCurrentUserRedactionOptions() { return { enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs, @@ -3416,103 +3016,6 @@ export function agentRoutes( res.json(detected); }); - // The environment drivers the adapter Test route accepts. A local, SSH, or - // sandbox environment can host a probe; a plugin environment cannot. - const ADAPTER_TEST_ALLOWED_ENVIRONMENT_DRIVERS = ["local", "ssh", "sandbox"]; - - // The fail-closed tenant-binding guard for the adapter Test route. A caller - // may name any instance environment by id, so the route must reject an - // environment that binds to another company before it resolves secrets, - // merges env, resolves the target, leases a sandbox, or runs the adapter - // test. The guard checks the company binding BEFORE it validates the status - // or the driver, so it never reveals the status or the driver of a foreign - // environment. A same-company or an instance-global environment then gets the - // shared driver and status validation. - async function assertAdapterTestEnvironmentForCompany( - companyId: string, - environmentId: string, - ): Promise { - const environment = await environmentsSvc.getById(environmentId); - if (!environment) { - // A missing environment leaks no tenant state. The execution-context - // resolver surfaces the existing environment_not_found check. - return; - } - const boundCompanyIds = await environmentsSvc.listBoundCompanyIds(environmentId); - if (boundCompanyIds.length > 0 && !boundCompanyIds.includes(companyId)) { - throw forbidden("The selected environment belongs to another company.", { - code: "environment_company_mismatch", - }); - } - await assertEnvironmentSelectionForCompany(environmentsSvc, companyId, environmentId, { - allowedDrivers: ADAPTER_TEST_ALLOWED_ENVIRONMENT_DRIVERS, - }); - } - - async function testManagedEnvironment(adapterType: string, context: Parameters["testEnvironment"]>[0], binding: AiConnectionBinding, managed: Awaited>, agentId?: string) { - const startedAt = new Date(); - const result = await probeManagedEnvironment(adapterType, context, binding); - // A provider rejection invalidates the tested credential generation. A - // missing CLI, unavailable environment, or other runtime error does not. - if (result.status === "fail" && result.checks.some(check => - check.code === ADAPTER_AUTH_MISSING_CHECK_CODE || /_hello_probe_auth_required$/.test(check.code) - || check.code === "ai_connection_api_key_rejected", - )) { - await aiConnectionService(db).markAuthenticationFailed({ - companyId: context.companyId, agentId, runStartedAt: startedAt, - attribution: { ...managed.attribution, identity: managed.identity }, - }); - } - return result; - } - - async function probeManagedEnvironment(adapterType: string, context: Parameters["testEnvironment"]>[0], binding: AiConnectionBinding) { - await assertManagedAiProjectAuth(context.config, binding.provider, context.executionTarget); - const result = await requireServerAdapter(adapterType).testEnvironment(context); - if (result.status === "fail") return result; - // The resolved method, not binding.method — on a responsible_user binding - // that field is wire-compat only and the default connection decides. - const resolvedMethod = (context.config as { managedAiConnection?: { method?: string } }).managedAiConnection?.method; - // An api_key account does not take the CLI hello probe below: a key - // travels as an env var any engine understands, and the engine's own test - // above judged whether this runtime can execute with it — the ACP lane - // deliberately runs no hello probe when a key is configured. Demanding one - // anyway forced the CLI lane, whose probe needs a provider CLI on PATH, - // and a clean install has none: that walled off onboarding's API-key path - // on exactly the machines the release smoke exists to guard. The account - // is still verified live here — the same provider-endpoint check the save - // performed — so a key revoked since its save fails adoption rather than - // producing an agent that cannot authenticate at runtime. The hello-probe - // requirement stays for subscriptions: a stored login is a file layout - // only a provider CLI reads, so proving the runtime lane can consume it - // takes a real hello turn. - if (resolvedMethod === "api_key" && !context.config.managedAiRouting) { - const envKey = AI_CONNECTION_CAPABILITIES[binding.provider].methods.api_key?.envKey; - const key = envKey ? parseObject(context.config.env)[envKey] : undefined; - try { - if (typeof key !== "string" || !key) throw unprocessable("The selected account's API key was not available to verify."); - await validateAiApiKey(binding.provider, key); - result.checks.push({ code: "ai_connection_api_key_reverified", level: "info", message: "The provider verified this API key for adoption." }); - } catch (error) { - result.status = "fail"; - const rejected = error instanceof HttpError && asRecord(error.details)?.code === "ai_connection_api_key_rejected"; - result.checks.push({ code: rejected ? "ai_connection_api_key_rejected" : "ai_connection_verification_failed", level: "error", message: error instanceof HttpError ? error.message : "Could not verify the account. Try again." }); - } - return result; - } - if (!result.checks.some(check => check.code.includes("hello_probe"))) { - const providerAdapter = context.config.managedAiRouting ? aiRoutingHarness(adapterType, context.config.provider, context.config.acpxAgent) : { anthropic: "claude_local", openai: "codex_local", openrouter: "opencode_local", xai: "grok_local", google: "gemini_local" }[binding.provider]; - const probe = await requireServerAdapter(providerAdapter).testEnvironment({ ...context, adapterType: providerAdapter, config: { ...context.config, engine: "cli" } }); - result.checks.push(...probe.checks); - result.status = probe.status === "fail" ? "fail" : result.status === "warn" || probe.status === "warn" ? "warn" : "pass"; - } - if (!result.checks.some(check => /hello_probe_(passed|succeeded)$/.test(check.code))) { - result.status = "fail"; - result.checks.push({ code: "ai_connection_validation_incomplete", level: "error", message: "The selected account has not completed a provider hello test. Retry before adopting it." }); - } - return result; - } - async function validateManagedAgentBinding(req: Request, companyId: string, agentId: string, adapterType: string, config: Record, binding: AiRuntimeConnectionBinding, environmentId: string | null | undefined, test: boolean, newAgent = false) { if (binding.mode === "router") { const settings = await import("../services/instance-settings.js"); @@ -4834,7 +4337,7 @@ export function agentRoutes( const status = requiresApproval ? "pending_approval" : "idle"; const managedHireBinding = normalizedHireInput.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(normalizedHireInput.runtimeConfig.aiConnection) : undefined; const managedHireConnection = managedHireBinding ? await validateManagedAgentBinding(req, companyId, hiredAgentId, normalizedHireInput.adapterType, normalizedHireInput.adapterConfig, managedHireBinding, normalizedHireInput.defaultEnvironmentId, false, true) : undefined; - const createdAgent = await svc.create( + const createdAgent = await svcLifecycle.requestHire( companyId, { id: hiredAgentId, @@ -4845,6 +4348,7 @@ export function agentRoutes( }, { createdByUserId: req.actor.type === "board" ? req.actor.userId : null, + responsibleUserId: responsibleUserForAiRequest(req), aiConnectionInstall: managedHireConnection ? { ...managedHireConnection, createdByUserId: responsibleUserForAiRequest(req) } : undefined, claudeLogin: { storedSessionId: hireStoredSessionId ?? null, @@ -5088,7 +4592,7 @@ export function agentRoutes( const managedBinding = normalizedRuntimeConfig.aiConnection ? aiRuntimeConnectionBindingSchema.parse(normalizedRuntimeConfig.aiConnection) : undefined; const managedConnection = managedBinding ? await validateManagedAgentBinding(req, companyId, agentId, createInput.adapterType, normalizedAdapterConfig, managedBinding, createInput.defaultEnvironmentId, false, true) : undefined; - const createdAgent = await svc.create( + const createdAgent = await svcLifecycle.requestHire( companyId, { id: agentId, @@ -5101,6 +4605,7 @@ export function agentRoutes( }, { createdByUserId: req.actor.type === "board" ? req.actor.userId : null, + responsibleUserId: responsibleUserForAiRequest(req), aiConnectionInstall: managedConnection ? { ...managedConnection, createdByUserId: responsibleUserForAiRequest(req) } : undefined, claudeLogin: { storedSessionId: createStoredSessionId ?? null, @@ -5643,6 +5148,9 @@ export function agentRoutes( } const patchData = { ...(req.body as Record) }; + const requestedStatus = patchData.status; + delete patchData.status; + if (requestedStatus !== undefined && !["paused", "idle", "terminated"].includes(String(requestedStatus))) throw badRequest("Use an agent lifecycle command to change status"); const replaceAdapterConfig = patchData.replaceAdapterConfig === true; delete patchData.replaceAdapterConfig; // The apply-existing flag is not an agent column. The server binds the fixed @@ -5825,7 +5333,7 @@ export function agentRoutes( } const actor = getActorInfo(req); - const agent = await svc.update(id, patchData, { + const updateOptions = { recordRevision: { createdByAgentId: actor.agentId, createdByUserId: actor.actorType === "user" ? actor.actorId : null, @@ -5838,7 +5346,11 @@ export function agentRoutes( applyExistingWithoutClaim: req.actor.type !== "agent" && applyStoredClaudeLogin, }, - }); + }; + const command = requestedStatus === "paused" ? "pause" : requestedStatus === "idle" ? "resume" : requestedStatus === "terminated" ? "terminate" : null; + const agent = command + ? await svcLifecycle.updateAndTransition(id, command, patchData, updateOptions) + : await svc.update(id, patchData, updateOptions); if (!agent) { res.status(404).json({ error: "Agent not found" }); return; @@ -5860,13 +5372,28 @@ export function agentRoutes( res.json(redactAgentRowForResponse(agent)); }); + router.post("/agents/:id/lifecycle/retry", async (req, res) => { + assertBoard(req); + const id = req.params.id as string; + const existing = await getAccessibleAgent(req, res, id); + if (!existing) return; + await assertCanUpdateAgent(req, existing); + const agent = await svcLifecycle.retry(id); + if (!agent) { res.status(404).json({ error: "Agent not found" }); return; } + const actor = getActorInfo(req); + await logActivity(db, { companyId: agent.companyId, actorType: actor.actorType, actorId: actor.actorId, + agentId: actor.agentId, runId: actor.runId, agentApiKeyId: actor.agentApiKeyId, + action: "agent.lifecycle_retried", entityType: "agent", entityId: agent.id }); + res.json(redactAgentRowForResponse(agent)); + }); + router.post("/agents/:id/pause", async (req, res) => { assertBoard(req); const id = req.params.id as string; if (!(await getAccessibleAgent(req, res, id))) { return; } - const agent = await svc.pause(id); + const agent = await svcLifecycle.pauseAgent(id); if (!agent) { res.status(404).json({ error: "Agent not found" }); return; @@ -5899,7 +5426,7 @@ export function agentRoutes( }); return; } - const agent = await svc.resume(id); + const agent = await svcLifecycle.resumeAgent(id); if (!agent) { res.status(404).json({ error: "Agent not found" }); return; @@ -5965,34 +5492,16 @@ export function agentRoutes( return; } - // Resolve the linked hire approval (clears it from the inbox) and run the - // shared approval side effects: agent activation, budget policy, and the - // hire-approved notification. Fall back to direct activation if no open - // approval record exists (e.g. agents created before approvals were tracked). - const decidedByUserId = req.actor.userId ?? "board"; - const openApproval = await approvalsSvc.findOpenHireApprovalForAgent(existing.companyId, id); - - let agent: Awaited> | null = null; - if (openApproval) { - await approvalsSvc.approve(openApproval.id, decidedByUserId); - agent = await svc.getById(id); - } else { - const approval = await svc.activatePendingApproval(id); - if (!approval) { - res.status(404).json({ error: "Agent not found" }); - return; - } - if (!approval.activated) { - res.status(409).json({ error: "Only pending approval agents can be approved" }); - return; - } - agent = approval.agent; - } - + const decision = await approvalsSvc.approveHire(id, req.actor.userId ?? "board"); + const agent = decision?.agent; if (!agent) { res.status(404).json({ error: "Agent not found" }); return; } + if (!decision.applied) { + res.status(409).json({ error: "Only pending approval agents can be approved" }); + return; + } await logActivity(db, { companyId: agent.companyId, @@ -6001,7 +5510,7 @@ export function agentRoutes( action: "agent.approved", entityType: "agent", entityId: agent.id, - details: { source: "agent_detail", approvalId: openApproval?.id ?? null }, + details: { source: "agent_detail", approvalId: decision.approval?.id ?? null }, }); res.json(redactAgentRowForResponse(agent)); @@ -6015,23 +5524,9 @@ export function agentRoutes( return; } - // Terminating an agent that is still awaiting approval is the agent-detail - // equivalent of rejecting the hire. When a linked hire approval is still - // open, delegate to approvalsSvc.reject(), which both resolves the approval - // (clearing the inbox "Approve/Reject" card) and terminates the agent. - // Mirror the approve path's branch-or-fallback so we never terminate twice: - // reject() already calls agentsSvc.terminate() internally. - let agent: Awaited> = null; - if (existing.status === "pending_approval") { - const openApproval = await approvalsSvc.findOpenHireApprovalForAgent(existing.companyId, id); - if (openApproval) { - await approvalsSvc.reject(openApproval.id, req.actor.userId ?? "board"); - agent = await svc.getById(id); - } - } - if (!agent) { - agent = await svc.terminate(id); - } + const agent = existing.status === "pending_approval" + ? (await approvalsSvc.rejectHire(id, req.actor.userId ?? "board"))?.agent + : await svcLifecycle.terminateAgent(id); if (!agent) { res.status(404).json({ error: "Agent not found" }); return; diff --git a/server/src/routes/ai-connections.ts b/server/src/routes/ai-connections.ts index ba40020e46..ef0498b610 100644 --- a/server/src/routes/ai-connections.ts +++ b/server/src/routes/ai-connections.ts @@ -1,3 +1,5 @@ +import { validateAiApiKey } from "../services/ai-api-key-test.js"; +export { validateAiApiKey } from "../services/ai-api-key-test.js"; import { supportsLocalAiLogin } from "../services/local-ai-login-policy.js"; import { localAiLoginService } from "../services/local-ai-login.js"; import { z } from "zod"; @@ -133,41 +135,6 @@ export async function canInstallSharedAiConnectionForNewAgent( connection.creator === userId || await accessService(db).hasPermission(companyId, "user", userId, "tools:manage_connections"); } -/** Fixed provider endpoints; credentials are never sent to a caller-supplied URL or through a redirect. */ -export async function validateAiApiKey( - provider: AiProvider, - key: string, - request: typeof fetch = fetch, -) { - const endpoints = { - anthropic: "https://api.anthropic.com/v1/models?limit=1", - openai: "https://api.openai.com/v1/models", - openrouter: "https://openrouter.ai/api/v1/key", - xai: "https://api.x.ai/v1/models", - google: "https://generativelanguage.googleapis.com/v1beta/models?pageSize=1", - }; - let response: Response; - try { - response = await request(endpoints[provider], { - redirect: "error", - signal: AbortSignal.timeout(15000), - headers: - provider === "google" ? { "x-goog-api-key": key } : provider === "anthropic" - ? { "x-api-key": key, "anthropic-version": "2023-06-01" } - : { Authorization: `Bearer ${key}` }, - }); - } catch { - throw unprocessable("Could not verify the account. Try again.", { code: "ai_connection_verification_failed" }); - } - await response.body?.cancel(); - if (!response.ok) - throw unprocessable( - response.status === 401 || response.status === 403 - ? "The provider rejected this API key." - : "The provider could not verify this account. Try again.", - { code: response.status === 401 || response.status === 403 ? "ai_connection_api_key_rejected" : "ai_connection_verification_failed" }, - ); -} export function aiConnectionRoutes(db: Db, options: Parameters[0] = {}) { function assertLocalLoginAvailable() { diff --git a/server/src/routes/environment-selection.ts b/server/src/routes/environment-selection.ts index eba3d8772c..6718be5624 100644 --- a/server/src/routes/environment-selection.ts +++ b/server/src/routes/environment-selection.ts @@ -1,52 +1 @@ -import { unprocessable } from "../errors.js"; - -export async function assertEnvironmentSelectionForCompany( - environmentsSvc: { - getById(environmentId: string): Promise<{ - id: string; - driver: string; - status?: string | null; - config: Record | null; - } | null>; - }, - _companyId: string, - environmentId: string | null | undefined, - options?: { - allowedDrivers?: string[]; - allowedSandboxProviders?: string[]; - }, -) { - if (environmentId === undefined || environmentId === null) return; - const environment = await environmentsSvc.getById(environmentId); - if (!environment) { - throw unprocessable("Environment not found."); - } - if (environment.status === "archived") { - throw unprocessable("Environment is archived."); - } - if (options?.allowedDrivers && !options.allowedDrivers.includes(environment.driver)) { - throw unprocessable( - `Environment driver "${environment.driver}" is not allowed here. Allowed drivers: ${options.allowedDrivers.join(", ")}`, - ); - } - if (environment.driver === "sandbox") { - const config = environment.config && typeof environment.config === "object" - ? environment.config as Record - : {}; - const provider = typeof config.provider === "string" ? config.provider : ""; - if (provider === "fake") { - throw unprocessable( - `Environment sandbox provider "${provider}" is not allowed here. The built-in fake provider is probe-only and cannot execute runs.`, - ); - } - if ( - options?.allowedSandboxProviders - && options.allowedSandboxProviders.length > 0 - && !options.allowedSandboxProviders.includes(provider) - ) { - throw unprocessable( - `Environment sandbox provider "${provider || "unknown"}" is not allowed here. Allowed providers: ${options.allowedSandboxProviders.join(", ")}`, - ); - } - } -} +export { assertEnvironmentSelectionForCompany } from "../services/environment-selection.js"; diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 9883c2aa6f..108ba69944 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -3384,14 +3384,6 @@ export function issueRoutes( ChatChannelService, "prepareFailedChatRunRetry" | "processFailedChatRunRetry" >; - feedbackExportService?: { - flushPendingFeedbackTraces(input?: { - companyId?: string; - traceId?: string; - limit?: number; - now?: Date; - }): Promise; - }; searchService?: CompanySearchService; searchRateLimiter?: CompanySearchRateLimiter; pluginWorkerManager?: PluginWorkerManager; @@ -3647,7 +3639,6 @@ export function issueRoutes( const treeControlSvc = issueTreeControlFactory?.(db) ?? { getActivePauseHoldGate: async () => null, }; - const feedbackExportService = opts?.feedbackExportService; const environmentsSvc = environmentService(db); async function queueTaskWatchdogEvaluation( @@ -19005,21 +18996,8 @@ export function issueRoutes( ); } - if (result.sharingEnabled && result.traceId && feedbackExportService) { - try { - await feedbackExportService.flushPendingFeedbackTraces({ - companyId: issue.companyId, - traceId: result.traceId, - limit: 1, - }); - } catch (err) { - logger.warn( - { err, issueId: issue.id, traceId: result.traceId }, - "failed to flush shared feedback trace immediately", - ); - } - } - + // saveIssueVote has committed the export and notified its worker. + // Return the saved vote without waiting for an upload or backlog. res.status(201).json(result.vote); }, ); diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 915e28ce50..b9fe16afcc 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -3578,7 +3578,7 @@ registry.registerPath({ summary: "Update an agent", request: { params: z.object({ id: z.string() }), - body: jsonBody(updateAgentSchema.omit({ permissions: true })), + body: jsonBody(updateAgentSchema.omit({ permissions: true, spentMonthlyCents: true })), }, responses: { 200: r.ok(), @@ -3850,6 +3850,15 @@ registry.registerPath({ }, }); +registry.registerPath({ + method: "post", + path: "/api/agents/{id}/lifecycle/retry", + tags: ["agents"], + summary: "Retry an incomplete agent lifecycle step", + request: { params: z.object({ id: z.string() }) }, + responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict }, +}); + registry.registerPath({ method: "post", path: "/api/agents/{id}/pause", diff --git a/server/src/services/activity-log.ts b/server/src/services/activity-log.ts index 7021b3e382..a7e4376dbd 100644 --- a/server/src/services/activity-log.ts +++ b/server/src/services/activity-log.ts @@ -1,3 +1,5 @@ +import type { ActivityPublication } from "../types/activity-publication.js"; +export type { ActivityPublication } from "../types/activity-publication.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -66,11 +68,6 @@ export interface LogActivityInput { responsibleUserIdOverride?: string | null; } -export interface ActivityPublication { - companyId: string; - payload: Record; - pluginEvent: PluginEvent | null; -} export async function createActivityDetailsRedactor(db: Db) { const currentUserRedactionOptions = { diff --git a/server/src/services/agent-configuration-transaction.ts b/server/src/services/agent-configuration-transaction.ts new file mode 100644 index 0000000000..24bcdea897 --- /dev/null +++ b/server/src/services/agent-configuration-transaction.ts @@ -0,0 +1,12 @@ +import type { Db } from "@paperclipai/db"; +import type { AgentConfigurationPatch, UpdateAgentOptions } from "../lib/agent-records.js"; +import { conflict } from "../errors.js"; +import { updateAgentConfiguration } from "../modules/agent-lifecycle/configuration.js"; +import { createAgentLifecycleEffects } from "./agent-lifecycle.js"; + +export function updateAgentConfigurationInTransaction(tx: Db, id: string, data: Omit, options?: UpdateAgentOptions) { + if (Object.prototype.hasOwnProperty.call(data, "budgetMonthlyCents")) { + throw conflict("Budget changes require a root lifecycle command"); + } + return updateAgentConfiguration(tx, createAgentLifecycleEffects(), id, data, options); +} diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts new file mode 100644 index 0000000000..8037f150d0 --- /dev/null +++ b/server/src/services/agent-configuration.ts @@ -0,0 +1,150 @@ +import { and, eq } from "drizzle-orm"; +import { assets, agentConfigRevisions, agentTaskSessions, agentRuntimeState, type Db } from "@paperclipai/db"; +import { normalizeAgentUrlKey } from "@paperclipai/shared"; +import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils"; +import { conflict, notFound, unprocessable } from "../errors.js"; +import { agentRecordQueries, type UpdateAgentOptions, type AgentConfigurationPatch, type AgentConfigurationRecord, isPlainRecord, buildConfigSnapshot, + hasConfigPatchFields, changedPendingApprovalConfigFields, diffConfigSnapshot } from "../lib/agent-records.js"; +import { normalizeAgentPermissions } from "../lib/agent-permissions.js"; +import type { ActivityPublication } from "../types/activity-publication.js"; +import { secretService, assertClaudeOAuthBindingInvariant } from "./secrets.js"; +import { agentCredentialService } from "./agent-credentials.js"; +import { clearPrimaryAgent } from "./primary-agent.js"; +import { budgetServiceInTransaction } from "./budgets.js"; + +export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise { + const id = existing.id; + const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db); + if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) { + const changedFields = changedPendingApprovalConfigFields(agentRecordQueries(db).normalizeAgentRow(existing), data); + if (changedFields.length > 0) { + throw conflict("Pending approval agent configuration cannot be changed before board approval", { + code: "pending_approval_agent_config_frozen", + agentId: id, + fields: changedFields, + }); + } + } + + if (data.reportsTo !== undefined) { + if (data.reportsTo) { + await ensureManager(existing.companyId, data.reportsTo); + } + await assertNoCycle(id, data.reportsTo); + } + + if (data.name !== undefined) { + const previousShortname = normalizeAgentUrlKey(existing.name); + const nextShortname = normalizeAgentUrlKey(data.name); + if (previousShortname !== nextShortname) { + await assertCompanyShortnameAvailable(existing.companyId, data.name, { excludeAgentId: id }); + } + } + + if (Object.prototype.hasOwnProperty.call(data, "metadata")) { + assertBuiltInAgentMetadataMutationAllowed(existing.metadata, data.metadata, options); + } + + if (data.appearance?.customAvatarAssetId) { + const [asset] = await db.select().from(assets).where(and( + eq(assets.id, data.appearance.customAvatarAssetId), eq(assets.companyId, existing.companyId), + eq(assets.createdByAgentId, id), + )); + if (!asset || asset.contentType !== "image/png" || !asset.objectKey.startsWith(`${existing.companyId}/agent-avatars/${id}/`)) { + throw unprocessable("Use the avatar upload endpoint to set this agent's image"); + } + } + const normalizedPatch = { ...data } as AgentConfigurationPatch; + if (data.permissions !== undefined) { + normalizedPatch.permissions = normalizeAgentPermissions(data.permissions); + } + if ( + Object.prototype.hasOwnProperty.call(normalizedPatch, "adapterConfig") && + isPlainRecord(normalizedPatch.adapterConfig) + ) { + const normalizedAdapterConfig = await secretService(db).normalizeAdapterConfigForPersistence( + existing.companyId, + normalizedPatch.adapterConfig, + { adapterType: (normalizedPatch.adapterType ?? existing.adapterType) as string }, + ); + normalizedPatch.adapterConfig = normalizePaperclipRunnerAdapterConfig( + (normalizedPatch.adapterType ?? existing.adapterType) as string, + normalizedAdapterConfig, + ); + } else if ( + Object.prototype.hasOwnProperty.call(normalizedPatch, "adapterType") + && isPlainRecord(existing.adapterConfig) + ) { + normalizedPatch.adapterConfig = normalizePaperclipRunnerAdapterConfig( + normalizedPatch.adapterType as string, + existing.adapterConfig, + ); + } + return normalizedPatch; +} + +export async function completeAgentConfiguration(txDb: Db, existing: AgentConfigurationRecord, updated: AgentConfigurationRecord, normalizedPatch: AgentConfigurationPatch, options: UpdateAgentOptions | undefined, publications: ActivityPublication[]) { + const id = updated.id; + const shouldRecordRevision = Boolean(options?.recordRevision) && hasConfigPatchFields(normalizedPatch); + const beforeConfig = shouldRecordRevision + ? buildConfigSnapshot(agentRecordQueries(txDb).normalizeAgentRow(existing)) + : null; + if (updated.status === "terminated") { + await clearPrimaryAgent(txDb, updated.companyId, id); + } + + const priorAdapterConfig = isPlainRecord(existing.adapterConfig) ? existing.adapterConfig : {}; + const afterConfig = isPlainRecord(updated.adapterConfig) ? updated.adapterConfig : {}; + const changedExecution = updated.adapterType !== existing.adapterType + || (updated.adapterType === "paperclip_runner" && ["provider", "acpxAgent", "model"].some( + (key) => priorAdapterConfig[key] !== afterConfig[key], + )); + if (changedExecution) { + await txDb.delete(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, existing.companyId), eq(agentTaskSessions.agentId, id))); + await txDb.update(agentRuntimeState).set({ adapterType: updated.adapterType, sessionId: null, stateJson: {}, updatedAt: new Date() }) + .where(and(eq(agentRuntimeState.companyId, existing.companyId), eq(agentRuntimeState.agentId, id))); + } + + if (Object.prototype.hasOwnProperty.call(normalizedPatch, "adapterConfig")) { + await agentCredentialService(txDb).enforceClaudeOAuthBindingClaim(txDb, { + companyId: existing.companyId, consume: false, environmentId: null, + childAdapterConfig: updated.adapterConfig, claudeLogin: options?.claudeLogin, + decision: assertClaudeOAuthBindingInvariant({ adapterType: updated.adapterType, + nextConfig: updated.adapterConfig, priorConfig: existing.adapterConfig }), + }); + await agentCredentialService(txDb).syncAgentSecretBindings( + updated, txDb, + existing.adapterConfig, + options?.recordRevision, + ); + } + + if (normalizedPatch.budgetMonthlyCents !== undefined) { + await budgetServiceInTransaction(txDb, publications).upsertPolicy(existing.companyId, { + scopeType: "agent", scopeId: id, amount: normalizedPatch.budgetMonthlyCents, + isActive: normalizedPatch.budgetMonthlyCents > 0, windowKind: "calendar_month_utc", + }, options?.recordRevision?.createdByUserId ?? null); + } + const normalizedUpdated = await agentRecordQueries(txDb).getById(updated.id); + if (!normalizedUpdated) { + throw notFound("Agent not found"); + } + + if (shouldRecordRevision && beforeConfig) { + const afterConfig = buildConfigSnapshot(normalizedUpdated); + const changedKeys = diffConfigSnapshot(beforeConfig, afterConfig); + if (changedKeys.length > 0) { + await txDb.insert(agentConfigRevisions).values({ + companyId: normalizedUpdated.companyId, + agentId: normalizedUpdated.id, + createdByAgentId: options?.recordRevision?.createdByAgentId ?? null, + createdByUserId: options?.recordRevision?.createdByUserId ?? null, + source: options?.recordRevision?.source ?? "patch", + rolledBackFromRevisionId: options?.recordRevision?.rolledBackFromRevisionId ?? null, + changedKeys, + beforeConfig: beforeConfig as unknown as Record, + afterConfig: afterConfig as unknown as Record, + }); + } + } +} diff --git a/server/src/services/agent-credentials.ts b/server/src/services/agent-credentials.ts new file mode 100644 index 0000000000..656fc18d3b --- /dev/null +++ b/server/src/services/agent-credentials.ts @@ -0,0 +1,197 @@ +import { eq } from "drizzle-orm"; +import { agents, type Db } from "@paperclipai/db"; +import { collectSecretRefs, collectUserSecretRefs, syncAgentAdapterEnvBindings } from "./agent-secret-bindings.js"; +import { logActivity } from "./activity-log.js"; +import { claudeOAuthBindingsMatchExactly, claudeOAuthClaimRejectedError, + CLAUDE_LOCAL_ADAPTER_TYPE, readClaudeOAuthBinding, secretService, type ClaudeOAuthBindingInvariantDecision } from "./secrets.js"; +import { createDbSetupTokenCleanupStore } from "./setup-token-session.js"; +import type { ClaudeLoginContext, RevisionMetadata } from "../lib/agent-records.js"; + +export function agentCredentialService(db: Db) { + const secretsSvc = secretService(db); + async function syncAgentSecretBindings( + agent: { id: string; companyId: string; adapterConfig: unknown }, + dbClient: Db = db, + previousAdapterConfig: unknown = null, + actor: RevisionMetadata = {}, + ) { + const scopedSecretsSvc = dbClient === db ? secretsSvc : secretService(dbClient); + await syncAgentAdapterEnvBindings({ + secretsSvc: scopedSecretsSvc, + companyId: agent.companyId, + agentId: agent.id, + adapterConfig: agent.adapterConfig, + }); + const previousRefs = new Set([ + ...collectSecretRefs(previousAdapterConfig).map((ref) => `secret:${ref.secretId}:${ref.configPath}`), + ...collectUserSecretRefs(previousAdapterConfig).map((ref) => `user:${ref.definitionKey}:${ref.configPath}`), + ]); + const createdRefs = [ + ...collectSecretRefs(agent.adapterConfig).map((ref) => ({ + key: `secret:${ref.secretId}:${ref.configPath}`, + configPath: ref.configPath, + bindingType: "secret_ref", + secretId: ref.secretId, + definitionKey: null, + })), + ...collectUserSecretRefs(agent.adapterConfig).map((ref) => ({ + key: `user:${ref.definitionKey}:${ref.configPath}`, + configPath: ref.configPath, + bindingType: "user_secret_ref", + secretId: null, + definitionKey: ref.definitionKey, + })), + ].filter((ref) => !previousRefs.has(ref.key)); + const actorType = actor.createdByUserId ? "user" as const : actor.createdByAgentId ? "agent" as const : "system" as const; + const actorId = actor.createdByUserId ?? actor.createdByAgentId ?? "system"; + for (const ref of createdRefs) { + await logActivity(dbClient, { + companyId: agent.companyId, + actorType, + actorId, + agentId: actor.createdByAgentId ?? null, + action: "secret.binding.created", + entityType: "agent", + entityId: agent.id, + details: { + targetType: "agent", + targetId: agent.id, + configPath: ref.configPath, + bindingType: ref.bindingType, + secretId: ref.secretId, + definitionKey: ref.definitionKey, + }, + }); + } + } + + /** + * Enforces the Claude OAuth binding claim inside a write transaction. It runs + * after {@link assertClaudeOAuthBindingInvariant} decided that the write + * introduces or keeps the fixed binding. + * + * When the write introduces the fixed binding: + * * A create or hire path (`consume: true`) consumes a stored-session claim + * with one conditional write. It builds the claim scope from the company, + * the owner user, the fixed adapter, the environment, and the + * `storedSessionId`. It inserts the binding only when the write returns one + * row; otherwise it raises the fixed claim error, which rolls back the + * whole transaction and inserts no binding. + * * An update, approval, or rollback path (`consume: false`) carries no + * claim, so it raises the same fixed claim error at once. + * + * The user-actor apply-existing path (`applyExistingWithoutClaim`) binds the + * fixed reference with no login round trip. The route sets the flag only for + * an authenticated user actor and derives the owner from that actor. The gate + * permits the no-claim bind only when the owner already has a stored value for + * the company. It reads the owner value status; it reads no token. A missing + * owner or a missing stored value raises the same fixed claim error, so the + * caller cannot tell the reasons apart. + * + * The hire-inheritance path (`inheritedFromAgentId`) binds the fixed + * reference with no login round trip and no stored owner value, because the + * owning user resolves per run, not from a value stored against this agent. + * The route copies the parent's reference onto the child before this + * transaction starts, so a concurrent version change on the parent can + * leave the child holding a stale version. The gate re-reads the named + * parent agent inside this transaction and permits the bind only when the + * parent exists, is in the same company, is a `claude_local` agent, and its + * current reference matches the child's copied reference exactly, including + * the version selector. The gate locks the parent row with `SELECT ... + * FOR UPDATE` before it reads the reference. The lock blocks a concurrent + * credential rotation on the same parent row until this transaction + * commits or rolls back, so the compare-and-bind check stays atomic with + * the parent's current state. The route derives the parent identifier + * from the authenticated agent actor, never from the request body, so the + * gate treats it as a claim to verify, not a trusted value. + * + * A controlled internal override skips the claim for a migration or an + * administrator repair. The function creates the fixed user-secret definition + * before the caller runs the declaration synchronization, so the synchronized + * declaration always references an existing definition. + */ + async function enforceClaudeOAuthBindingClaim( + txDb: Db, + input: { + companyId: string; + decision: ClaudeOAuthBindingInvariantDecision; + consume: boolean; + environmentId: string | null; + claudeLogin?: ClaudeLoginContext; + /** + * The adapter config the write is about to persist. The + * `inheritedFromAgentId` path reads the child's copied + * `CLAUDE_CODE_OAUTH_TOKEN` reference from it, to compare against the + * parent's current reference. + */ + childAdapterConfig?: unknown; + }, + ): Promise { + const ownerUserId = input.claudeLogin?.ownerUserId ?? null; + if (input.decision.introducesBinding && !input.claudeLogin?.allowInternalBindingOverride) { + if (input.claudeLogin?.applyExistingWithoutClaim) { + // The user-actor apply-existing path. The route derived the owner from + // the authenticated user actor. The gate binds the fixed reference only + // when that owner already has a stored value. It reads no token. + if (!ownerUserId) { + throw claudeOAuthClaimRejectedError(); + } + const stored = await secretService(txDb).readClaudeOAuthUserSecretStatus( + input.companyId, + ownerUserId, + ); + if (!stored) { + throw claudeOAuthClaimRejectedError(); + } + } else if (input.claudeLogin?.inheritedFromAgentId) { + // The hire-inheritance path. Re-read the named parent inside this + // transaction; a caller-supplied identifier never binds on its own. + // Compare the parent's current reference against the reference + // already copied onto the child, including the version selector, so + // a concurrent version change on the parent cannot leave the child + // bound to a stale version. + const parentId = input.claudeLogin.inheritedFromAgentId; + const parent = await txDb + .select({ + companyId: agents.companyId, + adapterType: agents.adapterType, + adapterConfig: agents.adapterConfig, + }) + .from(agents) + .where(eq(agents.id, parentId)) + .for("update") + .then((rows) => rows[0] ?? null); + const parentBinding = readClaudeOAuthBinding(parent?.adapterConfig ?? null); + const childBinding = readClaudeOAuthBinding(input.childAdapterConfig ?? null); + if ( + !parent || + parent.companyId !== input.companyId || + parent.adapterType !== CLAUDE_LOCAL_ADAPTER_TYPE || + !claudeOAuthBindingsMatchExactly(parentBinding, childBinding) + ) { + throw claudeOAuthClaimRejectedError(); + } + } else if (!input.consume) { + throw claudeOAuthClaimRejectedError(); + } else { + const consumed = await createDbSetupTokenCleanupStore(txDb).consumeStoredClaim({ + sessionId: input.claudeLogin?.storedSessionId ?? "", + companyId: input.companyId, + ownerUserId: ownerUserId ?? "", + adapterType: CLAUDE_LOCAL_ADAPTER_TYPE, + }); + if (!consumed) { + throw claudeOAuthClaimRejectedError(); + } + } + } + if (input.decision.introducesBinding || input.decision.keepsBinding) { + // Create the fixed definition before declaration synchronization. + await secretService(txDb).ensureClaudeOAuthUserSecretDefinition(input.companyId, { + userId: ownerUserId, + }); + } + } + + return { syncAgentSecretBindings, enforceClaudeOAuthBindingClaim }; +} diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts new file mode 100644 index 0000000000..7819af773e --- /dev/null +++ b/server/src/services/agent-deletion.ts @@ -0,0 +1,41 @@ +import { and, eq, or, sql } from "drizzle-orm"; +import { agents, agentApiKeys, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, + budgetReservations, heartbeatRunEvents, heartbeatRuns, issueExecutionDecisions, issues, issueComments, + principalPermissionGrants, type Db } from "@paperclipai/db"; +import { conflict } from "../errors.js"; +import { issueThreadInteractionService } from "./issue-thread-interactions.js"; + +export async function deleteAgentDependencies(tx: Db, companyId: string, id: string) { + const [decisionHold] = await tx.select({ id: budgetReservations.id }).from(budgetReservations).where(and( + eq(budgetReservations.companyId, companyId), eq(budgetReservations.agentId, id), + eq(budgetReservations.state, "held"), sql`${budgetReservations.decisionInvocationId} is not null`, + )).limit(1); + if (decisionHold) throw conflict("Wait for active decisions or resolve their unknown charges in Costs before deleting this agent", { + code: "agent_decision_accounting_pending", + }); + await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id); + await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id)); + await tx + .update(issues) + .set({ assigneeAgentId: null, createdByAgentId: null }) + .where(or(eq(issues.assigneeAgentId, id), eq(issues.createdByAgentId, id))); + await tx.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.agentId, id)); + await tx.delete(agentTaskSessions).where(eq(agentTaskSessions.agentId, id)); + await tx.delete(activityLog).where( + or( + eq(activityLog.agentId, id), + sql`${activityLog.runId} in (select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.agentId} = ${id})`, + ), + ); + await tx.delete(issueExecutionDecisions).where(eq(issueExecutionDecisions.actorAgentId, id)); + await tx.delete(issueComments).where(eq(issueComments.authorAgentId, id)); + await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.agentId, id)); + await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, id)); + await tx.delete(agentApiKeys).where(eq(agentApiKeys.agentId, id)); + await tx.delete(agentRuntimeState).where(eq(agentRuntimeState.agentId, id)); + await tx.delete(principalPermissionGrants).where(and( + eq(principalPermissionGrants.companyId, companyId), + eq(principalPermissionGrants.principalType, "agent"), + eq(principalPermissionGrants.principalId, id), + )); +} diff --git a/server/src/services/agent-environment-test.ts b/server/src/services/agent-environment-test.ts new file mode 100644 index 0000000000..ae26b719ba --- /dev/null +++ b/server/src/services/agent-environment-test.ts @@ -0,0 +1,533 @@ +import type { Db } from "@paperclipai/db"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { AdapterEnvironmentCheck } from "@paperclipai/adapter-utils"; +import { parseObject } from "@paperclipai/adapter-utils/server-utils"; +import { aiRoutingHarness, ADAPTER_AUTH_MISSING_CHECK_CODE, AI_CONNECTION_CAPABILITIES, type AiConnectionBinding } from "@paperclipai/shared"; +import { requireServerAdapter } from "../adapters/index.js"; +import { forbidden, HttpError, unprocessable } from "../errors.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { environmentService } from "./environments.js"; +import { environmentRuntimeService } from "./environment-runtime.js"; +import { resolveEnvironmentExecutionTarget } from "./environment-execution-target.js"; +import { prepareManagedAiRuntime, assertManagedAiProjectAuth } from "./ai-connection-runtime.js"; +import { aiConnectionService } from "./ai-connections.js"; +import { validateAiApiKey } from "./ai-api-key-test.js"; +import type { PluginWorkerManager } from "./plugin-worker-manager.js"; +import { assertEnvironmentSelectionForCompany } from "./environment-selection.js"; +const asRecord = (value: unknown): Record | null => value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; +export function agentEnvironmentTestService(db: Db, pluginWorkerManager?: PluginWorkerManager) { + const instanceSettings = instanceSettingsService(db); + const environmentsSvc = environmentService(db); + const environmentRuntime = environmentRuntimeService(db, { pluginWorkerManager }); + // A null agent override inherits the instance default, just like dispatch. + // Resolve this before secrets or probes so a default remote environment can + // never accidentally validate the account on the control-plane host. + async function resolveAdapterTestEnvironmentId(companyId: string, environmentId: string | null | undefined) { + if (environmentId) return environmentId; + const settings = await instanceSettings.get(); + if (settings.defaultEnvironmentId) return settings.defaultEnvironmentId; + if ((await instanceSettings.getExperimental()).enableManagedSandboxOnly === true) { + const managed = await environmentsSvc.findManagedSandboxEnvironment(companyId); + if (!managed) { + throw unprocessable("The managed sandbox is unavailable. Restore Paperclip Computer and retry.", { + code: "managed_sandbox_unavailable", + }); + } + return managed.id; + } + return null; + } + + /** + * Resolve the execution target the adapter should run its test probes against. + * + * - No environmentId / local environment → returns a local target so the + * adapter probes the Paperclip host (legacy behavior). + * - SSH environment → builds an SSH execution target from the environment + * config so the adapter probes the remote box. No lease is required: + * the SSH spec is fully derived from the saved environment config. + * - Sandbox / plugin environments → acquires an ad-hoc lease, realizes the + * workspace, and resolves a sandbox execution target wired to the runtime + * so the adapter probe runs inside the sandbox the same way a heartbeat + * would. The returned `release` callback rolls the lease back when the + * route is done. + * + * The caller MUST always invoke `release()` (typically in a `finally` block). + */ + async function resolveAdapterTestExecutionContext(input: { + agentId?: string; + companyId: string; + adapterType: string; + environmentId: string | null; + }): Promise<{ + executionTarget: AdapterExecutionTarget | null; + environmentName: string | null; + fallbackChecks: AdapterEnvironmentCheck[]; + sandboxIdentityCheck?: AdapterEnvironmentCheck | null; + release: (status?: "released" | "failed") => Promise; + }> { + const noopRelease = async () => {}; + + if (!input.environmentId) { + return { + executionTarget: null, + environmentName: null, + fallbackChecks: [], + release: noopRelease, + }; + } + + const requestedEnvironment = await environmentsSvc.getById(input.environmentId); + if (!requestedEnvironment) { + return { + executionTarget: null, + environmentName: null, + fallbackChecks: [ + { + code: "environment_not_found", + level: "warn", + message: "Selected environment was not found. The test did not run.", + }, + ], + release: noopRelease, + }; + } + + // Managed-sandbox-only policy: redirect a Test that would run on the local + // host onto the platform-managed sandbox, the same as a real run does + // (resolveExecutionWorkspaceEnvironmentId in heartbeat). Without this + // redirect the Test probes the local host while the run executes in the + // managed sandbox, so a passing Test validates the wrong execution target. + // With no active managed sandbox the Test fails closed — never local. + let environment = requestedEnvironment; + if (requestedEnvironment.driver === "local") { + const managedSandboxOnly = + (await instanceSettings.getExperimental()).enableManagedSandboxOnly === true; + if (managedSandboxOnly) { + const managedSandboxEnvironment = await environmentsSvc.findManagedSandboxEnvironment( + input.companyId, + ); + if (!managedSandboxEnvironment) { + return { + executionTarget: null, + environmentName: requestedEnvironment.name, + fallbackChecks: [ + { + code: "managed_sandbox_unavailable", + level: "error", + message: + "This instance runs agents only in its platform-managed sandbox, but no active managed sandbox environment exists. The test did not run.", + hint: "Restore the managed sandbox environment, then test again.", + }, + ], + release: noopRelease, + }; + } + environment = managedSandboxEnvironment; + } + } + + if (environment.driver === "local") { + return { + executionTarget: null, + environmentName: environment.name, + fallbackChecks: [], + release: noopRelease, + }; + } + + if (environment.driver === "ssh") { + try { + const target = await resolveEnvironmentExecutionTarget({ + db, + companyId: input.companyId, + adapterType: input.adapterType, + environment: { + id: environment.id, + driver: environment.driver, + config: environment.config ?? null, + }, + leaseMetadata: null, + }); + if (target) { + return { + executionTarget: target, + environmentName: environment.name, + fallbackChecks: [], + release: noopRelease, + }; + } + return { + executionTarget: null, + environmentName: environment.name, + fallbackChecks: [ + { + code: "environment_target_unavailable", + level: "warn", + message: + `Could not resolve an execution target for environment "${environment.name}". The test did not run.`, + }, + ], + release: noopRelease, + }; + } catch (err) { + return { + executionTarget: null, + environmentName: environment.name, + fallbackChecks: [ + { + code: "environment_target_failed", + level: "warn", + message: + `Could not connect to environment "${environment.name}" to run the test.`, + detail: err instanceof Error ? err.message : String(err), + }, + ], + release: noopRelease, + }; + } + } + + // sandbox / plugin / other remote drivers: spin up an ad-hoc lease, realize + // the workspace inside the box, and run the same probe SSH uses against + // a sandbox execution target wired to the environment runtime. + // + // We pass `heartbeatRunId: null` because there's no heartbeat run for an + // operator-initiated `Test` invocation — the leases table FKs heartbeat + // run id to heartbeat_runs.id, and we don't want to manufacture a fake + // run row. Cleanup goes through the driver's `releaseRunLease` directly + // (by lease record), since the batch helper queries by heartbeatRunId. + // + // Sandbox tests boot a fresh throwaway sandbox (never resume a retained + // agent lease) and archive it on release instead of deleting it, so the + // operator can inspect the exact sandbox from the provider dashboard while + // provider-side expiry reaps it later. + const testEnvironment = environment.driver === "sandbox" + ? { + ...environment, + config: { + ...(environment.config ?? {}), + reuseLease: false, + archiveOnRelease: true, + }, + } + : environment; + let leaseRecord: Awaited>; + try { + leaseRecord = await environmentRuntime.acquireRunLease({ + companyId: input.companyId, + agentId: input.agentId, + environment: testEnvironment, + issueId: null, + heartbeatRunId: null, + persistedExecutionWorkspace: null, + // Re-check the company binding atomically at lease time. The route + // guard already rejected a foreign environment, but the binding could + // change between the guard check and the lease acquire. This closes + // that check-to-lease race so a foreign sandbox never gets a lease. + assertCompanyBinding: true, + // Apply the active custom-image template so the Test boots with the + // operator's captured sandbox customizations and prepared image state, + // matching what real agent runs use. Without this the test would + // silently fall back to the base image. + applyCustomImageTemplate: true, + }); + } catch (err) { + return { + executionTarget: null, + environmentName: environment.name, + fallbackChecks: [ + { + code: "environment_lease_acquire_failed", + level: "error", + message: `Could not acquire a lease for environment "${environment.name}".`, + detail: err instanceof Error ? err.message : String(err), + hint: "Check the environment's provider credentials and quota.", + }, + ], + release: noopRelease, + }; + } + + const driver = environmentRuntime.getDriver(environment.driver); + const releaseLease = async (status: "released" | "failed" = "released") => { + try { + if (driver) { + await driver.releaseRunLease({ + environment: testEnvironment, + lease: leaseRecord.lease, + status, + }); + } else { + await environmentsSvc.releaseLease(leaseRecord.lease.id, status); + } + } catch (err) { + // Cleanup failures must not mask the test result. + // eslint-disable-next-line no-console + console.warn( + `[adapter-test] Failed to release lease ${leaseRecord.lease.id}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + }; + + let realizedCwd: string | null = null; + try { + const realized = await environmentRuntime.realizeWorkspace({ + environment: testEnvironment, + lease: leaseRecord.lease, + // No host workspace to copy for a Test invocation; sandbox/plugin + // realize implementations use the lease metadata's remoteCwd to + // create the working directory inside the box. + workspace: {}, + }); + realizedCwd = + typeof realized.cwd === "string" && realized.cwd.trim().length > 0 + ? realized.cwd.trim() + : null; + } catch (err) { + await releaseLease("failed"); + return { + executionTarget: null, + environmentName: environment.name, + fallbackChecks: [ + { + code: "environment_workspace_realize_failed", + level: "error", + message: `Could not realize a workspace inside "${environment.name}".`, + detail: err instanceof Error ? err.message : String(err), + }, + ], + release: noopRelease, + }; + } + + let target: AdapterExecutionTarget | null; + try { + // Prefer the cwd the realize step returned; fall back to lease metadata. + const leaseMetadataForTarget: Record | null = + realizedCwd + ? { ...(leaseRecord.lease.metadata ?? {}), remoteCwd: realizedCwd } + : (leaseRecord.lease.metadata as Record | null) ?? null; + + target = await resolveEnvironmentExecutionTarget({ + db, + companyId: input.companyId, + adapterType: input.adapterType, + environment: { + id: testEnvironment.id, + driver: testEnvironment.driver, + config: testEnvironment.config ?? null, + }, + leaseId: leaseRecord.lease.id, + leaseMetadata: leaseMetadataForTarget, + lease: leaseRecord.lease, + environmentRuntime, + }); + } catch (err) { + await releaseLease("failed"); + return { + executionTarget: null, + environmentName: environment.name, + fallbackChecks: [ + { + code: "environment_target_failed", + level: "error", + message: `Could not resolve an execution target for "${environment.name}".`, + detail: err instanceof Error ? err.message : String(err), + }, + ], + release: noopRelease, + }; + } + + if (!target) { + await releaseLease("failed"); + return { + executionTarget: null, + environmentName: environment.name, + fallbackChecks: [ + { + code: "environment_target_unsupported", + level: "warn", + message: + `Adapter "${input.adapterType}" is not allowed in "${environment.name}" environments.`, + }, + ], + release: noopRelease, + }; + } + + return { + executionTarget: target, + environmentName: environment.name, + fallbackChecks: [], + sandboxIdentityCheck: buildSandboxIdentityCheck({ + environmentName: environment.name, + lease: leaseRecord.lease, + }), + release: releaseLease, + }; + } + + function readMetadataString(metadata: Record, keys: string[]): string | null { + for (const key of keys) { + const value = metadata[key]; + if (typeof value === "string" && value.trim().length > 0) return value.trim(); + } + return null; + } + + function buildSandboxIdentityCheck(input: { + environmentName: string; + lease: { + id: string; + provider?: string | null; + providerLeaseId?: string | null; + metadata?: Record | null; + }; + }): AdapterEnvironmentCheck { + const metadata = input.lease.metadata ?? {}; + const provider = input.lease.provider ?? readMetadataString(metadata, ["provider"]); + const sandboxId = readMetadataString(metadata, ["sandboxId", "sandboxID", "sandbox_id", "id"]); + const sandboxName = readMetadataString(metadata, ["sandboxName", "sandbox_name", "name"]); + const snapshotRef = readMetadataString(metadata, [ + "snapshot", + "snapshotId", + "snapshotID", + "snapshotRef", + "snapshot_ref", + "templateRef", + "template_ref", + "templateId", + "templateID", + "image", + "imageId", + "imageID", + "imageRef", + "image_ref", + ]); + const templateKind = readMetadataString(metadata, [ + "templateKind", + "template_kind", + "templateRefKind", + "template_ref_kind", + ]); + const detailParts = [ + `paperclipLeaseId=${input.lease.id}`, + input.lease.providerLeaseId ? `providerLeaseId=${input.lease.providerLeaseId}` : null, + provider ? `provider=${provider}` : null, + sandboxId ? `sandboxId=${sandboxId}` : null, + sandboxName ? `sandboxName=${sandboxName}` : null, + snapshotRef ? `${templateKind ? `${templateKind}Ref` : "snapshotOrTemplateRef"}=${snapshotRef}` : null, + ].filter((part): part is string => Boolean(part)); + + return { + code: "sandbox_test_identity", + level: "info", + message: `Environment test identity for "${input.environmentName}".`, + detail: detailParts.join("; "), + hint: "Use these provider-neutral IDs when comparing model-test output with provider logs or refreshed environment snapshots.", + }; + } + + + // The environment drivers the adapter Test route accepts. A local, SSH, or + // sandbox environment can host a probe; a plugin environment cannot. + const ADAPTER_TEST_ALLOWED_ENVIRONMENT_DRIVERS = ["local", "ssh", "sandbox"]; + + // The fail-closed tenant-binding guard for the adapter Test route. A caller + // may name any instance environment by id, so the route must reject an + // environment that binds to another company before it resolves secrets, + // merges env, resolves the target, leases a sandbox, or runs the adapter + // test. The guard checks the company binding BEFORE it validates the status + // or the driver, so it never reveals the status or the driver of a foreign + // environment. A same-company or an instance-global environment then gets the + // shared driver and status validation. + async function assertAdapterTestEnvironmentForCompany( + companyId: string, + environmentId: string, + ): Promise { + const environment = await environmentsSvc.getById(environmentId); + if (!environment) { + // A missing environment leaks no tenant state. The execution-context + // resolver surfaces the existing environment_not_found check. + return; + } + const boundCompanyIds = await environmentsSvc.listBoundCompanyIds(environmentId); + if (boundCompanyIds.length > 0 && !boundCompanyIds.includes(companyId)) { + throw forbidden("The selected environment belongs to another company.", { + code: "environment_company_mismatch", + }); + } + await assertEnvironmentSelectionForCompany(environmentsSvc, companyId, environmentId, { + allowedDrivers: ADAPTER_TEST_ALLOWED_ENVIRONMENT_DRIVERS, + }); + } + + async function testManagedEnvironment(adapterType: string, context: Parameters["testEnvironment"]>[0], binding: AiConnectionBinding, managed: Awaited>, agentId?: string) { + const startedAt = new Date(); + const result = await probeManagedEnvironment(adapterType, context, binding); + // A provider rejection invalidates the tested credential generation. A + // missing CLI, unavailable environment, or other runtime error does not. + if (result.status === "fail" && result.checks.some(check => + check.code === ADAPTER_AUTH_MISSING_CHECK_CODE || /_hello_probe_auth_required$/.test(check.code) + || check.code === "ai_connection_api_key_rejected", + )) { + await aiConnectionService(db).markAuthenticationFailed({ + companyId: context.companyId, agentId, runStartedAt: startedAt, + attribution: { ...managed.attribution, identity: managed.identity }, + }); + } + return result; + } + + async function probeManagedEnvironment(adapterType: string, context: Parameters["testEnvironment"]>[0], binding: AiConnectionBinding) { + await assertManagedAiProjectAuth(context.config, binding.provider, context.executionTarget); + const result = await requireServerAdapter(adapterType).testEnvironment(context); + if (result.status === "fail") return result; + // The resolved method, not binding.method — on a responsible_user binding + // that field is wire-compat only and the default connection decides. + const resolvedMethod = (context.config as { managedAiConnection?: { method?: string } }).managedAiConnection?.method; + // An api_key account does not take the CLI hello probe below: a key + // travels as an env var any engine understands, and the engine's own test + // above judged whether this runtime can execute with it — the ACP lane + // deliberately runs no hello probe when a key is configured. Demanding one + // anyway forced the CLI lane, whose probe needs a provider CLI on PATH, + // and a clean install has none: that walled off onboarding's API-key path + // on exactly the machines the release smoke exists to guard. The account + // is still verified live here — the same provider-endpoint check the save + // performed — so a key revoked since its save fails adoption rather than + // producing an agent that cannot authenticate at runtime. The hello-probe + // requirement stays for subscriptions: a stored login is a file layout + // only a provider CLI reads, so proving the runtime lane can consume it + // takes a real hello turn. + if (resolvedMethod === "api_key" && !context.config.managedAiRouting) { + const envKey = AI_CONNECTION_CAPABILITIES[binding.provider].methods.api_key?.envKey; + const key = envKey ? parseObject(context.config.env)[envKey] : undefined; + try { + if (typeof key !== "string" || !key) throw unprocessable("The selected account's API key was not available to verify."); + await validateAiApiKey(binding.provider, key); + result.checks.push({ code: "ai_connection_api_key_reverified", level: "info", message: "The provider verified this API key for adoption." }); + } catch (error) { + result.status = "fail"; + const rejected = error instanceof HttpError && asRecord(error.details)?.code === "ai_connection_api_key_rejected"; + result.checks.push({ code: rejected ? "ai_connection_api_key_rejected" : "ai_connection_verification_failed", level: "error", message: error instanceof HttpError ? error.message : "Could not verify the account. Try again." }); + } + return result; + } + if (!result.checks.some(check => check.code.includes("hello_probe"))) { + const providerAdapter = context.config.managedAiRouting ? aiRoutingHarness(adapterType, context.config.provider, context.config.acpxAgent) : { anthropic: "claude_local", openai: "codex_local", openrouter: "opencode_local", xai: "grok_local", google: "gemini_local" }[binding.provider]; + const probe = await requireServerAdapter(providerAdapter).testEnvironment({ ...context, adapterType: providerAdapter, config: { ...context.config, engine: "cli" } }); + result.checks.push(...probe.checks); + result.status = probe.status === "fail" ? "fail" : result.status === "warn" || probe.status === "warn" ? "warn" : "pass"; + } + if (!result.checks.some(check => /hello_probe_(passed|succeeded)$/.test(check.code))) { + result.status = "fail"; + result.checks.push({ code: "ai_connection_validation_incomplete", level: "error", message: "The selected account has not completed a provider hello test. Retry before adopting it." }); + } + return result; + } + + + return { resolveAdapterTestEnvironmentId, resolveAdapterTestExecutionContext, assertAdapterTestEnvironmentForCompany, testManagedEnvironment }; +} diff --git a/server/src/services/agent-execution-stop.ts b/server/src/services/agent-execution-stop.ts new file mode 100644 index 0000000000..3fad14e89b --- /dev/null +++ b/server/src/services/agent-execution-stop.ts @@ -0,0 +1,74 @@ +import { readProcessStartedAt } from "./hot-restart.js"; +import { runUsedConversationAdapter } from "./conversation-continuation.js"; +import { and, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { hasNativeLocalProcessStop } from "./native-local-process-stop.js"; +import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; +import { isCancelledNativeStartup } from "./cancelled-native-startup.js"; + +/** Cancellation changes admission. Process and lease evidence controls cleanup. */ +export async function agentExecutionsHaveStopped(db: Db, agentIds: string[]) { + if (!agentIds.length) return true; + const runs = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, agentIds)); + const runIds = db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, agentIds)); + const leases = await db.select().from(environmentLeases).where(or( + inArray(environmentLeases.heartbeatRunId, runIds), + inArray(sql`${environmentLeases.metadata}->>'agentId'`, agentIds), + )); + if (leases.some(lease => (!lease.releasedAt && !(lease.status === "retained" && lease.cleanupStatus === "success")) || lease.status === "pending_cleanup" || lease.cleanupStatus === "failed")) return false; + const coordinators = await db.select().from(nativeRunFinalizations) + .where(inArray(nativeRunFinalizations.runId, runIds)); + const now = new Date(); + const abandoned = coordinators.filter(row => row.leaseOwner); + if (abandoned.some(row => !row.leaseExpiresAt || row.leaseExpiresAt > now || row.resultId || + runs.find(run => run.id === row.runId)?.status !== "cancelled")) return false; + const absent = (pid: number) => { + try { process.kill(pid, 0); return false; } + catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH"; } + }; + for (const coordinator of abandoned) { + if (!coordinator.controllerPid || absent(coordinator.controllerPid)) continue; + const observed = await readProcessStartedAt(coordinator.controllerPid).catch(() => null); + if (!observed || !coordinator.controllerProcessStartedAt || + new Date(observed).getTime() === coordinator.controllerProcessStartedAt.getTime()) return false; + } + for (const run of runs) { + if (!["succeeded", "failed", "cancelled", "timed_out", "interrupted"].includes(run.status)) return false; + if (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()) return false; + const runLeases = leases.filter(lease => lease.heartbeatRunId === run.id); + if (runLeases.some(lease => lease.provider && lease.provider !== "local")) { + if (run.status === "cancelled" && !runLeases.every(hasRemoteTerminationReceipt)) return false; + continue; + } + let processAlive = Boolean(run.processPid && !absent(run.processPid)); + let groupAlive = Boolean(run.processGroupId && !absent(-run.processGroupId)); + if (processAlive && run.processStartedAt) { + const observed = await readProcessStartedAt(run.processPid!).catch(() => null); + if (observed && new Date(observed).getTime() !== run.processStartedAt.getTime()) { + processAlive = false; + if (run.processGroupId === run.processPid) groupAlive = false; + } + } + if (processAlive || groupAlive) return false; + if (run.status !== "cancelled" || !run.startedAt) continue; + if (run.runtimeMode === "native" && !run.processPid && !run.processGroupId && + !await hasNativeLocalProcessStop(db, run.companyId, run.id) && + !await isCancelledNativeStartup(db, run, coordinators.find(row => row.runId === run.id))) return false; + if (run.runtimeMode === "legacy" && !run.processPid && !run.processGroupId && + (run.resultJson?.executionCancellation as Record | undefined)?.state !== "acknowledged" && + !(run.executionStage === "settled" && !await runUsedConversationAdapter(db, run)) && + !await isCancelledNativeStartup(db, run, coordinators.find(row => row.runId === run.id))) return false; + } + // Release an expired owner only after its cancelled execution has stop proof. + for (const coordinator of abandoned) { + const [cleared] = await db.update(nativeRunFinalizations).set({ leaseOwner: null, leaseExpiresAt: null, updatedAt: now }) + .where(and(eq(nativeRunFinalizations.runId, coordinator.runId), + eq(nativeRunFinalizations.leaseOwner, coordinator.leaseOwner!), + eq(nativeRunFinalizations.leaseExpiresAt, coordinator.leaseExpiresAt!), + eq(nativeRunFinalizations.controllerGeneration, coordinator.controllerGeneration), + eq(nativeRunFinalizations.phase, coordinator.phase), isNull(nativeRunFinalizations.resultId))) + .returning({ runId: nativeRunFinalizations.runId }); + if (!cleared) return false; + } + return true; +} diff --git a/server/src/services/agent-harness-verification.ts b/server/src/services/agent-harness-verification.ts new file mode 100644 index 0000000000..b7133488f9 --- /dev/null +++ b/server/src/services/agent-harness-verification.ts @@ -0,0 +1,76 @@ +import { and, eq } from "drizzle-orm"; +import { agents, companies, type Db } from "@paperclipai/db"; +import type { AdapterEnvironmentTestResult } from "@paperclipai/adapter-utils"; +import { ADAPTER_AUTH_MISSING_CHECK_CODE, aiRuntimeConnectionBindingSchema } from "@paperclipai/shared"; +import { isForbiddenConfigEnvKey, parseObject } from "@paperclipai/adapter-utils/server-utils"; +import { requireServerAdapter } from "../adapters/index.js"; +import { agentEnvironmentTestService } from "./agent-environment-test.js"; +import { aiConnectionRouterService } from "./ai-connection-router.js"; +import { withManagedAiProbe, stripAiAuthBindings } from "./ai-connection-runtime.js"; +import { environmentService } from "./environments.js"; +import { secretService } from "./secrets.js"; +import { dotRunnerBroker } from "./dot-runner-broker.js"; +import type { PluginWorkerManager } from "./plugin-worker-manager.js"; +import type { LifecycleAgent } from "../modules/agent-lifecycle/index.js"; + +export function assertHarnessTestPassed(result: AdapterEnvironmentTestResult) { + const verified = result.checks.some(check => check.level === "info" && + (/hello_probe_(passed|succeeded)$/.test(check.code) || check.code === "ai_connection_api_key_reverified")); + const authenticationFailed = result.checks.some(check => + check.code === ADAPTER_AUTH_MISSING_CHECK_CODE || /_hello_probe_auth_required$/.test(check.code)); + const incomplete = !verified && (result.status === "warn" || result.checks.some(check => check.code.includes("hello_probe"))); + if (result.status === "fail" || authenticationFailed || incomplete) throw Object.assign(new Error("The harness test failed"), { code: "harness_test_failed" }); +} + +export function agentHarnessVerificationService(db: Db, manager: PluginWorkerManager) { + const tests = agentEnvironmentTestService(db, manager); + const secrets = secretService(db); + const environments = environmentService(db); + async function verify(snapshot: LifecycleAgent): Promise<"complete" | "pending"> { + const [agent] = await db.select().from(agents).where(and(eq(agents.id, snapshot.id), eq(agents.lifecycleVersion, snapshot.lifecycleVersion))); + if (!agent) return "pending"; + if (agent.adapterType === "paperclip_runner" && agent.adapterConfig.provider === "openai_dot") { + const binding = await dotRunnerBroker(db).bindingForAgent(agent.companyId, agent.id); + return binding?.status === "ready" && binding.subscriptionVerified && binding.id === agent.adapterConfig.dotBindingId ? "complete" : "pending"; + } + const [company] = await db.select().from(companies).where(eq(companies.id, agent.companyId)); + const responsibleUserId = agent.lifecycleOperation?.responsibleUserId ?? company.defaultResponsibleUserId; + const context = { consumerType: "agent" as const, consumerId: agent.id, actorType: "system" as const, + actorId: "agent-lifecycle", responsibleUserId }; + let binding = agent.runtimeConfig.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined; + let config = agent.adapterConfig; + if (binding?.mode === "router") { + const selected = await aiConnectionRouterService(db, manager).resolve({ companyId: agent.companyId, + poolId: binding.connectionId, agentId: agent.id, userId: responsibleUserId, + adapterType: agent.adapterType, taskKey: `lifecycle:${agent.lifecycleOperation!.id}` }); + binding = selected.binding; + config = { ...config, ...selected.runtimeConfig }; + } + if (binding) config = { ...config, env: stripAiAuthBindings(config.env) }; + config = (await secrets.resolveAdapterConfigForRuntime(agent.companyId, config, context, { adapterType: agent.adapterType })).config; + const environmentId = await tests.resolveAdapterTestEnvironmentId(agent.companyId, agent.defaultEnvironmentId); + if (environmentId) { + await tests.assertAdapterTestEnvironmentForCompany(agent.companyId, environmentId); + const environment = await environments.getById(environmentId); + const env = Object.fromEntries(Object.entries(binding ? stripAiAuthBindings(environment?.envVars) : parseObject(environment?.envVars)) + .filter(([key]) => !isForbiddenConfigEnvKey(key))); + const resolved = await secrets.resolveEnvBindings(agent.companyId, env, + { ...context, consumerType: "environment", consumerId: environmentId }); + config = { ...config, env: { ...resolved.env, ...parseObject(config.env) } }; + } + const target = await tests.resolveAdapterTestExecutionContext({ agentId: agent.id, companyId: agent.companyId, adapterType: agent.adapterType, environmentId }); + let status: "released" | "failed" = "failed"; + try { + if (target.fallbackChecks.length) throw Object.assign(new Error("The test environment is unavailable"), { code: "test_environment_unavailable" }); + const input = { companyId: agent.companyId, adapterType: agent.adapterType, config, + executionTarget: target.executionTarget, environmentName: target.environmentName }; + const result = binding ? await withManagedAiProbe(db, { ...input, agentId: agent.id, responsibleUserId, binding }, + managed => tests.testManagedEnvironment(agent.adapterType, { ...input, config: managed.config }, binding!, managed, agent.id)) + : await requireServerAdapter(agent.adapterType).testEnvironment(input); + assertHarnessTestPassed(result); + status = "released"; + return "complete"; + } finally { await target.release(status); } + } + return { verify }; +} diff --git a/server/src/services/agent-hiring.ts b/server/src/services/agent-hiring.ts new file mode 100644 index 0000000000..c35299352e --- /dev/null +++ b/server/src/services/agent-hiring.ts @@ -0,0 +1,135 @@ +import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared"; +import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils"; +import { agents, toolConnectionInstalls, principalPermissionGrants, type Db } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { unprocessable } from "../errors.js"; +import { agentRecordQueries, isPlainRecord, configPatchFromApprovalPayload, normalizeRuntimeConfigForNewAgent, + deduplicateAgentName, type CreateAgentData, type CreateAgentOptions, type AgentConfigurationPatch, type AgentHireRecord } from "../lib/agent-records.js"; +import { normalizeAgentPermissions, permissionsImplyLowTrust, NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, + newStandardAgentGrantScope } from "../lib/agent-permissions.js"; +import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; +import { secretService, assertClaudeOAuthBindingInvariant } from "./secrets.js"; +import { agentCredentialService } from "./agent-credentials.js"; +import { agentIdentityService } from "./agent-identity.js"; +import { initializePrimaryAgent } from "./primary-agent.js"; + +export async function prepareAgentHire(db: Db, companyId: string, data: CreateAgentData, options?: CreateAgentOptions): Promise { + const { ensureManager, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db); + if (data.appearance?.customAvatarAssetId) throw unprocessable("Create the agent before uploading its avatar"); + assertBuiltInAgentMetadataMutationAllowed(null, data.metadata, options); + if (data.reportsTo) { + await ensureManager(companyId, data.reportsTo); + } + + const existingAgents = await db + .select({ id: agents.id, name: agents.name, status: agents.status }) + .from(agents) + .where(eq(agents.companyId, companyId)); + const uniqueName = deduplicateAgentName(data.name, existingAgents); + + const role = data.role ?? "general"; + const normalizedPermissions = normalizeAgentPermissions(data.permissions, { context: "create" }); + const runtimeConfig = normalizeRuntimeConfigForNewAgent(data.runtimeConfig); + const adapterType = data.adapterType ?? "process"; + const rawAdapterConfig = isPlainRecord(data.adapterConfig) + ? await secretService(db).normalizeAdapterConfigForPersistence(companyId, data.adapterConfig, { adapterType }) + : {}; + const adapterConfig = normalizePaperclipRunnerAdapterConfig(adapterType, rawAdapterConfig); + await agentCredentialService(db).enforceClaudeOAuthBindingClaim(db, { + companyId, consume: true, environmentId: data.defaultEnvironmentId ?? null, + childAdapterConfig: adapterConfig, claudeLogin: options?.claudeLogin, + decision: assertClaudeOAuthBindingInvariant({ adapterType, nextConfig: adapterConfig }), + }); + return { ...data, name: uniqueName, + appearance: data.appearance == null ? randomAgentAppearance() : agentAppearanceSchema.parse(data.appearance), + role, adapterType, adapterConfig, permissions: normalizedPermissions, runtimeConfig }; +} + +export async function initializeAgentHire(txDb: Db, created: AgentHireRecord, options?: CreateAgentOptions) { + const tx = txDb; + const companyId = created.companyId; + await agentIdentityService(txDb).ensureAgentIdentity(companyId, created.id); + // New standard agents receive the standard direct grants at activation. + // Low-trust and bundled agents keep their explicit, narrower grants. + if (created.status !== "pending_approval" && !permissionsImplyLowTrust(created.permissions) && + !readBuiltInAgentMarker(created.metadata)) { + await tx.insert(principalPermissionGrants).values( + NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({ + companyId, + principalType: "agent" as const, + principalId: created.id, + permissionKey, + scope: newStandardAgentGrantScope(permissionKey, created.id), + })), + ).onConflictDoNothing(); + } + if (options?.aiConnectionInstall) { + const install = options.aiConnectionInstall; + const connectionIds = [...new Set([install.connectionId, ...(install.memberConnectionIds ?? [])])]; + await tx.insert(toolConnectionInstalls).values(connectionIds.map(connectionId => ({ + companyId, connectionId, + targetType: "agent" as const, targetId: created.id, + createdByUserId: install.createdByUserId, + }))).onConflictDoNothing(); + } + await agentCredentialService(txDb).syncAgentSecretBindings(created, txDb); + if (options?.createdByUserId && !readBuiltInAgentMarker(created.metadata)) { + await initializePrimaryAgent(txDb, companyId, options.createdByUserId, created.id); + } +} + +export async function prepareAgentHireApproval(txDb: Db, existing: AgentHireRecord, approvedPayload?: Record | null): Promise { + const approvedPatch = approvedPayload ? configPatchFromApprovalPayload(approvedPayload) : {}; + const patch = { ...approvedPatch } as AgentConfigurationPatch; + const hasApprovedAdapterConfig = Object.prototype.hasOwnProperty.call(patch, "adapterConfig") && isPlainRecord(patch.adapterConfig); + if ( + Object.prototype.hasOwnProperty.call(patch, "adapterConfig") && + isPlainRecord(patch.adapterConfig) + ) { + const normalizedAdapterConfig = await secretService(txDb).normalizeAdapterConfigForPersistence( + existing.companyId, patch.adapterConfig, { adapterType: (patch.adapterType ?? existing.adapterType) as string }, + ); + patch.adapterConfig = normalizePaperclipRunnerAdapterConfig( + (patch.adapterType ?? existing.adapterType) as string, + normalizedAdapterConfig, + ); + } else if ( + Object.prototype.hasOwnProperty.call(patch, "adapterType") + && isPlainRecord(existing.adapterConfig) + ) { + patch.adapterConfig = normalizePaperclipRunnerAdapterConfig( + patch.adapterType as string, + existing.adapterConfig, + ); + } + if (patch.permissions !== undefined) { + // The pending-approval activation replays the original hire + // request, so the new-agent creation default applies. + patch.permissions = normalizeAgentPermissions(patch.permissions, { context: "create" }); + } + if (hasApprovedAdapterConfig) { + await agentCredentialService(txDb).enforceClaudeOAuthBindingClaim(txDb, { + companyId: existing.companyId, consume: false, environmentId: null, + childAdapterConfig: patch.adapterConfig, + decision: assertClaudeOAuthBindingInvariant({ adapterType: patch.adapterType ?? existing.adapterType, + nextConfig: patch.adapterConfig, priorConfig: existing.adapterConfig }), + }); + } + return patch; +} + +export async function completeAgentHireApproval(txDb: Db, updated: AgentHireRecord, existing: AgentHireRecord) { + const tx = txDb; + await agentCredentialService(txDb).syncAgentSecretBindings(updated, txDb, existing.adapterConfig); + if (!permissionsImplyLowTrust(updated.permissions) && !readBuiltInAgentMarker(updated.metadata)) { + await tx.insert(principalPermissionGrants).values( + NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({ + companyId: updated.companyId, + principalType: "agent" as const, + principalId: updated.id, + permissionKey, + scope: newStandardAgentGrantScope(permissionKey, updated.id), + })), + ).onConflictDoNothing(); + } +} diff --git a/server/src/services/agent-instruction-revisions.ts b/server/src/services/agent-instruction-revisions.ts index 6c07b0d08f..8a60208e31 100644 --- a/server/src/services/agent-instruction-revisions.ts +++ b/server/src/services/agent-instruction-revisions.ts @@ -1,3 +1,4 @@ +import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js"; import { createHash } from "node:crypto"; import { adoptAgentFiles, agentFileToken } from "./agent-file-store.js"; import { and, desc, eq, getTableColumns, lt, or, sql } from "drizzle-orm"; @@ -252,19 +253,15 @@ export function agentInstructionRevisionService(db: Db) { !state.agent.adapterConfig.instructionsBundleMode || (bound.type === "plugin" && configuredEntryFile !== input.entryFile) ) { - await tx - .update(agents) - .set({ - adapterConfig: { - ...state.agent.adapterConfig, - instructionsBundleMode: "managed", - instructionsRootPath: state.root, - instructionsEntryFile: state.entryFile, - instructionsFilePath: `${state.root}/${state.entryFile}`, - }, - updatedAt: new Date(), - }) - .where(eq(agents.id, state.agent.id)); + await updateAgentConfigurationInTransaction(tx as unknown as Db, state.agent.id, { + adapterConfig: { + ...state.agent.adapterConfig, + instructionsBundleMode: "managed", + instructionsRootPath: state.root, + instructionsEntryFile: state.entryFile, + instructionsFilePath: `${state.root}/${state.entryFile}`, + }, + }, { allowPendingApprovalConfigUpdate: true }); } }; // An exact replay can safely return the durable receipt even after its base diff --git a/server/src/services/agent-lifecycle-driver.ts b/server/src/services/agent-lifecycle-driver.ts new file mode 100644 index 0000000000..68cbea2999 --- /dev/null +++ b/server/src/services/agent-lifecycle-driver.ts @@ -0,0 +1,47 @@ +import { and, eq, inArray } from "drizzle-orm"; +import { agents, plugins, pluginCompanySettings, type Db } from "@paperclipai/db"; +import { heartbeatService } from "./heartbeat.js"; +import { listInvalidOrgChainDescendantIds } from "./agent-invokability.js"; +import type { PluginWorkerManager } from "./plugin-worker-manager.js"; +import type { LifecycleDriver } from "../modules/agent-lifecycle/index.js"; +import { agentHarnessVerificationService } from "./agent-harness-verification.js"; + +export function createLifecycleDriver(db: Db, manager: PluginWorkerManager): LifecycleDriver { + const { verify } = agentHarnessVerificationService(db, manager); + const heartbeat = heartbeatService(db, { pluginWorkerManager: manager }); + return { + async requiredPluginIds(agent) { + if (agent.lifecycleRequiredPluginIds !== null) return agent.lifecycleRequiredPluginIds; + const installed = await db.select().from(plugins).where(inArray(plugins.status, ["ready", "error", "installed", "upgrade_pending"])); + const disabled = await db.select({ pluginId: pluginCompanySettings.pluginId }).from(pluginCompanySettings) + .where(and(eq(pluginCompanySettings.companyId, agent.companyId), eq(pluginCompanySettings.enabled, false))); + return installed.filter(plugin => !disabled.some(row => row.pluginId === plugin.id) && + plugin.manifestJson.agentLifecycle && plugin.manifestJson.capabilities.includes("agents.lifecycle.manage")).map(plugin => plugin.id); + }, + async runPlugin(agent, pluginId) { + const [plugin] = await db.select().from(plugins).where(eq(plugins.id, pluginId)); + const [settings] = await db.select().from(pluginCompanySettings).where(and( + eq(pluginCompanySettings.pluginId, pluginId), eq(pluginCompanySettings.companyId, agent.companyId))); + if (settings?.enabled === false || !plugin || plugin.status !== "ready" || !plugin.manifestJson.agentLifecycle || !plugin.manifestJson.capabilities.includes("agents.lifecycle.manage")) { + throw Object.assign(new Error("A required lifecycle plugin is unavailable"), { code: "required_plugin_unavailable" }); + } + const operationId = agent.lifecycleOperation!.id; + const result = await manager.call(plugin.id, "agentLifecycle", { companyId: agent.companyId, + agentId: agent.id, operationId, version: agent.lifecycleVersion, phase: agent.lifecycleState }, 30_000); + if (!result || result.operationId !== operationId || result.version !== agent.lifecycleVersion || !["complete", "pending"].includes(result.status)) { + throw Object.assign(new Error("Invalid lifecycle result"), { code: "invalid_lifecycle_result" }); + } + return result.status; + }, + async runHost(agent) { + if (agent.lifecycleState === "verifying") return verify(agent); + if (["pausing", "terminating", "cleaning_up"].includes(agent.lifecycleState)) { + const companyAgents = await db.select().from(agents).where(eq(agents.companyId, agent.companyId)); + const ids = agent.lifecycleState === "pausing" ? [agent.id] + : [agent.id, ...listInvalidOrgChainDescendantIds(agent.id, companyAgents)]; + if (!await heartbeat.stopInvocationsForAgents(ids, "Agent lifecycle stop requested")) return "pending"; + } + return "complete"; + }, + }; +} diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts new file mode 100644 index 0000000000..25ef5e5873 --- /dev/null +++ b/server/src/services/agent-lifecycle.ts @@ -0,0 +1,61 @@ +import { logger } from "../middleware/logger.js"; +import type { Db } from "@paperclipai/db"; +import * as lifecycle from "../modules/agent-lifecycle/index.js"; +import { withAccountingTransaction } from "./accounting-transaction.js"; +import { budgetServiceInTransaction, deliverBudgetEnforcement, policyBlocks, type BudgetServiceHooks } from "./budgets.js"; +import { recordAgentStatusEvent, recordResourceCreationEvent } from "./resource-lifecycle-events.js"; +import { clearPrimaryAgent } from "./primary-agent.js"; +import { deleteAgentDependencies } from "./agent-deletion.js"; +import { prepareAgentConfiguration, completeAgentConfiguration } from "./agent-configuration.js"; +import { prepareAgentHire, initializeAgentHire, prepareAgentHireApproval, completeAgentHireApproval } from "./agent-hiring.js"; +import { trackIdleWork } from "./task-admission.js"; + +export { scheduleAgentLifecycle } from "../modules/agent-lifecycle/index.js"; +export type { LifecycleDriver, LifecycleAgent } from "../modules/agent-lifecycle/index.js"; + +export function createAgentLifecycleEffects(hooks: BudgetServiceHooks = {}): lifecycle.LifecycleEffects { + return { + reportFailure(context, error) { + // Only known codes are safe; provider errors can contain credentials or private data. + const allowedCodes = new Set(["required_plugin_unavailable", "invalid_lifecycle_result", "harness_test_failed", + "test_environment_unavailable", "ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]); + const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null; + const code = [candidate?.code, candidate?.cause?.code].find(value => typeof value === "string" && allowedCodes.has(value)); + logger.warn({ ...context, failureCode: code ?? "unclassified" }, "Agent lifecycle operation failed"); + }, + transaction: withAccountingTransaction, + deleteDependencies: deleteAgentDependencies, + policyBlocks, + setAgentBudget: (db, publications, companyId, agentId, amount, userId, isActive) => + budgetServiceInTransaction(db, publications).upsertPolicy(companyId, { + scopeType: "agent", scopeId: agentId, amount, isActive, windowKind: "calendar_month_utc", + }, userId), + enforceBudget: (db, companyId, agentId) => deliverBudgetEnforcement(db, hooks, companyId, agentId), + recordCreation: (db, companyId, agentId) => recordResourceCreationEvent(db, companyId, "agent", agentId), + recordStatus: recordAgentStatusEvent, + clearPrimary: clearPrimaryAgent, + prepareConfiguration: prepareAgentConfiguration, + completeConfiguration: completeAgentConfiguration, + prepareHire: prepareAgentHire, + initializeHire: initializeAgentHire, + prepareHireApproval: prepareAgentHireApproval, + completeHireApproval: completeAgentHireApproval, + }; +} + +export function createAgentLifecycle(db: Db, hooks: BudgetServiceHooks = {}) { + return lifecycle.createAgentLifecycle(db, createAgentLifecycleEffects(hooks)); +} + +function trackedDriver(driver: lifecycle.LifecycleDriver): lifecycle.LifecycleDriver { + return { ...driver, runHost: agent => trackIdleWork(driver.runHost(agent)), + runPlugin: (agent, pluginId) => trackIdleWork(driver.runPlugin(agent, pluginId)) }; +} + +export function configureAgentLifecycle(db: Db, driver: lifecycle.LifecycleDriver, canRun = () => true) { + return lifecycle.configureAgentLifecycle(db, createAgentLifecycleEffects(), trackedDriver(driver), canRun); +} + +export function startAgentLifecycle(db: Db, driver: lifecycle.LifecycleDriver, canRun: () => boolean) { + return lifecycle.startAgentLifecycle(db, createAgentLifecycleEffects(), trackedDriver(driver), canRun); +} diff --git a/server/src/services/agent-profile-avatar.ts b/server/src/services/agent-profile-avatar.ts index 64d0b7d3d9..24b5f68c7d 100644 --- a/server/src/services/agent-profile-avatar.ts +++ b/server/src/services/agent-profile-avatar.ts @@ -1,3 +1,4 @@ +import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js"; import { createHash } from "node:crypto"; import { and, eq } from "drizzle-orm"; import sharp from "sharp"; @@ -5,7 +6,6 @@ import { agents, assets, companies, type Db } from "@paperclipai/db"; import { MAX_AGENT_AVATAR_BYTES, resolveAgentAppearance, agentAvatarUrl, setAgentAvatarSchema, type SetAgentAvatarInput } from "@paperclipai/shared"; import { conflict, notFound, unprocessable } from "../errors.js"; import type { StorageService } from "../storage/types.js"; -import { agentService } from "./agents.js"; import { logActivity, publishActivity, type ActivityPublication, type LogActivityInput } from "./activity-log.js"; /** Decode and re-encode: exclude SVG, animation, metadata and unbounded pixel counts. No URL fetching. */ @@ -60,7 +60,7 @@ export async function setAgentProfileAvatar(db: Db, storage: StorageService, com createdByUserId: actor.actorType === "user" ? actor.actorId : null }).returning(); appearance.customAvatarAssetId = asset!.id; } - await agentService(txDb).update(agentId, { appearance }, { recordRevision: { + await updateAgentConfigurationInTransaction(txDb, agentId, { appearance }, { recordRevision: { createdByAgentId: actor.agentId, createdByUserId: actor.actorType === "user" ? actor.actorId : null, source: "avatar-upload", } }); await logActivity(txDb, { ...actor, companyId, action: "agent.avatar_updated", entityType: "agent", entityId: agentId, diff --git a/server/src/services/agents.ts b/server/src/services/agents.ts index 59e3a90512..d3bcaca2b1 100644 --- a/server/src/services/agents.ts +++ b/server/src/services/agents.ts @@ -1,911 +1,34 @@ -import { budgetServiceInTransaction, deliverBudgetEnforcement, type BudgetServiceHooks } from "./budgets.js"; -import { withAccountingTransaction } from "./accounting-transaction.js"; -import type { ActivityPublication } from "./activity-log.js"; -import { agentAppearanceSchema, randomAgentAppearance, resolveAgentAppearance, agentAvatarUrl } from "@paperclipai/shared"; -import { createHash, randomBytes } from "node:crypto"; -import { and, desc, eq, gte, inArray, lt, ne, or, sql } from "drizzle-orm"; +import { type BudgetServiceHooks } from "./budgets.js"; + +import { and, desc, eq, inArray, ne } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { - agents, - assets, - toolConnectionInstalls, - agentConfigRevisions, - agentApiKeys, - agentRuntimeState, - agentTaskSessions, - agentWakeupRequests, - activityLog, - costEvents, - budgetReservations, - heartbeatRunEvents, - heartbeatRuns, - issueExecutionDecisions, - issues, - issueComments, - principalPermissionGrants, -} from "@paperclipai/db"; -import { - AGENT_DEFAULT_MAX_CONCURRENT_RUNS, - agentRuntimeConfigSchema, - getAgentWorkEligibility, - isUuidLike, - normalizeAgentApiKeyScope, - normalizeAgentUrlKey, - type AgentEligibilityAgent, - type AgentApiKeyScope, -} from "@paperclipai/shared"; -import { - normalizePaperclipRunnerAdapterConfig, -} from "@paperclipai/adapter-utils/server-utils"; +import { agents, agentConfigRevisions, agentApiKeys, heartbeatRuns } from "@paperclipai/db"; +import { isUuidLike, normalizeAgentApiKeyScope, normalizeAgentUrlKey, type AgentApiKeyScope } from "@paperclipai/shared"; + import { conflict, notFound, unprocessable } from "../errors.js"; + +import { normalizeAgentPermissions } from "../lib/agent-permissions.js"; + import { - collectSecretRefs, - collectUserSecretRefs, - syncAgentAdapterEnvBindings, -} from "./agent-secret-bindings.js"; -import { logActivity } from "./activity-log.js"; -import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope, normalizeAgentPermissions, permissionsImplyLowTrust } from "./agent-permissions.js"; -import { recordAgentStatusEvent, recordResourceCreationEvent } from "./resource-lifecycle-events.js"; -import { REDACTED_EVENT_VALUE, sanitizeRecord } from "../redaction.js"; -import { - assertClaudeOAuthBindingInvariant, - claudeOAuthBindingsMatchExactly, - claudeOAuthClaimRejectedError, - CLAUDE_LOCAL_ADAPTER_TYPE, - readClaudeOAuthBinding, - secretService, - type ClaudeOAuthBindingInvariantDecision, -} from "./secrets.js"; -import { createDbSetupTokenCleanupStore } from "./setup-token-session.js"; -import { - builtInAgentMarkersEqual, - readBuiltInAgentMarker, -} from "./built-in-agent-metadata.js"; -import { issueThreadInteractionService } from "./issue-thread-interactions.js"; + agentRecordQueries, + hashToken, + createToken, + containsRedactedMarker, + configPatchFromSnapshot, +} from "../lib/agent-records.js"; +import { createAgentLifecycle } from "./agent-lifecycle.js"; -import { clearPrimaryAgent, initializePrimaryAgent } from "./primary-agent.js"; -import { agentIdentityService } from "./agent-identity.js"; - -function hashToken(token: string) { - return createHash("sha256").update(token).digest("hex"); -} - -function createToken() { - return `pcp_${randomBytes(24).toString("hex")}`; -} - -const CONFIG_REVISION_FIELDS = [ - "name", - "role", - "title", - "icon", - "appearance", - "reportsTo", - "capabilities", - "adapterType", - "adapterConfig", - "runtimeConfig", - "defaultEnvironmentId", - "budgetMonthlyCents", - "metadata", -] as const; - -type ConfigRevisionField = (typeof CONFIG_REVISION_FIELDS)[number]; -type AgentConfigSnapshot = Pick; - -interface RevisionMetadata { - createdByAgentId?: string | null; - createdByUserId?: string | null; - source?: string; - rolledBackFromRevisionId?: string | null; -} - -/** - * The Claude login context for an agent write. The route derives the owner user - * from the authenticated actor, not from the request body, and forwards the - * non-secret `storedSessionId` claim from a completed Claude login session. A - * controlled internal override permits a migration or an administrator repair to - * bind or unbind the fixed OAuth token without a claim. - * - * The `applyExistingWithoutClaim` field is the user-actor apply-existing path. - * The route sets it only for an authenticated user actor and derives the owner - * from that actor. The path binds the fixed reference to the owner stored value - * with no login round trip. It is distinct from `allowInternalBindingOverride`, - * which does no ownership check. - * - * The `inheritedFromAgentId` field is the hire-inheritance path. The route - * sets it only for an authenticated agent actor whose hire request inherited - * the fixed reference from that named parent. The service re-reads the parent - * agent inside the write transaction and binds the fixed reference only when - * the parent exists, is in the same company, is a `claude_local` agent, and - * already holds the exact fixed binding. - */ -interface ClaudeLoginContext { - storedSessionId?: string | null; - ownerUserId?: string | null; - allowInternalBindingOverride?: boolean; - applyExistingWithoutClaim?: boolean; - inheritedFromAgentId?: string | null; -} - -interface UpdateAgentOptions { - recordRevision?: RevisionMetadata; - allowBuiltInAgentMetadata?: boolean; - allowPendingApprovalConfigUpdate?: boolean; - claudeLogin?: ClaudeLoginContext; -} - -interface CreateAgentOptions { - createdByUserId?: string | null; - aiConnectionInstall?: { connectionId: string; memberConnectionIds?: string[]; createdByUserId: string | null }; - allowBuiltInAgentMetadata?: boolean; - claudeLogin?: ClaudeLoginContext; -} - -interface AgentShortnameRow { - id: string; - name: string; - status: string; -} - -interface AgentShortnameCollisionOptions { - excludeAgentId?: string | null; -} - -function isPlainRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function jsonEqual(left: unknown, right: unknown): boolean { - return JSON.stringify(left) === JSON.stringify(right); -} - -function buildConfigSnapshot( - row: Pick, -): AgentConfigSnapshot { - const adapterConfig = - typeof row.adapterConfig === "object" && row.adapterConfig !== null && !Array.isArray(row.adapterConfig) - ? sanitizeRecord(row.adapterConfig as Record) - : {}; - const runtimeConfig = - typeof row.runtimeConfig === "object" && row.runtimeConfig !== null && !Array.isArray(row.runtimeConfig) - ? sanitizeRecord(row.runtimeConfig as Record) - : {}; - const metadata = - typeof row.metadata === "object" && row.metadata !== null && !Array.isArray(row.metadata) - ? sanitizeRecord(row.metadata as Record) - : row.metadata ?? null; - return { - name: row.name, - role: row.role, - title: row.title, - icon: row.icon, - appearance: row.appearance, - reportsTo: row.reportsTo, - capabilities: row.capabilities, - adapterType: row.adapterType, - adapterConfig, - runtimeConfig, - defaultEnvironmentId: row.defaultEnvironmentId, - budgetMonthlyCents: row.budgetMonthlyCents, - metadata, - }; -} - -function containsRedactedMarker(value: unknown): boolean { - if (value === REDACTED_EVENT_VALUE) return true; - if (Array.isArray(value)) return value.some((item) => containsRedactedMarker(item)); - if (typeof value !== "object" || value === null) return false; - return Object.values(value as Record).some((entry) => containsRedactedMarker(entry)); -} - -function hasConfigPatchFields(data: Partial) { - return CONFIG_REVISION_FIELDS.some((field) => Object.prototype.hasOwnProperty.call(data, field)); -} - -function changedPendingApprovalConfigFields( - existing: typeof agents.$inferSelect, - data: Partial, -) { - return CONFIG_REVISION_FIELDS.filter((field) => - Object.prototype.hasOwnProperty.call(data, field) && !jsonEqual(data[field], existing[field]), - ); -} - -function configPatchFromApprovalPayload(payload: Record) { - const patch: Partial = {}; - if (typeof payload.name === "string") patch.name = payload.name; - if (typeof payload.role === "string") patch.role = payload.role; - if (payload.appearance != null) patch.appearance = agentAppearanceSchema.parse(payload.appearance); - if (Object.prototype.hasOwnProperty.call(payload, "title")) { - patch.title = typeof payload.title === "string" ? payload.title : null; - } - if (Object.prototype.hasOwnProperty.call(payload, "icon")) { - patch.icon = typeof payload.icon === "string" ? payload.icon : null; - } - if (Object.prototype.hasOwnProperty.call(payload, "reportsTo")) { - patch.reportsTo = typeof payload.reportsTo === "string" ? payload.reportsTo : null; - } - if (Object.prototype.hasOwnProperty.call(payload, "capabilities")) { - patch.capabilities = typeof payload.capabilities === "string" ? payload.capabilities : null; - } - if (typeof payload.adapterType === "string") patch.adapterType = payload.adapterType; - if (isPlainRecord(payload.adapterConfig)) patch.adapterConfig = payload.adapterConfig; - if (isPlainRecord(payload.runtimeConfig)) patch.runtimeConfig = payload.runtimeConfig; - if (Object.prototype.hasOwnProperty.call(payload, "defaultEnvironmentId")) { - patch.defaultEnvironmentId = - typeof payload.defaultEnvironmentId === "string" ? payload.defaultEnvironmentId : null; - } - if (typeof payload.budgetMonthlyCents === "number") { - patch.budgetMonthlyCents = payload.budgetMonthlyCents; - } - if (Object.prototype.hasOwnProperty.call(payload, "metadata")) { - patch.metadata = isPlainRecord(payload.metadata) ? payload.metadata : null; - } - if (isPlainRecord(payload.permissions)) { - patch.permissions = payload.permissions; - } - return patch; -} - -function parseFiniteNumberLike(value: unknown): number | null { - if (typeof value === "number" && Number.isFinite(value)) return value; - if (typeof value !== "string") return null; - const parsed = Number(value.trim()); - return Number.isFinite(parsed) ? parsed : null; -} - -function normalizeRuntimeConfigForNewAgent(runtimeConfig: unknown): Record { - const normalizedRuntimeConfig = isPlainRecord(runtimeConfig) ? { ...runtimeConfig } : {}; - const heartbeat = isPlainRecord(normalizedRuntimeConfig.heartbeat) - ? { ...normalizedRuntimeConfig.heartbeat } - : {}; - if (parseFiniteNumberLike(heartbeat.maxConcurrentRuns) == null) { - heartbeat.maxConcurrentRuns = AGENT_DEFAULT_MAX_CONCURRENT_RUNS; - } - normalizedRuntimeConfig.heartbeat = heartbeat; - return normalizedRuntimeConfig; -} - -function diffConfigSnapshot( - before: AgentConfigSnapshot, - after: AgentConfigSnapshot, -): string[] { - return CONFIG_REVISION_FIELDS.filter((field) => !jsonEqual(before[field], after[field])); -} - -function configPatchFromSnapshot(snapshot: unknown): Partial { - if (!isPlainRecord(snapshot)) throw unprocessable("Invalid revision snapshot"); - - if (typeof snapshot.name !== "string" || snapshot.name.length === 0) { - throw unprocessable("Invalid revision snapshot: name"); - } - if (typeof snapshot.role !== "string" || snapshot.role.length === 0) { - throw unprocessable("Invalid revision snapshot: role"); - } - if (typeof snapshot.adapterType !== "string" || snapshot.adapterType.length === 0) { - throw unprocessable("Invalid revision snapshot: adapterType"); - } - if (typeof snapshot.budgetMonthlyCents !== "number" || !Number.isFinite(snapshot.budgetMonthlyCents)) { - throw unprocessable("Invalid revision snapshot: budgetMonthlyCents"); - } - const runtimeConfig = agentRuntimeConfigSchema.safeParse( - isPlainRecord(snapshot.runtimeConfig) ? snapshot.runtimeConfig : {}, - ); - if (!runtimeConfig.success) { - throw unprocessable("Invalid revision snapshot: runtimeConfig"); - } - - return { - name: snapshot.name, - role: snapshot.role, - title: typeof snapshot.title === "string" || snapshot.title === null ? snapshot.title : null, - reportsTo: - typeof snapshot.reportsTo === "string" || snapshot.reportsTo === null ? snapshot.reportsTo : null, - capabilities: - typeof snapshot.capabilities === "string" || snapshot.capabilities === null - ? snapshot.capabilities - : null, - adapterType: snapshot.adapterType, - adapterConfig: isPlainRecord(snapshot.adapterConfig) ? snapshot.adapterConfig : {}, - runtimeConfig: runtimeConfig.data, - defaultEnvironmentId: - typeof snapshot.defaultEnvironmentId === "string" || snapshot.defaultEnvironmentId === null - ? snapshot.defaultEnvironmentId - : null, - budgetMonthlyCents: Math.max(0, Math.floor(snapshot.budgetMonthlyCents)), - metadata: isPlainRecord(snapshot.metadata) || snapshot.metadata === null ? snapshot.metadata : null, - }; -} - -export function hasAgentShortnameCollision( - candidateName: string, - existingAgents: AgentShortnameRow[], - options?: AgentShortnameCollisionOptions, -): boolean { - const candidateShortname = normalizeAgentUrlKey(candidateName); - if (!candidateShortname) return false; - - return existingAgents.some((agent) => { - if (agent.status === "terminated") return false; - if (options?.excludeAgentId && agent.id === options.excludeAgentId) return false; - return normalizeAgentUrlKey(agent.name) === candidateShortname; - }); -} - -export function deduplicateAgentName( - candidateName: string, - existingAgents: AgentShortnameRow[], -): string { - if (!hasAgentShortnameCollision(candidateName, existingAgents)) { - return candidateName; - } - for (let i = 2; i <= 100; i++) { - const suffixed = `${candidateName} ${i}`; - if (!hasAgentShortnameCollision(suffixed, existingAgents)) { - return suffixed; - } - } - return `${candidateName} ${Date.now()}`; -} +export { hasAgentShortnameCollision, deduplicateAgentName } from "../lib/agent-records.js"; export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { - const secretsSvc = secretService(db); - - function currentUtcMonthWindow(now = new Date()) { - const year = now.getUTCFullYear(); - const month = now.getUTCMonth(); - return { - start: new Date(Date.UTC(year, month, 1, 0, 0, 0, 0)), - end: new Date(Date.UTC(year, month + 1, 1, 0, 0, 0, 0)), - }; - } - - function withUrlKey(row: T) { - return { - ...row, - urlKey: normalizeAgentUrlKey(row.name) ?? row.id, - }; - } - - function normalizeAgentBaseRow(row: typeof agents.$inferSelect) { - return withUrlKey({ - ...row, - permissions: normalizeAgentPermissions(row.permissions), - }); - } - - function toEligibilityAgent(row: Pick): AgentEligibilityAgent { - return { - id: row.id, - companyId: row.companyId, - name: row.name, - status: row.status, - reportsTo: row.reportsTo, - }; - } - - function normalizeAgentRows(rows: (typeof agents.$inferSelect)[], allCompanyRows = rows) { - const eligibilityAgents = allCompanyRows.map(toEligibilityAgent); - return rows.map((row) => { - const base = normalizeAgentBaseRow(row); - const appearance = resolveAgentAppearance(row.appearance, row.id); - return { - ...base, - appearance, - avatarUrl: agentAvatarUrl(appearance), - orgChainHealth: getAgentWorkEligibility({ - agent: toEligibilityAgent(row), - agents: eligibilityAgents, - }).orgChainHealth, - }; - }); - } - - function normalizeAgentRow(row: typeof agents.$inferSelect, allCompanyRows?: (typeof agents.$inferSelect)[]) { - return normalizeAgentRows([row], allCompanyRows)[0]!; - } - - async function listCompanyAgentRows(companyId: string) { - return db.select().from(agents).where(eq(agents.companyId, companyId)); - } - - async function getMonthlySpendByAgentIds(companyId: string, agentIds: string[]) { - if (agentIds.length === 0) return new Map(); - const { start, end } = currentUtcMonthWindow(); - const rows = await db - .select({ - agentId: costEvents.agentId, - spentMonthlyCents: sql`coalesce(sum(${costEvents.costCents}), 0)::double precision`, - }) - .from(costEvents) - .where( - and( - eq(costEvents.companyId, companyId), - inArray(costEvents.agentId, agentIds), - gte(costEvents.occurredAt, start), - lt(costEvents.occurredAt, end), - ), - ) - .groupBy(costEvents.agentId); - return new Map(rows.map((row) => [row.agentId, Number(row.spentMonthlyCents ?? 0)])); - } - - async function hydrateAgentSpend(rows: T[]) { - const agentIds = rows.map((row) => row.id); - const companyId = rows[0]?.companyId; - if (!companyId || agentIds.length === 0) return rows; - const spendByAgentId = await getMonthlySpendByAgentIds(companyId, agentIds); - return rows.map((row) => ({ - ...row, - spentMonthlyCents: spendByAgentId.get(row.id) ?? 0, - })); - } - - async function getById(id: string) { - const row = await db - .select() - .from(agents) - .where(eq(agents.id, id)) - .then((rows) => rows[0] ?? null); - if (!row) return null; - const [companyRows, hydrated] = await Promise.all([ - listCompanyAgentRows(row.companyId), - hydrateAgentSpend([row]).then((rows) => rows[0]!), - ]); - return normalizeAgentRow(hydrated, companyRows); - } - - async function requireGetById(id: string) { - const agent = await getById(id); - if (!agent) throw notFound("Agent not found"); - return agent; - } - - async function ensureManager(companyId: string, managerId: string) { - const manager = await getById(managerId); - if (!manager) throw notFound("Manager not found"); - if (manager.companyId !== companyId) { - throw unprocessable("Manager must belong to same company"); - } - return manager; - } - - async function assertNoCycle(agentId: string, reportsTo: string | null | undefined) { - if (!reportsTo) return; - if (reportsTo === agentId) throw unprocessable("Agent cannot report to itself"); - - let cursor: string | null = reportsTo; - while (cursor) { - if (cursor === agentId) throw unprocessable("Reporting relationship would create cycle"); - const next = await getById(cursor); - cursor = next?.reportsTo ?? null; - } - } - - async function assertCompanyShortnameAvailable( - companyId: string, - candidateName: string, - options?: AgentShortnameCollisionOptions, - ) { - const candidateShortname = normalizeAgentUrlKey(candidateName); - if (!candidateShortname) return; - - const existingAgents = await db - .select({ - id: agents.id, - name: agents.name, - status: agents.status, - }) - .from(agents) - .where(eq(agents.companyId, companyId)); - - const hasCollision = hasAgentShortnameCollision(candidateName, existingAgents, options); - if (hasCollision) { - throw conflict( - `Agent shortname '${candidateShortname}' is already in use in this company`, - ); - } - } - - async function syncAgentSecretBindings( - agent: { id: string; companyId: string; adapterConfig: unknown }, - dbClient: Db = db, - previousAdapterConfig: unknown = null, - actor: RevisionMetadata = {}, - ) { - const scopedSecretsSvc = dbClient === db ? secretsSvc : secretService(dbClient); - await syncAgentAdapterEnvBindings({ - secretsSvc: scopedSecretsSvc, - companyId: agent.companyId, - agentId: agent.id, - adapterConfig: agent.adapterConfig, - }); - const previousRefs = new Set([ - ...collectSecretRefs(previousAdapterConfig).map((ref) => `secret:${ref.secretId}:${ref.configPath}`), - ...collectUserSecretRefs(previousAdapterConfig).map((ref) => `user:${ref.definitionKey}:${ref.configPath}`), - ]); - const createdRefs = [ - ...collectSecretRefs(agent.adapterConfig).map((ref) => ({ - key: `secret:${ref.secretId}:${ref.configPath}`, - configPath: ref.configPath, - bindingType: "secret_ref", - secretId: ref.secretId, - definitionKey: null, - })), - ...collectUserSecretRefs(agent.adapterConfig).map((ref) => ({ - key: `user:${ref.definitionKey}:${ref.configPath}`, - configPath: ref.configPath, - bindingType: "user_secret_ref", - secretId: null, - definitionKey: ref.definitionKey, - })), - ].filter((ref) => !previousRefs.has(ref.key)); - const actorType = actor.createdByUserId ? "user" as const : actor.createdByAgentId ? "agent" as const : "system" as const; - const actorId = actor.createdByUserId ?? actor.createdByAgentId ?? "system"; - for (const ref of createdRefs) { - await logActivity(dbClient, { - companyId: agent.companyId, - actorType, - actorId, - agentId: actor.createdByAgentId ?? null, - action: "secret.binding.created", - entityType: "agent", - entityId: agent.id, - details: { - targetType: "agent", - targetId: agent.id, - configPath: ref.configPath, - bindingType: ref.bindingType, - secretId: ref.secretId, - definitionKey: ref.definitionKey, - }, - }); - } - } - - /** - * Enforces the Claude OAuth binding claim inside a write transaction. It runs - * after {@link assertClaudeOAuthBindingInvariant} decided that the write - * introduces or keeps the fixed binding. - * - * When the write introduces the fixed binding: - * * A create or hire path (`consume: true`) consumes a stored-session claim - * with one conditional write. It builds the claim scope from the company, - * the owner user, the fixed adapter, the environment, and the - * `storedSessionId`. It inserts the binding only when the write returns one - * row; otherwise it raises the fixed claim error, which rolls back the - * whole transaction and inserts no binding. - * * An update, approval, or rollback path (`consume: false`) carries no - * claim, so it raises the same fixed claim error at once. - * - * The user-actor apply-existing path (`applyExistingWithoutClaim`) binds the - * fixed reference with no login round trip. The route sets the flag only for - * an authenticated user actor and derives the owner from that actor. The gate - * permits the no-claim bind only when the owner already has a stored value for - * the company. It reads the owner value status; it reads no token. A missing - * owner or a missing stored value raises the same fixed claim error, so the - * caller cannot tell the reasons apart. - * - * The hire-inheritance path (`inheritedFromAgentId`) binds the fixed - * reference with no login round trip and no stored owner value, because the - * owning user resolves per run, not from a value stored against this agent. - * The route copies the parent's reference onto the child before this - * transaction starts, so a concurrent version change on the parent can - * leave the child holding a stale version. The gate re-reads the named - * parent agent inside this transaction and permits the bind only when the - * parent exists, is in the same company, is a `claude_local` agent, and its - * current reference matches the child's copied reference exactly, including - * the version selector. The gate locks the parent row with `SELECT ... - * FOR UPDATE` before it reads the reference. The lock blocks a concurrent - * credential rotation on the same parent row until this transaction - * commits or rolls back, so the compare-and-bind check stays atomic with - * the parent's current state. The route derives the parent identifier - * from the authenticated agent actor, never from the request body, so the - * gate treats it as a claim to verify, not a trusted value. - * - * A controlled internal override skips the claim for a migration or an - * administrator repair. The function creates the fixed user-secret definition - * before the caller runs the declaration synchronization, so the synchronized - * declaration always references an existing definition. - */ - async function enforceClaudeOAuthBindingClaim( - txDb: Db, - input: { - companyId: string; - decision: ClaudeOAuthBindingInvariantDecision; - consume: boolean; - environmentId: string | null; - claudeLogin?: ClaudeLoginContext; - /** - * The adapter config the write is about to persist. The - * `inheritedFromAgentId` path reads the child's copied - * `CLAUDE_CODE_OAUTH_TOKEN` reference from it, to compare against the - * parent's current reference. - */ - childAdapterConfig?: unknown; - }, - ): Promise { - const ownerUserId = input.claudeLogin?.ownerUserId ?? null; - if (input.decision.introducesBinding && !input.claudeLogin?.allowInternalBindingOverride) { - if (input.claudeLogin?.applyExistingWithoutClaim) { - // The user-actor apply-existing path. The route derived the owner from - // the authenticated user actor. The gate binds the fixed reference only - // when that owner already has a stored value. It reads no token. - if (!ownerUserId) { - throw claudeOAuthClaimRejectedError(); - } - const stored = await secretService(txDb).readClaudeOAuthUserSecretStatus( - input.companyId, - ownerUserId, - ); - if (!stored) { - throw claudeOAuthClaimRejectedError(); - } - } else if (input.claudeLogin?.inheritedFromAgentId) { - // The hire-inheritance path. Re-read the named parent inside this - // transaction; a caller-supplied identifier never binds on its own. - // Compare the parent's current reference against the reference - // already copied onto the child, including the version selector, so - // a concurrent version change on the parent cannot leave the child - // bound to a stale version. - const parentId = input.claudeLogin.inheritedFromAgentId; - const parent = await txDb - .select({ - companyId: agents.companyId, - adapterType: agents.adapterType, - adapterConfig: agents.adapterConfig, - }) - .from(agents) - .where(eq(agents.id, parentId)) - .for("update") - .then((rows) => rows[0] ?? null); - const parentBinding = readClaudeOAuthBinding(parent?.adapterConfig ?? null); - const childBinding = readClaudeOAuthBinding(input.childAdapterConfig ?? null); - if ( - !parent || - parent.companyId !== input.companyId || - parent.adapterType !== CLAUDE_LOCAL_ADAPTER_TYPE || - !claudeOAuthBindingsMatchExactly(parentBinding, childBinding) - ) { - throw claudeOAuthClaimRejectedError(); - } - } else if (!input.consume) { - throw claudeOAuthClaimRejectedError(); - } else { - const consumed = await createDbSetupTokenCleanupStore(txDb).consumeStoredClaim({ - sessionId: input.claudeLogin?.storedSessionId ?? "", - companyId: input.companyId, - ownerUserId: ownerUserId ?? "", - adapterType: CLAUDE_LOCAL_ADAPTER_TYPE, - }); - if (!consumed) { - throw claudeOAuthClaimRejectedError(); - } - } - } - if (input.decision.introducesBinding || input.decision.keepsBinding) { - // Create the fixed definition before declaration synchronization. - await secretService(txDb).ensureClaudeOAuthUserSecretDefinition(input.companyId, { - userId: ownerUserId, - }); - } - } - - function assertBuiltInAgentMetadataMutationAllowed( - beforeMetadata: unknown, - afterMetadata: unknown, - options?: { allowBuiltInAgentMetadata?: boolean }, - ) { - if (options?.allowBuiltInAgentMetadata) return; - const beforeMarker = readBuiltInAgentMarker(beforeMetadata); - const afterMarker = readBuiltInAgentMarker(afterMetadata); - if (builtInAgentMarkersEqual(beforeMarker, afterMarker)) return; - throw conflict("Built-in agent marker is managed by Paperclip and cannot be edited directly", { - code: "built_in_agent_marker_readonly", - key: beforeMarker?.key ?? afterMarker?.key ?? null, - }); - } - - async function updateAgent( - id: string, - data: Partial, - options?: UpdateAgentOptions, - ) { - const existing = await getById(id); - if (!existing) return null; - - if (existing.status === "terminated" && data.status && data.status !== "terminated") { - throw conflict("Terminated agents cannot be resumed"); - } - if ( - existing.status === "pending_approval" && - data.status && - data.status !== "pending_approval" && - data.status !== "terminated" - ) { - throw conflict("Pending approval agents cannot be activated directly"); - } - if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) { - const changedFields = changedPendingApprovalConfigFields(existing as typeof agents.$inferSelect, data); - if (changedFields.length > 0) { - throw conflict("Pending approval agent configuration cannot be changed before board approval", { - code: "pending_approval_agent_config_frozen", - agentId: id, - fields: changedFields, - }); - } - } - - if (data.reportsTo !== undefined) { - if (data.reportsTo) { - await ensureManager(existing.companyId, data.reportsTo); - } - await assertNoCycle(id, data.reportsTo); - } - - if (data.name !== undefined) { - const previousShortname = normalizeAgentUrlKey(existing.name); - const nextShortname = normalizeAgentUrlKey(data.name); - if (previousShortname !== nextShortname) { - await assertCompanyShortnameAvailable(existing.companyId, data.name, { excludeAgentId: id }); - } - } - - if (Object.prototype.hasOwnProperty.call(data, "metadata")) { - assertBuiltInAgentMetadataMutationAllowed(existing.metadata, data.metadata, options); - } - - if (data.appearance?.customAvatarAssetId) { - const [asset] = await db.select().from(assets).where(and( - eq(assets.id, data.appearance.customAvatarAssetId), eq(assets.companyId, existing.companyId), - eq(assets.createdByAgentId, id), - )); - if (!asset || asset.contentType !== "image/png" || !asset.objectKey.startsWith(`${existing.companyId}/agent-avatars/${id}/`)) { - throw unprocessable("Use the avatar upload endpoint to set this agent's image"); - } - } - const normalizedPatch = { ...data } as Partial; - if (data.permissions !== undefined) { - normalizedPatch.permissions = normalizeAgentPermissions(data.permissions); - } - if ( - Object.prototype.hasOwnProperty.call(normalizedPatch, "adapterConfig") && - isPlainRecord(normalizedPatch.adapterConfig) - ) { - const normalizedAdapterConfig = await secretsSvc.normalizeAdapterConfigForPersistence( - existing.companyId, - normalizedPatch.adapterConfig, - { adapterType: (normalizedPatch.adapterType ?? existing.adapterType) as string }, - ); - normalizedPatch.adapterConfig = normalizePaperclipRunnerAdapterConfig( - (normalizedPatch.adapterType ?? existing.adapterType) as string, - normalizedAdapterConfig, - ); - } else if ( - Object.prototype.hasOwnProperty.call(normalizedPatch, "adapterType") - && isPlainRecord(existing.adapterConfig) - ) { - normalizedPatch.adapterConfig = normalizePaperclipRunnerAdapterConfig( - normalizedPatch.adapterType as string, - existing.adapterConfig, - ); - } - // Run the server-enforced binding invariant when the patch touches the - // adapter config. The update, approval, and rollback paths keep an existing - // fixed binding but reject a newly introduced binding, because they carry no - // stored-session claim. - const bindingDecision = Object.prototype.hasOwnProperty.call(normalizedPatch, "adapterConfig") - ? assertClaudeOAuthBindingInvariant({ - adapterType: (normalizedPatch.adapterType ?? existing.adapterType) as string, - nextConfig: normalizedPatch.adapterConfig, - priorConfig: existing.adapterConfig, - }) - : null; - - const shouldRecordRevision = Boolean(options?.recordRevision) && hasConfigPatchFields(normalizedPatch); - const beforeConfig = shouldRecordRevision ? buildConfigSnapshot(existing) : null; - - type AgentUpdateResult = Awaited>; - const applyUpdate = async (txDb: Db, publications: ActivityPublication[] = []): Promise => { - const current = data.status !== undefined - ? await txDb.select().from(agents).where(eq(agents.id, id)).for("update").then(rows => rows[0] ?? null) - : existing; - if (!current) return null; - if (current.status === "terminated" && data.status && data.status !== "terminated") { - throw conflict("Terminated agents cannot be resumed"); - } - if (current.status === "pending_approval" && data.status && data.status !== "pending_approval" && data.status !== "terminated") { - throw conflict("Pending approval agents cannot be activated directly"); - } - const updated = await txDb - .update(agents) - .set({ ...normalizedPatch, updatedAt: new Date() }) - .where(eq(agents.id, id)) - .returning() - .then((rows) => rows[0] ?? null); - if (!updated) return null; - if (updated.status === "terminated") { - await clearPrimaryAgent(txDb, updated.companyId, id); - } - if (data.status !== undefined) { - await recordAgentStatusEvent(txDb, updated.companyId, id, current.status, updated.status); - } - - const priorAdapterConfig = isPlainRecord(existing.adapterConfig) ? existing.adapterConfig : {}; - const afterConfig = isPlainRecord(updated.adapterConfig) ? updated.adapterConfig : {}; - const changedExecution = updated.adapterType !== existing.adapterType - || (updated.adapterType === "paperclip_runner" && ["provider", "acpxAgent", "model"].some( - (key) => priorAdapterConfig[key] !== afterConfig[key], - )); - if (changedExecution) { - await txDb.delete(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, existing.companyId), eq(agentTaskSessions.agentId, id))); - await txDb.update(agentRuntimeState).set({ adapterType: updated.adapterType, sessionId: null, stateJson: {}, updatedAt: new Date() }) - .where(and(eq(agentRuntimeState.companyId, existing.companyId), eq(agentRuntimeState.agentId, id))); - } - - if (Object.prototype.hasOwnProperty.call(normalizedPatch, "adapterConfig")) { - if (bindingDecision) { - await enforceClaudeOAuthBindingClaim(txDb, { - companyId: existing.companyId, - decision: bindingDecision, - consume: false, - environmentId: null, - claudeLogin: options?.claudeLogin, - }); - } - await syncAgentSecretBindings( - updated, - txDb, - existing.adapterConfig, - options?.recordRevision, - ); - } - - if (normalizedPatch.budgetMonthlyCents !== undefined) { - await budgetServiceInTransaction(txDb, publications).upsertPolicy(existing.companyId, { - scopeType: "agent", scopeId: id, amount: normalizedPatch.budgetMonthlyCents, isActive: normalizedPatch.budgetMonthlyCents > 0, windowKind: "calendar_month_utc", - }, options?.recordRevision?.createdByUserId ?? null); - } - const normalizedUpdated = await agentService(txDb).getById(updated.id); - if (!normalizedUpdated) { - throw notFound("Agent not found"); - } - - if (shouldRecordRevision && beforeConfig) { - const afterConfig = buildConfigSnapshot(normalizedUpdated); - const changedKeys = diffConfigSnapshot(beforeConfig, afterConfig); - if (changedKeys.length > 0) { - await txDb.insert(agentConfigRevisions).values({ - companyId: normalizedUpdated.companyId, - agentId: normalizedUpdated.id, - createdByAgentId: options?.recordRevision?.createdByAgentId ?? null, - createdByUserId: options?.recordRevision?.createdByUserId ?? null, - source: options?.recordRevision?.source ?? "patch", - rolledBackFromRevisionId: options?.recordRevision?.rolledBackFromRevisionId ?? null, - changedKeys, - beforeConfig: beforeConfig as unknown as Record, - afterConfig: afterConfig as unknown as Record, - }); - } - } - - return normalizedUpdated; - }; - - if (normalizedPatch.budgetMonthlyCents !== undefined) { - const result = await withAccountingTransaction(db, existing.companyId, applyUpdate); - await deliverBudgetEnforcement(db, budgetHooks, existing.companyId); - return result; - } - - const transaction = (db as unknown as { - transaction?: (callback: (tx: unknown) => Promise) => Promise; - }).transaction; - if (typeof transaction !== "function") return applyUpdate(db); - return transaction.call(db, async (tx) => applyUpdate(tx as unknown as Db)); - } - + const { + normalizeAgentRows, + listCompanyAgentRows, + hydrateAgentSpend, + getById + } = agentRecordQueries(db); + const updateAgent = (...args: Parameters["updateConfiguration"]>) => + createAgentLifecycle(db, budgetHooks).updateConfiguration(...args); return { list: async (companyId: string, options?: { includeTerminated?: boolean }) => { const conditions = [eq(agents.companyId, companyId)]; @@ -922,326 +45,6 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { getById, - create: async (companyId: string, data: Omit, options?: CreateAgentOptions) => { - if (data.appearance?.customAvatarAssetId) throw unprocessable("Create the agent before uploading its avatar"); - assertBuiltInAgentMetadataMutationAllowed(null, data.metadata, options); - if (data.reportsTo) { - await ensureManager(companyId, data.reportsTo); - } - - const existingAgents = await db - .select({ id: agents.id, name: agents.name, status: agents.status }) - .from(agents) - .where(eq(agents.companyId, companyId)); - const uniqueName = deduplicateAgentName(data.name, existingAgents); - - const role = data.role ?? "general"; - const normalizedPermissions = normalizeAgentPermissions(data.permissions, { context: "create" }); - const runtimeConfig = normalizeRuntimeConfigForNewAgent(data.runtimeConfig); - const adapterType = data.adapterType ?? "process"; - const rawAdapterConfig = isPlainRecord(data.adapterConfig) - ? await secretsSvc.normalizeAdapterConfigForPersistence(companyId, data.adapterConfig, { adapterType }) - : {}; - const adapterConfig = normalizePaperclipRunnerAdapterConfig(adapterType, rawAdapterConfig); - // Run the server-enforced binding invariant after generic normalization - // and before any database write. A create has no prior config. - const bindingDecision = assertClaudeOAuthBindingInvariant({ - adapterType, - nextConfig: adapterConfig, - priorConfig: null, - }); - return db.transaction(async (tx) => { - const txDb = tx as unknown as Db; - // Consume the stored-session claim and create the fixed definition inside - // the same transaction that inserts the binding. A rejected claim rolls - // back the whole transaction and inserts no binding. - await enforceClaudeOAuthBindingClaim(txDb, { - companyId, - decision: bindingDecision, - consume: true, - environmentId: (data.defaultEnvironmentId as string | null | undefined) ?? null, - claudeLogin: options?.claudeLogin, - childAdapterConfig: adapterConfig, - }); - const created = await tx - .insert(agents) - .values({ - ...data, - name: uniqueName, - appearance: data.appearance == null ? randomAgentAppearance() : agentAppearanceSchema.parse(data.appearance), - companyId, - role, - adapterType, - adapterConfig, - permissions: normalizedPermissions, - runtimeConfig, - }) - .returning() - .then((rows) => rows[0]); - await agentIdentityService(txDb).ensureAgentIdentity(companyId, created.id); - // New standard agents receive the standard direct grants at activation. - // Low-trust and bundled agents keep their explicit, narrower grants. - if (created.status !== "pending_approval" && !permissionsImplyLowTrust(normalizedPermissions) && - !readBuiltInAgentMarker(created.metadata)) { - await tx.insert(principalPermissionGrants).values( - NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({ - companyId, - principalType: "agent" as const, - principalId: created.id, - permissionKey, - scope: newStandardAgentGrantScope(permissionKey, created.id), - })), - ).onConflictDoNothing(); - } - if (options?.aiConnectionInstall) { - const install = options.aiConnectionInstall; - const connectionIds = [...new Set([install.connectionId, ...(install.memberConnectionIds ?? [])])]; - await tx.insert(toolConnectionInstalls).values(connectionIds.map(connectionId => ({ - companyId, connectionId, - targetType: "agent" as const, targetId: created.id, - createdByUserId: install.createdByUserId, - }))).onConflictDoNothing(); - } - await syncAgentSecretBindings(created, txDb); - if (options?.createdByUserId && !readBuiltInAgentMarker(created.metadata)) { - await initializePrimaryAgent(txDb, companyId, options.createdByUserId, created.id); - } - if (created.status !== "pending_approval" && created.status !== "terminated") { - await recordResourceCreationEvent(txDb, companyId, "agent", created.id); - } - const normalizedCreated = await agentService(txDb).getById(created.id); - if (!normalizedCreated) { - throw notFound("Agent not found"); - } - return normalizedCreated; - }); - }, - - update: updateAgent, - - pause: async (id: string, reason: "manual" | "budget" | "system" = "manual") => { - const existing = await getById(id); - if (!existing) return null; - if (existing.status === "terminated") throw conflict("Cannot pause terminated agent"); - - return updateAgent(id, { - status: "paused", - pauseReason: reason, - pausedAt: new Date(), - errorReason: null, - }); - }, - - resume: async (id: string) => { - const existing = await getById(id); - if (!existing) return null; - if (existing.status === "terminated") throw conflict("Cannot resume terminated agent"); - if (existing.status === "pending_approval") { - throw conflict("Pending approval agents cannot be resumed"); - } - - return updateAgent(id, { - status: "idle", - pauseReason: null, - pausedAt: null, - errorReason: null, - }); - }, - - clearError: async (id: string) => { - const existing = await getById(id); - if (!existing) return null; - if (existing.status === "terminated") throw conflict("Cannot clear error on terminated agent"); - if (existing.status === "pending_approval") { - throw conflict("Pending approval agents cannot have errors cleared"); - } - if (existing.status !== "error") { - throw conflict("Only agents in error status can have their error cleared"); - } - - const updated = await db - .update(agents) - .set({ - status: "idle", - pauseReason: null, - pausedAt: null, - errorReason: null, - updatedAt: new Date(), - }) - .where(and(eq(agents.id, id), eq(agents.status, "error"))) - .returning() - .then((rows) => rows[0] ?? null); - - if (!updated) { - throw conflict("Only agents in error status can have their error cleared"); - } - return getById(updated.id); - }, - - terminate: async (id: string) => { - const existing = await getById(id); - if (!existing) return null; - - return db.transaction(async tx => { - const txDb = tx as unknown as Db; - const updated = await agentService(txDb).update(id, { - status: "terminated", - pauseReason: null, - pausedAt: null, - errorReason: null, - updatedAt: new Date(), - }); - if (!updated) return null; - await tx.update(agentApiKeys).set({ revokedAt: new Date() }).where(eq(agentApiKeys.agentId, id)); - return updated; - }); - }, - - remove: async (id: string) => { - const existing = await getById(id); - if (!existing) return null; - const builtInMarker = readBuiltInAgentMarker(existing.metadata); - if (builtInMarker) { - throw conflict("Built-in agents cannot be deleted; pause them instead", { - code: "built_in_agent_undeletable", - key: builtInMarker.key, - featureKeys: builtInMarker.featureKeys, - }); - } - - return withAccountingTransaction(db, existing.companyId, async (tx) => { - const [decisionHold] = await tx.select({ id: budgetReservations.id }).from(budgetReservations).where(and( - eq(budgetReservations.companyId, existing.companyId), eq(budgetReservations.agentId, id), - eq(budgetReservations.state, "held"), sql`${budgetReservations.decisionInvocationId} is not null`, - )).limit(1); - if (decisionHold) throw conflict("Wait for active decisions or resolve their unknown charges in Costs before deleting this agent", { - code: "agent_decision_accounting_pending", - }); - await tx - .select({ id: agents.id }) - .from(agents) - .where(eq(agents.id, id)) - .for("update"); - await issueThreadInteractionService(tx as unknown as Db) - .cancelPendingForDeletedAddressee(existing.companyId, id); - await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id)); - await tx - .update(issues) - .set({ assigneeAgentId: null, createdByAgentId: null }) - .where(or(eq(issues.assigneeAgentId, id), eq(issues.createdByAgentId, id))); - await tx.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.agentId, id)); - await tx.delete(agentTaskSessions).where(eq(agentTaskSessions.agentId, id)); - await tx.delete(activityLog).where( - or( - eq(activityLog.agentId, id), - sql`${activityLog.runId} in (select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.agentId} = ${id})`, - ), - ); - await tx.delete(issueExecutionDecisions).where(eq(issueExecutionDecisions.actorAgentId, id)); - await tx.delete(issueComments).where(eq(issueComments.authorAgentId, id)); - await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.agentId, id)); - await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, id)); - await tx.delete(agentApiKeys).where(eq(agentApiKeys.agentId, id)); - await tx.delete(agentRuntimeState).where(eq(agentRuntimeState.agentId, id)); - await tx.delete(principalPermissionGrants).where(and( - eq(principalPermissionGrants.companyId, existing.companyId), - eq(principalPermissionGrants.principalType, "agent"), - eq(principalPermissionGrants.principalId, id), - )); - const deleted = await tx - .delete(agents) - .where(eq(agents.id, id)) - .returning() - .then((rows) => rows[0] ?? null); - return deleted ? normalizeAgentRow(deleted) : null; - }); - }, - - activatePendingApproval: async (id: string, approvedPayload?: Record | null) => { - const activatedAgent = await db.transaction(async (tx) => { - const txDb = tx as unknown as Db; - const existing = await agentService(txDb).getById(id); - if (!existing || existing.status !== "pending_approval") return null; - const approvedPatch = approvedPayload ? configPatchFromApprovalPayload(approvedPayload) : {}; - let patch = { ...approvedPatch } as Partial; - let approvalBindingDecision: ClaudeOAuthBindingInvariantDecision | null = null; - if ( - Object.prototype.hasOwnProperty.call(patch, "adapterConfig") && - isPlainRecord(patch.adapterConfig) - ) { - const normalizedAdapterConfig = await secretService(txDb).normalizeAdapterConfigForPersistence( - existing.companyId, - patch.adapterConfig, - { adapterType: (patch.adapterType ?? existing.adapterType) as string }, - ); - patch.adapterConfig = normalizePaperclipRunnerAdapterConfig( - (patch.adapterType ?? existing.adapterType) as string, - normalizedAdapterConfig, - ); - // The approval activation keeps an existing fixed binding but rejects a - // newly introduced binding, because it carries no stored-session claim. - approvalBindingDecision = assertClaudeOAuthBindingInvariant({ - adapterType: (patch.adapterType ?? existing.adapterType) as string, - nextConfig: patch.adapterConfig, - priorConfig: existing.adapterConfig, - }); - } else if ( - Object.prototype.hasOwnProperty.call(patch, "adapterType") - && isPlainRecord(existing.adapterConfig) - ) { - patch.adapterConfig = normalizePaperclipRunnerAdapterConfig( - patch.adapterType as string, - existing.adapterConfig, - ); - } - if (patch.permissions !== undefined) { - // The pending-approval activation replays the original hire - // request, so the new-agent creation default applies. - patch.permissions = normalizeAgentPermissions(patch.permissions, { context: "create" }); - } - const updated = await tx - .update(agents) - .set({ ...patch, status: "idle", updatedAt: new Date() }) - .where(and(eq(agents.id, id), eq(agents.status, "pending_approval"))) - .returning() - .then((rows) => rows[0] ?? null); - if (!updated) return null; - if (approvalBindingDecision) { - await enforceClaudeOAuthBindingClaim(txDb, { - companyId: existing.companyId, - decision: approvalBindingDecision, - consume: false, - environmentId: null, - }); - } - await syncAgentSecretBindings(updated, txDb, existing.adapterConfig); - if (!permissionsImplyLowTrust(updated.permissions) && !readBuiltInAgentMarker(updated.metadata)) { - await tx.insert(principalPermissionGrants).values( - NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({ - companyId: updated.companyId, - principalType: "agent" as const, - principalId: updated.id, - permissionKey, - scope: newStandardAgentGrantScope(permissionKey, updated.id), - })), - ).onConflictDoNothing(); - } - await recordResourceCreationEvent(txDb, existing.companyId, "agent", updated.id); - const agent = await agentService(txDb).getById(updated.id); - if (!agent) { - throw notFound("Agent not found"); - } - return agent; - }); - - if (activatedAgent) { - return { agent: activatedAgent, activated: true }; - } - - const existing = await getById(id); - return existing ? { agent: existing, activated: false } : null; - }, - updatePermissions: async (id: string, permissions: Record & { canCreateAgents: boolean }) => { const existing = await getById(id); if (!existing) return null; @@ -1474,5 +277,8 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { } return { agent: null, ambiguous: false } as const; }, + update: updateAgent, + remove: (id: string) => createAgentLifecycle(db, budgetHooks).purgeAgent(id), + clearError: (id: string) => createAgentLifecycle(db, budgetHooks).clearError(id), }; } diff --git a/server/src/services/ai-api-key-test.ts b/server/src/services/ai-api-key-test.ts new file mode 100644 index 0000000000..1e5405d708 --- /dev/null +++ b/server/src/services/ai-api-key-test.ts @@ -0,0 +1,38 @@ +import type { AiProvider } from "@paperclipai/shared"; +import { unprocessable } from "../errors.js"; + +/** Fixed provider endpoints; credentials are never sent to a caller-supplied URL or through a redirect. */ +export async function validateAiApiKey( + provider: AiProvider, + key: string, + request: typeof fetch = fetch, +) { + const endpoints = { + anthropic: "https://api.anthropic.com/v1/models?limit=1", + openai: "https://api.openai.com/v1/models", + openrouter: "https://openrouter.ai/api/v1/key", + xai: "https://api.x.ai/v1/models", + google: "https://generativelanguage.googleapis.com/v1beta/models?pageSize=1", + }; + let response: Response; + try { + response = await request(endpoints[provider], { + redirect: "error", + signal: AbortSignal.timeout(15000), + headers: + provider === "google" ? { "x-goog-api-key": key } : provider === "anthropic" + ? { "x-api-key": key, "anthropic-version": "2023-06-01" } + : { Authorization: `Bearer ${key}` }, + }); + } catch { + throw unprocessable("Could not verify the account. Try again.", { code: "ai_connection_verification_failed" }); + } + await response.body?.cancel(); + if (!response.ok) + throw unprocessable( + response.status === 401 || response.status === 403 + ? "The provider rejected this API key." + : "The provider could not verify this account. Try again.", + { code: response.status === 401 || response.status === 403 ? "ai_connection_api_key_rejected" : "ai_connection_verification_failed" }, + ); +} diff --git a/server/src/services/approvals.ts b/server/src/services/approvals.ts index f90542846c..dffd11b72e 100644 --- a/server/src/services/approvals.ts +++ b/server/src/services/approvals.ts @@ -1,21 +1,19 @@ -import { agentAppearanceSchema } from "@paperclipai/shared"; +import { approvalResolutionRecords, resolvableApprovalStatuses as resolvableStatuses } from "../lib/approval-records.js"; +import { budgetService } from "./budgets.js"; +import { notifyHireApproved } from "./hire-hook.js"; + import { and, asc, eq, inArray, sql, type SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { approvalComments, approvals } from "@paperclipai/db"; -import { notFound, unprocessable } from "../errors.js"; +import { unprocessable } from "../errors.js"; import { redactCurrentUserText } from "../log-redaction.js"; -import { agentService } from "./agents.js"; -import { budgetService, budgetServiceInTransaction } from "./budgets.js"; -import { withAccountingTransaction } from "./accounting-transaction.js"; -import { notifyHireApproved } from "./hire-hook.js"; +import { createAgentLifecycle } from "./agent-lifecycle.js"; + import { instanceSettingsService } from "./instance-settings.js"; -export function approvalService(db: Db) { +export function approvalRecords(db: Db) { const instanceSettings = instanceSettingsService(db); - const canResolveStatuses = new Set(["pending", "revision_requested"]); - const resolvableStatuses = Array.from(canResolveStatuses); - type ApprovalRecord = typeof approvals.$inferSelect; - type ResolutionResult = { approval: ApprovalRecord; applied: boolean }; + const { getExistingApproval, resolveApproval } = approvalResolutionRecords(db); function redactApprovalComment(comment: T, censorUsernameInLogs: boolean): T { return { @@ -24,69 +22,8 @@ export function approvalService(db: Db) { }; } - async function reconcileApprovedBuiltInAgent(companyId: string, payload: Record, database: Db) { - const sourceBuiltInAgentKey = typeof payload.sourceBuiltInAgentKey === "string" ? payload.sourceBuiltInAgentKey : null; - if (!sourceBuiltInAgentKey) return; - const { builtInAgentService } = await import("./built-in-agents.js"); - await builtInAgentService(database).ensure(companyId, sourceBuiltInAgentKey); - } - - async function getExistingApproval(id: string, database: Db = db) { - const existing = await database - .select() - .from(approvals) - .where(eq(approvals.id, id)) - .then((rows) => rows[0] ?? null); - if (!existing) throw notFound("Approval not found"); - return existing; - } - - async function resolveApproval( - id: string, - targetStatus: "approved" | "rejected", - decidedByUserId: string, - decisionNote: string | null | undefined, - database: Db = db, - ): Promise { - const existing = await getExistingApproval(id, database); - if (!canResolveStatuses.has(existing.status)) { - if (existing.status === targetStatus) { - return { approval: existing, applied: false }; - } - throw unprocessable( - `Only pending or revision requested approvals can be ${targetStatus === "approved" ? "approved" : "rejected"}`, - ); - } - - const now = new Date(); - const updated = await database - .update(approvals) - .set({ - status: targetStatus, - decidedByUserId, - decisionNote: decisionNote ?? null, - decidedAt: now, - updatedAt: now, - }) - .where(and(eq(approvals.id, id), inArray(approvals.status, resolvableStatuses))) - .returning() - .then((rows) => rows[0] ?? null); - - if (updated) { - return { approval: updated, applied: true }; - } - - const latest = await getExistingApproval(id, database); - if (latest.status === targetStatus) { - return { approval: latest, applied: false }; - } - - throw unprocessable( - `Only pending or revision requested approvals can be ${targetStatus === "approved" ? "approved" : "rejected"}`, - ); - } - return { + getExistingApproval, resolveApproval, list: (companyId: string, status?: string, readCondition?: SQL) => { const conditions = [eq(approvals.companyId, companyId)]; if (status) conditions.push(eq(approvals.status, status)); @@ -142,113 +79,6 @@ export function approvalService(db: Db) { return updated; }, - approve: async (id: string, decidedByUserId: string, decisionNote?: string | null) => { - const now = new Date(); - const existing = await getExistingApproval(id); - // Receipt writers lock company before agent. Hire approval must use that - // order too, including activation of an existing pending agent. - const result = await withAccountingTransaction(db, existing.companyId, async (txDb, publications) => { - const agentsSvc = agentService(txDb); - const budgets = budgetServiceInTransaction(txDb, publications); - const { approval: updated, applied } = await resolveApproval( - id, - "approved", - decidedByUserId, - decisionNote, - txDb, - ); - - let hireApprovedAgentId: string | null = null; - if (applied && updated.type === "hire_agent") { - const payload = updated.payload as Record; - const payloadAgentId = typeof payload.agentId === "string" ? payload.agentId : null; - if (payloadAgentId) { - await agentsSvc.activatePendingApproval(payloadAgentId, payload); - await reconcileApprovedBuiltInAgent(updated.companyId, payload, txDb); - hireApprovedAgentId = payloadAgentId; - } else { - const created = await agentsSvc.create(updated.companyId, { - name: String(payload.name ?? "New Agent"), - appearance: payload.appearance == null ? undefined : agentAppearanceSchema.parse(payload.appearance), - role: String(payload.role ?? "general"), - title: typeof payload.title === "string" ? payload.title : null, - reportsTo: typeof payload.reportsTo === "string" ? payload.reportsTo : null, - capabilities: typeof payload.capabilities === "string" ? payload.capabilities : null, - adapterType: String(payload.adapterType ?? "process"), - adapterConfig: - typeof payload.adapterConfig === "object" && payload.adapterConfig !== null - ? (payload.adapterConfig as Record) - : {}, - budgetMonthlyCents: - typeof payload.budgetMonthlyCents === "number" ? payload.budgetMonthlyCents : 0, - metadata: - typeof payload.metadata === "object" && payload.metadata !== null - ? (payload.metadata as Record) - : null, - status: "idle", - spentMonthlyCents: 0, - permissions: undefined, - lastHeartbeatAt: null, - }, { createdByUserId: updated.requestedByAgentId ? null : updated.requestedByUserId }); - hireApprovedAgentId = created?.id ?? null; - } - if (hireApprovedAgentId) { - const budgetMonthlyCents = - typeof payload.budgetMonthlyCents === "number" ? payload.budgetMonthlyCents : 0; - if (budgetMonthlyCents > 0) { - await budgets.upsertPolicy( - updated.companyId, - { - scopeType: "agent", - scopeId: hireApprovedAgentId, - amount: budgetMonthlyCents, - windowKind: "calendar_month_utc", - }, - decidedByUserId, - ); - } - - } - } - - return { approval: updated, applied, hireApprovedAgentId }; - }); - if (result.hireApprovedAgentId) { - await budgetService(db).deliverPendingEnforcement(result.approval.companyId); - void notifyHireApproved(db, { - companyId: result.approval.companyId, - agentId: result.hireApprovedAgentId, - source: "approval", - sourceId: id, - approvedAt: now, - }).catch(() => {}); - } - return { approval: result.approval, applied: result.applied }; - }, - - reject: async (id: string, decidedByUserId: string, decisionNote?: string | null) => { - return db.transaction(async tx => { - const txDb = tx as unknown as Db; - const { approval: updated, applied } = await resolveApproval( - id, - "rejected", - decidedByUserId, - decisionNote, - txDb, - ); - - if (applied && updated.type === "hire_agent") { - const payload = updated.payload as Record; - const payloadAgentId = typeof payload.agentId === "string" ? payload.agentId : null; - if (payloadAgentId) { - await agentService(txDb).terminate(payloadAgentId); - } - } - - return { approval: updated, applied }; - }); - }, - requestRevision: async (id: string, decidedByUserId: string, decisionNote?: string | null) => { const existing = await getExistingApproval(id); if (existing.status !== "pending") { @@ -332,3 +162,40 @@ export function approvalService(db: Db) { }, }; } + +export function approvalService(db: Db) { + const { getExistingApproval, resolveApproval, ...records } = approvalRecords(db); + async function reconcileApprovedBuiltInAgent(companyId: string, payload: Record, database: Db) { + const sourceBuiltInAgentKey = typeof payload.sourceBuiltInAgentKey === "string" ? payload.sourceBuiltInAgentKey : null; + if (!sourceBuiltInAgentKey) return; + const { builtInAgentService } = await import("./built-in-agents.js"); + await builtInAgentService(database).ensure(companyId, sourceBuiltInAgentKey); + } + + async function decideHire(target: string | { approvalId: string }, status: "approved" | "rejected", userId: string, note?: string | null) { + const lifecycle = createAgentLifecycle(db); + const result = await (status === "approved" ? lifecycle.approveHire : lifecycle.rejectHire)(target, userId, note); + if (result?.hireApprovedAgentId && result.approval) { + await reconcileApprovedBuiltInAgent(result.approval.companyId, result.approval.payload, db); + await budgetService(db).deliverPendingEnforcement(result.approval.companyId); + void notifyHireApproved(db, { companyId: result.approval.companyId, agentId: result.hireApprovedAgentId, + source: "approval", sourceId: result.approval.id, approvedAt: result.approval.decidedAt ?? new Date() }).catch(() => {}); + } + return result; + } + + async function decide(id: string, status: "approved" | "rejected", userId: string, note?: string | null) { + const approval = await getExistingApproval(id); + if (approval.type === "hire_agent") { + const result = await decideHire({ approvalId: id }, status, userId, note); + return { approval: result!.approval!, applied: result!.applied }; + } + return resolveApproval(id, status, userId, note); + } + return { ...records, + approveHire: (agentId: string, userId: string, note?: string | null) => decideHire(agentId, "approved", userId, note), + rejectHire: (agentId: string, userId: string, note?: string | null) => decideHire(agentId, "rejected", userId, note), + approve: (id: string, userId: string, note?: string | null) => decide(id, "approved", userId, note), + reject: (id: string, userId: string, note?: string | null) => decide(id, "rejected", userId, note), + }; +} diff --git a/server/src/services/authorization.ts b/server/src/services/authorization.ts index 47732acf76..533cf186f5 100644 --- a/server/src/services/authorization.ts +++ b/server/src/services/authorization.ts @@ -45,7 +45,7 @@ import { type TrustPresetResolution, } from "./trust-preset-resolver.js"; import { logger } from "../middleware/logger.js"; -import { normalizeAgentPermissions } from "./agent-permissions.js"; +import { normalizeAgentPermissions } from "../lib/agent-permissions.js"; import { grantsForHumanRole, normalizeHumanRole } from "./company-member-roles.js"; export type AuthorizationActor = diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 9de0a2dd09..f938b168ee 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,7 +1,7 @@ import { compareCents, normalizeCents } from "@paperclipai/shared"; import { and, desc, eq, gte, inArray, lt, ne, sql, or, isNull, type SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { recordAgentStatusEvent } from "./resource-lifecycle-events.js"; + import { agents, approvals, @@ -225,7 +225,7 @@ function observedBlocks(policy: PolicyRow, observed: Awaited= 0 || (observed.pendingRunCount - observed.recoveringRunCount) > 0 || (policy.unpricedUsagePolicy !== "allow" && observed.unpricedEventCount > 0); } -async function policyBlocks(db: Db, policy: PolicyRow) { +export async function policyBlocks(db: Db, policy: PolicyRow) { if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false; return observedBlocks(policy, await computeObservedSpend(db, policy)); } @@ -282,18 +282,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const recordActivity = (input: LogActivityInput) => logActivity(db, input, publications); async function pauseScopeForBudget(policy: PolicyRow) { const now = new Date(); - if (policy.scopeType === "agent") { - await db.transaction(async tx => { - const [agent] = await tx.select().from(agents) - .where(and(eq(agents.id, policy.scopeId), eq(agents.companyId, policy.companyId))).for("update"); - if (agent && ["active", "idle", "running", "error"].includes(agent.status)) { - await tx.update(agents).set({ status: "paused", pauseReason: "budget", pausedAt: now, updatedAt: now }) - .where(eq(agents.id, agent.id)); - await recordAgentStatusEvent(tx as unknown as Db, agent.companyId, agent.id, agent.status, "paused"); - } - }); - return; - } + if (policy.scopeType === "agent") return; if (policy.scopeType === "project") { await db @@ -334,18 +323,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (await policyBlocks(db, candidate)) return; } const now = new Date(); - if (policy.scopeType === "agent") { - await db.transaction(async tx => { - const [agent] = await tx.select().from(agents) - .where(and(eq(agents.id, policy.scopeId), eq(agents.companyId, policy.companyId))).for("update"); - if (agent?.status === "paused" && agent.pauseReason === "budget") { - await tx.update(agents).set({ status: "idle", pauseReason: null, pausedAt: null, updatedAt: now }) - .where(eq(agents.id, agent.id)); - await recordAgentStatusEvent(tx as unknown as Db, agent.companyId, agent.id, agent.status, "idle"); - } - }); - return; - } + if (policy.scopeType === "agent") return; if (policy.scopeType === "project") { await db @@ -993,7 +971,10 @@ export async function withCurrentBudgetEnforcement(db: Db, scope: BudgetEnfor /** Cancellation is an at-least-once external effect. A failed delivery never * rolls back committed spend, and its version remains pending for recovery. */ -export async function deliverBudgetEnforcement(db: Db, hooks: BudgetServiceHooks, companyId?: string) { +export async function deliverBudgetEnforcement(db: Db, hooks: BudgetServiceHooks, companyId?: string, agentId?: string | null) { + const { createAgentLifecycle } = await import("./agent-lifecycle.js"); + // Global recovery uses the bounded lifecycle sweep for policy holds. + if (companyId) await createAgentLifecycle(db).reconcilePolicyHolds(companyId, agentId); if (!hooks.cancelWorkForScope) return; const pending = await db.select().from(budgetPolicies).where(and( sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`, @@ -1017,9 +998,9 @@ export async function deliverBudgetEnforcement(db: Db, hooks: BudgetServiceHooks export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) { const reads = budgetServiceInTransaction(db); - async function mutate(companyId: string, work: (service: ReturnType) => Promise) { + async function mutate(companyId: string, work: (service: ReturnType) => Promise, agentId?: string | null) { const result = await withAccountingTransaction(db, companyId, (tx, publications) => work(budgetServiceInTransaction(tx, publications))); - await deliverBudgetEnforcement(db, hooks, companyId); + await deliverBudgetEnforcement(db, hooks, companyId, agentId); return result; } return { @@ -1044,7 +1025,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) { evaluateCostEvent: (event: typeof costEvents.$inferSelect) => mutate(event.companyId, (service) => service.evaluateCostEvent(event)), getInvocationBlock: (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) => - mutate(companyId, (service) => service.getInvocationBlock(companyId, agentId, context)), + mutate(companyId, (service) => service.getInvocationBlock(companyId, agentId, context), agentId), resolveIncident: (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string) => mutate(companyId, (service) => service.resolveIncident(companyId, incidentId, input, actorUserId)), }; diff --git a/server/src/services/built-in-agents.ts b/server/src/services/built-in-agents.ts index 1b5f70308d..989f7b93b3 100644 --- a/server/src/services/built-in-agents.ts +++ b/server/src/services/built-in-agents.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "./agent-lifecycle.js"; import fs from "node:fs/promises"; import { readFileSync } from "node:fs"; import { createRequire } from "node:module"; @@ -21,7 +22,7 @@ import { approvalService } from "./approvals.js"; import { readBuiltInAgentMarker, withBuiltInAgentMarker, -} from "./built-in-agent-metadata.js"; +} from "../lib/built-in-agent-metadata.js"; import { companySkillService } from "./company-skills.js"; import { routineService } from "./routines.js"; import { accessService } from "./access.js"; @@ -805,6 +806,7 @@ function isBuiltInAgentMarkerConflict(error: unknown): boolean { export function builtInAgentService(db: Db) { const agentSvc = agentService(db); + const agentSvcLifecycle = createAgentLifecycle(db); const accessSvc = accessService(db); const approvalSvc = approvalService(db); const instructionsSvc = agentInstructionsService(db); @@ -1403,8 +1405,8 @@ export function builtInAgentService(db: Db) { } async function ensureBuiltInAgentAssignable(agent: Agent) { - if (agent.status !== "paused") return agent; - const resumed = await agentSvc.resume(agent.id); + if (agent.status !== "paused" || ["preparing", "verifying", "resuming"].includes(agent.lifecycleState ?? "")) return agent; + const resumed = await agentSvcLifecycle.resumeAgent(agent.id); if (!resumed) throw notFound("Built-in agent not found"); return resumed as Agent; } @@ -1664,7 +1666,7 @@ export function builtInAgentService(db: Db) { const [keep, ...duplicates] = markedRows; for (const duplicate of duplicates) { const openApproval = await approvalSvc.findOpenHireApprovalForAgent(companyId, duplicate.id); - await agentSvc.terminate(duplicate.id); + await agentSvcLifecycle.terminateAgent(duplicate.id); if (openApproval) { await approvalSvc.cancel( openApproval.id, @@ -1785,7 +1787,7 @@ export function builtInAgentService(db: Db) { : null; let created: Agent; try { - created = await agentSvc.create(companyId, { + created = await agentSvcLifecycle.requestHire(companyId, { ...definitionPatch(definition, resolvedInput), status: definition.defaultStatus ?? "idle", pauseReason: definition.defaultStatus === "paused" @@ -1890,7 +1892,7 @@ export function builtInAgentService(db: Db) { : null; let pending: Agent; try { - pending = await agentSvc.create(companyId, { + pending = await agentSvcLifecycle.requestHire(companyId, { ...definitionPatch(definition, input), status: "pending_approval", reportsTo, diff --git a/server/src/services/chat-completion-delivery.ts b/server/src/services/chat-completion-delivery.ts index 1cbd95b500..18ddabd254 100644 --- a/server/src/services/chat-completion-delivery.ts +++ b/server/src/services/chat-completion-delivery.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { and, asc, eq, inArray, lte, sql } from "drizzle-orm"; import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs, heartbeatRuns, issueComments, issueDocuments, issues, type Db } from "@paperclipai/db"; @@ -31,13 +32,16 @@ export async function recordChatHandoff(tx: Connection, task: Issue, actorRunId: } /** Must run on the same transaction as status projection (including native arbitration). */ -export async function recordChatCompletion(tx: Connection, before: Issue, after: Issue) { - if (before.status === after.status) return; +export async function recordChatCompletion(tx: Connection, before: Issue, after: Issue): Promise { + if (before.status === after.status) return false; await tx.update(deliveries).set({ status: "superseded" }).where(and(eq(deliveries.taskId, after.id), eq(deliveries.companyId, after.companyId), inArray(deliveries.status, [...pending]))); - if (after.status !== "done") return; + if (after.status !== "done") return false; const [handoff] = await tx.select().from(handoffs).where(and(eq(handoffs.taskId, after.id), eq(handoffs.companyId, after.companyId))); - if (handoff) await tx.insert(deliveries).values({ companyId: after.companyId, taskId: after.id, statusVersion: after.statusVersion }).onConflictDoNothing(); + if (!handoff) return false; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatCompletion); + await tx.insert(deliveries).values({ companyId: after.companyId, taskId: after.id, statusVersion: after.statusVersion }).onConflictDoNothing(); + return true; } async function loadAudience(tx: Connection, deliveryId: string) { @@ -243,5 +247,9 @@ export function chatCompletionDeliveryService(db: Db, heartbeat: { wakeup(agentI .orderBy(asc(deliveries.nextAttemptAt)).limit(100); for (const row of due) await deliver(row.id); } - return { deliver, sweepPending }; + async function hasPending() { + return (await db.select({ id: deliveries.id }).from(deliveries) + .where(inArray(deliveries.status, [...pending])).limit(1)).length > 0; + } + return { deliver, sweepPending, hasPending }; } diff --git a/server/src/services/chat-publication-reconciliation.test.ts b/server/src/services/chat-publication-reconciliation.test.ts index 46c27ab8ce..3af863db2d 100644 --- a/server/src/services/chat-publication-reconciliation.test.ts +++ b/server/src/services/chat-publication-reconciliation.test.ts @@ -220,18 +220,18 @@ describe("createCoalescedAsyncTrigger", () => { describe("chat publication commit signals", () => { it("emits the accepted signals only after their durable source writes", () => { - const heartbeatSource = readFileSync( - new URL("./heartbeat.ts", import.meta.url), + const lifecycleSource = readFileSync( + new URL("./heartbeat/run-lifecycle.ts", import.meta.url), "utf8", ); - const appendRunEventStart = heartbeatSource.indexOf( + const appendRunEventStart = lifecycleSource.indexOf( "async function appendRunEvent(", ); - const persistedEvent = heartbeatSource.indexOf( + const persistedEvent = lifecycleSource.indexOf( "await appendHeartbeatRunEvent", appendRunEventStart, ); - const emittedEvent = heartbeatSource.indexOf( + const emittedEvent = lifecycleSource.indexOf( "publishLiveEvent({", persistedEvent, ); @@ -239,6 +239,10 @@ describe("chat publication commit signals", () => { expect(persistedEvent).toBeGreaterThan(appendRunEventStart); expect(emittedEvent).toBeGreaterThan(persistedEvent); + const heartbeatSource = readFileSync( + new URL("./heartbeat.ts", import.meta.url), + "utf8", + ); const presentationMarker = heartbeatSource.indexOf( 'eventType: "run.presentation.resolved"', ); diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 94c058e71e..149b30b4fe 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,3 +1,5 @@ +import { deleteCompany } from "./company-deletion.js"; +import { createAgentLifecycle } from "./agent-lifecycle.js"; import { publishAccountingActivities } from "./accounting-transaction.js"; import { budgetServiceInTransaction, deliverBudgetEnforcement, type BudgetServiceHooks } from "./budgets.js"; import { and, count, eq, gte, inArray, isNull, lt, notInArray, sql } from "drizzle-orm"; @@ -7,37 +9,10 @@ import { companyLogos, assets, agents, - agentApiKeys, - agentRuntimeState, - agentTaskSessions, agentWakeupRequests, issues, - issueComments, - projects, - goals, heartbeatRuns, - runIdentityContexts, - heartbeatRunEvents, costEvents, - decisionInvocations, - financeEvents, - budgetPolicies, - budgetIncidents, - issueReadStates, - approvalComments, - approvals, - activityLog, - companySecrets, - joinRequests, - invites, - principalPermissionGrants, - companyMemberships, - companySkills, - documents, - routineRuns, - routineTriggers, - routineRevisions, - routines, } from "@paperclipai/db"; import { notFound, unprocessable } from "../errors.js"; import { isCloudManagedInstance } from "./cloud-instance.js"; @@ -79,19 +54,7 @@ export function companyService(db: Db, budgetHooks: BudgetServiceHooks = {}) { type CompanyTx = Parameters[0]>[0]; async function applyArchiveCascadeInTx(tx: CompanyTx, id: string) { - const pausedAgentRows = await tx - .update(agents) - .set({ - status: "paused", - pauseReason: "company_archived", - pausedAt: new Date(), - updatedAt: new Date(), - }) - .where(and( - eq(agents.companyId, id), - notInArray(agents.status, ["paused", "terminated", "pending_approval"]), - )) - .returning({ id: agents.id }); + const pausedAgentRows = await tx.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, id), notInArray(agents.status, ["paused", "terminated", "pending_approval"]))); const activeRunIds = await tx .select({ id: heartbeatRuns.id }) @@ -399,20 +362,7 @@ export function companyService(db: Db, budgetHooks: BudgetServiceHooks = {}) { let agentsRestored = 0; if (willReactivate) { - const restoredRows = await tx - .update(agents) - .set({ - status: "idle", - pauseReason: null, - pausedAt: null, - updatedAt: new Date(), - }) - .where(and( - eq(agents.companyId, id), - eq(agents.status, "paused"), - eq(agents.pauseReason, "company_archived"), - )) - .returning({ id: agents.id }); + const restoredRows = await tx.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, id), eq(agents.pauseReason, "company_archived"))); agentsRestored = restoredRows.length; } @@ -464,6 +414,7 @@ export function companyService(db: Db, budgetHooks: BudgetServiceHooks = {}) { }; }); if (!result) return null; + await createAgentLifecycle(db).reconcilePolicyHolds(id); publishAccountingActivities(id, budgetPublications); if (data.budgetMonthlyCents !== undefined) await deliverBudgetEnforcement(db, budgetHooks, id); // Post-commit, fire-and-forget, and BEFORE any finalization that @@ -542,6 +493,7 @@ export function companyService(db: Db, budgetHooks: BudgetServiceHooks = {}) { }; }); if (!result) return null; + await createAgentLifecycle(db).reconcilePolicyHolds(id); // Same doorbell rule as update(): the archive is committed, so ring // before finalization, which can throw without undoing it. @@ -553,67 +505,7 @@ export function companyService(db: Db, budgetHooks: BudgetServiceHooks = {}) { return result.company; }, - remove: (id: string) => - db.transaction(async (tx) => { - // Exclude accounting writers before taking child locks. KEY SHARE must - // remain compatible: native writers can already hold a child row while - // saving a company-scoped result. FOR UPDATE would deadlock that save. - const [existing] = await tx.select({ id: companies.id }).from(companies) - .where(eq(companies.id, id)).for("no key update"); - if (!existing) return null; - // Finance can reference costs and both can reference runs. Incidents - // reference policies and approvals; delete these dependents first. - await tx.delete(financeEvents).where(eq(financeEvents.companyId, id)); - await tx.delete(decisionInvocations).where(eq(decisionInvocations.companyId, id)); - await tx.delete(costEvents).where(eq(costEvents.companyId, id)); - await tx.delete(budgetIncidents).where(eq(budgetIncidents.companyId, id)); - await tx.delete(budgetPolicies).where(eq(budgetPolicies.companyId, id)); - // Delete from child tables in dependency order - const companyRunIds = await tx - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.companyId, id)); - - await tx.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, id)); - if (companyRunIds.length > 0) { - await tx - .delete(heartbeatRunEvents) - .where(inArray(heartbeatRunEvents.runId, companyRunIds.map((run) => run.id))); - } - await tx.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, id)); - await tx.delete(activityLog).where(eq(activityLog.companyId, id)); - await tx.delete(runIdentityContexts).where(eq(runIdentityContexts.companyId, id)); - await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, id)); - await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, id)); - await tx.delete(agentApiKeys).where(eq(agentApiKeys.companyId, id)); - await tx.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, id)); - await tx.delete(issueComments).where(eq(issueComments.companyId, id)); - await tx.delete(approvalComments).where(eq(approvalComments.companyId, id)); - await tx.delete(approvals).where(eq(approvals.companyId, id)); - await tx.delete(companySecrets).where(eq(companySecrets.companyId, id)); - await tx.delete(joinRequests).where(eq(joinRequests.companyId, id)); - await tx.delete(invites).where(eq(invites.companyId, id)); - await tx.delete(principalPermissionGrants).where(eq(principalPermissionGrants.companyId, id)); - await tx.delete(companyMemberships).where(eq(companyMemberships.companyId, id)); - await tx.delete(companySkills).where(eq(companySkills.companyId, id)); - await tx.delete(routineRuns).where(eq(routineRuns.companyId, id)); - await tx.delete(routineTriggers).where(eq(routineTriggers.companyId, id)); - await tx.delete(routineRevisions).where(eq(routineRevisions.companyId, id)); - await tx.delete(routines).where(eq(routines.companyId, id)); - await tx.delete(issueReadStates).where(eq(issueReadStates.companyId, id)); - await tx.delete(documents).where(eq(documents.companyId, id)); - await tx.delete(issues).where(eq(issues.companyId, id)); - await tx.delete(companyLogos).where(eq(companyLogos.companyId, id)); - await tx.delete(assets).where(eq(assets.companyId, id)); - await tx.delete(goals).where(eq(goals.companyId, id)); - await tx.delete(projects).where(eq(projects.companyId, id)); - await tx.delete(agents).where(eq(agents.companyId, id)); - const rows = await tx - .delete(companies) - .where(eq(companies.id, id)) - .returning(); - return rows[0] ?? null; - }), + remove: (id: string) => deleteCompany(db, id), stats: () => Promise.all([ diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts new file mode 100644 index 0000000000..4ab40f03e6 --- /dev/null +++ b/server/src/services/company-deletion.ts @@ -0,0 +1,110 @@ +import type { Db } from "@paperclipai/db"; +import { eq, inArray } from "drizzle-orm"; +import { createAgentLifecycle, scheduleAgentLifecycle } from "./agent-lifecycle.js"; +import { agentLifecycleCompanyDeletion } from "../modules/agent-lifecycle/company-deletion.js"; +import { + companies, + agents, + companyLogos, + assets, + agentApiKeys, + agentRuntimeState, + agentTaskSessions, + agentWakeupRequests, + issues, + issueComments, + projects, + goals, + heartbeatRuns, + runIdentityContexts, + heartbeatRunEvents, + costEvents, + decisionInvocations, + financeEvents, + budgetPolicies, + budgetIncidents, + issueReadStates, + approvalComments, + approvals, + activityLog, + companySecrets, + joinRequests, + invites, + principalPermissionGrants, + companyMemberships, + companySkills, + documents, + routineRuns, + routineTriggers, + routineRevisions, + routines, +} from "@paperclipai/db"; + +export async function deleteCompany(db: Db, id: string) { + const lifecycle = createAgentLifecycle(db); + const companyAgents = await db.select({ id: agents.id }).from(agents).where(eq(agents.companyId, id)); + for (const agent of companyAgents) { + await lifecycle.terminateAgent(agent.id); + await scheduleAgentLifecycle(db, agent.id); + } + return db.transaction(async (tx) => { + // Exclude accounting writers before taking child locks. KEY SHARE must + // remain compatible: native writers can already hold a child row while + // saving a company-scoped result. FOR UPDATE would deadlock that save. + const [existing] = await tx.select({ id: companies.id }).from(companies) + .where(eq(companies.id, id)).for("no key update"); + if (!existing) return null; + // Finance can reference costs and both can reference runs. Incidents + // reference policies and approvals; delete these dependents first. + await tx.delete(financeEvents).where(eq(financeEvents.companyId, id)); + await tx.delete(decisionInvocations).where(eq(decisionInvocations.companyId, id)); + await tx.delete(costEvents).where(eq(costEvents.companyId, id)); + await tx.delete(budgetIncidents).where(eq(budgetIncidents.companyId, id)); + await tx.delete(budgetPolicies).where(eq(budgetPolicies.companyId, id)); + // Delete from child tables in dependency order + const companyRunIds = await tx + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.companyId, id)); + + await tx.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, id)); + if (companyRunIds.length > 0) { + await tx + .delete(heartbeatRunEvents) + .where(inArray(heartbeatRunEvents.runId, companyRunIds.map((run) => run.id))); + } + await tx.delete(agentTaskSessions).where(eq(agentTaskSessions.companyId, id)); + await tx.delete(activityLog).where(eq(activityLog.companyId, id)); + await tx.delete(runIdentityContexts).where(eq(runIdentityContexts.companyId, id)); + await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, id)); + await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, id)); + await tx.delete(agentApiKeys).where(eq(agentApiKeys.companyId, id)); + await tx.delete(agentRuntimeState).where(eq(agentRuntimeState.companyId, id)); + await tx.delete(issueComments).where(eq(issueComments.companyId, id)); + await tx.delete(approvalComments).where(eq(approvalComments.companyId, id)); + await tx.delete(approvals).where(eq(approvals.companyId, id)); + await tx.delete(companySecrets).where(eq(companySecrets.companyId, id)); + await tx.delete(joinRequests).where(eq(joinRequests.companyId, id)); + await tx.delete(invites).where(eq(invites.companyId, id)); + await tx.delete(principalPermissionGrants).where(eq(principalPermissionGrants.companyId, id)); + await tx.delete(companyMemberships).where(eq(companyMemberships.companyId, id)); + await tx.delete(companySkills).where(eq(companySkills.companyId, id)); + await tx.delete(routineRuns).where(eq(routineRuns.companyId, id)); + await tx.delete(routineTriggers).where(eq(routineTriggers.companyId, id)); + await tx.delete(routineRevisions).where(eq(routineRevisions.companyId, id)); + await tx.delete(routines).where(eq(routines.companyId, id)); + await tx.delete(issueReadStates).where(eq(issueReadStates.companyId, id)); + await tx.delete(documents).where(eq(documents.companyId, id)); + await tx.delete(issues).where(eq(issues.companyId, id)); + await tx.delete(companyLogos).where(eq(companyLogos.companyId, id)); + await tx.delete(assets).where(eq(assets.companyId, id)); + await tx.delete(goals).where(eq(goals.companyId, id)); + await tx.delete(projects).where(eq(projects.companyId, id)); + await agentLifecycleCompanyDeletion.deleteCompanyData(tx, id); + const rows = await tx + .delete(companies) + .where(eq(companies.id, id)) + .returning(); + return rows[0] ?? null; + }); +} diff --git a/server/src/services/company-portability-agent-selection.ts b/server/src/services/company-portability-agent-selection.ts index e258eb81e7..4f3110edfb 100644 --- a/server/src/services/company-portability-agent-selection.ts +++ b/server/src/services/company-portability-agent-selection.ts @@ -1,5 +1,5 @@ import { normalizeAgentUrlKey } from "@paperclipai/shared"; -import { readBuiltInAgentMarker } from "./built-in-agent-metadata.js"; +import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; interface ExportAgentCandidate { id: string; diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index ad438f8829..1d7b983f34 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "./agent-lifecycle.js"; import { agentAppearanceSchema } from "@paperclipai/shared"; import { createHash, randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; @@ -3559,6 +3560,7 @@ export function parseGitHubSourceUrl(rawUrl: string) { export function companyPortabilityService(db: Db, storage?: StorageService) { const companies = companyService(db); const agents = agentService(db); + const agentsLifecycle = createAgentLifecycle(db); const assetRecords = assetService(db); const instructions = agentInstructionsService(db); const access = accessService(db); @@ -5642,9 +5644,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { : {}; if (planAgent.action === "update" && planAgent.existingAgentId) { + if (pauseAutomations) await agentsLifecycle.pauseAgent(planAgent.existingAgentId, "import"); let updated = await agents.update(planAgent.existingAgentId, { ...patch, - ...automationPausePatch, }); if (!updated) { warnings.push(`Skipped update for missing agent ${planAgent.existingAgentId}.`); @@ -5690,7 +5692,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { continue; } - let created = await agents.create(targetCompany.id, { + let created = await agentsLifecycle.requestHire(targetCompany.id, { ...patch, ...automationPausePatch, status: pauseAutomations ? "paused" : "idle", diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index 0db8d650ff..fc07bf1ebe 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -1,3 +1,4 @@ +import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js"; import { logger } from "../middleware/logger.js"; import { removeRuntimeSkillCache, resolveRuntimeSkillCache, runtimeSkillCacheSpec } from "./runtime-skill-cache.js"; import { createHash, randomUUID } from "node:crypto"; @@ -3925,15 +3926,9 @@ export function companySkillService(db: Db) { ? { key: forkKey, versionId: null } : entry ); - const updated = await tx - .update(agentsTable) - .set({ - adapterConfig: writePaperclipSkillSyncPreference(adapterConfig, nextEntries), - updatedAt: new Date(), - }) - .where(and(eq(agentsTable.companyId, companyId), eq(agentsTable.id, item.agentId))) - .returning({ id: agentsTable.id }) - .then((rows) => rows[0] ?? null); + const updated = await updateAgentConfigurationInTransaction(tx as unknown as Db, item.agentId, { + adapterConfig: writePaperclipSkillSyncPreference(adapterConfig, nextEntries), + }, { allowPendingApprovalConfigUpdate: true }); if (!updated) throw notFound(`Agent not found for skill reassignment: ${item.agentId}`); } }); @@ -4196,13 +4191,9 @@ export function companySkillService(db: Db) { entry.key === previousKey ? { key: newKey, versionId: entry.versionId ?? null } : entry); - await tx - .update(agentsTable) - .set({ - adapterConfig: writePaperclipSkillSyncPreference(adapterConfig, nextEntries), - updatedAt: new Date(), - }) - .where(and(eq(agentsTable.companyId, companyId), eq(agentsTable.id, item.agentId))); + await updateAgentConfigurationInTransaction(tx as unknown as Db, item.agentId, { + adapterConfig: writePaperclipSkillSyncPreference(adapterConfig, nextEntries), + }, { allowPendingApprovalConfigUpdate: true }); } } catch (error) { // Restore while the name locks are still held. diff --git a/server/src/services/connection-intent-delivery.ts b/server/src/services/connection-intent-delivery.ts index 540524bf4e..c128d309fc 100644 --- a/server/src/services/connection-intent-delivery.ts +++ b/server/src/services/connection-intent-delivery.ts @@ -167,7 +167,11 @@ export function connectionIntentDeliveryService(db: Db, heartbeat: Pick { try { await deliver(id); } catch { /* Persisted delivery remains due after its lease. */ } }, sweepPending: async () => { + async function hasPending() { + return (await db.select({ id: connectionIntentDeliveries.interactionId }).from(connectionIntentDeliveries) + .where(isNull(connectionIntentDeliveries.deliveredAt)).limit(1)).length > 0; + } + return { deliver, hasPending, tryDeliver: async (id: string) => { try { await deliver(id); } catch { /* Persisted delivery remains due after its lease. */ } }, sweepPending: async () => { const rows = await db.select().from(connectionIntentDeliveries).where(and(isNull(connectionIntentDeliveries.deliveredAt), lte(connectionIntentDeliveries.nextAttemptAt, new Date()))) .orderBy(asc(connectionIntentDeliveries.nextAttemptAt)).limit(50); let failed = 0; diff --git a/server/src/services/connection-intents.ts b/server/src/services/connection-intents.ts index 68005f19f4..d75743e50d 100644 --- a/server/src/services/connection-intents.ts +++ b/server/src/services/connection-intents.ts @@ -1,3 +1,4 @@ +import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js"; import { emailChannelService } from "./email-channels.js"; import { emailConnectionService } from "./email-connections.js"; import { grantConnectionAgentTools } from "./connection-agent-access.js"; @@ -1035,7 +1036,7 @@ export function connectionIntentService(db: Db) { } const binding = options.validatedAdoption.binding; if (binding.provider !== payload.serviceSlug || binding.mode !== "responsible_user") throw conflict("Invalid legacy adoption binding"); - const updated = await agentService(txDb).update(agent.id, { + const updated = await updateAgentConfigurationInTransaction(txDb, agent.id, { runtimeConfig: { ...agent.runtimeConfig, aiConnection: binding }, }, { recordRevision: { createdByUserId: userId, source: "patch" } }); if (!updated) throw notFound("Agent not found"); diff --git a/server/src/services/delivery-work-coordinator.ts b/server/src/services/delivery-work-coordinator.ts new file mode 100644 index 0000000000..34a198671b --- /dev/null +++ b/server/src/services/delivery-work-coordinator.ts @@ -0,0 +1,100 @@ +import { subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "@paperclipai/db"; +import { createWorkScheduler } from "./work-scheduler.js"; +import { type DeliveryQueue } from "./delivery-work-notifications.js"; +import { beginIdleTrackedWork } from "./task-admission.js"; + +export function createDeliveryWorkCoordinator(input: { + owner: object; + canRun: () => boolean; + // Idle drain may reconcile admitted writes without dispatching new work. + // Warm standby can disable both operations. + canReconcile?: () => boolean; + onError: (error: unknown, queue: DeliveryQueue) => void; +}) { + const scheduler = createWorkScheduler(); + const workers = new Map void; stop: () => Promise }>(); + let stopped = false; + + return { + // Only these registered tasks are represented; not yet the whole instance. + nextWakeAt: scheduler.nextWakeAt, + register(queue: DeliveryQueue, task: { + retryMs: number; + run: (signal: AbortSignal) => Promise; + hasPending: () => Promise; + }) { + if (stopped) throw new Error("Delivery coordinator is stopped"); + if (workers.has(queue)) throw new Error(`Delivery worker already registered: ${queue}`); + let dirty = false; + let scheduledAt: number | null = null; + let running: Promise | null = null; + let workerStopped = false; + let controller: AbortController | null = null; + let finishIntent: (() => void) | undefined; + function schedule(delay: number) { + if (workerStopped) return; + const at = Date.now() + delay; + if (scheduledAt !== null && scheduledAt <= at) return; + scheduledAt = at; + scheduler.schedule(queue, at, start); + } + function start() { + if (workerStopped || running) return; + scheduler.cancel(queue); + scheduledAt = null; + if (!input.canRun() && !(input.canReconcile?.() ?? false)) { schedule(task.retryMs); return; } + dirty = false; + const finish = beginIdleTrackedWork(); + const attempt = new AbortController(); + controller = attempt; + running = Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue)) + .then(() => workerStopped || !input.canRun() ? undefined : task.run(attempt.signal)).then(() => workerStopped ? false : task.hasPending()).then(pending => { + if (pending || databaseWorkPending(input.owner, queue)) schedule(task.retryMs); + else { finishIntent?.(); finishIntent = undefined; } + }).catch(error => { + schedule(task.retryMs); + // Logging must never turn a recoverable sweep failure into an + // unhandled rejection or prevent other queues from running. + try { if (!workerStopped) input.onError(error, queue); } catch { /* Recovery remains scheduled. */ } + }).finally(() => { + running = null; + controller = null; + finish(); + if (dirty) schedule(0); + }); + } + function wake() { + if (workerStopped) return; + dirty = true; + if (!running) schedule(0); + } + const unsubscribe = subscribeDatabaseWork(input.owner, queue, event => { + if (workerStopped) return; + if (databaseWorkPending(input.owner, queue)) { + finishIntent ??= beginIdleTrackedWork(); + // Give the transaction time to settle. Its settlement notification + // replaces this deadline with an immediate wake on the normal path. + if (!running) schedule(task.retryMs); + } + if (event === "settled") wake(); + }); + const worker = { wake, async stop() { + workerStopped = true; + controller?.abort(new Error("Delivery worker stopped")); + unsubscribe(); + scheduler.cancel(queue); + await running; + finishIntent?.(); finishIntent = undefined; + } }; + workers.set(queue, worker); + const ready = Promise.resolve().then(() => { start(); return running; }); + return { ready, wake }; + }, + async stop() { + stopped = true; + scheduler.stop(); + await Promise.all(Array.from(workers.values(), worker => worker.stop())); + workers.clear(); + }, + }; +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts new file mode 100644 index 0000000000..14b67085d1 --- /dev/null +++ b/server/src/services/delivery-work-notifications.ts @@ -0,0 +1,22 @@ +import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; + +/** Topics for existing durable queues; register intent before their writes. */ +export const DELIVERY_QUEUES = { + feedback: "feedback-exports", + chatCompletion: "chat-completions", + connection: "connection-continuations", + question: "question-responses", + toolAction: "tool-action-receipts", +} as const; +export type DeliveryQueue = typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]; + +export async function notifyDeliveryWork(transaction: object, queue: DeliveryQueue): Promise { + await signalDatabaseWork(transaction, queue); +} +// Public delivery notification observers only see settled work. The coordinator +// uses the lower-level lifecycle subscription to also fence in-flight writes. +export function subscribeDeliveryWork(owner: object, queue: DeliveryQueue, wake: () => void): () => void { + return subscribeDatabaseWork(owner, queue, event => { + if (event === "settled") wake(); + }); +} diff --git a/server/src/services/dot-invitations.ts b/server/src/services/dot-invitations.ts index 975e7b0bc4..735b9e669f 100644 --- a/server/src/services/dot-invitations.ts +++ b/server/src/services/dot-invitations.ts @@ -1,6 +1,8 @@ import { and, desc, eq, isNull, ne, sql } from "drizzle-orm"; import { activityLog, agents, companies, dotAgentBindings, type Db } from "@paperclipai/db"; import { agentService } from "./agents.js"; +import { createAgentLifecycle, scheduleAgentLifecycle } from "./agent-lifecycle.js"; +import { withDedicatedDbConnection } from "@paperclipai/db"; import { approvalService } from "./approvals.js"; import { accessService } from "./access.js"; import { agentInstructionsService } from "./agent-instructions.js"; @@ -11,21 +13,22 @@ import { notFound, unprocessable } from "../errors.js"; /** One unfinished invitation per operator/company. Only non-secret references are persisted. */ export function dotInvitationService(db: Db) { - async function pending(database: Db, companyId: string, operatorId: string) { + async function pending(database: Db, companyId: string, operatorId: string, recoverIncomplete = false) { const rows = await database.select({ agent: agents, binding: dotAgentBindings }) .from(agents).leftJoin(dotAgentBindings, and(eq(dotAgentBindings.agentId, agents.id), isNull(dotAgentBindings.revokedAt))) .where(and(eq(agents.companyId, companyId), ne(agents.status, "terminated"), eq(agents.adapterType, "paperclip_runner"), sql`${agents.adapterConfig}->>'provider' = 'openai_dot'`, - sql`exists (select 1 from ${activityLog} where ${activityLog.companyId} = ${agents.companyId} + sql`((${recoverIncomplete} and ${agents.metadata}->'dotInvitation'->>'operatorId' = ${operatorId} + and ${agents.lifecycleOperation}->>'responsibleUserId' = ${operatorId}) or exists (select 1 from ${activityLog} where ${activityLog.companyId} = ${agents.companyId} and ${activityLog.entityId} = ${agents.id}::text and ${activityLog.entityType} = 'agent' and ${activityLog.actorType} = 'user' and ${activityLog.actorId} = ${operatorId} - and ${activityLog.action} = 'agent.hire_created' and ${activityLog.details}->>'source' = 'dot-invitation')`)).orderBy(desc(agents.createdAt)); + and ${activityLog.action} = 'agent.hire_created' and ${activityLog.details}->>'source' = 'dot-invitation'))`)).orderBy(desc(agents.createdAt)); return rows.find(row => row.binding?.status !== "ready")?.agent ?? null; } - async function describe(agent: typeof agents.$inferSelect) { - const approval = await approvalService(db).findOpenHireApprovalForAgent(agent.companyId, agent.id); + async function describe(agent: Pick, database: Db = db) { + const approval = await approvalService(database).findOpenHireApprovalForAgent(agent.companyId, agent.id); return { agent: { id: agent.id, name: agent.name, status: agent.status }, approvalId: approval?.id ?? null, - binding: await dotRunnerBroker(db).bindingForAgent(agent.companyId, agent.id) }; + binding: await dotRunnerBroker(database).bindingForAgent(agent.companyId, agent.id) }; } return { async resume(companyId: string, operatorId: string) { @@ -34,16 +37,24 @@ export function dotInvitationService(db: Db) { }, async create(companyId: string, operatorId: string) { if (!await dotRunnerBroker(db).enabled()) throw unprocessable("Enable OpenAI Dot and Assistant connections (MCP) in experimental settings."); - const agent = await db.transaction(async tx => { - const database = tx as unknown as Db; + const database = db; + const lockKey = `paperclip:dot-invitation:${companyId}:${operatorId}`; + return withDedicatedDbConnection(db, lockDb => lockDb.transaction(async lock => { + await lock.execute(sql`select pg_advisory_xact_lock(hashtextextended(${lockKey}, 0))`); // SQL locking also serializes retries on different control-plane replicas. - const [company] = await tx.select().from(companies).where(eq(companies.id, companyId)).for("update"); + const [company] = await database.select().from(companies).where(eq(companies.id, companyId)); if (!company) throw notFound("Company not found"); if (company.status !== "active") throw unprocessable("Activate this company before inviting Dot."); - const existing = await pending(database, companyId, operatorId); - if (existing) return existing; + const existing = await pending(database, companyId, operatorId, true); + if (existing) { + const [completed] = await database.select({ id: activityLog.id }).from(activityLog).where(and( + eq(activityLog.companyId, companyId), eq(activityLog.entityId, existing.id), + eq(activityLog.action, "agent.hire_created"), eq(activityLog.actorId, operatorId), + sql`${activityLog.details}->>'source' = 'dot-invitation'`)).limit(1); + if (completed) return describe(existing, database); + } const svc = agentService(database); - let created = await svc.create(companyId, { + let created = existing ?? await createAgentLifecycle(database).requestHire(companyId, { name: "Dot", role: "general", adapterType: "paperclip_runner", adapterConfig: { provider: "openai_dot", lifecycleMode: "per_turn", allowUnmeteredProvider: true, dotWorkspaceAccess: false, dotAttachmentAccess: false }, @@ -54,25 +65,28 @@ export function dotInvitationService(db: Db) { const bundle = await agentInstructionsService(database).materializeManagedBundle(created, await loadDefaultAgentInstructionsBundle("default"), { entryFile: "AGENTS.md", replaceExisting: false }); created = (await svc.update(created.id, { adapterConfig: bundle.adapterConfig }, { allowPendingApprovalConfigUpdate: true }))!; - const access = accessService(database); - await access.ensureMembership(companyId, "agent", created.id, "member", "active"); - await access.setPrincipalPermission(companyId, "agent", created.id, "tasks:assign", true, operatorId); - const approval = company.requireBoardApprovalForNewAgents ? await approvalService(database).create(companyId, { - type: "hire_agent", status: "pending", requestedByUserId: operatorId, - payload: { agentId: created.id, name: created.name, role: created.role, adapterType: created.adapterType, - adapterConfig: created.adapterConfig, runtimeConfig: created.runtimeConfig, metadata: created.metadata, - budgetMonthlyCents: created.budgetMonthlyCents, requestedConfigurationSnapshot: { - adapterType: created.adapterType, runtimeConfig: created.runtimeConfig, adapterConfig: created.adapterConfig, - } }, - }) : null; - await logActivity(database, { companyId, actorType: "user", actorId: operatorId, action: "agent.hire_created", - entityType: "agent", entityId: created.id, details: { name: created.name, role: created.role, - requiresApproval: !!approval, approvalId: approval?.id ?? null, source: "dot-invitation" } }); - if (approval) await logActivity(database, { companyId, actorType: "user", actorId: operatorId, action: "approval.created", - entityType: "approval", entityId: approval.id, details: { type: "hire_agent", agentId: created.id } }); - return created; - }); - return describe(agent); + await database.transaction(async tx => { + const txDb = tx as unknown as Db; + const access = accessService(txDb); + await access.ensureMembership(companyId, "agent", created.id, "member", "active"); + await access.setPrincipalPermission(companyId, "agent", created.id, "tasks:assign", true, operatorId); + const approval = await approvalService(txDb).findOpenHireApprovalForAgent(companyId, created.id) ?? (company.requireBoardApprovalForNewAgents ? await approvalService(txDb).create(companyId, { + type: "hire_agent", status: "pending", requestedByUserId: operatorId, + payload: { agentId: created.id, name: created.name, role: created.role, adapterType: created.adapterType, + adapterConfig: created.adapterConfig, runtimeConfig: created.runtimeConfig, metadata: created.metadata, + budgetMonthlyCents: created.budgetMonthlyCents, requestedConfigurationSnapshot: { + adapterType: created.adapterType, runtimeConfig: created.runtimeConfig, adapterConfig: created.adapterConfig, + } }, + }) : null); + await logActivity(txDb, { companyId, actorType: "user", actorId: operatorId, action: "agent.hire_created", + entityType: "agent", entityId: created.id, details: { name: created.name, role: created.role, + requiresApproval: !!approval, approvalId: approval?.id ?? null, source: "dot-invitation" } }); + if (approval) await logActivity(txDb, { companyId, actorType: "user", actorId: operatorId, action: "approval.created", + entityType: "approval", entityId: approval.id, details: { type: "hire_agent", agentId: created.id } }); + }); + scheduleAgentLifecycle(db, created.id); + return describe(created, database); + })); }, }; } diff --git a/server/src/services/dot-runner-broker.ts b/server/src/services/dot-runner-broker.ts index a65107e51d..9650cf3caf 100644 --- a/server/src/services/dot-runner-broker.ts +++ b/server/src/services/dot-runner-broker.ts @@ -1,3 +1,4 @@ +import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js"; import { createHash, randomBytes, randomUUID } from "node:crypto"; import { and, asc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; import { z } from "zod"; @@ -13,7 +14,7 @@ import type { PublicMcpToolExtension } from "./public-mcp/dot-runner.js"; import { boardAuthService } from "./board-auth.js"; import { logActivity } from "./activity-log.js"; import { authorizationService } from "./authorization.js"; -import { agentService } from "./agents.js"; +import { canConfigureAgentConnection } from "../modules/agent-lifecycle/index.js"; import { issueService } from "./issues.js"; import { instanceSettingsService } from "./instance-settings.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js"; @@ -43,7 +44,9 @@ function createBroker(db: Db) { const [agent] = await db.select().from(agents).where(and(eq(agents.id, binding.agentId), eq(agents.companyId, binding.companyId))); const access = grant ? await boardAuthService(db).resolveBoardAccess(grant.userId) : null; const membership = access?.memberships.find(m => m.companyId === binding.companyId && m.status === "active"); - if (!access?.user || !membership || membership.membershipRole === "viewer" || !grant || grant.agentId !== binding.agentId || !agent || ["terminated", "pending_approval", ...(allowPaused ? [] : ["paused"])].includes(agent.status)) throw fail("Agent connection authority is unavailable."); + if (!access?.user || !membership || membership.membershipRole === "viewer" || !grant || grant.agentId !== binding.agentId || !agent + || ["terminated", "pending_approval"].includes(agent.status) + || (!allowPaused && (requireReady ? agent.status === "paused" : !canConfigureAgentConnection(agent)))) throw fail("Agent connection authority is unavailable."); return binding; } @@ -115,7 +118,7 @@ function createBroker(db: Db) { const code = randomBytes(24).toString("base64url"); const binding = await db.transaction(async tx => { const [agent] = await tx.select().from(agents).where(and(eq(agents.id, input.agentId), eq(agents.companyId, input.companyId))).for("update"); - if (!agent || agent.adapterType !== "paperclip_runner" || agent.adapterConfig.provider !== "openai_dot" || ["pending_approval", "terminated", "paused"].includes(agent.status)) throw fail("Choose an approved Paperclip Runner agent."); + if (!agent || agent.adapterType !== "paperclip_runner" || agent.adapterConfig.provider !== "openai_dot" || !canConfigureAgentConnection(agent)) throw fail("Choose an approved Paperclip Runner agent."); const [existing] = await tx.select().from(bindings).where(and(eq(bindings.companyId, input.companyId), eq(bindings.agentId, input.agentId), isNull(bindings.revokedAt))); if (existing) { if (existing.id !== input.replaceBindingId || existing.status !== "pairing" || existing.operatorId !== input.operatorId) { @@ -143,10 +146,10 @@ function createBroker(db: Db) { if (!b || b.status !== "pairing" || !b.pairingExpiresAt || b.pairingExpiresAt <= new Date()) throw fail("Pairing code expired or was consumed."); const [agent] = await tx.select().from(agents).where(and(eq(agents.id, b.agentId), eq(agents.companyId, b.companyId))).for("update"); const [grant] = await tx.select().from(mcpOauthGrants).where(and(eq(mcpOauthGrants.id, principal.grant.id), isNull(mcpOauthGrants.revokedAt))).for("update"); - if (!grant || grant.agentId || !agent || ["paused", "terminated", "pending_approval"].includes(agent.status)) throw fail("Pairing authority is unavailable."); + if (!grant || grant.agentId || !agent || !canConfigureAgentConnection(agent)) throw fail("Pairing authority is unavailable."); await tx.update(bindings).set({ grantId: grant.id, status: "connected", pairingCodeHash: null, pairingExpiresAt: null, updatedAt: new Date() }).where(eq(bindings.id, b.id)); await tx.update(mcpOauthGrants).set({ agentId: b.agentId }).where(eq(mcpOauthGrants.id, grant.id)); - await agentService(tx as unknown as Db).update(agent.id, { adapterConfig: { ...agent.adapterConfig, dotBindingId: b.id } }, + await updateAgentConfigurationInTransaction(tx as unknown as Db, agent.id, { adapterConfig: { ...agent.adapterConfig, dotBindingId: b.id } }, { recordRevision: { createdByUserId: b.operatorId, source: "dot-pairing" } }); await logActivity(tx as unknown as Db, { companyId: b.companyId, actorType: "user", actorId: b.operatorId, action: "dot.paired", entityType: "agent", entityId: b.agentId, details: { bindingId: b.id, generation: b.generation } }); @@ -197,9 +200,10 @@ function createBroker(db: Db) { const [agent] = await tx.select().from(agents).where(eq(agents.id, b.agentId)); if (!agent || ["terminated", "pending_approval"].includes(agent.status)) throw fail("Agent connection authority is unavailable."); const paused = agent.status === "paused"; + const visibleKinds: Array = canConfigureAgentConnection(agent) ? ["authority_revoked", "readiness_challenge"] : ["authority_revoked"]; const items = await tx.select().from(mailbox).where(and(eq(mailbox.companyId, b.companyId), eq(mailbox.bindingId, b.id), - eq(mailbox.bindingGeneration, b.generation), gt(mailbox.id, after), paused ? eq(mailbox.kind, "authority_revoked") : undefined)).orderBy(asc(mailbox.id)).limit(50); - // Paused reads expose fences only and never consume hidden task items. + eq(mailbox.bindingGeneration, b.generation), gt(mailbox.id, after), paused ? inArray(mailbox.kind, visibleKinds) : undefined)).orderBy(asc(mailbox.id)).limit(50); + // Setup reads expose challenges and fences without consuming hidden work. return { bindingId: b.id, generation: b.generation, items, nextCursor: paused ? after : items.at(-1)?.id ?? after }; }); }, @@ -256,7 +260,7 @@ function createBroker(db: Db) { const access = await boardAuthService(db).resolveBoardAccess(b.operatorId); return { companyId: b.companyId, agentId: b.agentId, agentName: agent!.name, responsibleUser: access.user ? { id: access.user.id, name: access.user.name } : null, - permissions: agent!.permissions, ready: state?.status === "ready" && state.subscriptionVerified, + permissions: agent!.permissions, ready: agent!.lifecycleState === "ready" && state?.status === "ready" && state.subscriptionVerified, assignment: state?.assignment ?? null, idle: { read: ["paperclip_dot_capabilities", "paperclip_dot_tasks", "paperclip_dot_inbox"], profile: "paperclip_dot_set_avatar", @@ -429,7 +433,7 @@ function createBroker(db: Db) { const [agent] = await tx.select().from(agents).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))).for("update"); if (agent?.adapterConfig.dotBindingId === b.id) { const { dotBindingId: _removed, ...adapterConfig } = agent.adapterConfig; - await agentService(tx as unknown as Db).update(agentId, { adapterConfig }, + await updateAgentConfigurationInTransaction(tx as unknown as Db, agentId, { adapterConfig }, { recordRevision: { createdByUserId: operatorId, source: "dot-revoke" } }); } await tx.update(assignments).set({ status: "fenced" }).where(and(eq(assignments.bindingId, b.id), inArray(assignments.status, ["offered", "accepted"]))); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index e0cd734df4..06816cbd2c 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -4043,7 +4043,10 @@ export function environmentRuntimeService( and( eq(environmentLeases.heartbeatRunId, heartbeatRunId), or(eq(environmentLeases.status, "active"), and(eq(environmentLeases.status, "pending_cleanup"), - sql`(${environmentLeases.metadata} ? 'nativeWorkspaceExportResume' or ${environmentLeases.metadata} ? 'sandboxStopAndRetain')`)), + sql`(${environmentLeases.metadata} ? 'nativeWorkspaceExportResume' or ${environmentLeases.metadata} ? 'sandboxStopAndRetain')`), + providerResourceDisposition === "destroy" + ? and(eq(environmentLeases.status, "retained"), eq(environmentLeases.cleanupStatus, "failed")) + : undefined), ), ); if (leaseRows.length === 0) { diff --git a/server/src/services/environment-selection.ts b/server/src/services/environment-selection.ts new file mode 100644 index 0000000000..eba3d8772c --- /dev/null +++ b/server/src/services/environment-selection.ts @@ -0,0 +1,52 @@ +import { unprocessable } from "../errors.js"; + +export async function assertEnvironmentSelectionForCompany( + environmentsSvc: { + getById(environmentId: string): Promise<{ + id: string; + driver: string; + status?: string | null; + config: Record | null; + } | null>; + }, + _companyId: string, + environmentId: string | null | undefined, + options?: { + allowedDrivers?: string[]; + allowedSandboxProviders?: string[]; + }, +) { + if (environmentId === undefined || environmentId === null) return; + const environment = await environmentsSvc.getById(environmentId); + if (!environment) { + throw unprocessable("Environment not found."); + } + if (environment.status === "archived") { + throw unprocessable("Environment is archived."); + } + if (options?.allowedDrivers && !options.allowedDrivers.includes(environment.driver)) { + throw unprocessable( + `Environment driver "${environment.driver}" is not allowed here. Allowed drivers: ${options.allowedDrivers.join(", ")}`, + ); + } + if (environment.driver === "sandbox") { + const config = environment.config && typeof environment.config === "object" + ? environment.config as Record + : {}; + const provider = typeof config.provider === "string" ? config.provider : ""; + if (provider === "fake") { + throw unprocessable( + `Environment sandbox provider "${provider}" is not allowed here. The built-in fake provider is probe-only and cannot execute runs.`, + ); + } + if ( + options?.allowedSandboxProviders + && options.allowedSandboxProviders.length > 0 + && !options.allowedSandboxProviders.includes(provider) + ) { + throw unprocessable( + `Environment sandbox provider "${provider || "unknown"}" is not allowed here. Allowed providers: ${options.allowedSandboxProviders.join(", ")}`, + ); + } + } +} diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index c278f8408b..b2a0d488dc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -200,6 +200,7 @@ export async function reconcileAbandonedExecutionControl( eq(agents.id, run.agentId), eq(agents.companyId, run.companyId), eq(agents.status, "running"), + eq(agents.lifecycleState, "ready"), sql`not exists (select 1 from ${heartbeatRuns} where ${heartbeatRuns.agentId} = ${run.agentId} and ${heartbeatRuns.status} = 'running')`, ), ); diff --git a/server/src/services/feedback-share-client.ts b/server/src/services/feedback-share-client.ts index 7fdf97ca4a..ae5a308829 100644 --- a/server/src/services/feedback-share-client.ts +++ b/server/src/services/feedback-share-client.ts @@ -2,6 +2,8 @@ import { gzipSync } from "node:zlib"; import type { FeedbackTraceBundle } from "@paperclipai/shared"; import type { Config } from "../config.js"; +export const FEEDBACK_UPLOAD_TIMEOUT_MS = 30_000; + const DEFAULT_FEEDBACK_EXPORT_BACKEND_URL = "https://telemetry.paperclip.ing"; function buildFeedbackShareObjectKey(bundle: FeedbackTraceBundle, exportedAt: Date) { @@ -12,7 +14,7 @@ function buildFeedbackShareObjectKey(bundle: FeedbackTraceBundle, exportedAt: Da } export interface FeedbackTraceShareClient { - uploadTraceBundle(bundle: FeedbackTraceBundle): Promise<{ objectKey: string }>; + uploadTraceBundle(bundle: FeedbackTraceBundle, signal?: AbortSignal): Promise<{ objectKey: string }>; } export function createFeedbackTraceShareClientFromConfig( @@ -23,7 +25,10 @@ export function createFeedbackTraceShareClientFromConfig( const endpoint = new URL("/feedback-traces", baseUrl).toString(); return { - async uploadTraceBundle(bundle) { + async uploadTraceBundle(bundle, shutdownSignal) { + const deadline = AbortSignal.timeout(FEEDBACK_UPLOAD_TIMEOUT_MS); + const signal = shutdownSignal ? AbortSignal.any([shutdownSignal, deadline]) : deadline; + signal.throwIfAborted(); const exportedAt = new Date(); const objectKey = buildFeedbackShareObjectKey(bundle, exportedAt); const requestBody = JSON.stringify({ @@ -33,6 +38,7 @@ export function createFeedbackTraceShareClientFromConfig( }); const response = await fetch(endpoint, { method: "POST", + signal, headers: { "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}), @@ -49,6 +55,8 @@ export function createFeedbackTraceShareClientFromConfig( } const payload = await response.json().catch(() => null) as { objectKey?: unknown } | null; + // A timed-out response body is not a successful upload acknowledgement. + signal.throwIfAborted(); return { objectKey: typeof payload?.objectKey === "string" && payload.objectKey.trim().length > 0 ? payload.objectKey diff --git a/server/src/services/feedback.ts b/server/src/services/feedback.ts index 2bbc0361a3..1f9ceb7872 100644 --- a/server/src/services/feedback.ts +++ b/server/src/services/feedback.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { readFile, readdir } from "node:fs/promises"; import path from "node:path"; import { and, asc, desc, eq, getTableColumns, gte, isNull, lte, ne, or } from "drizzle-orm"; @@ -111,7 +112,7 @@ const feedbackExportColumns = getTableColumns(feedbackExports); const instructionsSvc = agentInstructionsService(); type FeedbackTraceShareClient = { - uploadTraceBundle(bundle: FeedbackTraceBundle): Promise<{ objectKey: string }>; + uploadTraceBundle(bundle: FeedbackTraceBundle, signal?: AbortSignal): Promise<{ objectKey: string }>; }; type FeedbackServiceOptions = { @@ -1715,6 +1716,15 @@ async function buildFeedbackTraceBundleFromRow( } export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { + // Serialize explicit flushes and the worker's batches so callers cannot + // upload the same trace concurrently. HTTP vote handlers only save and notify. + let exportFlush: Promise = Promise.resolve(); + function serializeExportFlush(operation: () => Promise): Promise { + const result = exportFlush.then(operation, operation); + exportFlush = result.catch(() => undefined); + return result; + } + return { listIssueVotesForUser: async (issueId: string, authorUserId: string) => db @@ -1786,13 +1796,23 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { return row ? buildFeedbackTraceBundleFromRow(db, row) : null; }, + hasPendingFeedbackTraces: async () => { + const pending = options.shareClient + ? or(eq(feedbackExports.status, "pending"), eq(feedbackExports.status, "failed")) + : eq(feedbackExports.status, "pending"); + return (await db.select({ id: feedbackExports.id }).from(feedbackExports) + .where(pending).limit(1)).length > 0; + }, + flushPendingFeedbackTraces: async (input?: { companyId?: string; traceId?: string; limit?: number; now?: Date; - }) => { + signal?: AbortSignal; + }) => serializeExportFlush(async () => { const shareClient = options.shareClient; + input?.signal?.throwIfAborted(); if (!shareClient) { const filters = [eq(feedbackExports.status, "pending")]; if (input?.companyId) { @@ -1861,12 +1881,13 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { let failed = 0; for (const row of rows) { + if (input?.signal?.aborted) break; const attemptAt = input?.now ?? new Date(); attempted += 1; try { const bundle = await buildFeedbackTraceBundleFromRow(db, row); - await shareClient.uploadTraceBundle(bundle); + await shareClient.uploadTraceBundle(bundle, input?.signal); await db .update(feedbackExports) @@ -1900,7 +1921,7 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { sent, failed, }; - }, + }), saveIssueVote: async (input: { issueId: string; @@ -2071,6 +2092,7 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { }) .where(eq(feedbackVotes.id, savedVote.id)); + if (sharedWithLabs) await notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback); const [savedTrace] = await tx .insert(feedbackExports) .values({ diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8d84f116e6..7a0c1436c8 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,10 +1,35 @@ +import { agentExecutionsHaveStopped } from "./agent-execution-stop.js"; +import { + cancelHeartbeatNativeRun, + terminateHeartbeatRunProcess, + providerResourceDispositionForTerminalRun, + createHeartbeatRunControl, + type CancelRunOptions, +} from "./heartbeat/run-control.js"; +export { + leaseReleaseStatusForRunStatus, + providerResourceDispositionForTerminalRun, + cancelHeartbeatNativeRun, +} from "./heartbeat/run-control.js"; +import { + NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, + isHeartbeatRunTerminalStatus, + isHeartbeatRunRuntimeStatusActive, + publishHeartbeatRunRuntimeProgress, + decorateHeartbeatRunRuntimeStatus, + createHeartbeatLifecycle, +} from "./heartbeat/run-lifecycle.js"; +export { + redactDetectedSuccessfulRunProgressSummaryForBoard, + redactSuccessfulRunHandoffEvidence, + persistHeartbeatRunProcessMetadata, + resolveSkillTestRunCompletionForHeartbeatOutcome, +} from "./heartbeat/run-lifecycle.js"; import { ChatControlRecoveryUnresolvedError, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, DEFERRED_WAKE_CONTEXT_KEY, - EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, - shouldRequireIssueCommentForWake, createHeartbeatQueue, } from "./heartbeat/queue.js"; export { @@ -12,15 +37,11 @@ export { shouldQueueFollowupForRunningIssueWake, } from "./heartbeat/queue.js"; import { - HEARTBEAT_RUN_TERMINAL_STATUSES, - DETACHED_PROCESS_ERROR_CODE, isProcessAlive, - CANCELLABLE_HEARTBEAT_RUN_STATUSES, createHeartbeatRecovery, } from "./heartbeat/recovery.js"; import { BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, - normalizeAgentNameKey, WorkspaceBusyDeferral, readTransientRecoveryContractFromRun, readHeartbeatRunErrorFamily, @@ -44,7 +65,6 @@ import { type RunSessionOutcome, EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, deriveTaskKeyWithHeartbeatFallback, - type UsageTotals, getAdapterSessionCodec, shouldResetTaskSessionForWake, normalizeSessionParams, @@ -119,7 +139,6 @@ export { buildPaperclipWakePayload, } from "./heartbeat/run-preparation.js"; import { - WORKSPACE_VALIDATION_FAILURE_CODE, isWorkspaceValidationFailedRun, resolveNativeRecoveryExecutionWorkspaceBinding, resolveExecutionWorkspaceReuseRequestForIssue, @@ -224,13 +243,13 @@ export { import { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js"; export { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js"; import { preserveWorkspaceRestoreRecoveryMetadataSql } from "./legacy-workspace-restore-recovery.js"; -import { preserveWorkspaceRestoreRecoveryMetadata } from "./workspace-restore-recovery-state.js"; + import { recordLegacyWorkspaceRestoreFailure } from "./legacy-execution-recovery.js"; import { configuredEnvironmentProjection, } from "../vendor/paperclip-runner/index.js"; import { decisionModelService } from "./decision-models.js"; -import { activeIssueInteractionCondition, TASK_QUESTION_GUIDANCE } from "./issue-question-context.js"; +import { TASK_QUESTION_GUIDANCE } from "./issue-question-context.js"; import { createAgentIdentityRedactor } from "./agent-identity-redaction.js"; import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-identity.js"; import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils"; @@ -240,8 +259,10 @@ import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/as import { externalObjectService } from "./external-objects.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { dotRunnerBroker } from "./dot-runner-broker.js"; -import { isAiAuthenticationBlocked } from "./ai-auth-failure.js"; -import { nativeRetryCancellationCommitCondition, rethrowNativeCancellationLockConflict, claimCancellationRequest, startupCancellationFence } from "./native-runtime/native-cancellation-request.js"; + +import { + startupCancellationFence, +} from "./native-runtime/native-cancellation-request.js"; import { prepareChatCompletionTurn, chatCompletionInstruction, @@ -251,7 +272,10 @@ import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./age import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js"; -import { cancellationResultJson, canContinueCancelledRun, readRunCancellation, requestedRunCancellation } from "./run-cancellation.js"; +import { + cancellationResultJson, + readRunCancellation, +} from "./run-cancellation.js"; import { withNativeWorkspaceFinalizationOwnership, NativeWorkspaceFinalizationBusyError, @@ -267,40 +291,30 @@ import { hasWorkspaceRestoreFailure, } from "@paperclipai/shared"; -import { settleSlackConversation } from "./slack-conversation-lifecycle.js"; import { toolActionDeliveryService } from "./tool-action-delivery.js"; - -import { readQueuedInteractionResponse } from "./queued-interaction-response.js"; import { isConversation, - isWaitingConversation, prepareConversationTurn, settleConversationTurn, } from "./agent-conversations.js"; import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-phase"; import { getConversationConfirmationContext } from "./conversation-confirmation-context.js"; -import { PROCESS_IDENTITY_RECORDED, recordNativeLocalProcessStop } from "./native-local-process-stop.js"; -import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent, isAcknowledgedNativeStop, acknowledgedNativeStopExecutionHasStopped } from "./acknowledged-native-stop.js"; +import { + recordNativeLocalProcessStop, +} from "./native-local-process-stop.js"; + import { legacyControllerBootId, renewLegacyControllerLease, watchLegacyControllerLease, } from "./legacy-controller-lease.js"; -import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js"; + import { - hasRemoteTerminationReceipt, remoteExecutionHasStopped, - stoppedRemoteCleanupScopes, } from "./remote-execution-termination.js"; import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js"; -import { - admitExplicitNativeContinuation, - undeliveredLegacyUserCommentIds, -} from "./explicit-native-continuation.js"; -import { - isCancelledNativeStartup, -} from "./cancelled-native-startup.js"; + import { connectionIntentService } from "./connection-intents.js"; import { prepareManagedAiRuntime, @@ -312,8 +326,12 @@ import { import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiConnectionRouterSelection } from "@paperclipai/shared"; import { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js"; import { aiConnectionSessionCompatibilityInputs, managedAiSessionIdentityCompatible } from "./ai-connection-session.js"; -import { executionBlockerPredicate, getExecutionBlocker } from "./execution-blocker.js"; -import { CONVERSATION_CONTINUATION_POLICY, claimedAdapterType, runUsedConversationAdapter, hasConversationContinuationPolicy, isConversationAdapter } from "./conversation-continuation.js"; +import { + getExecutionBlocker, +} from "./execution-blocker.js"; +import { + claimedAdapterType, +} from "./conversation-continuation.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js"; @@ -325,10 +343,8 @@ import { } from "./legacy-execution-recovery.js"; import { adapterExecutionControls, - captureAdapterStopOwnership, createAdapterExecutionControl, registerAdapterExecutionControl, - waitForAdapterStop, } from "./adapter-execution-control.js"; import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; @@ -366,14 +382,11 @@ import { eq, exists, getTableColumns, - gt, gte, inArray, isNull, - isNotNull, lt, lte, - ne, notInArray, or, sql, @@ -381,48 +394,31 @@ import { import type { Db } from "@paperclipai/db"; import { CHAT_PROVIDERS, - ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, ISSUE_DISPOSITION_REPAIR_RETRY_REASON, PROVIDER_QUOTA_MONITOR_SERVICE_NAME, isEnvironmentDriverSupportedForAdapter, - type EnvironmentLeaseStatus, type ExecutionWorkspace, - type HeartbeatRunStatusPhase, type IssueExecutionMonitorClearReason, type IssueExecutionMonitorPolicy, type IssueExecutionMonitorRecoveryPolicy, - type RequestConfirmationResult, - type RunLivenessState, } from "@paperclipai/shared"; import { agents, agentConfigRevisions, - agentRuntimeState, agentSessionGoalActions, agentTaskSessions, agentWakeupRequests, - activityLog, - approvals, chatActions, - companySkillTestRuns, companies, completionContracts, - documentRevisions, - environmentLeases, - issueDocuments, heartbeatRunEvents, heartbeatRuns, - issueApprovals, + environmentLeases, issueComments, - issueRecoveryActions, - issueRelations, - issueThreadInteractions, issues, - issueWorkProducts, nativeRunFinalizations, projects, projectWorkspaces, - routines, workspaceOperations, } from "@paperclipai/db"; import { conflict, HttpError, notFound } from "../errors.js"; @@ -442,15 +438,9 @@ import { // git-credentials module became its canonical home; existing importers keep working. export { scrubGitCredentialText }; import { publishLiveEvent } from "./live-events.js"; + import { - appendHeartbeatRunEvent, - type AppendHeartbeatRunEventInput, -} from "./heartbeat-run-events.js"; -import { - queuedCommentIdsFromWakePayload, queuedCommentIdsFromRunContext, - withQueuedCommentIdsInWakePayload, - withQueuedCommentIdsInRunContext, } from "./issue-queued-comment-queue.js"; import { documentService } from "./documents.js"; import { getTaskPlanContext } from "./task-plan-context.js"; @@ -461,10 +451,8 @@ import { buildNativeExecutionInput, buildNativeExecutionWithCheckpoint, buildNativeRuntimeContext, - cancelNativeSession, claimWarmNativeInstructionCopy, nativeSessionWorkspaceScope, - closeWarmNativeSessionsForEnvironment, reserveWarmNativeInstructionDirectory, dispatchNativeSessionResumptions, ensureNativeCompletionContract, @@ -528,7 +516,6 @@ import { createLocalAgentJwt } from "../agent-auth-jwt.js"; import { createRuntimeToolsToken } from "../runtime-tools-token.js"; import { parseObject, - asBoolean, asNumber, } from "../adapters/utils.js"; @@ -539,7 +526,6 @@ import { materializeExternalChatQuestionResponseInput } from "./native-runtime/e import { NativeRunnerOwnershipUnverifiedError, isNativeRunnerOwnershipHeld, - nativeRunnerOwnershipNotHeldCondition, NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE, } from "./native-runtime/native-runner-ownership.js"; import { @@ -560,7 +546,9 @@ import { reportRunFailure } from "./run-failure-report.js"; import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js"; import { companySkillService } from "./company-skills.js"; -import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js"; +import { + budgetService, +} from "./budgets.js"; import { secretService, } from "./secrets.js"; @@ -570,32 +558,21 @@ import { import { buildHeartbeatRunIssueComment, CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON, - findHeartbeatRunCompletionComment, - hasAcceptedSemanticResult, isExternalChatPresentationContext, mergeHeartbeatRunResultJson, - readCompletedAssistantMessageCandidate, resolveHeartbeatRunResponse, - selectHeartbeatRunFinalAgentMessage, type RunPresentationDecision, } from "./heartbeat-run-summary.js"; import { - buildHeartbeatRunStopMetadata, - mergeHeartbeatRunStopMetadata, normalizeMaxTurnStopReason, } from "./heartbeat-stop-metadata.js"; import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, } from "./chat-control-recovery-stop.js"; -import { - classifyRunLiveness, - type RunLivenessClassificationInput, -} from "./run-liveness.js"; import { logActivity, - publishPluginDomainEvent, type LogActivityInput, } from "./activity-log.js"; import { @@ -646,7 +623,6 @@ import { import { continuationSummaryParksExecutor, getIssueContinuationSummaryDocument, - refreshIssueContinuationSummary, } from "./issue-continuation-summary.js"; import { buildPlanReviewContext, @@ -659,7 +635,6 @@ import { } from "./workspace-operations.js"; import { isProcessGroupAlive, - terminateLocalService, } from "./local-service-supervisor.js"; import { isRuntimeOwnedGitBranch, @@ -694,19 +669,6 @@ import { } from "./execution-allowlist.js"; import { RECOVERY_ORIGIN_KINDS, - FINISH_SUCCESSFUL_RUN_HANDOFF_REASON, - SUCCESSFUL_RUN_MISSING_STATE_REASON, - RUN_LIVENESS_CONTINUATION_REASON, - buildRunLivenessContinuationIdempotencyKey, - buildFinishSuccessfulRunHandoffIdempotencyKey, - buildSuccessfulRunHandoffRequiredNotice, - decideRunLivenessContinuation, - decideSuccessfulRunHandoff, - findExistingFinishSuccessfulRunHandoffWake, - findExistingRunLivenessContinuationWake, - isSuccessfulRunHandoffValidPathSkip, - SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, - readContinuationAttempt, } from "./recovery/index.js"; import { buildConfigurationIncompleteRecoveryNoticeSeed, @@ -734,21 +696,15 @@ import { } from "../modules/run-dispatch/index.js"; import { createWakeQueue, - WakeQueueApplicationError, type IssueSnapshot as WakeQueueIssueSnapshot, type PostCommitEffect as WakeQueuePostCommitEffect, type ReleaseRecoveryBlockedNoticeKind, type RunSnapshot as WakeQueueRunSnapshot, } from "../modules/wake-queue/index.js"; import { - buildIssueReviewPathLostIdempotencyKey, - decideIssueReviewPathRecovery, - ISSUE_REVIEW_PATH_LOST_WAKE_REASON, - isReviewPathRecoveryIdempotencyConflict, REVIEW_PATH_RECOVERY_INSTRUCTION, - reviewPathConsumedRefFromRun, } from "./recovery/review-path-recovery.js"; -import { resolveRequiredSuccessfulRunHandoffOnValidPath } from "./successful-run-handoff-state.js"; + import { taskWatchdogService } from "./task-watchdogs.js"; import { @@ -763,20 +719,15 @@ import { } from "./source-trust.js"; import { redactCurrentUserText, - redactCurrentUserValue, - type CurrentUserRedactionOptions, } from "../log-redaction.js"; -import { redactEventPayload, redactSensitiveText } from "../redaction.js"; + import { createRunSecretRedactionRegistry } from "./run-secret-redaction.js"; import { resolvePaperclipRunnerIdleTimeoutMs, - type RuntimeStatusUpdate, } from "@paperclipai/adapter-utils"; import { readPaperclipSkillSyncPreference, selectPaperclipTaskMarkdown, - UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, - UNMANAGED_BACKGROUND_TASK_STOP_REASON, } from "@paperclipai/adapter-utils/server-utils"; import { isUuidLike, @@ -794,16 +745,10 @@ import { environmentRunOrchestrator } from "./environment-run-orchestrator.js"; import { clearHeartbeatRunRuntimeStatus, - getHeartbeatRunRuntimeStatus, - MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, - MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, - setHeartbeatRunRuntimeStatus, sweepExpiredHeartbeatRunRuntimeStatuses, touchHeartbeatRunRuntimeStatus, } from "./heartbeat-run-runtime-status.js"; -import { - readProcessStartedAt, -} from "./hot-restart.js"; + import { assertLowTrustRuntimeServicesAllowed, } from "./low-trust-runtime-containment.js"; @@ -816,32 +761,6 @@ import { computeTaskDrain, applyTaskDrain, startTaskDrain, stopTaskDrain, readTa const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024; -export function redactDetectedSuccessfulRunProgressSummaryForBoard( - summary: string, - currentUserRedactionOptions?: CurrentUserRedactionOptions, -) { - const normalized = summary.replace(/\s+/g, " ").trim(); - const redacted = redactSensitiveText( - redactCurrentUserText(normalized, currentUserRedactionOptions), - ); - return redacted.length <= 280 ? redacted : `${redacted.slice(0, 277)}...`; -} - -export function redactSuccessfulRunHandoffEvidence( - value: string | null, - currentUserRedactionOptions?: CurrentUserRedactionOptions, -) { - if (!value) return null; - return redactSensitiveText( - redactCurrentUserText(value, currentUserRedactionOptions), - ); -} - -const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [ - "environment.lease_acquired", - "environment.lease_released", - "cost.reported", -]; const ACCEPTED_PLAN_CONVERSION_SKILL_KEY = "paperclipai/paperclip/paperclip-converting-plans-to-tasks"; @@ -873,25 +792,6 @@ const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery"; -const NON_RETRYABLE_PREFLIGHT_FAILURE_CODES = new Set([ - "low_trust_isolation_unavailable", - "low_trust_requires_isolated_workspace", - "low_trust_boundary_mismatch", - "low_trust_requires_sandbox_environment", - "low_trust_runtime_services_denied", - "chat_failed_run_retry_not_authorized", - CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, -]); -// Error codes that mark a pre-dispatch setup failure. The adapter process never -// started, so no agent could post an issue comment. The setup catch writes one -// of these codes when a failure happens before `adapter.execute`. -const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set([ - "setup_failed", - CONFIGURATION_INCOMPLETE_FAILURE_CODE, - WORKSPACE_VALIDATION_FAILURE_CODE, - ...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, -]); - function nonRetryablePreflightFailureCode(error: unknown): string | null { if (error instanceof ChatControlRecoveryUnresolvedError) return CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE; @@ -908,19 +808,9 @@ function nonRetryablePreflightFailureCode(error: unknown): string | null { } export { MAX_TURN_CONTINUATION_RETRY_REASON }; export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry"; -const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2; -const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10; -const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000; -const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000; export { WORKSPACE_BUSY_RETRY_REASON }; -interface MaxTurnContinuationPolicy { - enabled: boolean; - maxAttempts: number; - delayMs: number; -} - // Build the configuration-incomplete result payload for a workspace base ref // that never resolved to a commit. The setup catch maps this to errorCode // `configuration_incomplete`, so the recovery path routes it to a human owner @@ -1114,21 +1004,6 @@ export function getTaskDrainStatus(): { }; } -export function leaseReleaseStatusForRunStatus( - status: string | null | undefined, -): Extract { - if (status === "cancelled") return "expired"; - return status === "failed" || status === "timed_out" ? "failed" : "released"; -} - -export function providerResourceDispositionForTerminalRun( - desired: ProviderResourceDisposition | undefined, - status: string | null | undefined, -): ProviderResourceDisposition | undefined { - if (desired !== "keep_running") return desired; - return status === "succeeded" ? desired : "stop_and_retain"; -} - export interface NativeSandboxLifecycle { runnerProcess: "per_turn" | "warm"; sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; @@ -1379,377 +1254,6 @@ function isSameTaskScope(left: string | null, right: string | null) { return (left ?? null) === (right ?? null); } -function isHeartbeatRunTerminalStatus( - status: string | null | undefined, -): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] { - return HEARTBEAT_RUN_TERMINAL_STATUSES.includes( - status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number], - ); -} - -function isHeartbeatRunRuntimeStatusActive( - status: string | null | undefined, -): boolean { - return status === "queued" || status === "running"; -} - -type HeartbeatRunRuntimeStatusRunLike = { - id: string; - status?: string | null; - companyId?: string | null; - agentId?: string | null; - issueId?: string | null; - contextSnapshot?: Record | null; -}; - -function readRuntimeStatusIssueIdCandidate( - run: HeartbeatRunRuntimeStatusRunLike, -): string | null | undefined { - if ("issueId" in run) return readNonEmptyString(run.issueId) ?? null; - if ("contextSnapshot" in run) { - return readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? null; - } - return undefined; -} - -function decorateHeartbeatRunRuntimeStatus< - T extends HeartbeatRunRuntimeStatusRunLike, ->( - run: T, - expected: { - companyId?: string | null; - issueId?: string | null; - agentId?: string | null; - } = {}, -): T & { - currentStatusMessage: string | null; - currentStatusUpdatedAt: Date | null; - currentToolName: string | null; - lastAssistantSnippet: string | null; - lastEventAt: Date | null; -} { - if (isHeartbeatRunTerminalStatus(run.status)) { - clearHeartbeatRunRuntimeStatus(run.id); - } - - const companyId = expected.companyId ?? run.companyId ?? null; - const agentId = expected.agentId ?? run.agentId ?? null; - const issueId = - expected.issueId !== undefined - ? expected.issueId - : readRuntimeStatusIssueIdCandidate(run); - const currentStatus = - isHeartbeatRunRuntimeStatusActive(run.status) && companyId && agentId - ? getHeartbeatRunRuntimeStatus(run.id, { - companyId, - agentId, - ...(issueId !== undefined ? { issueId } : {}), - }) - : null; - - return { - ...run, - currentStatusMessage: currentStatus?.message ?? null, - currentStatusUpdatedAt: currentStatus?.updatedAt ?? null, - currentToolName: currentStatus?.currentToolName ?? null, - lastAssistantSnippet: currentStatus?.lastAssistantSnippet ?? null, - lastEventAt: currentStatus?.lastEventAt ?? null, - }; -} - -function publishHeartbeatRunRuntimeProgress(status: { - companyId: string; - runId: string; - agentId: string; - issueId: string | null; - phase: HeartbeatRunStatusPhase; - message: string; - updatedAt: Date; - currentToolName?: string | null; - lastAssistantSnippet?: string | null; - lastEventAt?: Date | null; -}) { - publishLiveEvent({ - companyId: status.companyId, - type: "heartbeat.run.progress", - payload: { - runId: status.runId, - agentId: status.agentId, - issueId: status.issueId, - phase: status.phase, - message: status.message, - currentToolName: status.currentToolName ?? null, - lastAssistantSnippet: status.lastAssistantSnippet ?? null, - lastEventAt: (status.lastEventAt ?? status.updatedAt).toISOString(), - updatedAt: status.updatedAt.toISOString(), - }, - }); -} - -function recordHeartbeatRunRuntimeProgress( - run: Pick< - typeof heartbeatRuns.$inferSelect, - "id" | "companyId" | "agentId" | "status" | "contextSnapshot" - >, - update: RuntimeStatusUpdate, - issueId: string | null, -) { - if (!isHeartbeatRunRuntimeStatusActive(run.status)) return null; - const status = setHeartbeatRunRuntimeStatus({ - companyId: run.companyId, - issueId, - agentId: run.agentId, - runId: run.id, - phase: update.phase as HeartbeatRunStatusPhase, - message: update.message, - currentToolName: readNonEmptyString(update.currentToolName) ?? null, - lastAssistantSnippet: - readNonEmptyString(update.lastAssistantSnippet) ?? null, - lastEventAt: update.lastEventAt ? new Date(update.lastEventAt) : new Date(), - }); - if (!status) return null; - - publishHeartbeatRunRuntimeProgress(status); - return status; -} - -function sanitizeLiveRunProgressText( - value: string, - maxChars: number, -): string | null { - const normalized = value.replace(/\s+/g, " ").trim(); - if (!normalized) return null; - const redacted = redactSensitiveText(normalized); - if (redacted.length <= maxChars) return redacted; - return `${redacted.slice(0, maxChars - 3)}...`; -} - -function readLiveRunProgressString( - value: unknown, - maxChars: number, -): string | null { - return typeof value === "string" - ? sanitizeLiveRunProgressText(value, maxChars) - : null; -} - -function readFirstLiveRunProgressString( - maxChars: number, - values: unknown[], -): string | null { - for (const value of values) { - const text = readLiveRunProgressString(value, maxChars); - if (text) return text; - } - return null; -} - -function readLiveRunToolName( - payload: Record | null, - eventType: string, -): string | null { - const toolCall = - parseObject(payload?.tool_call) ?? parseObject(payload?.toolCall); - const message = parseObject(payload?.message); - const direct = readFirstLiveRunProgressString( - MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, - [ - payload?.toolName, - payload?.tool_name, - payload?.tool, - payload?.name, - payload?.title, - toolCall?.name, - toolCall?.toolName, - message?.name, - message?.toolName, - ], - ); - if (direct) return direct; - - const normalizedEventType = eventType.toLowerCase(); - if (!normalizedEventType.includes("tool")) return null; - return readLiveRunProgressString( - eventType.replace(/[._-]+/g, " "), - MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, - ); -} - -function readLiveRunAssistantSnippet( - payload: Record | null, - eventType: string, - message: string | null, -): string | null { - const normalizedEventType = eventType.toLowerCase(); - const messagePayload = parseObject(payload?.message); - const direct = readFirstLiveRunProgressString( - MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, - [ - payload?.text, - payload?.delta, - payload?.text_delta, - payload?.content, - payload?.summary, - messagePayload?.text, - messagePayload?.content, - ], - ); - if (direct) return direct; - - if ( - normalizedEventType.includes("assistant") || - normalizedEventType.includes("text_delta") || - normalizedEventType.includes("message.delta") || - normalizedEventType.includes("message_delta") - ) { - return message - ? sanitizeLiveRunProgressText( - message, - MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, - ) - : null; - } - - return null; -} - -function buildRunEventRuntimeProgress(input: { - eventType: string; - message: string | null; - payload: Record | null; - at: Date; -}) { - const normalizedEventType = input.eventType.toLowerCase(); - if ( - normalizedEventType === "lifecycle" || - normalizedEventType === "adapter.invoke" - ) { - return null; - } - - const currentToolName = readLiveRunToolName(input.payload, input.eventType); - const lastAssistantSnippet = readLiveRunAssistantSnippet( - input.payload, - input.eventType, - input.message, - ); - const fallbackMessage = - readLiveRunProgressString( - input.message, - MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, - ) ?? - sanitizeLiveRunProgressText( - input.eventType.replace(/[._-]+/g, " "), - MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, - ); - const message = currentToolName - ? `Using ${currentToolName}` - : (lastAssistantSnippet ?? fallbackMessage); - - if (!message) return null; - return { - phase: "run_activity" as const, - message, - currentToolName, - lastAssistantSnippet, - lastEventAt: input.at, - }; -} - -export async function persistHeartbeatRunProcessMetadata( - db: Db, - runId: string, - meta: { pid: number; processGroupId: number | null; startedAt: string }, -) { - const observedStartedAt = await readProcessStartedAt(meta.pid).catch( - () => null, - ); - const startedAt = new Date(observedStartedAt ?? meta.startedAt); - return db.transaction(async tx => { - const run = await tx - .update(heartbeatRuns) - .set({ - processPid: meta.pid, - processGroupId: meta.processGroupId, - processStartedAt: Number.isNaN(startedAt.getTime()) - ? new Date() - : startedAt, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, runId)) - .returning() - .then((rows) => rows[0] ?? null); - if (run?.runtimeMode === "native") await appendHeartbeatRunEvent(tx as unknown as Db, { - companyId: run.companyId, runId, agentId: run.agentId, - eventType: PROCESS_IDENTITY_RECORDED, stream: "system", level: "info", - message: "Process identity recorded; prior stop evidence no longer applies.", - }); - return run; - }); -} - -async function terminateHeartbeatRunProcess(input: { - pid: number | null | undefined; - processGroupId: number | null | undefined; - graceMs?: number; - signal?: NodeJS.Signals; -}) { - const pid = input.pid ?? null; - const processGroupId = input.processGroupId ?? null; - if (typeof pid !== "number" && typeof processGroupId !== "number") return; - - await terminateLocalService( - { - pid: - typeof pid === "number" && Number.isInteger(pid) && pid > 0 - ? pid - : (processGroupId ?? 0), - processGroupId: - typeof processGroupId === "number" && - Number.isInteger(processGroupId) && - processGroupId > 0 - ? processGroupId - : null, - }, - { forceAfterMs: input.graceMs, signal: input.signal }, - ); -} - -type SkillTestHeartbeatCompletion = { - outcome: "failed" | "cancelled"; - error: string | null; - heartbeatOutcome: RunSessionOutcome; -}; - -export function resolveSkillTestRunCompletionForHeartbeatOutcome( - outcome: RunSessionOutcome, - error: string | null | undefined, -): SkillTestHeartbeatCompletion | null { - if (outcome === "cancelled") { - return { - outcome: "cancelled", - error: error ?? "Harness run was cancelled", - heartbeatOutcome: outcome, - }; - } - if (outcome === "timed_out") { - return { - outcome: "failed", - error: error ?? "Timed out", - heartbeatOutcome: outcome, - }; - } - if (outcome === "failed") { - return { - outcome: "failed", - error: error ?? "Adapter failed", - heartbeatOutcome: outcome, - }; - } - return null; -} - - export type HeartbeatEnvironmentRuntime = ReturnType< typeof environmentRuntimeService >; @@ -1791,38 +1295,6 @@ export interface HeartbeatServiceOptions { }) => Promise; } -export async function cancelHeartbeatNativeRun(input: { - db: Db; - runId: string; - reason: string; - runtimeMode: string | null; - cancellationRequestId?: string; - cancel?: ( - runId: string, - reason: string, - options: { db: Db; scope: "run"; cancellationRequestId?: string }, - ) => Promise<{ decision: unknown | null; auditId: string | null }>; -}) { - if (input.runtimeMode !== "native") { - return { decision: null, auditId: null }; - } - const cancellation = input.cancel - ? await input.cancel(input.runId, input.reason, { - db: input.db, - scope: "run", - ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), - }) - : await cancelNativeSession(input.runId, input.reason, { - db: input.db, - scope: "run", - ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), - }); - if (!cancellation.decision || !cancellation.auditId) { - throw new Error("native_cancellation_outcome_not_audited"); - } - return cancellation; -} - class NativeSessionResumeScheduledError extends Error { constructor(readonly original: unknown) { super("Native session recovery has been scheduled for the same run."); @@ -2421,6 +1893,154 @@ export function heartbeatService( return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args); } + const runLifecycle = createHeartbeatLifecycle(db, { + issuesSvc, + recovery, + companySkills, + getRun, + emitTerminalAgentTaskRun, + getIssueExecutionContext, + budgets, + enqueueWakeup, + getCurrentUserRedactionOptions, + treeControlSvc, + getAgentInvokability, + resolveSessionBeforeForWakeup, + resolveResponsibleUserIdForRunContext, + ensureRuntimeState, + budgetHooks, + }); + + const { + persistRunProcessMetadata, + recordCurrentHeartbeatRunRuntimeProgress, + completeSkillTestRunForHeartbeatOutcome, + refreshContinuationSummaryForRun, + findRunIssueComment, + findLatestCompletedFinalAgentMessage, + parseMaxTurnContinuationPolicy, + finalizeIssueCommentPolicy, + handleIssueReviewPathDisposition, + handleRunLivenessContinuation, + handleSuccessfulRunHandoff, + updateRuntimeState, + terminalizeRunOnLeaseRelease, + clearDetachedRunWarning, + } = runLifecycle; + + // Earlier factories capture these callbacks before lifecycle construction. + function appendRunEvent(...args: Parameters) { + return runLifecycle.appendRunEvent(...args); + } + + function escalatePlanApprovalResumeFailureNeedsAttention(...args: Parameters) { + return runLifecycle.escalatePlanApprovalResumeFailureNeedsAttention(...args); + } + + function recordPlanApprovalResumeFailureRetry(...args: Parameters) { + return runLifecycle.recordPlanApprovalResumeFailureRetry(...args); + } + + function setRunStatusIfRunning(...args: Parameters) { + return runLifecycle.setRunStatusIfRunning(...args); + } + + function setWakeupStatus(...args: Parameters) { + return runLifecycle.setWakeupStatus(...args); + } + + function mergeRunStopMetadataForAgent(...args: Parameters) { + return runLifecycle.mergeRunStopMetadataForAgent(...args); + } + + function classifyAndPersistRunLiveness(...args: Parameters) { + return runLifecycle.classifyAndPersistRunLiveness(...args); + } + + function setRunStatusFromLive(...args: Parameters) { + return runLifecycle.setRunStatusFromLive(...args); + } + + function setRunStatus(...args: Parameters) { + return runLifecycle.setRunStatus(...args); + } + + function publishRunLifecyclePluginEventData(...args: Parameters) { + return runLifecycle.publishRunLifecyclePluginEventData(...args); + } + + function publishRunLifecyclePluginEvent(...args: Parameters) { + return runLifecycle.publishRunLifecyclePluginEvent(...args); + } + + const runControl = createHeartbeatRunControl(db, { + getRun, + options, + envOrchestrator, + activeRunExecutions, + enqueueWakeup, + getAgent, + sweepPendingCleanupLeases, + getSchedulingSuppression, + wakeQueue, + applyWakeQueuePostCommitEffects, + processRunCancellationSettlements, + mergeRunStopMetadataForAgent, + setRunStatusFromLive, + failedProcessRunCancellations, + setWakeupStatus, + appendRunEvent, + finalizeAgentStatus, + timerClaimWasFirstHeartbeat, + startNextQueuedRunForAgent, + setRunStatus, + }); + + const { + cancelInvocationsForAgentsInternal, + } = runControl; + + // Earlier factories capture these callbacks before run-control construction. + function cancelBudgetScopeWork(...args: Parameters) { + return runControl.cancelBudgetScopeWork(...args); + } + + function releaseIssueExecutionAndPromote(...args: Parameters) { + return runControl.releaseIssueExecutionAndPromote(...args); + } + + function releaseEnvironmentLeasesForRun(...args: Parameters) { + return runControl.releaseEnvironmentLeasesForRun(...args); + } + + function acknowledgeRemoteStop(...args: Parameters) { + return runControl.acknowledgeRemoteStop(...args); + } + + function resumeRemoteStopComments(...args: Parameters) { + return runControl.resumeRemoteStopComments(...args); + } + + function cancelRunInternal(...args: Parameters) { + return runControl.cancelRunInternal(...args); + } + + function resumeExecutionWaitComments(...args: Parameters) { + return runControl.resumeExecutionWaitComments(...args); + } + + function resumeQueuedCommentInterrupt(...args: Parameters) { + return runControl.resumeQueuedCommentInterrupt(...args); + } + + function resumeSavedLegacyComments(...args: Parameters) { + return runControl.resumeSavedLegacyComments(...args); + } + + function cancelActiveForAgentInternal(...args: Parameters) { + return runControl.cancelActiveForAgentInternal(...args); + } + // Applies the post-commit effects a wake-queue release returns, exactly as // the original release function did before its writes moved into that // module: publish + dispatch a promoted or recovery run, log a reopened @@ -2482,791 +2102,13 @@ export function heartbeatService( } } - function isPlanApprovalConfirmationPayload(payload: unknown) { - const target = parseObject(parseObject(payload).target); - return ( - readNonEmptyString(target.type) === "issue_document" && - readNonEmptyString(target.key) === "plan" - ); - } - - async function getAcceptedPlanApprovalInteractionForRun( - run: typeof heartbeatRuns.$inferSelect, - issueId: string | null, - ) { - const context = parseObject(run.contextSnapshot); - const interactionId = readNonEmptyString(context.interactionId); - if (!issueId || !interactionId) return null; - - const interaction = await db - .select({ - id: issueThreadInteractions.id, - kind: issueThreadInteractions.kind, - status: issueThreadInteractions.status, - payload: issueThreadInteractions.payload, - result: issueThreadInteractions.result, - }) - .from(issueThreadInteractions) - .where( - and( - eq(issueThreadInteractions.companyId, run.companyId), - eq(issueThreadInteractions.issueId, issueId), - eq(issueThreadInteractions.id, interactionId), - ), - ) - .then((rows) => rows[0] ?? null); - - if (!interaction) return null; - if ( - interaction.kind !== "request_confirmation" || - interaction.status !== "accepted" - ) - return null; - return isPlanApprovalConfirmationPayload(interaction.payload) - ? interaction - : null; - } - - function planApprovalResumeFailureErrorCode( - run: typeof heartbeatRuns.$inferSelect, - ) { - return readNonEmptyString(run.errorCode) ?? "unknown_error"; - } - - function buildPlanApprovalResumeFailureComment(input: { - run: typeof heartbeatRuns.$inferSelect; - status: "retrying" | "needs_attention"; - attempt: number; - maxAttempts: number; - }) { - const errorCode = planApprovalResumeFailureErrorCode(input.run); - if (input.status === "retrying") { - return `Agent failed to resume after approval: \`${errorCode}\` — retrying (attempt ${input.attempt}/${input.maxAttempts})`; - } - return `Agent failed to resume after approval: \`${errorCode}\` — needs attention`; - } - - function buildPlanApprovalResumeFailureResult(input: { - run: typeof heartbeatRuns.$inferSelect; - status: "retrying" | "needs_attention"; - attempt: number; - maxAttempts: number; - retryRunId?: string | null; - recoveryActionId?: string | null; - }): NonNullable { - return { - status: input.status, - errorCode: planApprovalResumeFailureErrorCode(input.run), - attempt: input.attempt, - maxAttempts: input.maxAttempts, - runId: input.run.id, - retryRunId: input.retryRunId ?? null, - recoveryActionId: input.recoveryActionId ?? null, - updatedAt: new Date().toISOString(), - }; - } - - async function updatePlanApprovalInteractionResumeFailure(input: { - interaction: NonNullable< - Awaited> - >; - failure: NonNullable; - }) { - const result = parseObject(input.interaction.result); - const nextResult = { - ...result, - version: 1 as const, - outcome: "accepted" as const, - resumeFailure: input.failure, - } satisfies RequestConfirmationResult; - - await db - .update(issueThreadInteractions) - .set({ - result: nextResult, - updatedAt: new Date(), - }) - .where(eq(issueThreadInteractions.id, input.interaction.id)); - } - - async function addPlanApprovalResumeFailureCommentOnce(input: { - issueId: string; - run: typeof heartbeatRuns.$inferSelect; - body: string; - }) { - const existing = await db - .select({ id: issueComments.id }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, input.run.companyId), - eq(issueComments.issueId, input.issueId), - or( - eq(issueComments.body, input.body), - sql`${issueComments.body} like ${`${input.body}\n%`}`, - ), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - if (existing) return null; - return issuesSvc.addComment( - input.issueId, - input.body, - { runId: input.run.id }, - { authorType: "system" }, - ); - } - - async function getActiveRecoveryActionId( - companyId: string, - sourceIssueId: string, - ) { - return db - .select({ id: issueRecoveryActions.id }) - .from(issueRecoveryActions) - .where( - and( - eq(issueRecoveryActions.companyId, companyId), - eq(issueRecoveryActions.sourceIssueId, sourceIssueId), - inArray(issueRecoveryActions.status, ["active", "escalated"]), - ), - ) - .orderBy(desc(issueRecoveryActions.updatedAt)) - .limit(1) - .then((rows) => rows[0]?.id ?? null); - } - - async function recordPlanApprovalResumeFailureRetry(input: { - run: typeof heartbeatRuns.$inferSelect; - issueId: string | null; - retryRunId: string | null; - attempt: number; - maxAttempts: number; - }) { - const interaction = await getAcceptedPlanApprovalInteractionForRun( - input.run, - input.issueId, - ); - if (!interaction || !input.issueId) return null; - - const body = buildPlanApprovalResumeFailureComment({ - run: input.run, - status: "retrying", - attempt: input.attempt, - maxAttempts: input.maxAttempts, - }); - await addPlanApprovalResumeFailureCommentOnce({ - issueId: input.issueId, - run: input.run, - body, - }); - await updatePlanApprovalInteractionResumeFailure({ - interaction, - failure: buildPlanApprovalResumeFailureResult({ - run: input.run, - status: "retrying", - attempt: input.attempt, - maxAttempts: input.maxAttempts, - retryRunId: input.retryRunId, - }), - }); - return interaction.id; - } - - async function escalatePlanApprovalResumeFailureNeedsAttention(input: { - run: typeof heartbeatRuns.$inferSelect; - issueId: string | null; - attempt: number; - maxAttempts: number; - }) { - const interaction = await getAcceptedPlanApprovalInteractionForRun( - input.run, - input.issueId, - ); - if (!interaction || !input.issueId) return null; - - const issue = await db - .select() - .from(issues) - .where( - and( - eq(issues.companyId, input.run.companyId), - eq(issues.id, input.issueId), - ), - ) - .then((rows) => rows[0] ?? null); - if (!issue) return null; - if ( - issue.status !== "todo" && - issue.status !== "in_progress" && - issue.status !== "in_review" - ) - return null; - - const body = buildPlanApprovalResumeFailureComment({ - run: input.run, - status: "needs_attention", - attempt: input.attempt, - maxAttempts: input.maxAttempts, - }); - await recovery.escalateStrandedAssignedIssue({ - issue, - previousStatus: issue.status, - latestRun: input.run, - comment: body, - }); - await addPlanApprovalResumeFailureCommentOnce({ - issueId: issue.id, - run: input.run, - body, - }); - - const recoveryActionId = await getActiveRecoveryActionId( - issue.companyId, - issue.id, - ); - await updatePlanApprovalInteractionResumeFailure({ - interaction, - failure: buildPlanApprovalResumeFailureResult({ - run: input.run, - status: "needs_attention", - attempt: input.attempt, - maxAttempts: input.maxAttempts, - recoveryActionId, - }), - }); - return interaction.id; - } - const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup }); - async function completeSkillTestRunForHeartbeatOutcome(input: { - run: typeof heartbeatRuns.$inferSelect; - issueId: string | null; - issueWorkMode?: string | null; - outcome: RunSessionOutcome; - error: string | null; - }) { - const completion = resolveSkillTestRunCompletionForHeartbeatOutcome( - input.outcome, - input.error, - ); - if (!completion || !input.issueId) return null; - - let isSkillTestIssue = input.issueWorkMode === "skill_test"; - if (!isSkillTestIssue && input.issueWorkMode === undefined) { - const issueRow = await db - .select({ - workMode: issues.workMode, - harnessKind: issues.harnessKind, - }) - .from(issues) - .where( - and( - eq(issues.companyId, input.run.companyId), - eq(issues.id, input.issueId), - ), - ) - .then((rows) => rows[0] ?? null); - isSkillTestIssue = - issueRow?.workMode === "skill_test" || - issueRow?.harnessKind === "skill_test"; - } - if (!isSkillTestIssue) return null; - - const existingRun = await db - .select({ - id: companySkillTestRuns.id, - status: companySkillTestRuns.status, - }) - .from(companySkillTestRuns) - .where( - and( - eq(companySkillTestRuns.companyId, input.run.companyId), - eq(companySkillTestRuns.issueId, input.issueId), - ), - ) - .then((rows) => rows[0] ?? null); - if ( - !existingRun || - ["succeeded", "failed", "cancelled"].includes(existingRun.status) - ) - return null; - - const completedRun = await companySkills.completeTestRunForIssue({ - companyId: input.run.companyId, - issueId: input.issueId, - outcome: completion.outcome, - error: completion.error, - }); - if (!completedRun) return null; - - await logActivity(db, { - companyId: input.run.companyId, - actorType: "system", - actorId: "heartbeat_finalize", - agentId: input.run.agentId, - runId: input.run.id, - action: "company.skill_test_run_completed", - entityType: "company_skill_test_run", - entityId: completedRun.id, - issueId: input.issueId, - details: { - issueId: input.issueId, - status: completedRun.status, - outputDocumentKey: completedRun.outputDocumentKey, - heartbeatOutcome: completion.heartbeatOutcome, - source: "heartbeat.run_finalized", - }, - }); - - return completedRun; - } - - async function releaseEnvironmentLeasesForRun(input: { - runId: string; - companyId: string; - agentId: string; - status: string | null | undefined; - failureReason?: string | null; - providerResourceDisposition?: ProviderResourceDisposition; - nativeLifecycleTelemetry?: { - provider: string; - harness: string; - lifecycleMode: "per_turn" | "warm"; - sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; - }; - }) { - const leaseOwnerRun = await getRun(input.runId); - if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return; - // Recovery can finish workspace copy-back outside the executor's finally. - // Successful copy-back does not earn warm retention for a failed turn. - const status = leaseOwnerRun?.status ?? input.status; - const providerResourceDisposition = providerResourceDispositionForTerminalRun( - input.providerResourceDisposition, - status, - ); - if (providerResourceDisposition === "destroy") { - const closeResult = await ( - options.closeWarmNativeSessionsForRun ?? - (async ({ runId, reason }) => { - const environmentIds = await db - .selectDistinct({ environmentId: environmentLeases.environmentId }) - .from(environmentLeases) - .where( - and( - eq(environmentLeases.heartbeatRunId, runId), - eq(environmentLeases.status, "active"), - ), - ) - .then((rows) => - rows.flatMap((row) => - typeof row.environmentId === "string" && - row.environmentId.length > 0 - ? [row.environmentId] - : [], - ), - ); - const aggregate = { closed: 0, busy: 0, failed: 0 }; - for (const environmentId of environmentIds) { - const result = await closeWarmNativeSessionsForEnvironment({ - environmentId, - reason, - }); - aggregate.closed += result.closed; - aggregate.busy += result.busy; - aggregate.failed += result.failed; - } - return aggregate; - }) - )({ - runId: input.runId, - reason: "terminal heartbeat run destroyed its environment lease", - }).catch((err) => { - logger.warn( - { err, runId: input.runId }, - "failed to close warm native sessions before environment lease destruction", - ); - return { closed: 0, busy: 0, failed: 1 }; - }); - // A failed remote checkpoint cannot veto destruction of the isolated - // sandbox after the run stopped. Provider destruction supplies the exit - // proof; it does not turn the interrupted checkpoint into a success. - const remoteLeases = closeResult.failed > 0 && leaseOwnerRun && - ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status) - ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and( - eq(environmentLeases.companyId, input.companyId), - eq(environmentLeases.heartbeatRunId, input.runId), - )) - : []; - const canDestroyRemote = remoteLeases.length > 0 && - remoteLeases.every(lease => lease.provider && lease.provider !== "local"); - if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) { - logger.warn( - { runId: input.runId, warmNativeSessions: closeResult }, - "deferred environment lease destruction until warm native sessions close", - ); - return; - } - } - const releaseResult = await envOrchestrator - .releaseForRun({ - heartbeatRunId: input.runId, - companyId: input.companyId, - agentId: input.agentId, - status: leaseReleaseStatusForRunStatus(status), - failureReason: input.failureReason ?? undefined, - providerResourceDisposition, - nativeLifecycleTelemetry: input.nativeLifecycleTelemetry, - }) - .catch((err) => { - logger.warn( - { err, runId: input.runId }, - "failed to release environment leases for heartbeat run", - ); - return null; - }); - for (const releaseError of releaseResult?.errors ?? []) { - logger.warn( - { - err: releaseError.error, - leaseId: releaseError.leaseId, - runId: input.runId, - }, - "failed to release environment lease for heartbeat run", - ); - } - await acknowledgeRemoteStop(input.runId, input.companyId); - } - - async function acknowledgeRemoteStop(runId: string, companyId: string) { - // The provider receipt arrives after adapter settlement. A remote ACP child - // has no host PID, so only this target-aware boundary can acknowledge Stop. - const stopped = await getRun(runId); - if (stopped?.runtimeMode === "native") { - const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId); - for (const remoteCleanupScope of scopes ?? []) { - completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope }); - } - } - if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" && - parseObject(stopped.resultJson?.executionCancellation).state === "requested" && - await runUsedConversationAdapter(db, stopped) && - await remoteExecutionHasStopped(db, companyId, runId)) { - await db.update(heartbeatRuns).set({ - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ - executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), - state: "acknowledged", acknowledgedAt: new Date().toISOString(), - proof: "provider_termination_receipt" }, - conversationContinuation: CONVERSATION_CONTINUATION_POLICY, - })}::jsonb`, - updatedAt: new Date(), - }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.status, "cancelled"))); - } - } - - async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) { - if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; - const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt - ? await db.select().from(nativeRunFinalizations).where(and( - eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), - )) : []; - const cancelledPreparation = run.runtimeMode === "legacy" && - await isCancelledNativeStartup(db, run, preparationCoordinator); - if (run.runtimeMode !== "native" && !cancelledPreparation && - parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true && - !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return; - const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null); - if (!issueId) return; - const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null; - const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase, - leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and( - eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), - )) : []; - // Stop ends one response. Saved user input can enter ordinary admission - // once its executor settles; it does not need a manufactured crash incident. - // The ordinary path still enforces task holds, ownership, and native session - // cleanup before starting a provider. - const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) && - !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner && - ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") && - await acknowledgedNativeStopExecutionHasStopped(db, currentRun) && - !(await getExecutionBlocker(db, run.companyId, issueId)); - const legacyContinuation = run.runtimeMode === "legacy" && - hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) && - !(await getExecutionBlocker(db, run.companyId, issueId)); - if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return; - const pending = await db.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - eq(agentWakeupRequests.requestedByActorType, "user"), - requestId ? eq(agentWakeupRequests.id, requestId) : undefined, - sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, - )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50); - for (const wake of pending) { - if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue; - let payload = parseObject(wake.payload); - if (payload.queuedCommentInterrupt) { - await resumeQueuedCommentInterrupt(wake.companyId, wake.id); - continue; - } - let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); - let commentId = deriveCommentId(context, payload); - let requestedByActorId = wake.requestedByActorId; - const reason = readNonEmptyString(context.wakeReason) ?? wake.reason; - if (run.runtimeMode === "native" || cancelledPreparation) { - const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId, - queuedCommentIdsFromWakePayload(payload)); - if (!ids.length) continue; - payload = withQueuedCommentIdsInWakePayload(payload, ids); - context = withQueuedCommentIdsInRunContext(context, ids); - commentId = ids.at(-1)!; - // A coalesced queue can contain several authors. Its saved comments, - // not the outer wake's first author, authorize the remaining input. - const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and( - eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId), - )); - requestedByActorId = author?.id ?? null; - } - if (legacyContinuation || stoppedNativeContinuation) { - if (!commentId || !run.finishedAt || !requestedByActorId || - !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue; - const [comment] = await db.select().from(issueComments).where(and( - eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), - sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"), - eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt), - isNull(issueComments.createdByRunId), - stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt), - )); - if (!comment?.body.trim()) continue; - } else { - let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; - const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId, - agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId, - reason, commentId, successorRunId: randomUUID(), dryRun: true, - ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), - onBlocked: (reason, message) => { wait = { reason, message }; }, - }); - if (!admitted) { - await db.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', - coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`, - updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); - continue; - } - } - // Re-enter ordinary admission with the original user's authority. It - // atomically adopts the deferred comments and still applies every gate. - await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"], - reason, payload, contextSnapshot: context, - requestedByActorType: "user", requestedByActorId, - ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), - idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id); - break; - } - } - - async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) { - return resumeSavedLegacyComments(companyId, queueId, true, opts); - } - - async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) { - const [wake] = await db.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - )); - if (!wake) return; - const payload = parseObject(wake.payload); - let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId); - let commentIds = queuedCommentIdsFromWakePayload(payload); - const issueId = readNonEmptyString(payload.issueId); - if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return; - const response = await readQueuedInteractionResponse(db, companyId, issueId, payload); - if (!commentIds.length && !response) return; - // Resolved cards are immutable input. Only an explicit Interrupt click can - // authorize continuation across a stopped execution; ordinary completion - // uses normal deferred-wake promotion. - if (response && !interrupted) return; - if (!interrupted) { - commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds); - if (!commentIds.length) return; - // The queue itself may have begun as a system wake. The saved human - // comment, not that wake's origin or mutable caller payload, is authority. - const [comment] = await db.select().from(issueComments).where(and( - eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), - eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"), - isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt), - )).orderBy(desc(issueComments.createdAt)).limit(1); - if (!comment?.body.trim() || !comment.authorUserId) return; - actorId = comment.authorUserId; - } - if (!actorId) return; - const agent = await getAgent(wake.agentId); - if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return; - const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, wake.agentId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), - )).limit(1); - if (active) return; - if (interrupted && opts?.retryCleanup) { - // Only the HTTP click grants an extra cleanup attempt. Periodic retries - // reuse the intent to deliver, never a fresh provider teardown budget. - const sourceRun = await db.transaction(async tx => { - const [task] = await tx.select().from(issues).where(and( - eq(issues.companyId, companyId), eq(issues.id, issueId), - )).for("update"); - const [current] = await tx.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, - sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`, - )); - if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) || - !current || (!queuedCommentIdsFromWakePayload(current.payload).length && - !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null; - const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, wake.agentId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), - )).limit(1); - if (successor) return null; - const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId); - const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId), - eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"), - inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - )).then(rows => rows[0]) : null; - if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null; - // Older ephemeral leases recorded successful cleanup without a provider - // receipt. Re-verify them through the recorded teardown path; a timestamp - // alone never certifies termination. Retained/reusable resources stay put. - const historical = await tx.select().from(environmentLeases).where(and( - eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), - eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt), - inArray(environmentLeases.status, ["released", "expired", "failed"]), - )).for("update"); - for (const lease of historical) { - if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue; - // Provider resource IDs identify physical sandboxes. A lease in any - // company can still own this resource; never destroy it on behalf of - // this company. This existence-only guard exposes no foreign data. - const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( - ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider), - eq(environmentLeases.providerLeaseId, lease.providerLeaseId), - or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])), - )).limit(1); - if (otherOwner) continue; - await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() }) - .where(eq(environmentLeases.id, lease.id)); - } - return run; - }); - if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: { - companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt", - } }); - } - const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds); - delete deliveryPayload.queuedCommentInterrupt; - await enqueueWakeup(wake.agentId, { - source: "on_demand", triggerDetail: "manual", reason: "issue_commented", - payload: deliveryPayload, contextSnapshot: response - ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId, - responsibleUserId: actorId } - : withQueuedCommentIdsInRunContext({ - issueId, triggeredBy: "board", actorId, responsibleUserId: actorId, - }, commentIds), - requestedByActorType: "user", requestedByActorId: actorId, - ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }), - issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] }, - idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`, - }, queueId); - } - - async function resumeExecutionWaitComments() { - if ((await getSchedulingSuppression()).suppressed) return; - const waits = await db.select({ wake: agentWakeupRequests }) - .from(agentWakeupRequests) - .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), - sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, - eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) - .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( - eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id), - executionBlockerPredicate(), - ))), eq(agentWakeupRequests.status, "deferred_issue_execution"), - eq(agentWakeupRequests.requestedByActorType, "user"), - sql`${agentWakeupRequests.payload}->'executionWait' is not null`, - sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, - lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), - notInArray(issues.status, ["done", "cancelled"]))) - .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); - const seen = new Set(); - for (const { wake } of waits) { - const issueId = String(wake.payload?.issueId); - if (seen.has(issueId)) continue; - seen.add(issueId); - // Advance the cursor even for invalid evidence so one damaged task cannot - // starve later requests in this bounded scan. Preserve concurrent edits. - const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( - eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), - )).returning({ id: agentWakeupRequests.id }); - if (!claimed) continue; - // Match normal admission's deterministic current blocker selection. An - // arbitrary historical action must not choose the retry's source run. - const blocker = await getExecutionBlocker(db, wake.companyId, issueId); - const sourceId = blocker?.runId; - if (!sourceId || !isUuidLike(sourceId)) continue; - const run = await getRun(sourceId, { includeExecutionEvidence: true }); - if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue; - if (canContinueCancelledRun(run)) { - await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { - logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation"); - }); - continue; - } - await resumeRemoteStopComments(run, wake.id).catch(err => { - logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message"); - }); - } - } - async function getAgentInvokability( agent: typeof agents.$inferSelect | null | undefined, ) { return evaluateAgentInvokabilityFromDb(db, agent); } - async function recordCurrentHeartbeatRunRuntimeProgress( - run: Pick< - typeof heartbeatRuns.$inferSelect, - "id" | "companyId" | "agentId" | "status" | "contextSnapshot" - >, - update: RuntimeStatusUpdate, - issueId: string | null, - ) { - if (!isHeartbeatRunRuntimeStatusActive(run.status)) { - clearHeartbeatRunRuntimeStatus(run.id); - return null; - } - - const currentRun = await getRun(run.id); - if (!currentRun || !isHeartbeatRunRuntimeStatusActive(currentRun.status)) { - clearHeartbeatRunRuntimeStatus(run.id); - return null; - } - - return recordHeartbeatRunRuntimeProgress(currentRun, update, issueId); - } - const issueMonitorDispatchColumns = { id: issues.id, companyId: issues.companyId, @@ -4157,1683 +2999,6 @@ export function heartbeatService( void reportRunFailure(db, updated, failureReport); } - async function setRunStatus( - runId: string, - status: string, - patch?: Partial, - ) { - const previousStatus = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null); - - if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch, - resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson), - }; - - // Preserve the receipt-source fence when finalization enriches usage. A - // late spool replay must still be able to complete an unfinished receipt. - if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } }; - - // Cancelling a queued run that never acquired provider execution is - // positive bootstrap evidence. It must not hold unrelated queued messages. - if (previousStatus && (status === "cancelled" || status === "interrupted")) { - patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) }; - } - if ( - status === "cancelled" && - previousStatus?.status === "queued" && - previousStatus.runtimeMode !== "native" && - !previousStatus.startedAt && - !previousStatus.processPid - ) { - patch = { - ...patch, - resultJson: { - ...previousStatus.resultJson, - ...patch?.resultJson, - executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, - }, - }; - } - const updated = - previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status) - ? await terminalizeLegacyExecution({ - db, - run: previousStatus, - status, - patch, - reconcileIfNeeded: true, - }) - : await db - .update(heartbeatRuns) - .set({ - status, - ...patch, - executionStatusDeliveryId: randomUUID(), - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, runId)) - .returning() - .then((rows) => rows[0] ?? null); - - if (updated) { - publishLiveEvent({ - companyId: updated.companyId, - type: "heartbeat.run.status", - payload: buildHeartbeatRunStatusLiveEventPayload(updated), - }); - publishRunLifecyclePluginEvent(updated); - emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null); - } - - return updated; - } - - async function setRunStatusIfRunning( - runId: string, - status: string, - patch?: Partial, - failureReport?: RunFailureReportOptions, - ) { - return setRunStatusFromLive(runId, status, ["running"], patch, failureReport); - } - - // Move a run to a new status only when its current status is one of - // `fromStatuses`. The compare-and-set is a single conditional update, so a - // concurrent path can win the race. When this update matches nothing, the - // function reads the current row and reports updated=false, so the caller can - // keep the terminal outcome that another path already wrote. - async function setRunStatusFromLive( - runId: string, - status: string, - fromStatuses: string[], - patch?: Partial, - failureReport?: RunFailureReportOptions, - cancellationCondition?: ReturnType, - ) { - // fromStatuses can name a terminal status as its own source (for example, - // an idempotent "still failed" patch), so the write below is not always a - // genuine transition. Read the pre-write status to tell the two apart. - const previousStatus = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null); - - if (previousStatus && (status === "cancelled" || status === "interrupted")) { - patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) }; - } - - if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch, - resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson), - }; - - // Preserve the receipt-source fence when finalization enriches usage. A - // late spool replay must still be able to complete an unfinished receipt. - if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } }; - - // Cancelling a queued run that never acquired provider execution is - // positive bootstrap evidence. It must not hold unrelated queued messages. - if ( - status === "cancelled" && - previousStatus?.status === "queued" && - previousStatus.runtimeMode !== "native" && - !previousStatus.startedAt && - !previousStatus.processPid - ) { - patch = { - ...patch, - resultJson: { - ...previousStatus.resultJson, - ...patch?.resultJson, - executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, - }, - }; - } - const updated = - previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status) - ? await terminalizeLegacyExecution({ - db, - run: previousStatus, - status, - patch, - fromStatuses, - reconcileIfNeeded: true, - writeConditions: [nativeRunnerOwnershipNotHeldCondition(), ...(cancellationCondition ? [cancellationCondition] : [])], - }) - : await db - .update(heartbeatRuns) - .set({ - status, - ...patch, - executionStatusDeliveryId: randomUUID(), - updatedAt: new Date(), - }) - .where( - and( - eq(heartbeatRuns.id, runId), - inArray(heartbeatRuns.status, fromStatuses), - ...(cancellationCondition ? [cancellationCondition] : []), - ...(isHeartbeatRunTerminalStatus(status) - ? [nativeRunnerOwnershipNotHeldCondition()] - : []), - ), - ) - .returning() - .then((rows) => rows[0] ?? null) - .catch((error: unknown) => { - if (cancellationCondition) rethrowNativeCancellationLockConflict(error); - throw error; - }); - - if (updated) { - publishLiveEvent({ - companyId: updated.companyId, - type: "heartbeat.run.status", - payload: buildHeartbeatRunStatusLiveEventPayload(updated), - }); - publishRunLifecyclePluginEvent(updated); - emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport); - return { run: updated, updated: true as const }; - } - - const current = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null); - - return { run: current, updated: false as const }; - } - - // Invariant: when a run releases its environment lease, the run row must be - // terminal. The finalizer writes the terminal status in a step that is - // separate from the agent status=done PATCH. If the sandbox or the run - // process stops between the two steps, heartbeat_runs.status stays "running". - // The UI reads liveness from that row, so a finished task shows "Live" - // forever. This function closes the gap in the run teardown: when the run is - // still running or queued, it forces a terminal status before the lease is - // released. It never overwrites a status that another path already made - // terminal. - async function terminalizeRunOnLeaseRelease( - run: typeof heartbeatRuns.$inferSelect, - ): Promise { - if (isNativeRunnerOwnershipHeld(run)) return run; - if (isHeartbeatRunTerminalStatus(run.status)) return run; - if (run.status !== "running" && run.status !== "queued") return run; - - // Choose the terminal status that reflects the true outcome. When the issue - // already reached a terminal status, the run reached its goal, so use the - // matching terminal run status. Otherwise the teardown cut the run short, - // so use "interrupted". - const issueId = readNonEmptyString( - parseObject(run.contextSnapshot).issueId, - ); - let terminalStatus: "succeeded" | "cancelled" | "interrupted" = - "interrupted"; - if (issueId) { - const issueStatus = await db - .select({ status: issues.status }) - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0]?.status ?? null); - if (issueStatus === "done") terminalStatus = "succeeded"; - else if (issueStatus === "cancelled") terminalStatus = "cancelled"; - } - - // Teardown can beat the cancellation finalizer. Preserve the acknowledged - // user intent instead of reporting an infrastructure interruption. - if (hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run)) terminalStatus = "cancelled"; - - const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`; - // Match both "running" and "queued". A queued run has released its lease but - // never reached "running", so a running-only update would miss it and leave - // a phantom live run behind. - const write = await setRunStatusFromLive( - run.id, - terminalStatus, - ["running", "queued"], - { - finishedAt: run.finishedAt ?? new Date(), - error: run.error ?? (terminalStatus === "interrupted" ? message : null), - errorCode: - run.errorCode ?? - (terminalStatus === "interrupted" - ? "lease_released_before_terminal" - : null), - }, - ); - if (!write.updated) { - // Another path already finalized the run. Keep that terminal outcome. - return write.run ?? run; - } - - const terminalRun = write.run; - if (terminalRun) { - await appendRunEvent(terminalRun, { - eventType: "lifecycle", - stream: "system", - level: terminalStatus === "interrupted" ? "warn" : "info", - message, - payload: { - previousStatus: run.status, - terminalStatus, - reason: "environment_lease_release", - ...(issueId ? { issueId } : {}), - }, - }).catch((eventErr) => { - logger.warn( - { err: eventErr, runId: run.id }, - "failed to append run event for lease-release terminalization", - ); - }); - } - return terminalRun ?? run; - } - - function publishRunLifecyclePluginEvent( - run: typeof heartbeatRuns.$inferSelect, - ) { - publishRunLifecyclePluginEventData({ - companyId: run.companyId, - runId: run.id, - agentId: run.agentId, - status: run.status, - invocationSource: run.invocationSource, - triggerDetail: run.triggerDetail, - error: run.error, - errorCode: run.errorCode, - issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId), - startedAt: run.startedAt, - finishedAt: run.finishedAt, - }); - } - - function publishRunLifecyclePluginEventData(run: { - companyId: string; - runId: string; - agentId: string; - status: string; - invocationSource: string; - triggerDetail: string | null; - error: string | null; - errorCode: string | null; - issueId: string | null; - startedAt: Date | null; - finishedAt: Date | null; - }) { - const eventType = - run.status === "running" - ? "agent.run.started" - : run.status === "succeeded" - ? "agent.run.finished" - : run.status === "failed" || run.status === "timed_out" - ? "agent.run.failed" - : run.status === "cancelled" - ? "agent.run.cancelled" - : null; - if (!eventType) return; - publishPluginDomainEvent({ - eventId: randomUUID(), - eventType, - occurredAt: new Date().toISOString(), - actorId: run.agentId, - actorType: "agent", - entityId: run.runId, - entityType: "heartbeat_run", - companyId: run.companyId, - payload: { - runId: run.runId, - agentId: run.agentId, - status: run.status, - invocationSource: run.invocationSource, - triggerDetail: run.triggerDetail, - error: run.error ?? null, - errorCode: run.errorCode ?? null, - issueId: run.issueId, - startedAt: run.startedAt ? new Date(run.startedAt).toISOString() : null, - finishedAt: run.finishedAt - ? new Date(run.finishedAt).toISOString() - : null, - }, - }); - } - - async function setWakeupStatus( - wakeupRequestId: string | null | undefined, - status: string, - patch?: Partial, - ) { - if (!wakeupRequestId) return; - await db - .update(agentWakeupRequests) - .set({ status, ...patch, updatedAt: new Date() }) - // Reassignment revokes requests transactionally. Late execution settlement - // must not turn a revoked request back into authority for a later retry. - .where(and(eq(agentWakeupRequests.id, wakeupRequestId), ne(agentWakeupRequests.status, "cancelled"))); - } - - async function addContinuationExhaustedCommentOnce(input: { - run: typeof heartbeatRuns.$inferSelect; - issueId: string; - comment: string; - }) { - const existing = await db - .select({ id: issueComments.id }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, input.run.companyId), - eq(issueComments.issueId, input.issueId), - eq(issueComments.createdByRunId, input.run.id), - sql`${issueComments.body} like 'Bounded liveness continuation exhausted%'`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - if (existing) return; - await issuesSvc.addComment(input.issueId, input.comment, { - agentId: input.run.agentId, - runId: input.run.id, - }); - } - - async function handleRunLivenessContinuation( - run: typeof heartbeatRuns.$inferSelect, - ) { - const context = parseObject(run.contextSnapshot); - if ( - readNonEmptyString(context.goalControlRequestId) || - context.resumeSessionGoalHeartbeat === true - ) - return; - const livenessState = run.livenessState as RunLivenessState | null; - if (livenessState !== "plan_only" && livenessState !== "empty_response") - return; - - const issueId = readNonEmptyString(context.issueId); - if (!issueId) return; - const waitingContext = await getIssueExecutionContext(run.companyId, issueId); - if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return; - - const [issue, agent] = await Promise.all([ - db - .select({ - id: issues.id, - companyId: issues.companyId, - identifier: issues.identifier, - title: issues.title, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - executionState: issues.executionState, - projectId: issues.projectId, - }) - .from(issues) - .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) - .then((rows) => rows[0] ?? null), - db - .select({ - id: agents.id, - companyId: agents.companyId, - status: agents.status, - }) - .from(agents) - .where(eq(agents.id, run.agentId)) - .then((rows) => rows[0] ?? null), - ]); - - const budgetBlock = - issue && agent - ? await budgets.getInvocationBlock(issue.companyId, agent.id, { - issueId: issue.id, - projectId: issue.projectId, - }) - : null; - - const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1; - const idempotencyKey = issue - ? buildRunLivenessContinuationIdempotencyKey({ - issueId: issue.id, - sourceRunId: run.id, - livenessState, - nextAttempt, - }) - : null; - const existingWake = idempotencyKey - ? await findExistingRunLivenessContinuationWake(db, { - companyId: run.companyId, - idempotencyKey, - }) - : null; - - const decision = decideRunLivenessContinuation({ - run, - issue, - agent, - livenessState, - livenessReason: run.livenessReason, - nextAction: run.nextAction, - budgetBlocked: Boolean(budgetBlock), - idempotentWakeExists: Boolean(existingWake), - }); - - if (decision.kind === "exhausted") { - await setRunStatus(run.id, run.status, { - livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation attempts exhausted`, - }); - await addContinuationExhaustedCommentOnce({ - run, - issueId, - comment: decision.comment, - }); - return; - } - - if (decision.kind !== "enqueue") return; - - const continuationRun = await enqueueWakeup(run.agentId, { - source: "automation", - triggerDetail: "system", - reason: RUN_LIVENESS_CONTINUATION_REASON, - payload: decision.payload, - contextSnapshot: { - ...decision.contextSnapshot, - originIdentityContextId: null, - parentRunId: run.id, - }, - idempotencyKey: decision.idempotencyKey, - requestedByActorType: "system", - requestedByActorId: "heartbeat", - }); - - if (continuationRun) { - await db - .update(heartbeatRuns) - .set({ - continuationAttempt: decision.nextAttempt, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)); - } - } - - function issueUiLink( - issue: Pick, - ) { - const label = issue.identifier ?? issue.id; - const prefix = issue.identifier?.split("-")[0] || "PAP"; - return `[${label}](/${prefix}/issues/${label})`; - } - - function hasUnmanagedBackgroundTaskEvidence( - resultJson: Record | null | undefined, - ) { - const evidence = parseObject(resultJson?.unmanagedBackgroundTask); - return ( - evidence.stopped === true && - (evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON || - evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON) - ); - } - - function withUnmanagedBackgroundTaskStopReason( - resultJson: Record | null | undefined, - ) { - return { - ...(resultJson ?? {}), - stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, - }; - } - - function buildDetectedSuccessfulRunProgressSummary( - run: typeof heartbeatRuns.$inferSelect, - currentUserRedactionOptions: CurrentUserRedactionOptions, - ) { - const resultJson = parseObject(run.resultJson); - const candidates = [ - hasUnmanagedBackgroundTaskEvidence(resultJson) - ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON - : null, - readNonEmptyString(run.nextAction) - ? `Next action noted: ${readNonEmptyString(run.nextAction)}` - : null, - readNonEmptyString(run.livenessReason), - readNonEmptyString(resultJson.summary), - readNonEmptyString(resultJson.result), - readNonEmptyString(resultJson.message), - ].filter((value): value is string => Boolean(value)); - const summary = candidates[0]; - if (!summary) return null; - return redactDetectedSuccessfulRunProgressSummaryForBoard( - summary, - currentUserRedactionOptions, - ); - } - - async function addSuccessfulRunHandoffCommentOnce(input: { - issue: Pick< - typeof issues.$inferSelect, - "id" | "identifier" | "title" | "status" - >; - run: typeof heartbeatRuns.$inferSelect; - agent: Pick; - detectedProgressSummary: string; - }) { - const existing = await db - .select({ id: issueComments.id }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, input.run.companyId), - eq(issueComments.issueId, input.issue.id), - eq(issueComments.createdByRunId, input.run.id), - sql`(${issueComments.body} = ${SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY} or ${issueComments.body} like '## This issue still needs a next step%' or ${issueComments.body} like '## Successful run missing issue disposition%')`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - if (existing) return null; - const notice = buildSuccessfulRunHandoffRequiredNotice(input); - return issuesSvc.addComment( - input.issue.id, - notice.body, - { runId: input.run.id }, - { - authorType: "system", - presentation: notice.presentation, - metadata: notice.metadata, - }, - ); - } - - async function handleSuccessfulRunHandoff( - run: typeof heartbeatRuns.$inferSelect, - agent: typeof agents.$inferSelect, - ) { - if (run.status !== "succeeded") return; - const context = parseObject(run.contextSnapshot); - const issueId = - readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); - if (!issueId) return; - const waitingContext = await getIssueExecutionContext(run.companyId, issueId); - if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return; - if ( - readNonEmptyString(context.goalControlRequestId) || - context.resumeSessionGoalHeartbeat === true - ) { - const goalProjection = await runnerGoalService(db).projection( - run.companyId, - issueId, - run.agentId, - ); - if (goalProjection?.goal?.status !== "complete") return; - } - - const issue = await db - .select({ - id: issues.id, - companyId: issues.companyId, - identifier: issues.identifier, - title: issues.title, - description: issues.description, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - assigneeUserId: issues.assigneeUserId, - executionState: issues.executionState, - monitorNextCheckAt: issues.monitorNextCheckAt, - projectId: issues.projectId, - originKind: issues.originKind, - }) - .from(issues) - .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) - .then((rows) => rows[0] ?? null); - const idempotencyKey = issue - ? buildFinishSuccessfulRunHandoffIdempotencyKey({ - issueId: issue.id, - sourceRunId: run.id, - }) - : null; - const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null); - const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); - const detectedProgressSummary = buildDetectedSuccessfulRunProgressSummary( - run, - currentUserRedactionOptions, - ); - const resultJson = parseObject(run.resultJson); - const finalReport = redactSuccessfulRunHandoffEvidence( - [ - readNonEmptyString(resultJson.summary), - readNonEmptyString(resultJson.result), - readNonEmptyString(resultJson.message), - ].find((value): value is string => Boolean(value)) ?? null, - currentUserRedactionOptions, - ); - const nextAction = redactSuccessfulRunHandoffEvidence( - readNonEmptyString(run.nextAction), - currentUserRedactionOptions, - ); - - const [ - activeExecutionPath, - queuedWake, - pendingInteraction, - pendingApproval, - explicitBlocker, - openRecoveryIssue, - existingWake, - budgetBlock, - pauseHold, - activeRoutineContinuation, - ] = await Promise.all([ - issue - ? db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, issue.companyId), - eq(heartbeatRuns.agentId, run.agentId), - inArray(heartbeatRuns.status, [ - ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, - ]), - sql`( - ${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id} - or ${heartbeatRuns.contextSnapshot} ->> 'taskId' = ${issue.id} - )`, - sql`${heartbeatRuns.id} <> ${run.id}`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) - : Promise.resolve(null), - issue - ? db - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, issue.companyId), - eq(agentWakeupRequests.agentId, run.agentId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - "claimed", - ]), - sql`( - ${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id} - or ${agentWakeupRequests.payload} ->> 'taskId' = ${issue.id} - or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issue.id} - or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issue.id} - )`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) - : Promise.resolve(null), - issue - ? db - .select({ id: issueThreadInteractions.id }) - .from(issueThreadInteractions) - .where( - and( - eq(issueThreadInteractions.companyId, issue.companyId), - eq(issueThreadInteractions.issueId, issue.id), - eq(issueThreadInteractions.status, "pending"), - activeIssueInteractionCondition(), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) - : Promise.resolve(null), - issue - ? db - .select({ id: issueApprovals.approvalId }) - .from(issueApprovals) - .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) - .where( - and( - eq(issueApprovals.companyId, issue.companyId), - eq(issueApprovals.issueId, issue.id), - inArray(approvals.status, ["pending", "revision_requested"]), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) - : Promise.resolve(null), - issue - ? db - .select({ id: issueRelations.issueId }) - .from(issueRelations) - .where( - and( - eq(issueRelations.companyId, issue.companyId), - eq(issueRelations.relatedIssueId, issue.id), - eq(issueRelations.type, "blocks"), - sql`exists ( - select 1 - from issues blocker - where blocker.id = ${issueRelations.issueId} - and blocker.company_id = ${issue.companyId} - and blocker.status not in ('done', 'cancelled') - and blocker.hidden_at is null - )`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) - : Promise.resolve(null), - issue - ? db - .select({ id: issues.id }) - .from(issues) - .where( - and( - eq(issues.companyId, issue.companyId), - inArray(issues.originKind, [ - RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, - RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, - ]), - eq(issues.originId, issue.id), - visibleIssueCondition(), - notInArray(issues.status, ["done", "cancelled"]), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) - : Promise.resolve(null), - idempotencyKey - ? findExistingFinishSuccessfulRunHandoffWake(db, { - companyId: run.companyId, - idempotencyKey, - }) - : Promise.resolve(null), - issue - ? budgets.getInvocationBlock(issue.companyId, run.agentId, { - issueId: issue.id, - projectId: issue.projectId, - }) - : Promise.resolve(null), - issue - ? treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id) - : Promise.resolve(null), - issue - ? db - .select({ id: routines.id }) - .from(routines) - .where( - and( - eq(routines.companyId, issue.companyId), - eq(routines.parentIssueId, issue.id), - eq(routines.status, "active"), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) - : Promise.resolve(null), - ]); - - const decision = decideSuccessfulRunHandoff({ - run, - issue, - agent, - livenessState: run.livenessState as RunLivenessState | null, - detectedProgressSummary, - finalReport, - nextAction, - taskKey, - hasActiveExecutionPath: Boolean(activeExecutionPath), - hasQueuedWake: Boolean(queuedWake), - hasPendingInteractionOrApproval: Boolean( - pendingInteraction || pendingApproval, - ), - hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt), - hasExplicitBlockerPath: Boolean(explicitBlocker), - hasOpenRecoveryIssue: Boolean(openRecoveryIssue), - hasPauseHold: Boolean(pauseHold), - hasActiveRoutineContinuation: Boolean(activeRoutineContinuation), - budgetBlocked: Boolean(budgetBlock), - idempotentWakeExists: Boolean(existingWake), - }); - - if (isSuccessfulRunHandoffValidPathSkip(decision) && issue) { - await resolveRequiredSuccessfulRunHandoffOnValidPath(db, { - companyId: issue.companyId, - issueId: issue.id, - issueIdentifier: issue.identifier, - agentId: run.agentId, - runId: run.id, - skipReason: decision.reason, - }); - } - - if (decision.kind !== "enqueue" || !issue) return; - - if (hasUnmanagedBackgroundTaskEvidence(parseObject(run.resultJson))) { - await db - .update(heartbeatRuns) - .set({ - livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || - ${JSON.stringify({ stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON })}::jsonb`, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)); - } - - const handoffRun = await enqueueWakeup(decision.targetAgentId, { - source: "automation", - triggerDetail: "system", - reason: FINISH_SUCCESSFUL_RUN_HANDOFF_REASON, - payload: decision.payload, - contextSnapshot: { - ...decision.contextSnapshot, - originIdentityContextId: null, - parentRunId: run.id, - }, - idempotencyKey: decision.idempotencyKey, - requestedByActorType: "system", - requestedByActorId: "heartbeat", - }); - if (!handoffRun) return; - - await addSuccessfulRunHandoffCommentOnce({ - issue, - run, - agent, - detectedProgressSummary: - detectedProgressSummary ?? - "The run reported progress, but did not choose a next step.", - }); - await logActivity(db, { - companyId: issue.companyId, - actorType: "system", - actorId: "heartbeat", - agentId: run.agentId, - runId: run.id, - action: "issue.successful_run_handoff_required", - entityType: "issue", - entityId: issue.id, - details: { - label: "Successful run missing issue disposition", - sourceRunId: run.id, - correctiveRunId: handoffRun.id, - handoffReason: SUCCESSFUL_RUN_MISSING_STATE_REASON, - missingDisposition: "clear_next_step", - detectedProgressSummary, - issue: issueUiLink(issue), - }, - }); - } - - async function handleIssueReviewPathDisposition( - run: typeof heartbeatRuns.$inferSelect, - ) { - const contextSnapshot = parseObject(run.contextSnapshot); - if (readNonEmptyString(contextSnapshot.goalControlRequestId)) return; - const issueId = - readNonEmptyString(contextSnapshot.issueId) ?? - readNonEmptyString(contextSnapshot.taskId); - if (!issueId) return; - const waitingContext = await getIssueExecutionContext(run.companyId, issueId); - if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return; - - const issue = await db - .select({ - id: issues.id, - companyId: issues.companyId, - identifier: issues.identifier, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - }) - .from(issues) - .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) - .then((rows) => rows[0] ?? null); - if (!issue || issue.status !== "in_review" || !issue.assigneeAgentId) - return; - - const reviewAttention = await issuesSvc - .listReviewAttention(issue.companyId, [issue]) - .then( - (map) => - map.get(issue.id) ?? { - state: "none" as const, - paths: [], - reason: null, - }, - ); - if (reviewAttention.state !== "stalled") return; - - const consumedPathRef = reviewPathConsumedRefFromRun({ - runId: run.id, - issueId: issue.id, - contextSnapshot, - }); - const idempotencyKey = buildIssueReviewPathLostIdempotencyKey({ - issueId: issue.id, - consumedPathRef, - }); - const existingWake = await db - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, issue.companyId), - eq(agentWakeupRequests.idempotencyKey, idempotencyKey), - notInArray(agentWakeupRequests.status, ["skipped"]), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - - const decision = decideIssueReviewPathRecovery({ - issueId: issue.id, - sourceRunId: run.id, - assigneeAgentId: issue.assigneeAgentId, - contextSnapshot, - reviewAttention, - existingWake: Boolean(existingWake), - }); - if (decision.kind !== "enqueue") return; - - const recoveryRun = await enqueueWakeup(issue.assigneeAgentId, { - source: "automation", - triggerDetail: "system", - reason: ISSUE_REVIEW_PATH_LOST_WAKE_REASON, - idempotencyKey: decision.idempotencyKey, - payload: decision.payload, - contextSnapshot: decision.contextSnapshot, - requestedByActorType: "system", - requestedByActorId: "heartbeat", - }).catch((error: unknown) => { - if (isReviewPathRecoveryIdempotencyConflict(error)) return null; - throw error; - }); - if (!recoveryRun) return; - - await logActivity(db, { - companyId: issue.companyId, - actorType: "system", - actorId: "heartbeat", - agentId: issue.assigneeAgentId, - runId: run.id, - action: "issue.review_path_recovery_queued", - entityType: "issue", - entityId: issue.id, - details: { - sourceRunId: run.id, - recoveryRunId: recoveryRun.id, - consumedPathRef, - recoveryAttempt: 1, - maxRecoveryAttempts: 1, - }, - }); - } - - async function appendRunEvent( - run: typeof heartbeatRuns.$inferSelect, - event: { - eventType: string; - stream?: "system" | "stdout" | "stderr"; - level?: "info" | "warn" | "error"; - color?: string; - message?: string; - payload?: Record; - retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"]; - }, - ) { - const eventAt = new Date(); - const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); - const sanitizedMessage = event.message - ? redactSensitiveText( - redactCurrentUserText(event.message, currentUserRedactionOptions), - ) - : event.message; - const boundedPayload = event.payload - ? boundHeartbeatRunEventPayloadForStorage(event.payload) - : event.payload; - const secretSanitizedPayload = boundedPayload - ? redactEventPayload(boundedPayload) - : boundedPayload; - const sanitizedPayload = secretSanitizedPayload - ? redactCurrentUserValue( - secretSanitizedPayload, - currentUserRedactionOptions, - ) - : secretSanitizedPayload; - const issueId = readRuntimeStatusIssueIdCandidate(run) ?? null; - const progress = buildRunEventRuntimeProgress({ - eventType: event.eventType, - message: sanitizedMessage ?? null, - payload: sanitizedPayload ?? null, - at: eventAt, - }); - const persistedEvent = await appendHeartbeatRunEvent(db, { - companyId: run.companyId, - runId: run.id, - agentId: run.agentId, - eventType: event.eventType, - stream: event.stream, - level: event.level, - color: event.color, - message: sanitizedMessage, - payload: sanitizedPayload, - retryExhaustion: event.retryExhaustion, - }); - if (persistedEvent.disposition === "duplicate") return; - const seq = persistedEvent.row.seq; - - publishLiveEvent({ - companyId: run.companyId, - type: "heartbeat.run.event", - payload: { - runId: run.id, - agentId: run.agentId, - issueId, - seq, - eventType: event.eventType, - stream: event.stream ?? null, - level: event.level ?? null, - color: event.color ?? null, - message: sanitizedMessage ?? null, - currentToolName: progress?.currentToolName ?? null, - lastAssistantSnippet: progress?.lastAssistantSnippet ?? null, - lastEventAt: (progress?.lastEventAt ?? eventAt).toISOString(), - payload: sanitizedPayload ?? null, - }, - }); - if (progress && isHeartbeatRunRuntimeStatusActive(run.status)) { - const status = setHeartbeatRunRuntimeStatus({ - companyId: run.companyId, - issueId, - agentId: run.agentId, - runId: run.id, - phase: progress.phase, - message: progress.message, - updatedAt: eventAt, - currentToolName: progress.currentToolName, - lastAssistantSnippet: progress.lastAssistantSnippet, - lastEventAt: progress.lastEventAt, - }); - if (status) publishHeartbeatRunRuntimeProgress(status); - } - } - - async function persistRunProcessMetadata( - runId: string, - meta: { pid: number; processGroupId: number | null; startedAt: string }, - ) { - return persistHeartbeatRunProcessMetadata(db, runId, meta); - } - - async function clearDetachedRunWarning(runId: string) { - const updated = await db - .update(heartbeatRuns) - .set({ - error: null, - errorCode: null, - updatedAt: new Date(), - }) - .where( - and( - eq(heartbeatRuns.id, runId), - eq(heartbeatRuns.status, "running"), - eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE), - ), - ) - .returning() - .then((rows) => rows[0] ?? null); - if (!updated) return null; - - await appendRunEvent(updated, { - eventType: "lifecycle", - stream: "system", - level: "info", - message: - "Detached child process reported activity; cleared detached warning", - }); - return updated; - } - - async function patchRunIssueCommentStatus( - runId: string, - patch: Partial< - Pick< - typeof heartbeatRuns.$inferInsert, - | "issueCommentStatus" - | "issueCommentSatisfiedByCommentId" - | "issueCommentRetryQueuedAt" - > - >, - ) { - return db - .update(heartbeatRuns) - .set({ ...patch, updatedAt: new Date() }) - .where(eq(heartbeatRuns.id, runId)) - .returning() - .then((rows) => rows[0] ?? null); - } - - async function findRunIssueComment( - runId: string, - companyId: string, - issueId: string, - resultJson?: Record | null, - ) { - const comments = await db - .select({ - id: issueComments.id, - body: issueComments.body, - }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, companyId), - eq(issueComments.issueId, issueId), - eq(issueComments.createdByRunId, runId), - ), - ) - .orderBy(desc(issueComments.createdAt), desc(issueComments.id)); - return findHeartbeatRunCompletionComment(comments, resultJson); - } - - async function findLatestCompletedFinalAgentMessage( - runId: string, - companyId: string, - ) { - const rows = await db - .select({ - seq: heartbeatRunEvents.seq, - payload: heartbeatRunEvents.payload, - }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.companyId, companyId), - eq(heartbeatRunEvents.runId, runId), - eq(heartbeatRunEvents.eventType, "item.completed"), - ), - ) - .orderBy(desc(heartbeatRunEvents.seq)) - .limit(200); - const candidates: Array<{ - seq: number; - text: string; - sourceEventId: string | null; - channel: "final" | "unknown"; - }> = []; - for (const row of rows) { - const prpEvent = parseObject(parseObject(row.payload).prpEvent); - const candidate = readCompletedAssistantMessageCandidate({ - seq: row.seq, - prpEvent, - }); - if (candidate) candidates.push(candidate); - } - const recoveryBoundary = await db - .select({ - seq: heartbeatRunEvents.seq, - payload: heartbeatRunEvents.payload, - }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.companyId, companyId), - eq(heartbeatRunEvents.runId, runId), - eq(heartbeatRunEvents.eventType, "lifecycle"), - ), - ) - .orderBy(heartbeatRunEvents.seq) - .limit(200) - .then( - (lifecycleRows) => - lifecycleRows.find( - (row) => - parseObject(row.payload).retryReasonCode === - "semantic_result_missing", - )?.seq ?? null, - ); - return selectHeartbeatRunFinalAgentMessage({ - candidates, - semanticResultRecoveryAfterSeq: recoveryBoundary, - }); - } - - async function refreshContinuationSummaryForRun( - run: typeof heartbeatRuns.$inferSelect, - agent: typeof agents.$inferSelect, - ) { - const contextSnapshot = parseObject(run.contextSnapshot); - const issueId = readNonEmptyString(contextSnapshot.issueId); - if (!issueId) return null; - try { - return await refreshIssueContinuationSummary({ - db, - issueId, - run: { - id: run.id, - status: run.status, - error: run.error, - errorCode: run.errorCode, - resultJson: run.resultJson as Record | null, - stdoutExcerpt: run.stdoutExcerpt, - stderrExcerpt: run.stderrExcerpt, - finishedAt: run.finishedAt, - }, - agent: { - id: agent.id, - name: agent.name, - adapterType: agent.adapterType, - }, - }); - } catch (err) { - logger.warn( - { - err, - runId: run.id, - issueId, - agentId: agent.id, - }, - "failed to refresh issue continuation summary", - ); - return null; - } - } - - async function enqueueMissingIssueCommentRetry( - run: typeof heartbeatRuns.$inferSelect, - agent: typeof agents.$inferSelect, - issueId: string, - ) { - const invokability = await getAgentInvokability(agent); - if (!invokability.invokable) { - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: - "Missing-comment retry suppressed because the agent is not invokable", - payload: { - reason: invokability.reason, - invalidOrgChain: invokability.invalidOrgChain, - ...invokability.details, - }, - }); - return null; - } - - const contextSnapshot = parseObject(run.contextSnapshot); - const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); - const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); - const retryContextSnapshot = withRecoveryContext( - { - ...contextSnapshot, - retryOfRunId: run.id, - wakeReason: "missing_issue_comment", - retryReason: "missing_issue_comment", - missingIssueCommentForRunId: run.id, - }, - "status_only", - ); - const responsibleUserId = await resolveResponsibleUserIdForRunContext( - run, - retryContextSnapshot, - ); - const now = new Date(); - - const retryRun = await db.transaction(async (tx) => { - await tx.execute( - sql`select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`, - ); - - const issue = await tx - .select({ id: issues.id }) - .from(issues) - .where( - and( - eq(issues.companyId, run.companyId), - eq(issues.executionRunId, run.id), - ), - ) - .then((rows) => rows[0] ?? null); - if (!issue) return null; - - const wakeupRequest = await tx - .insert(agentWakeupRequests) - .values({ - companyId: run.companyId, - agentId: run.agentId, - source: "automation", - triggerDetail: "system", - reason: "missing_issue_comment", - payload: withRecoveryContext( - { - issueId, - retryOfRunId: run.id, - retryReason: "missing_issue_comment", - }, - "status_only", - ), - status: "queued", - requestedByActorType: "system", - requestedByActorId: null, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - const queuedRun = await tx - .insert(heartbeatRuns) - .values({ - companyId: run.companyId, - agentId: run.agentId, - scopeKind: "issue", - issueId, - invocationSource: "automation", - triggerDetail: "system", - status: "queued", - wakeupRequestId: wakeupRequest.id, - contextSnapshot: retryContextSnapshot, - responsibleUserId, - sessionIdBefore: sessionBefore, - retryOfRunId: run.id, - issueCommentStatus: "not_applicable", - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - await tx - .update(agentWakeupRequests) - .set({ - runId: queuedRun.id, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, wakeupRequest.id)); - - await tx - .update(issues) - .set({ - executionRunId: queuedRun.id, - executionAgentNameKey: normalizeAgentNameKey(agent.name), - executionLockedAt: now, - updatedAt: now, - }) - .where(eq(issues.id, issue.id)); - - await tx - .update(heartbeatRuns) - .set({ - issueCommentStatus: "retry_queued", - issueCommentRetryQueuedAt: now, - updatedAt: now, - }) - .where(eq(heartbeatRuns.id, run.id)); - - return queuedRun; - }); - - if (!retryRun) return null; - - publishLiveEvent({ - companyId: retryRun.companyId, - type: "heartbeat.run.queued", - payload: { - runId: retryRun.id, - agentId: retryRun.agentId, - invocationSource: retryRun.invocationSource, - triggerDetail: retryRun.triggerDetail, - wakeupRequestId: retryRun.wakeupRequestId, - }, - }); - - return retryRun; - } - - async function hasDeferredIssueCommentWake( - companyId: string, - issueId: string, - agentId: string, - ) { - const deferredPayloads = await db - .select({ payload: agentWakeupRequests.payload }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.agentId, agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, - ), - ); - - return deferredPayloads.some(({ payload }) => { - const parsedPayload = parseObject(payload); - const deferredContext = parseObject( - parsedPayload[DEFERRED_WAKE_CONTEXT_KEY], - ); - return Boolean(deriveCommentId(deferredContext, parsedPayload)); - }); - } - - async function finalizeIssueCommentPolicy( - run: typeof heartbeatRuns.$inferSelect, - agent: typeof agents.$inferSelect, - presentationDecision?: RunPresentationDecision | null, - ) { - const contextSnapshot = parseObject(run.contextSnapshot); - // The explicit receipt admitted one turn. A prose-only follow-up cannot - // reuse it or renew it under the separate transient-failure retry policy. - if (readNonEmptyString(contextSnapshot.goalControlRequestId) || contextSnapshot.explicitUserContinuation) { - if (run.issueCommentStatus !== "not_applicable") { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "not_applicable", - issueCommentSatisfiedByCommentId: null, - issueCommentRetryQueuedAt: null, - }); - } - return { outcome: "not_applicable" as const, queuedRun: null }; - } - const issueId = readNonEmptyString(contextSnapshot.issueId); - if (!issueId) { - if (run.issueCommentStatus !== "not_applicable") { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "not_applicable", - issueCommentSatisfiedByCommentId: null, - issueCommentRetryQueuedAt: null, - }); - } - return { outcome: "not_applicable" as const, queuedRun: null }; - } - - // A failed/timed-out/cancelled run may immediately enqueue the normal - // assignment or continuation recovery below. Do not let the lower-priority - // status-only missing-comment wake claim the issue first. The dedicated - // missing-comment retry still records exhaustion if that retry itself - // fails, but an original failed execution remains on the established - // direct-adapter recovery path. - if ( - run.status !== "succeeded" && - readNonEmptyString(contextSnapshot.retryReason) !== - "missing_issue_comment" - ) { - if (run.issueCommentStatus !== "not_applicable") { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "not_applicable", - issueCommentSatisfiedByCommentId: null, - issueCommentRetryQueuedAt: null, - }); - } - return { outcome: "not_applicable" as const, queuedRun: null }; - } - - // A settled run may legitimately have no user-facing prose. The response - // resolver owns that decision; do not wake the agent again merely to force - // an artificial comment into the issue thread. - if ( - presentationDecision?.chosenSource === "none" && - (hasAcceptedSemanticResult(parseObject(run.resultJson)) || - presentationDecision.reasonCodes.includes( - "legacy_adapter_summary_ambiguous", - )) - ) { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "not_applicable", - issueCommentSatisfiedByCommentId: null, - issueCommentRetryQueuedAt: null, - }); - return { outcome: "not_applicable" as const, queuedRun: null }; - } - - // A pre-dispatch setup failure means the adapter process never started (for - // example an unresolved workspace base ref). No agent could run, so no agent - // could post an issue comment. A missing-comment retry cannot help and would - // loop the identical pre-adapter failure, so mark the policy not_applicable - // and queue nothing. - if ( - run.errorCode != null && - PRE_ADAPTER_SETUP_FAILURE_CODES.has(run.errorCode) - ) { - if (run.issueCommentStatus !== "not_applicable") { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "not_applicable", - issueCommentSatisfiedByCommentId: null, - issueCommentRetryQueuedAt: null, - }); - } - return { outcome: "not_applicable" as const, queuedRun: null }; - } - - const postedComment = await findRunIssueComment( - run.id, - run.companyId, - issueId, - parseObject(run.resultJson), - ); - if (postedComment) { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "satisfied", - issueCommentSatisfiedByCommentId: postedComment.id, - issueCommentRetryQueuedAt: null, - }); - return { outcome: "satisfied" as const, queuedRun: null }; - } - - // Missing-comment recovery is a legacy compatibility path for otherwise - // successful runs. A failed, timed-out, or cancelled run is already owned - // by lifecycle recovery and its terminal system presentation. Queuing a - // prose-only retry here can seize the issue execution lock before the - // authoritative continuation is materialized, replacing real recovery - // with a cheap status-only turn. - if (run.status !== "succeeded") { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "not_applicable", - issueCommentSatisfiedByCommentId: null, - issueCommentRetryQueuedAt: null, - }); - return { outcome: "not_applicable" as const, queuedRun: null }; - } - - if ( - readNonEmptyString(contextSnapshot.retryReason) === - "missing_issue_comment" - ) { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "retry_exhausted", - issueCommentSatisfiedByCommentId: null, - }); - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: - "Run ended without an issue comment after one retry; no further comment wake will be queued", - }); - return { outcome: "retry_exhausted" as const, queuedRun: null }; - } - - if (!shouldRequireIssueCommentForWake(contextSnapshot)) { - if (run.issueCommentStatus !== "not_applicable") { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "not_applicable", - issueCommentSatisfiedByCommentId: null, - issueCommentRetryQueuedAt: null, - }); - } - return { outcome: "not_applicable" as const, queuedRun: null }; - } - - if ( - await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId) - ) { - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "not_applicable", - issueCommentSatisfiedByCommentId: null, - issueCommentRetryQueuedAt: null, - }); - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "info", - message: - "Run ended without an issue comment; a deferred comment wake already exists for this issue", - }); - return { outcome: "not_applicable" as const, queuedRun: null }; - } - - const queuedRun = await enqueueMissingIssueCommentRetry( - run, - agent, - issueId, - ); - if (queuedRun) { - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: - "Run ended without an issue comment; queued one follow-up wake to require a comment", - }); - return { outcome: "retry_queued" as const, queuedRun }; - } - - await patchRunIssueCommentStatus(run.id, { - issueCommentStatus: "retry_exhausted", - issueCommentSatisfiedByCommentId: null, - }); - return { outcome: "retry_exhausted" as const, queuedRun: null }; - } - - function parseMaxTurnContinuationPolicy( - agent: typeof agents.$inferSelect, - ): MaxTurnContinuationPolicy { - const runtimeConfig = parseObject(agent.runtimeConfig); - const heartbeat = parseObject(runtimeConfig.heartbeat); - const configured = parseObject(heartbeat.maxTurnContinuation); - const rawMaxAttempts = Math.floor( - asNumber( - configured.maxAttempts, - MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS, - ), - ); - const rawDelayMs = Math.floor( - asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS), - ); - - return { - enabled: asBoolean(configured.enabled, true), - maxAttempts: Math.max( - 0, - Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts), - ), - delayMs: Math.max( - 0, - Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs), - ), - }; - } - function truncateAgentErrorReason( reason: string | null | undefined, ): string | null { @@ -5884,7 +3049,7 @@ export function heartbeatService( lastHeartbeatAt: new Date(), updatedAt: new Date(), }) - .where(eq(agents.id, agentId)) + .where(and(eq(agents.id, agentId), eq(agents.lifecycleState, "ready"))) .returning() .then((rows) => rows[0] ?? null); @@ -5913,266 +3078,6 @@ export function heartbeatService( } } - function mergeRunStopMetadataForAgent( - agent: Pick, - outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out", - options?: { - resultJson?: Record | null; - conversationContinuationEligible?: boolean; - errorCode?: string | null; - errorMessage?: string | null; - }, - ) { - const stopMetadata = buildHeartbeatRunStopMetadata({ - adapterType: agent.adapterType, - adapterConfig: parseObject(agent.adapterConfig), - outcome, - errorCode: options?.errorCode ?? null, - errorMessage: options?.errorMessage ?? null, - }); - const result = mergeHeartbeatRunStopMetadata( - options?.resultJson ?? null, - stopMetadata, - ); - const cancellationAcknowledged = - parseObject(result?.executionCancellation).state === "acknowledged"; - return options?.conversationContinuationEligible !== false && outcome !== "succeeded" && - (outcome !== "cancelled" || cancellationAcknowledged) && - isConversationAdapter(agent.adapterType) - ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY } - : result; - } - - function countValue(value: unknown) { - const parsed = Number(value ?? 0); - return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0; - } - - function dateValue(value: unknown) { - if (value instanceof Date) - return Number.isNaN(value.getTime()) ? null : value; - if (typeof value === "string" || typeof value === "number") { - const parsed = new Date(value); - return Number.isNaN(parsed.getTime()) ? null : parsed; - } - return null; - } - - function latestDate(...values: unknown[]) { - let latest: Date | null = null; - for (const value of values) { - const parsed = dateValue(value); - if (!parsed) continue; - if (!latest || parsed.getTime() > latest.getTime()) latest = parsed; - } - return latest; - } - - async function buildRunLivenessInput( - run: typeof heartbeatRuns.$inferSelect, - resultJson: Record | null | undefined, - ): Promise { - const context = parseObject(run.contextSnapshot); - const contextIssueId = readNonEmptyString(context.issueId); - const continuationAttempt = asNumber( - context.continuationAttempt, - run.continuationAttempt ?? 0, - ); - - const issue = contextIssueId - ? await db - .select({ - status: issues.status, - title: issues.title, - description: issues.description, - workMode: issues.workMode, - }) - .from(issues) - .where( - and( - eq(issues.companyId, run.companyId), - eq(issues.id, contextIssueId), - ), - ) - .then((rows) => rows[0] ?? null) - : null; - - const [commentStats] = contextIssueId - ? await db - .select({ - count: sql`count(*)::int`, - latestAt: sql`max(${issueComments.createdAt})`, - }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, run.companyId), - eq(issueComments.issueId, contextIssueId), - eq(issueComments.createdByRunId, run.id), - isNull(issueComments.deletedAt), - ), - ) - : [{ count: 0, latestAt: null }]; - - const issueCommentBodies = contextIssueId - ? await db - .select({ body: issueComments.body }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, run.companyId), - eq(issueComments.issueId, contextIssueId), - eq(issueComments.createdByRunId, run.id), - ), - ) - .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) - .limit(5) - .then((rows) => rows.reverse().map((row) => row.body)) - : []; - - const continuationSummary = contextIssueId - ? await getIssueContinuationSummaryDocument(db, contextIssueId) - : null; - - const [documentStats] = contextIssueId - ? await db - .select({ - count: sql`count(*)::int`, - planCount: sql`count(*) filter (where ${issueDocuments.key} = 'plan')::int`, - latestAt: sql`max(${documentRevisions.createdAt})`, - }) - .from(documentRevisions) - .innerJoin( - issueDocuments, - eq(documentRevisions.documentId, issueDocuments.documentId), - ) - .where( - and( - eq(documentRevisions.companyId, run.companyId), - eq(documentRevisions.createdByRunId, run.id), - eq(issueDocuments.companyId, run.companyId), - eq(issueDocuments.issueId, contextIssueId), - sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`, - ), - ) - : [{ count: 0, planCount: 0, latestAt: null }]; - - const [workProductStats] = contextIssueId - ? await db - .select({ - count: sql`count(*)::int`, - latestAt: sql`max(${issueWorkProducts.createdAt})`, - }) - .from(issueWorkProducts) - .where( - and( - eq(issueWorkProducts.companyId, run.companyId), - eq(issueWorkProducts.issueId, contextIssueId), - eq(issueWorkProducts.createdByRunId, run.id), - ), - ) - : [{ count: 0, latestAt: null }]; - - const [workspaceOperationStats] = await db - .select({ - count: sql`count(*)::int`, - latestAt: sql`max(${workspaceOperations.startedAt})`, - }) - .from(workspaceOperations) - .where( - and( - eq(workspaceOperations.companyId, run.companyId), - eq(workspaceOperations.heartbeatRunId, run.id), - ), - ); - - const [activityStats] = await db - .select({ - count: sql`count(*)::int`, - latestAt: sql`max(${activityLog.createdAt})`, - }) - .from(activityLog) - .where( - and( - eq(activityLog.companyId, run.companyId), - eq(activityLog.runId, run.id), - notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS), - ), - ); - - const [eventStats] = await db - .select({ - count: sql`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`, - latestAt: sql`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`, - }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.companyId, run.companyId), - eq(heartbeatRunEvents.runId, run.id), - ), - ); - - return { - runStatus: run.status, - issue, - resultJson: resultJson ?? run.resultJson ?? null, - issueCommentBodies, - continuationSummaryBody: continuationSummary?.body ?? null, - stdoutExcerpt: run.stdoutExcerpt ?? null, - stderrExcerpt: run.stderrExcerpt ?? null, - error: run.error ?? null, - errorCode: run.errorCode ?? null, - continuationAttempt, - evidence: { - issueCommentsCreated: countValue(commentStats?.count), - documentRevisionsCreated: countValue(documentStats?.count), - planDocumentRevisionsCreated: countValue(documentStats?.planCount), - workProductsCreated: countValue(workProductStats?.count), - workspaceOperationsCreated: countValue(workspaceOperationStats?.count), - activityEventsCreated: countValue(activityStats?.count), - toolOrActionEventsCreated: countValue(eventStats?.count), - latestEvidenceAt: latestDate( - commentStats?.latestAt, - documentStats?.latestAt, - workProductStats?.latestAt, - workspaceOperationStats?.latestAt, - activityStats?.latestAt, - eventStats?.latestAt, - ), - }, - }; - } - - async function classifyAndPersistRunLiveness( - run: typeof heartbeatRuns.$inferSelect, - resultJson?: Record | null, - ) { - const authRepair = run.status === "failed" - ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => { - logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available"); - return null; - }) - : null; - const classification = classifyRunLiveness({ - ...await buildRunLivenessInput(run, resultJson), - authenticationRepairRequested: Boolean(authRepair?.interactionId), - }); - return db - .update(heartbeatRuns) - .set({ - livenessState: classification.livenessState, - livenessReason: classification.livenessReason, - continuationAttempt: classification.continuationAttempt, - lastUsefulActionAt: classification.lastUsefulActionAt, - nextAction: classification.nextAction, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)) - .returning() - .then((rows) => rows[0] ?? null); - } - async function recoverActiveSessionGoals() { if ((await getSchedulingSuppression()).suppressed) { return { scanned: 0, enqueued: 0 }; @@ -6406,29 +3311,6 @@ export function heartbeatService( return recovery.reconcileResolvedDependencyWakeBackstop(opts); } - async function updateRuntimeState( - agent: typeof agents.$inferSelect, - run: typeof heartbeatRuns.$inferSelect, - result: AdapterExecutionResult, - session: { legacySessionId: string | null }, - normalizedUsage?: UsageTotals | null, - ) { - await ensureRuntimeState(agent); - await accountRunCost(db, run.id, budgetHooks); - await db - .update(agentRuntimeState) - .set({ - adapterType: agent.adapterType, - sessionId: session.legacySessionId, - lastRunId: run.id, - lastRunStatus: run.status, - lastError: run.error ?? null, - updatedAt: new Date(), - }) - .where(eq(agentRuntimeState.agentId, agent.id)); - - } - // Await every background heartbeat execution that is currently in flight. A // draining run can, in its finally block, promote and dispatch the next queued // run for the same agent — that follow-up execution is registered in the set @@ -9634,6 +6516,7 @@ export function heartbeatService( and( eq(agents.id, agent.id), notInArray(agents.status, [...DIRECT_NON_INVOKABLE_STATUSES]), + eq(agents.lifecycleState, "ready"), ), ) .returning() @@ -13245,6 +10128,13 @@ export function heartbeatService( ${JSON.stringify({ startupPreparationSettledAt: new Date().toISOString() })}::jsonb`, }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "cancelled"))); } + await db.update(heartbeatRuns).set({ + executionStage: sql`case when ${heartbeatRuns.status} = 'cancelled' then 'settled' else ${heartbeatRuns.executionStage} end`, + controllerLeaseExpiresAt: null, + }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.runtimeMode, "legacy"), + eq(heartbeatRuns.controllerBootId, legacyControllerBootId), + inArray(heartbeatRuns.status, ["succeeded", "failed", "cancelled", "timed_out", "interrupted"]))); + } finally { controllerLease.stop(); activeRunExecutions.delete(run.id); @@ -13300,684 +10190,30 @@ export function heartbeatService( } } - async function releaseIssueExecutionAndPromote( - run: Pick, - options: { suppressImmediateRecovery?: boolean } = {}, - ) { - try { - const source = await getRun(run.id); - const { postCommitEffects } = await wakeQueue.releaseIssueExecution({ - companyId: run.companyId, - runId: run.id, - now: new Date(), - // A durable authentication card owns recovery. This covers the review path, - // while continuation classification blocks periodic generic retries. - suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source), - }); - await applyWakeQueuePostCommitEffects(postCommitEffects); - const completed = await getRun(run.id); - const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId) - ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId; - if (completed?.status === "succeeded" && issueId) { - await settleSlackConversation(db, run.companyId, issueId).catch((err) => { - logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation"); - }); - } - } catch (error) { - if ( - error instanceof WakeQueueApplicationError && - error.code === "responsible_user_unresolved" - ) { - // Every other `responsible_user_unresolved` HttpError in this file - // carries `code` inside its own `details`; match that shape here too. - throw new HttpError(422, error.message, { code: error.code, ...error.details }); - } - throw error; - } - } - - async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) { - const runIssueId = sql< - string | null - >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; - const effectiveProjectId = sql< - string | null - >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; - - const rows = await db - .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id }) - .from(heartbeatRuns) - .leftJoin( - issues, - and( - eq(issues.companyId, companyId), - sql`${issues.id}::text = ${runIssueId}`, - ), - ) - .where( - and( - eq(heartbeatRuns.companyId, companyId), - createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined, - inArray(heartbeatRuns.status, [ - ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, - ]), - sql`${effectiveProjectId} = ${projectId}`, - ), - ); - - return rows.map((row) => row.id); - } - - async function listProjectScopedWakeupIds( - companyId: string, - projectId: string, - database: Db = db, - ) { - const wakeIssueId = sql< - string | null - >`${agentWakeupRequests.payload} ->> 'issueId'`; - const effectiveProjectId = sql< - string | null - >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; - - const rows = await database - .selectDistinctOn([agentWakeupRequests.id], { - id: agentWakeupRequests.id, - }) - .from(agentWakeupRequests) - .leftJoin( - issues, - and( - eq(issues.companyId, companyId), - sql`${issues.id}::text = ${wakeIssueId}`, - ), - ) - .where( - and( - eq(agentWakeupRequests.companyId, companyId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - sql`${effectiveProjectId} = ${projectId}`, - ), - ); - - return rows.map((row) => row.id); - } - - async function cancelPendingWakeupsForBudgetScope( - scope: BudgetEnforcementScope, - ) { - return withCurrentBudgetEnforcement(db, scope, async (tx) => { - const now = new Date(); - let wakeupIds: string[] = []; - - if (scope.scopeType === "company") { - wakeupIds = await tx - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, scope.companyId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - } else if (scope.scopeType === "agent") { - wakeupIds = await tx - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, scope.companyId), - eq(agentWakeupRequests.agentId, scope.scopeId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - } else { - wakeupIds = await listProjectScopedWakeupIds( - scope.companyId, - scope.scopeId, - tx, - ); - } - - if (wakeupIds.length === 0) return 0; - - await tx - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: now, - error: "Cancelled due to budget pause", - updatedAt: now, - }) - .where(and( - inArray(agentWakeupRequests.id, wakeupIds), - inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), - isNull(agentWakeupRequests.runId), - scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined, - )); - - return wakeupIds.length; - }); - } - - type CancelRunOptions = { - budgetEnforcement?: BudgetEnforcementScope; - /** Optional board request identity, atomically reserved for native Stop. */ - cancellationRequestId?: string; - cancellationRequestedByUserId?: string | null; - errorCode?: string; - resultJson?: Record; - eventMessage?: string; - eventPayload?: Record; - /** Per-call graceful process shutdown window, bounded to a safe range. */ - terminationGraceMs?: number; - /** Caller is immediately scheduling an explicit successor path. */ - suppressImmediateRecovery?: boolean; - }; - - function cancellationTerminationGraceMs( - configuredGraceSec: number, - requestedGraceMs: number | undefined, - ) { - const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000; - if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) { - return configuredGraceMs; - } - return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs))); - } - - async function cancelRunInternal( - runId: string, - reason = "Cancelled by control plane", - options: CancelRunOptions = {}, - ) { - let run = await getRun(runId); - if (!run) throw notFound("Heartbeat run not found"); - if (options.cancellationRequestId) { - run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null); - } - // The caller claim checked retry eligibility under both durable row locks. - // This is only a cancellation candidate: dispatch rechecks the coordinator - // under lock, and the final CAS requires its own unchanged acknowledged - // retry fence. Re-reading only retryable_failure here would strand replay. - const pendingNativeRetry = - run.runtimeMode === "native" && run.status === "failed" && ( - Boolean(options.cancellationRequestId) || await db - .select({ runId: nativeRunFinalizations.runId }) - .from(nativeRunFinalizations) - .where( - and( - eq(nativeRunFinalizations.runId, run.id), - eq(nativeRunFinalizations.companyId, run.companyId), - eq(nativeRunFinalizations.phase, "retryable_failure"), - ), - ) - .then((rows) => rows.length > 0) - ); - if ( - !pendingNativeRetry && - !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes( - run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number], - ) - ) - return run; - const agent = await getAgent(run.agentId); - const errorCode = options.errorCode ?? "cancelled"; - const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason); - - const pendingProcessCancellation = processRunCancellationSettlements.get( - run.id, - ); - if (pendingProcessCancellation) { - await pendingProcessCancellation.settled; - if (pendingProcessCancellation.failed) - throw pendingProcessCancellation.error; - return getRun(run.id); - } - const running = runningProcesses.get(run.id); - const stopOwnership = - run.runtimeMode !== "native" - ? captureAdapterStopOwnership(run.id) - : undefined; - const control = stopOwnership?.control; - // Capture the existing adapter owner before waiting on the run lock. Then - // atomically fence preparation and refresh the selected runtime, so Stop - // cannot miss a native handoff that won after its first read. - // Established legacy processes must still be stopped if the database is - // unavailable. Only native or not-yet-dispatched preparation needs this - // additional durable fence before its existing cancellation path. - if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) { - const fence = async (tx: Db) => { - const [fenced] = await tx.update(heartbeatRuns).set({ - // Record handoff intent before native cancellation can finalize and release - // the run. Only its audited stop acknowledgement suppresses recovery. - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || - ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true - ? { reassignmentStopRequested: true } : {})}::jsonb || - ${JSON.stringify({ cancellation })}::jsonb || - jsonb_build_object('startupCancellation', jsonb_build_object( - 'requestedAt', ${new Date().toISOString()}::text, - 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy' - and ${heartbeatRuns.runtimeModeResolvedAt} is null - and ${heartbeatRuns.executionStage} = 'preparing' - and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false) - ))`, - }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status, - pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], - ))).returning(); - return fenced ?? null; - }; - let fenced: typeof run | null; - try { - fenced = options.budgetEnforcement - ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence) - : await fence(db); - } catch (error) { - stopOwnership?.release(); - throw error; - } - if (!fenced) { - stopOwnership?.release(); - return getRun(runId); - } - run = fenced; - } - const resultJson = { ...(agent - ? { - ...mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: parseObject(run.resultJson), - errorCode, - errorMessage: reason, - }), - ...(options.resultJson ?? {}), - } - : options.resultJson), cancellation }; - - try { - let releaseProcessCancellation: (() => void) | undefined; - const processCancellationSettlement = - run.runtimeMode !== "native" && - !control && - running - ? { - settled: new Promise((resolve) => { - releaseProcessCancellation = resolve; - }), - failed: false, - error: undefined as unknown, - } - : undefined; - if (processCancellationSettlement) { - // No await between joining an existing owner above and registering ours. - processRunCancellationSettlements.set( - run.id, - processCancellationSettlement, - ); - } - const cancellation = await (async () => { - try { - if (control) { - await db - .update(heartbeatRuns) - .set({ - error: reason, - errorCode, - // Stop may have read the row before adapter settlement saved - // a required-copyback receipt. Merge into current database JSON - // and leave those server-owned fields to the restore recorder. - resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({ - ...resultJson, - ...(!running ? { executionCancellation: { - state: "requested", requestedAt: new Date().toISOString(), - } } : {}), - }, true), - updatedAt: new Date(), - }) - .where( - and( - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.status, "running"), - ), - ); - control.controller.abort(new Error(reason)); - } - let terminationSettled = false; - try { - await cancelHeartbeatNativeRun({ - db, - runId: run.id, - reason, - runtimeMode: run.runtimeMode, - ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), - }); - if (running) { - await terminateHeartbeatRunProcess({ - pid: running.child.pid, - processGroupId: running.processGroupId, - // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM - // can leave commands in their separate process groups alive. - signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined, - graceMs: cancellationTerminationGraceMs( - running.graceSec, - options.terminationGraceMs, - ), - }); - } - terminationSettled = true; - } finally { - if ( - (!processCancellationSettlement || terminationSettled) && - runningProcesses.get(run.id) === running - ) { - runningProcesses.delete(run.id); - } - } - - if (control) { - await waitForAdapterStop(control.settled, undefined, { - runId: run.id, - adapterType: agent?.adapterType, - runtimeMode: run.runtimeMode, - abortRequested: control.controller.signal.aborted, - }, control); - const stopped = await getRun(run.id); - if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) { - if ( - parseObject(stopped.resultJson?.executionCancellation).state !== - "acknowledged" - ) { - throw conflict( - "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.", - ); - } - // The owned adapter already finalized this run and its lifecycle. - // Do not replay the process cancellation side effects below. - return { run: stopped, updated: false }; - } - } - - const finishedAt = new Date(); - const persistedCancellationResult = - run.runtimeMode === "native" - ? await getRun(run.id).then((current) => - parseObject(current?.resultJson), - ) - : {}; - return await setRunStatusFromLive( - run.id, - "cancelled", - pendingNativeRetry - ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] - : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], - { - finishedAt, - error: reason, - errorCode, - ...(resultJson || - Object.keys(persistedCancellationResult).length > 0 - ? { - resultJson: { - ...persistedCancellationResult, - ...(resultJson ?? {}), - // A scheduler placeholder has no process to acknowledge. - // Preserve its normal release policy instead of treating - // it as an operator stop of provider work. - ...(processCancellationSettlement && agent && running && ( - (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) || - (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0) - ) - ? mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: { - ...resultJson, - executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() }, - }, - errorCode, errorMessage: reason, - }) - : {}), - // The native cancellation helper may have advanced a durable - // pending intent to its acknowledged state after `run` was - // first read. Never let that stale snapshot overwrite the - // authoritative post-dispatch acknowledgement. - ...(Object.hasOwn( - persistedCancellationResult, - "nativeCancellation", - ) - ? { - nativeCancellation: - persistedCancellationResult.nativeCancellation, - } - : {}), - }, - } - : {}), - }, - undefined, - pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined, - ); - } catch (error) { - if (processCancellationSettlement) { - processCancellationSettlement.failed = true; - processCancellationSettlement.error = error; - if (activeRunExecutions.has(run.id)) { - failedProcessRunCancellations.set( - run.id, - processCancellationSettlement, - ); - } - } - throw error; - } finally { - if (processCancellationSettlement) { - if ( - processRunCancellationSettlements.get(run.id) === - processCancellationSettlement - ) { - processRunCancellationSettlements.delete(run.id); - } - // Always settle waiters, including termination and DB errors. A - // failed Stop remains an error to its caller, never a cancellation - // receipt; the signal-bearing executor may then record failure. - releaseProcessCancellation?.(); - } - } - })(); - const cancelled = cancellation.run; - - if (cancellation.updated && cancelled) { - await setWakeupStatus(run.wakeupRequestId, "cancelled", { - finishedAt: cancelled.finishedAt ?? new Date(), - error: reason, - }); - await appendRunEvent(cancelled, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: options.eventMessage ?? "run cancelled", - payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, - }); - await releaseIssueExecutionAndPromote(cancelled, { - suppressImmediateRecovery: options.suppressImmediateRecovery, - }); - await finalizeAgentStatus(run.agentId, "cancelled", undefined, { - wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - }); - await startNextQueuedRunForAgent(run.agentId); - } - return cancelled; - } finally { - stopOwnership?.release(); - } - } - - async function cancelActiveForAgentInternal( - agentId: string, - reason = "Cancelled due to agent pause", - errorCode = "cancelled", - ) { - const agent = await getAgent(agentId); - const runs = await db - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.agentId, agentId), - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), - ), - ); - + async function stopInvocationsForAgents(agentIds: string[], reason: string) { + await cancelInvocationsForAgentsInternal(agentIds, reason); + const runs = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, agentIds)); + const leasesToRelease = await db.select({ runId: environmentLeases.heartbeatRunId }).from(environmentLeases) + .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, environmentLeases.heartbeatRunId)) + .where(and(inArray(heartbeatRuns.agentId, agentIds), + or(and(eq(heartbeatRuns.status, "cancelled"), inArray(environmentLeases.status, ["active", "pending_cleanup"])), + and(eq(environmentLeases.status, "retained"), eq(environmentLeases.cleanupStatus, "failed"))))); + const needsRelease = new Set(leasesToRelease.map(lease => lease.runId)); for (const run of runs) { - const stopOwnership = - run.runtimeMode !== "native" - ? captureAdapterStopOwnership(run.id) - : undefined; - try { - if (stopOwnership?.control) { - await cancelRunInternal(run.id, reason, { errorCode }); - continue; - } - if (run.runtimeMode === "native") { - await db.update(heartbeatRuns).set({ resultJson: - sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`, - }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status))); - await cancelHeartbeatNativeRun({ - db, - runId: run.id, - reason, - runtimeMode: run.runtimeMode, - }); - } - const persistedCancellationResult = - run.runtimeMode === "native" - ? await getRun(run.id).then((current) => - parseObject(current?.resultJson), - ) - : parseObject(run.resultJson); - await setRunStatus(run.id, "cancelled", { - finishedAt: new Date(), - error: reason, - errorCode, - resultJson: { - ...persistedCancellationResult, - ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: persistedCancellationResult, errorCode, errorMessage: reason, - }) : {}), - cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason), - }, - }); - - await setWakeupStatus(run.wakeupRequestId, "cancelled", { - finishedAt: new Date(), - error: reason, - }); - - const running = runningProcesses.get(run.id); - if (running) { - await terminateHeartbeatRunProcess({ - pid: running.child.pid, - processGroupId: running.processGroupId, - graceMs: Math.max(1, running.graceSec) * 1000, - }); - } - runningProcesses.delete(run.id); - await releaseIssueExecutionAndPromote(run); - } finally { - stopOwnership?.release(); + if (!isHeartbeatRunTerminalStatus(run.status) || liveRunExecutions.has(run.id) || + adapterExecutionControls.has(run.id) || processRunCancellationSettlements.has(run.id) || + (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date())) return false; + if (needsRelease.has(run.id)) { + // Retry incomplete cleanup after a run becomes terminal. + await releaseEnvironmentLeasesForRun({ runId: run.id, companyId: run.companyId, + agentId: run.agentId, status: run.status, providerResourceDisposition: "destroy" }); } } - - return runs.length; - } - - async function cancelPendingWakeupsForAgentsInternal( - agentIds: string[], - reason: string, - ) { - const uniqueAgentIds = [...new Set(agentIds)].filter( - (agentId) => agentId.length > 0, - ); - if (uniqueAgentIds.length === 0) return 0; - - const now = new Date(); - const wakeupIds = await db - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - inArray(agentWakeupRequests.agentId, uniqueAgentIds), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - - if (wakeupIds.length === 0) return 0; - - await db - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: now, - error: reason, - updatedAt: now, - }) - .where(inArray(agentWakeupRequests.id, wakeupIds)); - - return wakeupIds.length; - } - - async function cancelInvocationsForAgentsInternal( - agentIds: string[], - reason: string, - ) { - const uniqueAgentIds = [...new Set(agentIds)].filter( - (agentId) => agentId.length > 0, - ); - let runsCancelled = 0; - for (const agentId of uniqueAgentIds) { - runsCancelled += await cancelActiveForAgentInternal(agentId, reason); - } - const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal( - uniqueAgentIds, - reason, - ); - return { - agentIds: uniqueAgentIds, - runsCancelled, - wakeupsCancelled, - }; - } - - async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) { - const runIds = scope.scopeType === "project" - ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore) - : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, scope.companyId), - scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined, - scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined, - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), - )).then((rows) => rows.map((row) => row.id)); - for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope }); - await cancelPendingWakeupsForBudgetScope(scope); + return agentExecutionsHaveStopped(db, agentIds); } return { + stopInvocationsForAgents, waitForRunExecutionDrain: async ( runId: string, options: { timeoutMs?: number; intervalMs?: number } = {}, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 04e51012bf..9332de4437 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; import { normalizeAgentNameKey } from "./retries.js"; import { type WakeupOptions, @@ -941,6 +942,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende ); return null; } + // Keep accepted work in the durable queue until setup completes. + if (isAgentAwaitingSetup(agent)) return null; const invokability = companyAgents ? evaluateAgentInvokability(toAgentOrgRow(agent), companyAgents) : await getAgentInvokability(agent); @@ -1862,6 +1865,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // Cancelled after the start lock is released: cancelRunInternal promotes the // agent's next queued run, which takes this same lock. const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = []; + let cancellationReason: string | undefined; return withAgentStartLock(agentId, async () => { const agent = await getAgent(agentId); @@ -1869,10 +1873,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const invokability = await getAgentInvokability(agent); if (!invokability.invokable) { if (shouldCancelRunsForNonInvokableAgent(invokability)) { - await cancelActiveForAgentInternal( - agentId, - `Cancelled because the agent is not invokable: ${invokability.reason}`, - ); + cancellationReason = `Cancelled because the agent is not invokable: ${invokability.reason}`; } return []; } @@ -2011,7 +2012,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }); } return claimedRuns; - }).finally(() => cancelRejectedQueuedRuns(rejectedClaims)); + }).finally(async () => { + if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason); + await cancelRejectedQueuedRuns(rejectedClaims); + }); } // Public wakeup entry point. Callers dispatch it fire-and-forget, so register @@ -2501,7 +2505,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }); } - const invokability = await getAgentInvokability(agent); + // Setup delays execution, but must not discard work accepted by a caller. + const invokability = await getAgentInvokability(isAgentAwaitingSetup(agent) ? { ...agent, status: "idle" } : agent); if (!invokability.invokable) { if (opts.requestedByActorType !== "user" || executionWaitRequestId) { await writeSkippedRequest("agent.not_invokable", { diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts new file mode 100644 index 0000000000..0234f326f4 --- /dev/null +++ b/server/src/services/heartbeat/run-control.test.ts @@ -0,0 +1,309 @@ +import { randomUUID } from "node:crypto"; +import type { Db } from "@paperclipai/db"; +import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, projects } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { runningProcesses } from "../../adapters/index.js"; +import { adapterExecutionControls } from "../adapter-execution-control.js"; +import { terminateLocalService } from "../local-service-supervisor.js"; +import { createHeartbeatRunState } from "./run-state.js"; +import { createHeartbeatRunPreparation } from "./run-preparation.js"; +import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js"; +import { createHeartbeatRunControl, type HeartbeatRunControlDependencies } from "./run-control.js"; + +vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() })); +vi.mock("../local-service-supervisor.js", async (importOriginal) => ({ + ...await importOriginal(), + terminateLocalService: vi.fn(async () => undefined), +})); + +type Run = typeof heartbeatRuns.$inferSelect; +function callbacks(db: Db) { + const state = createHeartbeatRunState(db); + const lifecycleDependencies = { + ...state, + ...createHeartbeatRunPreparation(db), + issuesSvc: { + addComment: vi.fn(), + listReviewAttention: vi.fn(), + }, + recovery: { escalateStrandedAssignedIssue: vi.fn() }, + companySkills: { completeTestRunForIssue: vi.fn() }, + budgets: { getInvocationBlock: vi.fn(async () => null) }, + treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) }, + enqueueWakeup: vi.fn(), + getCurrentUserRedactionOptions: vi.fn(async () => ({})), + getAgentInvokability: vi.fn(async () => ({ invokable: true })), + emitTerminalAgentTaskRun: vi.fn(), + budgetHooks: {}, + } satisfies HeartbeatLifecycleDependencies; + const lifecycle = createHeartbeatLifecycle(db, lifecycleDependencies); + return { + ...state, + ...lifecycle, + getRun: vi.fn(state.getRun), + getAgent: vi.fn(state.getAgent), + setRunStatusFromLive: vi.fn(lifecycle.setRunStatusFromLive), + setRunStatus: vi.fn(lifecycle.setRunStatus), + setWakeupStatus: vi.fn(lifecycle.setWakeupStatus), + appendRunEvent: vi.fn(lifecycle.appendRunEvent), + finalizeAgentStatus: vi.fn(), + options: { closeWarmNativeSessionsForRun: vi.fn(async () => ({ closed: 1, busy: 0, failed: 0 })) }, + envOrchestrator: { releaseForRun: vi.fn(async () => ({ released: [], errors: [] })) }, + enqueueWakeup: lifecycleDependencies.enqueueWakeup, + startNextQueuedRunForAgent: vi.fn(), + sweepPendingCleanupLeases: vi.fn(), + timerClaimWasFirstHeartbeat: vi.fn(() => undefined), + wakeQueue: { releaseIssueExecution: vi.fn(async () => ({ outcome: { kind: "released" }, postCommitEffects: [] })) }, + applyWakeQueuePostCommitEffects: vi.fn(async () => undefined), + getSchedulingSuppression: vi.fn(async () => ({ suppressed: false })), + activeRunExecutions: new Set(), + processRunCancellationSettlements: new Map; failed: boolean; error?: unknown }>(), + failedProcessRunCancellations: new Map; failed: boolean; error?: unknown }>(), + } satisfies HeartbeatRunControlDependencies; +} +function guardedDatabase() { + const access = vi.fn(() => { throw new Error("Unexpected run-control database access"); }); + return { db: new Proxy({}, { get: access }) as Db, access }; +} + +describe("heartbeat run-control module boundary", () => { + afterEach(() => adapterExecutionControls.clear()); + + it("constructs without database access, cancellation, or cleanup", () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + createHeartbeatRunControl(database.db, deps); + expect(database.access).not.toHaveBeenCalled(); + for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled(); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled(); + }); + + it("keeps an already completed run out of cancellation effects", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const run = { id: "finished", status: "succeeded", runtimeMode: "legacy" } as Run; + deps.getRun.mockResolvedValue(run); + expect(await createHeartbeatRunControl(database.db, deps).cancelRunInternal(run.id)).toBe(run); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.getAgent).not.toHaveBeenCalled(); + expect(deps.appendRunEvent).not.toHaveBeenCalled(); + }); + + it("keeps a suppressed saved-input scan out of the database and wake admission", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true }); + await createHeartbeatRunControl(database.db, deps).resumeExecutionWaitComments(); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.enqueueWakeup).not.toHaveBeenCalled(); + }); + + it("ignores an empty cancellation batch", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + expect(await createHeartbeatRunControl(database.db, deps).cancelInvocationsForAgentsInternal(["", ""], "pause")).toEqual({ agentIds: [], runsCancelled: 0, wakeupsCancelled: 0 }); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.getAgent).not.toHaveBeenCalled(); + }); + + it("waits for an owned adapter before admitting saved comments", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const run = { id: "owned", status: "cancelled", runtimeMode: "legacy" } as Run; + adapterExecutionControls.set(run.id, {} as never); + await createHeartbeatRunControl(database.db, deps).resumeRemoteStopComments(run); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.enqueueWakeup).not.toHaveBeenCalled(); + }); + + it("retains leases while native execution ownership is unverified", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ runtimeMode: "native", status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" } as Run); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "held", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" }); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled(); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + }); + + it("uses the persisted terminal outcome when choosing warm resource retention", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "failed", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "keep_running" }); + expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ heartbeatRunId: "failed", status: "failed", providerResourceDisposition: "stop_and_retain" })); + expect(database.access).not.toHaveBeenCalled(); + }); + + it("defers destruction when a warm session remains busy", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run); + deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: 1, failed: 0 }); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "busy", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" }); + expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith({ runId: "busy", reason: "terminal heartbeat run destroyed its environment lease" }); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + expect(database.access).not.toHaveBeenCalled(); + }); +}); + +const support = await getEmbeddedPostgresTestSupport(); +if (!support.supported) console.warn(`Skipping run-control database tests: ${support.reason}`); +describe.skipIf(!support.supported)("heartbeat run-control database wiring", () => { + let database: Awaited>; + let db: Db; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-run-control-module"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + beforeEach(async () => { + vi.mocked(terminateLocalService).mockReset().mockResolvedValue(undefined); + await db.execute(sql`truncate table companies restart identity cascade`); + }); + afterEach(() => runningProcesses.clear()); + + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Run control", issuePrefix: "RC" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Controlled agent", adapterType: "codex_local" }).returning(); + return { company, agent }; + } + async function liveRun(companyId: string, agentId: string) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", runtimeMode: "legacy", runtimeModeResolvedAt: new Date() }).returning(); + runningProcesses.set(run.id, { child: { pid: 424242 } as never, graceSec: 4, processGroupId: 424242 }); + return run; + } + + it("reports a missing run without entering cancellation", async () => { + const deps = callbacks(db); + await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(randomUUID())).rejects.toThrow("Heartbeat run not found"); + expect(deps.getAgent).not.toHaveBeenCalled(); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(deps.appendRunEvent).not.toHaveBeenCalled(); + }); + + it("joins concurrent Stop requests across factories through the shared process barrier", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get"); + let releaseTermination!: () => void; + const terminating = new Promise(resolve => { releaseTermination = resolve; }); + vi.mocked(terminateLocalService).mockReturnValue(terminating); + const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop", { terminationGraceMs: 1 }); + await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true)); + const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop again"); + await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2)); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(terminateLocalService).toHaveBeenCalledTimes(1); + releaseTermination(); + const results = await Promise.all([first, second]); + expect(results.map(result => result?.status)).toEqual(["cancelled", "cancelled"]); + expect(deps.setRunStatusFromLive).toHaveBeenCalledTimes(1); + expect(deps.appendRunEvent).toHaveBeenCalledTimes(1); + expect(deps.finalizeAgentStatus).toHaveBeenCalledTimes(1); + expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledTimes(1); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(runningProcesses.has(run.id)).toBe(false); + expect(terminateLocalService).toHaveBeenCalledWith({ pid: 424242, processGroupId: 424242 }, { forceAfterMs: 100, signal: "SIGINT" }); + }); + + it("settles every joined Stop caller with the same termination failure", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get"); + deps.activeRunExecutions.add(run.id); + let rejectTermination!: (error: Error) => void; + vi.mocked(terminateLocalService).mockReturnValue(new Promise((_resolve, reject) => { rejectTermination = reject; })); + const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id); + await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true)); + const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id); + const results = Promise.allSettled([first, second]); + await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2)); + const failure = new Error("provider did not stop"); + rejectTermination(failure); + expect(await results).toEqual([{ status: "rejected", reason: failure }, { status: "rejected", reason: failure }]); + expect(terminateLocalService).toHaveBeenCalledTimes(1); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect((await deps.getRun(run.id))?.status).toBe("running"); + }); + + it("retains failed Stop evidence and the owned process until a later request succeeds", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + deps.activeRunExecutions.add(run.id); + const failure = new Error("termination failed"); + vi.mocked(terminateLocalService).mockRejectedValueOnce(failure); + await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).rejects.toBe(failure); + expect(deps.failedProcessRunCancellations.get(run.id)).toMatchObject({ failed: true, error: failure }); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(runningProcesses.has(run.id)).toBe(true); + expect((await deps.getRun(run.id))?.status).toBe("running"); + expect(deps.finalizeAgentStatus).not.toHaveBeenCalled(); + expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).toMatchObject({ status: "cancelled" }); + expect(terminateLocalService).toHaveBeenCalledTimes(2); + // The executor still owns removal of this evidence after it observes Stop. + expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure); + }); + + it("deduplicates agent batches and cancels only unclaimed pending wakes", async () => { + const { company, agent } = await fixture(); + const [otherAgent] = await db.insert(agents).values({ companyId: company.id, name: "Other agent" }).returning(); + const [finished] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "succeeded" }).returning(); + const wakes = await db.insert(agentWakeupRequests).values([ + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "deferred_issue_execution" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", runId: finished.id }, + { companyId: company.id, agentId: otherAgent.id, source: "assignment", status: "queued" }, + ]).returning(); + const deps = callbacks(db); + expect(await createHeartbeatRunControl(db, deps).cancelInvocationsForAgentsInternal([agent.id, "", agent.id], "pause")).toEqual({ agentIds: [agent.id], runsCancelled: 0, wakeupsCancelled: 2 }); + const persisted = await db.select().from(agentWakeupRequests); + expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "cancelled", "claimed", "queued", "queued"]); + }); + + it("rechecks stale budget enforcement before cancelling runs or queued wakes", async () => { + const { company, agent } = await fixture(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "queued", runtimeMode: "legacy" }).returning(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" }).returning(); + const deps = callbacks(db); + await createHeartbeatRunControl(db, deps).cancelBudgetScopeWork({ companyId: company.id, scopeType: "agent", scopeId: agent.id, enforcement: { policyId: randomUUID(), version: 1 } }); + expect((await deps.getRun(run.id))?.status).toBe("queued"); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("queued"); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(deps.finalizeAgentStatus).not.toHaveBeenCalled(); + expect(terminateLocalService).not.toHaveBeenCalled(); + }); + + it("limits project budget cancellation by company, effective project, and admission cutoff", async () => { + const { company, agent } = await fixture(); + const [otherCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning(); + const [otherAgent] = await db.insert(agents).values({ companyId: otherCompany.id, name: "Other company agent" }).returning(); + const projectRows = await db.insert(projects).values([ + { companyId: company.id, name: "Target" }, { companyId: company.id, name: "Override" }, + ]).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, projectId: projectRows[0].id, title: "Bound task", issueNumber: 1, identifier: "RC-1" }).returning(); + const old = new Date("2026-01-01T00:00:00Z"); + const cutoff = new Date("2026-01-02T00:00:00Z"); + const wakes = await db.insert(agentWakeupRequests).values([ + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id } }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id, projectId: projectRows[1].id } }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: cutoff, payload: { projectId: projectRows[0].id } }, + { companyId: otherCompany.id, agentId: otherAgent.id, source: "assignment", status: "queued", createdAt: old, payload: { projectId: projectRows[0].id } }, + ]).returning(); + await createHeartbeatRunControl(db, callbacks(db)).cancelBudgetScopeWork({ companyId: company.id, scopeType: "project", scopeId: projectRows[0].id, createdBefore: cutoff }); + const persisted = await db.select().from(agentWakeupRequests); + expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "queued", "queued", "queued"]); + }); +}); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts new file mode 100644 index 0000000000..81a593a63d --- /dev/null +++ b/server/src/services/heartbeat/run-control.ts @@ -0,0 +1,1301 @@ +import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js"; +import { DEFERRED_WAKE_CONTEXT_KEY } from "./queue.js"; +import { CANCELLABLE_HEARTBEAT_RUN_STATUSES } from "./recovery.js"; +import { + type WakeupOptions, +} from "./run-preparation.js"; +import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js"; +import { isAiAuthenticationBlocked } from "../ai-auth-failure.js"; +import { + nativeRetryCancellationCommitCondition, + claimCancellationRequest, +} from "../native-runtime/native-cancellation-request.js"; +import { + canContinueCancelledRun, + readRunCancellation, + requestedRunCancellation, +} from "../run-cancellation.js"; +import { settleSlackConversation } from "../slack-conversation-lifecycle.js"; +import { readQueuedInteractionResponse } from "../queued-interaction-response.js"; +import { + isAcknowledgedNativeStop, + acknowledgedNativeStopExecutionHasStopped, +} from "../acknowledged-native-stop.js"; +import { completeTerminatedRemoteNativeSessionCleanup } from "../../vendor/paperclip-runner/index.js"; +import { + hasRemoteTerminationReceipt, + remoteExecutionHasStopped, + stoppedRemoteCleanupScopes, +} from "../remote-execution-termination.js"; +import { + admitExplicitNativeContinuation, + undeliveredLegacyUserCommentIds, +} from "../explicit-native-continuation.js"; +import { isCancelledNativeStartup } from "../cancelled-native-startup.js"; +import { + executionBlockerPredicate, + getExecutionBlocker, +} from "../execution-blocker.js"; +import { + CONVERSATION_CONTINUATION_POLICY, + runUsedConversationAdapter, + hasConversationContinuationPolicy, +} from "../conversation-continuation.js"; +import { + adapterExecutionControls, + captureAdapterStopOwnership, + waitForAdapterStop, +} from "../adapter-execution-control.js"; +import { randomUUID } from "node:crypto"; +import { + and, + asc, + desc, + eq, + exists, + gt, + inArray, + isNull, + isNotNull, + lt, + lte, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + type EnvironmentLeaseStatus, +} from "@paperclipai/shared"; +import { + agentWakeupRequests, + companies, + environmentLeases, + heartbeatRuns, + issueComments, + issueRecoveryActions, + issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { + conflict, + HttpError, + notFound, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInWakePayload, + withQueuedCommentIdsInRunContext, +} from "../issue-queued-comment-queue.js"; +import { + cancelNativeSession, + closeWarmNativeSessionsForEnvironment, +} from "../native-runtime/index.js"; +import { runningProcesses } from "../../adapters/index.js"; +import { parseObject } from "../../adapters/utils.js"; +import { isNativeRunnerOwnershipHeld } from "../native-runtime/native-runner-ownership.js"; +import { + withCurrentBudgetEnforcement, + type BudgetEnforcementScope, +} from "../budgets.js"; +import { terminateLocalService } from "../local-service-supervisor.js"; +import { deriveCommentId } from "../../modules/run-dispatch/index.js"; +import { WakeQueueApplicationError } from "../../modules/wake-queue/index.js"; +import { isUuidLike } from "@paperclipai/shared"; +import { + type ProviderResourceDisposition, +} from "../environment-runtime.js"; + +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatLifecycle } from "./run-lifecycle.js"; +import type { createHeartbeatRetries, HeartbeatRetryDependencies } from "./retries.js"; +import type { createHeartbeatQueue } from "./queue.js"; +import type { createHeartbeatRecovery } from "./recovery.js"; +import type { createWakeQueue, PostCommitEffect as WakeQueuePostCommitEffect } from "../../modules/wake-queue/index.js"; +import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js"; + +type ProcessRunCancellationSettlement = { + settled: Promise; + failed: boolean; + error?: unknown; +}; + +/** Cancellation shares executor ownership and delegates lifecycle and admission effects. */ +export interface HeartbeatRunControlDependencies extends Pick, + "getRun" | "getAgent">, + Pick, + "mergeRunStopMetadataForAgent" | "setRunStatusFromLive" | "setRunStatus" | "setWakeupStatus" | "appendRunEvent">, + Pick { + options: { + closeWarmNativeSessionsForRun?: (input: { + runId: string; + reason: string; + }) => Promise<{ closed: number; busy: number; failed: number }>; + }; + envOrchestrator: Pick, "releaseForRun">; + enqueueWakeup: ReturnType["enqueueWakeup"]; + startNextQueuedRunForAgent: ReturnType["startNextQueuedRunForAgent"]; + sweepPendingCleanupLeases: ReturnType["sweepPendingCleanupLeases"]; + timerClaimWasFirstHeartbeat: ReturnType["timerClaimWasFirstHeartbeat"]; + wakeQueue: Pick, "releaseIssueExecution">; + applyWakeQueuePostCommitEffects: (effects: WakeQueuePostCommitEffect[]) => Promise; + getSchedulingSuppression: () => Promise<{ suppressed: boolean }>; + // Scheduler and route service instances must retain the same in-process barriers. + activeRunExecutions: Set; + processRunCancellationSettlements: Map; + failedProcessRunCancellations: Map; +} + +export function leaseReleaseStatusForRunStatus( + status: string | null | undefined, +): Extract { + if (status === "cancelled") return "expired"; + return status === "failed" || status === "timed_out" ? "failed" : "released"; +} + +export function providerResourceDispositionForTerminalRun( + desired: ProviderResourceDisposition | undefined, + status: string | null | undefined, +): ProviderResourceDisposition | undefined { + if (desired !== "keep_running") return desired; + return status === "succeeded" ? desired : "stop_and_retain"; +} + +export async function terminateHeartbeatRunProcess(input: { + pid: number | null | undefined; + processGroupId: number | null | undefined; + graceMs?: number; + signal?: NodeJS.Signals; +}) { + const pid = input.pid ?? null; + const processGroupId = input.processGroupId ?? null; + if (typeof pid !== "number" && typeof processGroupId !== "number") return; + + await terminateLocalService( + { + pid: + typeof pid === "number" && Number.isInteger(pid) && pid > 0 + ? pid + : (processGroupId ?? 0), + processGroupId: + typeof processGroupId === "number" && + Number.isInteger(processGroupId) && + processGroupId > 0 + ? processGroupId + : null, + }, + { forceAfterMs: input.graceMs, signal: input.signal }, + ); +} + +export async function cancelHeartbeatNativeRun(input: { + db: Db; + runId: string; + reason: string; + runtimeMode: string | null; + cancellationRequestId?: string; + cancel?: ( + runId: string, + reason: string, + options: { db: Db; scope: "run"; cancellationRequestId?: string }, + ) => Promise<{ decision: unknown | null; auditId: string | null }>; +}) { + if (input.runtimeMode !== "native") { + return { decision: null, auditId: null }; + } + const cancellation = input.cancel + ? await input.cancel(input.runId, input.reason, { + db: input.db, + scope: "run", + ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), + }) + : await cancelNativeSession(input.runId, input.reason, { + db: input.db, + scope: "run", + ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), + }); + if (!cancellation.decision || !cancellation.auditId) { + throw new Error("native_cancellation_outcome_not_audited"); + } + return cancellation; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +export type CancelRunOptions = { + budgetEnforcement?: BudgetEnforcementScope; + /** Optional board request identity, atomically reserved for native Stop. */ + cancellationRequestId?: string; + cancellationRequestedByUserId?: string | null; + errorCode?: string; + resultJson?: Record; + eventMessage?: string; + eventPayload?: Record; + /** Per-call graceful process shutdown window, bounded to a safe range. */ + terminationGraceMs?: number; + /** Caller is immediately scheduling an explicit successor path. */ + suppressImmediateRecovery?: boolean; + }; + +/** Run control binds cancellation, resource release, and saved-input resumption. */ +export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunControlDependencies) { + const { + getRun, + options, + envOrchestrator, + activeRunExecutions, + enqueueWakeup, + getAgent, + sweepPendingCleanupLeases, + getSchedulingSuppression, + wakeQueue, + applyWakeQueuePostCommitEffects, + processRunCancellationSettlements, + mergeRunStopMetadataForAgent, + setRunStatusFromLive, + failedProcessRunCancellations, + setWakeupStatus, + appendRunEvent, + finalizeAgentStatus, + timerClaimWasFirstHeartbeat, + startNextQueuedRunForAgent, + setRunStatus, + } = dependencies; + + async function releaseEnvironmentLeasesForRun(input: { + runId: string; + companyId: string; + agentId: string; + status: string | null | undefined; + failureReason?: string | null; + providerResourceDisposition?: ProviderResourceDisposition; + nativeLifecycleTelemetry?: { + provider: string; + harness: string; + lifecycleMode: "per_turn" | "warm"; + sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; + }; + }) { + const leaseOwnerRun = await getRun(input.runId); + if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return; + // Recovery can finish workspace copy-back outside the executor's finally. + // Successful copy-back does not earn warm retention for a failed turn. + const status = leaseOwnerRun?.status ?? input.status; + const providerResourceDisposition = providerResourceDispositionForTerminalRun( + input.providerResourceDisposition, + status, + ); + if (providerResourceDisposition === "destroy") { + const closeResult = await ( + options.closeWarmNativeSessionsForRun ?? + (async ({ runId, reason }) => { + const environmentIds = await db + .selectDistinct({ environmentId: environmentLeases.environmentId }) + .from(environmentLeases) + .where( + and( + eq(environmentLeases.heartbeatRunId, runId), + eq(environmentLeases.status, "active"), + ), + ) + .then((rows) => + rows.flatMap((row) => + typeof row.environmentId === "string" && + row.environmentId.length > 0 + ? [row.environmentId] + : [], + ), + ); + const aggregate = { closed: 0, busy: 0, failed: 0 }; + for (const environmentId of environmentIds) { + const result = await closeWarmNativeSessionsForEnvironment({ + environmentId, + reason, + }); + aggregate.closed += result.closed; + aggregate.busy += result.busy; + aggregate.failed += result.failed; + } + return aggregate; + }) + )({ + runId: input.runId, + reason: "terminal heartbeat run destroyed its environment lease", + }).catch((err) => { + logger.warn( + { err, runId: input.runId }, + "failed to close warm native sessions before environment lease destruction", + ); + return { closed: 0, busy: 0, failed: 1 }; + }); + // A failed remote checkpoint cannot veto destruction of the isolated + // sandbox after the run stopped. Provider destruction supplies the exit + // proof; it does not turn the interrupted checkpoint into a success. + const remoteLeases = closeResult.failed > 0 && leaseOwnerRun && + ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status) + ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.heartbeatRunId, input.runId), + )) + : []; + const canDestroyRemote = remoteLeases.length > 0 && + remoteLeases.every(lease => lease.provider && lease.provider !== "local"); + if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) { + logger.warn( + { runId: input.runId, warmNativeSessions: closeResult }, + "deferred environment lease destruction until warm native sessions close", + ); + return; + } + } + const releaseResult = await envOrchestrator + .releaseForRun({ + heartbeatRunId: input.runId, + companyId: input.companyId, + agentId: input.agentId, + status: leaseReleaseStatusForRunStatus(status), + failureReason: input.failureReason ?? undefined, + providerResourceDisposition, + nativeLifecycleTelemetry: input.nativeLifecycleTelemetry, + }) + .catch((err) => { + logger.warn( + { err, runId: input.runId }, + "failed to release environment leases for heartbeat run", + ); + return null; + }); + for (const releaseError of releaseResult?.errors ?? []) { + logger.warn( + { + err: releaseError.error, + leaseId: releaseError.leaseId, + runId: input.runId, + }, + "failed to release environment lease for heartbeat run", + ); + } + await acknowledgeRemoteStop(input.runId, input.companyId); + } + + async function acknowledgeRemoteStop(runId: string, companyId: string) { + // The provider receipt arrives after adapter settlement. A remote ACP child + // has no host PID, so only this target-aware boundary can acknowledge Stop. + const stopped = await getRun(runId); + if (stopped?.runtimeMode === "native") { + const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId); + for (const remoteCleanupScope of scopes ?? []) { + completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope }); + } + } + if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" && + parseObject(stopped.resultJson?.executionCancellation).state === "requested" && + await runUsedConversationAdapter(db, stopped) && + await remoteExecutionHasStopped(db, companyId, runId)) { + await db.update(heartbeatRuns).set({ + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ + executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), + state: "acknowledged", acknowledgedAt: new Date().toISOString(), + proof: "provider_termination_receipt" }, + conversationContinuation: CONVERSATION_CONTINUATION_POLICY, + })}::jsonb`, + updatedAt: new Date(), + }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.status, "cancelled"))); + } + } + + async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) { + if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; + const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt + ? await db.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), + )) : []; + const cancelledPreparation = run.runtimeMode === "legacy" && + await isCancelledNativeStartup(db, run, preparationCoordinator); + if (run.runtimeMode !== "native" && !cancelledPreparation && + parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true && + !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return; + const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null); + if (!issueId) return; + const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null; + const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase, + leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), + )) : []; + // Stop ends one response. Saved user input can enter ordinary admission + // once its executor settles; it does not need a manufactured crash incident. + // The ordinary path still enforces task holds, ownership, and native session + // cleanup before starting a provider. + const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) && + !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner && + ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") && + await acknowledgedNativeStopExecutionHasStopped(db, currentRun) && + !(await getExecutionBlocker(db, run.companyId, issueId)); + const legacyContinuation = run.runtimeMode === "legacy" && + hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) && + !(await getExecutionBlocker(db, run.companyId, issueId)); + if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return; + const pending = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.requestedByActorType, "user"), + requestId ? eq(agentWakeupRequests.id, requestId) : undefined, + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50); + for (const wake of pending) { + if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue; + let payload = parseObject(wake.payload); + if (payload.queuedCommentInterrupt) { + await resumeQueuedCommentInterrupt(wake.companyId, wake.id); + continue; + } + let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + let commentId = deriveCommentId(context, payload); + let requestedByActorId = wake.requestedByActorId; + const reason = readNonEmptyString(context.wakeReason) ?? wake.reason; + if (run.runtimeMode === "native" || cancelledPreparation) { + const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId, + queuedCommentIdsFromWakePayload(payload)); + if (!ids.length) continue; + payload = withQueuedCommentIdsInWakePayload(payload, ids); + context = withQueuedCommentIdsInRunContext(context, ids); + commentId = ids.at(-1)!; + // A coalesced queue can contain several authors. Its saved comments, + // not the outer wake's first author, authorize the remaining input. + const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId), + )); + requestedByActorId = author?.id ?? null; + } + if (legacyContinuation || stoppedNativeContinuation) { + if (!commentId || !run.finishedAt || !requestedByActorId || + !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue; + const [comment] = await db.select().from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), + sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"), + eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt), + isNull(issueComments.createdByRunId), + stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt), + )); + if (!comment?.body.trim()) continue; + } else { + let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; + const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId, + agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId, + reason, commentId, successorRunId: randomUUID(), dryRun: true, + ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), + onBlocked: (reason, message) => { wait = { reason, message }; }, + }); + if (!admitted) { + await db.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', + coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + continue; + } + } + // Re-enter ordinary admission with the original user's authority. It + // atomically adopts the deferred comments and still applies every gate. + await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"], + reason, payload, contextSnapshot: context, + requestedByActorType: "user", requestedByActorId, + ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), + idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id); + break; + } + } + + async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) { + return resumeSavedLegacyComments(companyId, queueId, true, opts); + } + + async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) { + const [wake] = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + if (!wake) return; + const payload = parseObject(wake.payload); + let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId); + let commentIds = queuedCommentIdsFromWakePayload(payload); + const issueId = readNonEmptyString(payload.issueId); + if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return; + const response = await readQueuedInteractionResponse(db, companyId, issueId, payload); + if (!commentIds.length && !response) return; + // Resolved cards are immutable input. Only an explicit Interrupt click can + // authorize continuation across a stopped execution; ordinary completion + // uses normal deferred-wake promotion. + if (response && !interrupted) return; + if (!interrupted) { + commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds); + if (!commentIds.length) return; + // The queue itself may have begun as a system wake. The saved human + // comment, not that wake's origin or mutable caller payload, is authority. + const [comment] = await db.select().from(issueComments).where(and( + eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), + eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"), + isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt), + )).orderBy(desc(issueComments.createdAt)).limit(1); + if (!comment?.body.trim() || !comment.authorUserId) return; + actorId = comment.authorUserId; + } + if (!actorId) return; + const agent = await getAgent(wake.agentId); + if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return; + const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, wake.agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), + )).limit(1); + if (active) return; + if (interrupted && opts?.retryCleanup) { + // Only the HTTP click grants an extra cleanup attempt. Periodic retries + // reuse the intent to deliver, never a fresh provider teardown budget. + const sourceRun = await db.transaction(async tx => { + const [task] = await tx.select().from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.id, issueId), + )).for("update"); + const [current] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`, + )); + if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) || + !current || (!queuedCommentIdsFromWakePayload(current.payload).length && + !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null; + const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, wake.agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), + )).limit(1); + if (successor) return null; + const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId); + const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId), + eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"), + inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + )).then(rows => rows[0]) : null; + if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null; + // Older ephemeral leases recorded successful cleanup without a provider + // receipt. Re-verify them through the recorded teardown path; a timestamp + // alone never certifies termination. Retained/reusable resources stay put. + const historical = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), + eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt), + inArray(environmentLeases.status, ["released", "expired", "failed"]), + )).for("update"); + for (const lease of historical) { + if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue; + // Provider resource IDs identify physical sandboxes. A lease in any + // company can still own this resource; never destroy it on behalf of + // this company. This existence-only guard exposes no foreign data. + const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider), + eq(environmentLeases.providerLeaseId, lease.providerLeaseId), + or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])), + )).limit(1); + if (otherOwner) continue; + await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() }) + .where(eq(environmentLeases.id, lease.id)); + } + return run; + }); + if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: { + companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt", + } }); + } + const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds); + delete deliveryPayload.queuedCommentInterrupt; + await enqueueWakeup(wake.agentId, { + source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: deliveryPayload, contextSnapshot: response + ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId, + responsibleUserId: actorId } + : withQueuedCommentIdsInRunContext({ + issueId, triggeredBy: "board", actorId, responsibleUserId: actorId, + }, commentIds), + requestedByActorType: "user", requestedByActorId: actorId, + ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }), + issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] }, + idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`, + }, queueId); + } + + async function resumeExecutionWaitComments() { + if ((await getSchedulingSuppression()).suppressed) return; + const waits = await db.select({ wake: agentWakeupRequests }) + .from(agentWakeupRequests) + .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), + sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, + eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) + .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) + .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id), + executionBlockerPredicate(), + ))), eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.requestedByActorType, "user"), + sql`${agentWakeupRequests.payload}->'executionWait' is not null`, + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, + lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), + notInArray(issues.status, ["done", "cancelled"]))) + .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); + const seen = new Set(); + for (const { wake } of waits) { + const issueId = String(wake.payload?.issueId); + if (seen.has(issueId)) continue; + seen.add(issueId); + // Advance the cursor even for invalid evidence so one damaged task cannot + // starve later requests in this bounded scan. Preserve concurrent edits. + const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( + eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), + )).returning({ id: agentWakeupRequests.id }); + if (!claimed) continue; + // Match normal admission's deterministic current blocker selection. An + // arbitrary historical action must not choose the retry's source run. + const blocker = await getExecutionBlocker(db, wake.companyId, issueId); + const sourceId = blocker?.runId; + if (!sourceId || !isUuidLike(sourceId)) continue; + const run = await getRun(sourceId, { includeExecutionEvidence: true }); + if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue; + if (canContinueCancelledRun(run)) { + await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { + logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation"); + }); + continue; + } + await resumeRemoteStopComments(run, wake.id).catch(err => { + logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message"); + }); + } + } + + async function releaseIssueExecutionAndPromote( + run: Pick, + options: { suppressImmediateRecovery?: boolean } = {}, + ) { + try { + const source = await getRun(run.id); + const { postCommitEffects } = await wakeQueue.releaseIssueExecution({ + companyId: run.companyId, + runId: run.id, + now: new Date(), + // A durable authentication card owns recovery. This covers the review path, + // while continuation classification blocks periodic generic retries. + suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source), + }); + await applyWakeQueuePostCommitEffects(postCommitEffects); + const completed = await getRun(run.id); + const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId) + ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId; + if (completed?.status === "succeeded" && issueId) { + await settleSlackConversation(db, run.companyId, issueId).catch((err) => { + logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation"); + }); + } + } catch (error) { + if ( + error instanceof WakeQueueApplicationError && + error.code === "responsible_user_unresolved" + ) { + // Every other `responsible_user_unresolved` HttpError in this file + // carries `code` inside its own `details`; match that shape here too. + throw new HttpError(422, error.message, { code: error.code, ...error.details }); + } + throw error; + } + } + + async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) { + const runIssueId = sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; + const effectiveProjectId = sql< + string | null + >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; + + const rows = await db + .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id }) + .from(heartbeatRuns) + .leftJoin( + issues, + and( + eq(issues.companyId, companyId), + sql`${issues.id}::text = ${runIssueId}`, + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined, + inArray(heartbeatRuns.status, [ + ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, + ]), + sql`${effectiveProjectId} = ${projectId}`, + ), + ); + + return rows.map((row) => row.id); + } + + async function listProjectScopedWakeupIds( + companyId: string, + projectId: string, + database: Db = db, + ) { + const wakeIssueId = sql< + string | null + >`${agentWakeupRequests.payload} ->> 'issueId'`; + const effectiveProjectId = sql< + string | null + >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; + + const rows = await database + .selectDistinctOn([agentWakeupRequests.id], { + id: agentWakeupRequests.id, + }) + .from(agentWakeupRequests) + .leftJoin( + issues, + and( + eq(issues.companyId, companyId), + sql`${issues.id}::text = ${wakeIssueId}`, + ), + ) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + sql`${effectiveProjectId} = ${projectId}`, + ), + ); + + return rows.map((row) => row.id); + } + + async function cancelPendingWakeupsForBudgetScope( + scope: BudgetEnforcementScope, + ) { + return withCurrentBudgetEnforcement(db, scope, async (tx) => { + const now = new Date(); + let wakeupIds: string[] = []; + + if (scope.scopeType === "company") { + wakeupIds = await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, scope.companyId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + } else if (scope.scopeType === "agent") { + wakeupIds = await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, scope.companyId), + eq(agentWakeupRequests.agentId, scope.scopeId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + } else { + wakeupIds = await listProjectScopedWakeupIds( + scope.companyId, + scope.scopeId, + tx, + ); + } + + if (wakeupIds.length === 0) return 0; + + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: "Cancelled due to budget pause", + updatedAt: now, + }) + .where(and( + inArray(agentWakeupRequests.id, wakeupIds), + inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), + isNull(agentWakeupRequests.runId), + scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined, + )); + + return wakeupIds.length; + }); + } + + function cancellationTerminationGraceMs( + configuredGraceSec: number, + requestedGraceMs: number | undefined, + ) { + const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000; + if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) { + return configuredGraceMs; + } + return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs))); + } + + async function cancelRunInternal( + runId: string, + reason = "Cancelled by control plane", + options: CancelRunOptions = {}, + ) { + let run = await getRun(runId); + if (!run) throw notFound("Heartbeat run not found"); + if (options.cancellationRequestId) { + run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null); + } + // The caller claim checked retry eligibility under both durable row locks. + // This is only a cancellation candidate: dispatch rechecks the coordinator + // under lock, and the final CAS requires its own unchanged acknowledged + // retry fence. Re-reading only retryable_failure here would strand replay. + const pendingNativeRetry = + run.runtimeMode === "native" && run.status === "failed" && ( + Boolean(options.cancellationRequestId) || await db + .select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.runId, run.id), + eq(nativeRunFinalizations.companyId, run.companyId), + eq(nativeRunFinalizations.phase, "retryable_failure"), + ), + ) + .then((rows) => rows.length > 0) + ); + if ( + !pendingNativeRetry && + !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes( + run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number], + ) + ) + return run; + const agent = await getAgent(run.agentId); + const errorCode = options.errorCode ?? "cancelled"; + const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason); + + const pendingProcessCancellation = processRunCancellationSettlements.get( + run.id, + ); + if (pendingProcessCancellation) { + await pendingProcessCancellation.settled; + if (pendingProcessCancellation.failed) + throw pendingProcessCancellation.error; + return getRun(run.id); + } + const running = runningProcesses.get(run.id); + const stopOwnership = + run.runtimeMode !== "native" + ? captureAdapterStopOwnership(run.id) + : undefined; + const control = stopOwnership?.control; + // Capture the existing adapter owner before waiting on the run lock. Then + // atomically fence preparation and refresh the selected runtime, so Stop + // cannot miss a native handoff that won after its first read. + // Established legacy processes must still be stopped if the database is + // unavailable. Only native or not-yet-dispatched preparation needs this + // additional durable fence before its existing cancellation path. + if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) { + const fence = async (tx: Db) => { + const [fenced] = await tx.update(heartbeatRuns).set({ + // Record handoff intent before native cancellation can finalize and release + // the run. Only its audited stop acknowledgement suppresses recovery. + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || + ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true + ? { reassignmentStopRequested: true } : {})}::jsonb || + ${JSON.stringify({ cancellation })}::jsonb || + jsonb_build_object('startupCancellation', jsonb_build_object( + 'requestedAt', ${new Date().toISOString()}::text, + 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy' + and ${heartbeatRuns.runtimeModeResolvedAt} is null + and ${heartbeatRuns.executionStage} = 'preparing' + and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false) + ))`, + }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status, + pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], + ))).returning(); + return fenced ?? null; + }; + let fenced: typeof run | null; + try { + fenced = options.budgetEnforcement + ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence) + : await fence(db); + } catch (error) { + stopOwnership?.release(); + throw error; + } + if (!fenced) { + stopOwnership?.release(); + return getRun(runId); + } + run = fenced; + } + const resultJson = { ...(agent + ? { + ...mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: parseObject(run.resultJson), + errorCode, + errorMessage: reason, + }), + ...(options.resultJson ?? {}), + } + : options.resultJson), cancellation }; + + try { + let releaseProcessCancellation: (() => void) | undefined; + const processCancellationSettlement = + run.runtimeMode !== "native" && + !control && + running + ? { + settled: new Promise((resolve) => { + releaseProcessCancellation = resolve; + }), + failed: false, + error: undefined as unknown, + } + : undefined; + if (processCancellationSettlement) { + // No await between joining an existing owner above and registering ours. + processRunCancellationSettlements.set( + run.id, + processCancellationSettlement, + ); + } + const cancellation = await (async () => { + try { + if (control) { + await db + .update(heartbeatRuns) + .set({ + error: reason, + errorCode, + // Stop may have read the row before adapter settlement saved + // a required-copyback receipt. Merge into current database JSON + // and leave those server-owned fields to the restore recorder. + resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({ + ...resultJson, + ...(!running ? { executionCancellation: { + state: "requested", requestedAt: new Date().toISOString(), + } } : {}), + }, true), + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, "running"), + ), + ); + control.controller.abort(new Error(reason)); + } + let terminationSettled = false; + try { + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason, + runtimeMode: run.runtimeMode, + ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), + }); + if (running) { + await terminateHeartbeatRunProcess({ + pid: running.child.pid, + processGroupId: running.processGroupId, + // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM + // can leave commands in their separate process groups alive. + signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined, + graceMs: cancellationTerminationGraceMs( + running.graceSec, + options.terminationGraceMs, + ), + }); + } + terminationSettled = true; + } finally { + if ( + (!processCancellationSettlement || terminationSettled) && + runningProcesses.get(run.id) === running + ) { + runningProcesses.delete(run.id); + } + } + + if (control) { + await waitForAdapterStop(control.settled, undefined, { + runId: run.id, + adapterType: agent?.adapterType, + runtimeMode: run.runtimeMode, + abortRequested: control.controller.signal.aborted, + }, control); + const stopped = await getRun(run.id); + if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) { + if ( + parseObject(stopped.resultJson?.executionCancellation).state !== + "acknowledged" + ) { + throw conflict( + "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.", + ); + } + // The owned adapter already finalized this run and its lifecycle. + // Do not replay the process cancellation side effects below. + return { run: stopped, updated: false }; + } + } + + const finishedAt = new Date(); + const persistedCancellationResult = + run.runtimeMode === "native" + ? await getRun(run.id).then((current) => + parseObject(current?.resultJson), + ) + : {}; + return await setRunStatusFromLive( + run.id, + "cancelled", + pendingNativeRetry + ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] + : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], + { + finishedAt, + error: reason, + errorCode, + ...(resultJson || + Object.keys(persistedCancellationResult).length > 0 + ? { + resultJson: { + ...persistedCancellationResult, + ...(resultJson ?? {}), + // A scheduler placeholder has no process to acknowledge. + // Preserve its normal release policy instead of treating + // it as an operator stop of provider work. + ...(processCancellationSettlement && agent && running && ( + (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) || + (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0) + ) + ? mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: { + ...resultJson, + executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() }, + }, + errorCode, errorMessage: reason, + }) + : {}), + // The native cancellation helper may have advanced a durable + // pending intent to its acknowledged state after `run` was + // first read. Never let that stale snapshot overwrite the + // authoritative post-dispatch acknowledgement. + ...(Object.hasOwn( + persistedCancellationResult, + "nativeCancellation", + ) + ? { + nativeCancellation: + persistedCancellationResult.nativeCancellation, + } + : {}), + }, + } + : {}), + }, + undefined, + pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined, + ); + } catch (error) { + if (processCancellationSettlement) { + processCancellationSettlement.failed = true; + processCancellationSettlement.error = error; + if (activeRunExecutions.has(run.id)) { + failedProcessRunCancellations.set( + run.id, + processCancellationSettlement, + ); + } + } + throw error; + } finally { + if (processCancellationSettlement) { + if ( + processRunCancellationSettlements.get(run.id) === + processCancellationSettlement + ) { + processRunCancellationSettlements.delete(run.id); + } + // Always settle waiters, including termination and DB errors. A + // failed Stop remains an error to its caller, never a cancellation + // receipt; the signal-bearing executor may then record failure. + releaseProcessCancellation?.(); + } + } + })(); + const cancelled = cancellation.run; + + if (cancellation.updated && cancelled) { + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: cancelled.finishedAt ?? new Date(), + error: reason, + }); + await appendRunEvent(cancelled, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: options.eventMessage ?? "run cancelled", + payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, + }); + await releaseIssueExecutionAndPromote(cancelled, { + suppressImmediateRecovery: options.suppressImmediateRecovery, + }); + await finalizeAgentStatus(run.agentId, "cancelled", undefined, { + wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), + }); + await startNextQueuedRunForAgent(run.agentId); + } + return cancelled; + } finally { + stopOwnership?.release(); + } + } + + async function cancelActiveForAgentInternal( + agentId: string, + reason = "Cancelled due to agent pause", + errorCode = "cancelled", + ) { + const runs = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + ), + ); + + for (const run of runs) await cancelRunInternal(run.id, reason, { errorCode }); + + return runs.length; + } + + async function cancelPendingWakeupsForAgentsInternal( + agentIds: string[], + reason: string, + ) { + const uniqueAgentIds = [...new Set(agentIds)].filter( + (agentId) => agentId.length > 0, + ); + if (uniqueAgentIds.length === 0) return 0; + + const now = new Date(); + const wakeupIds = await db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + inArray(agentWakeupRequests.agentId, uniqueAgentIds), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + + if (wakeupIds.length === 0) return 0; + + await db + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: reason, + updatedAt: now, + }) + .where(inArray(agentWakeupRequests.id, wakeupIds)); + + return wakeupIds.length; + } + + async function cancelInvocationsForAgentsInternal( + agentIds: string[], + reason: string, + ) { + const uniqueAgentIds = [...new Set(agentIds)].filter( + (agentId) => agentId.length > 0, + ); + let runsCancelled = 0; + for (const agentId of uniqueAgentIds) { + runsCancelled += await cancelActiveForAgentInternal(agentId, reason); + } + const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal( + uniqueAgentIds, + reason, + ); + return { + agentIds: uniqueAgentIds, + runsCancelled, + wakeupsCancelled, + }; + } + + async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) { + const runIds = scope.scopeType === "project" + ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore) + : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, scope.companyId), + scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined, + scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined, + inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + )).then((rows) => rows.map((row) => row.id)); + for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope }); + await cancelPendingWakeupsForBudgetScope(scope); + } + + return { + cancelBudgetScopeWork, + releaseIssueExecutionAndPromote, + releaseEnvironmentLeasesForRun, + acknowledgeRemoteStop, + resumeRemoteStopComments, + cancelRunInternal, + resumeExecutionWaitComments, + resumeQueuedCommentInterrupt, + resumeSavedLegacyComments, + cancelActiveForAgentInternal, + cancelInvocationsForAgentsInternal, + }; +} diff --git a/server/src/services/heartbeat/run-lifecycle.test.ts b/server/src/services/heartbeat/run-lifecycle.test.ts new file mode 100644 index 0000000000..414315fbae --- /dev/null +++ b/server/src/services/heartbeat/run-lifecycle.test.ts @@ -0,0 +1,196 @@ +import type { Db } from "@paperclipai/db"; +import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import type { CurrentUserRedactionOptions } from "../../log-redaction.js"; +import { publishLiveEvent } from "../live-events.js"; +import { getHeartbeatRunRuntimeStatus } from "../heartbeat-run-runtime-status.js"; +import { createHeartbeatRunState } from "./run-state.js"; +import { createHeartbeatRunPreparation } from "./run-preparation.js"; +import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js"; + +vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() })); + +type Run = typeof heartbeatRuns.$inferSelect; +function callbacks(db: Db) { + const state = createHeartbeatRunState(db); + return { + ...state, + ...createHeartbeatRunPreparation(db), + getRun: vi.fn(state.getRun), + issuesSvc: { + addComment: vi.fn(), + listReviewAttention: vi.fn(), + }, + recovery: { escalateStrandedAssignedIssue: vi.fn() }, + companySkills: { completeTestRunForIssue: vi.fn() }, + budgets: { getInvocationBlock: vi.fn(async () => null) }, + treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) }, + enqueueWakeup: vi.fn(), + getCurrentUserRedactionOptions: vi.fn(async (): Promise => ({})), + getAgentInvokability: vi.fn(async () => ({ invokable: true })), + emitTerminalAgentTaskRun: vi.fn(), + budgetHooks: {}, + } satisfies HeartbeatLifecycleDependencies; +} +function guardedDatabase() { + const access = vi.fn(() => { throw new Error("Unexpected lifecycle database access"); }); + return { db: new Proxy({}, { get: access }) as Db, access }; +} + +describe("heartbeat lifecycle module boundary", () => { + it("constructs without database work or reporting effects", () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + createHeartbeatLifecycle(database.db, deps); + expect(database.access).not.toHaveBeenCalled(); + for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled(); + }); + + it("ignores absent wake requests without accessing the database", async () => { + const database = guardedDatabase(); + const lifecycle = createHeartbeatLifecycle(database.db, callbacks(database.db)); + await lifecycle.setWakeupStatus(null, "completed"); + await lifecycle.setWakeupStatus(undefined, "completed"); + expect(database.access).not.toHaveBeenCalled(); + }); + + it.each([ + { livenessState: "empty_response", contextSnapshot: { goalControlRequestId: "goal" } }, + { livenessState: "plan_only", contextSnapshot: { resumeSessionGoalHeartbeat: true } }, + { livenessState: "productive", contextSnapshot: { issueId: "issue" } }, + { livenessState: "empty_response", contextSnapshot: {} }, + ])("keeps ineligible continuations out of wake admission (%j)", async (input) => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + await createHeartbeatLifecycle(database.db, deps).handleRunLivenessContinuation(input as unknown as Run); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.enqueueWakeup).not.toHaveBeenCalled(); + expect(deps.budgets.getInvocationBlock).not.toHaveBeenCalled(); + }); + + it("preserves retained native ownership before lease-release teardown reads or writes", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const run = { runtimeMode: "native", status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" } as Run; + expect(await createHeartbeatLifecycle(database.db, deps).terminalizeRunOnLeaseRelease(run)).toBe(run); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.emitTerminalAgentTaskRun).not.toHaveBeenCalled(); + }); + + it("rejects late progress after re-reading a terminal run through the service callback", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const run = { id: "late-progress", companyId: "company", agentId: "agent", status: "running", contextSnapshot: {} } as Run; + deps.getRun.mockResolvedValue({ ...run, status: "succeeded" }); + expect(await createHeartbeatLifecycle(database.db, deps).recordCurrentHeartbeatRunRuntimeProgress(run, { phase: "finalize", message: "late update" }, null)).toBeNull(); + expect(deps.getRun).toHaveBeenCalledWith(run.id); + expect(getHeartbeatRunRuntimeStatus(run.id)).toBeNull(); + expect(database.access).not.toHaveBeenCalled(); + }); +}); + +const support = await getEmbeddedPostgresTestSupport(); +if (!support.supported) console.warn(`Skipping lifecycle module database tests: ${support.reason}`); +describe.skipIf(!support.supported)("heartbeat lifecycle database wiring", () => { + let database: Awaited>; + let db: Db; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-lifecycle-module"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + beforeEach(async () => { + vi.mocked(publishLiveEvent).mockClear(); + await db.execute(sql`truncate table companies restart identity cascade`); + }); + + async function fixture(patch: Partial = {}) { + const [company] = await db.insert(companies).values({ name: "Lifecycle module", issuePrefix: "LM" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Lifecycle agent" }).returning(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, wakeupRequestId: wake.id, status: "running", runtimeMode: "native", ...patch }).returning(); + return { company, agent, wake, run }; + } + + it("keeps one terminal outcome and reports only the winner of competing writes", async () => { + const { run } = await fixture(); + const deps = callbacks(db); + const lifecycle = createHeartbeatLifecycle(db, deps); + const results = await Promise.all([ + lifecycle.setRunStatusIfRunning(run.id, "succeeded", { finishedAt: new Date() }), + lifecycle.setRunStatusIfRunning(run.id, "failed", { errorCode: "late_failure", finishedAt: new Date() }), + ]); + const winner = results.find(result => result.updated)!; + expect(results.filter(result => result.updated)).toHaveLength(1); + const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(persisted.status).toBe(winner.run?.status); + expect(results.every(result => result.run?.status === persisted.status)).toBe(true); + expect(deps.emitTerminalAgentTaskRun).toHaveBeenCalledTimes(1); + expect(deps.emitTerminalAgentTaskRun).toHaveBeenCalledWith(winner.run, "running", undefined); + expect(vi.mocked(publishLiveEvent).mock.calls.filter(([event]) => event.type === "heartbeat.run.status")).toHaveLength(1); + }); + + it("rechecks native ownership atomically before a terminal write", async () => { + const { run } = await fixture({ errorCode: "native_execution_ownership_unverified", nativePhase: "terminal_failure" }); + const deps = callbacks(db); + const result = await createHeartbeatLifecycle(db, deps).setRunStatusIfRunning(run.id, "failed"); + expect(result).toMatchObject({ updated: false, run: { status: "running", errorCode: run.errorCode } }); + expect(deps.emitTerminalAgentTaskRun).not.toHaveBeenCalled(); + expect(publishLiveEvent).not.toHaveBeenCalled(); + }); + + it.each(["setRunStatus", "setRunStatusIfRunning"] as const)("preserves accounting metadata while %s enriches usage", async (operation) => { + const { run } = await fixture({ usageJson: { accountingReceiptReady: false, usageSource: "per_run", inputTokens: 5 } }); + await createHeartbeatLifecycle(db, callbacks(db))[operation](run.id, "succeeded", { usageJson: { inputTokens: 8, outputTokens: 3 } }); + const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(persisted.usageJson).toEqual({ accountingReceiptReady: false, usageSource: "per_run", inputTokens: 8, outputTokens: 3 }); + }); + + it.each(["claimed", "cancelled"] as const)("settles a %s wake without reviving a revoked request", async (status) => { + const { wake } = await fixture(); + await db.update(agentWakeupRequests).set({ status }).where(eq(agentWakeupRequests.id, wake.id)); + await createHeartbeatLifecycle(db, callbacks(db)).setWakeupStatus(wake.id, "completed", { reason: "settled" }); + const [persisted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); + expect(persisted.status).toBe(status === "cancelled" ? "cancelled" : "completed"); + expect(persisted.reason).toBe(status === "cancelled" ? wake.reason : "settled"); + }); + + it("publishes the persisted sequence for concurrent event writes", async () => { + const { run } = await fixture(); + const lifecycle = createHeartbeatLifecycle(db, callbacks(db)); + await Promise.all(["first", "second"].map(message => lifecycle.appendRunEvent(run, { eventType: "lifecycle", message }))); + const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id)).orderBy(heartbeatRunEvents.seq); + expect(rows.map(row => row.seq)).toEqual([1, 2]); + const published = vi.mocked(publishLiveEvent).mock.calls.map(([event]) => event).filter(event => event.type === "heartbeat.run.event"); + expect(published).toHaveLength(2); + for (const row of rows) expect(published.find(event => event.payload?.seq === row.seq)?.payload).toMatchObject({ runId: run.id, message: row.message }); + }); + + it("reads current redaction settings for each event and persists the published sanitized payload", async () => { + const { run } = await fixture(); + const deps = callbacks(db); + deps.getCurrentUserRedactionOptions.mockResolvedValueOnce({ enabled: true, userNames: ["alice"], homeDirs: ["/Users/alice"] }); + deps.getCurrentUserRedactionOptions.mockResolvedValueOnce({ enabled: true, userNames: ["bob"], homeDirs: ["/Users/bob"] }); + const lifecycle = createHeartbeatLifecycle(db, deps); + for (const name of ["alice", "bob"]) await lifecycle.appendRunEvent(run, { + eventType: "lifecycle", message: `Opening /Users/${name}/project`, payload: { path: `/Users/${name}/project`, token: "test-credential-material" }, + }); + const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id)).orderBy(heartbeatRunEvents.seq); + expect(deps.getCurrentUserRedactionOptions).toHaveBeenCalledTimes(2); + expect(rows).toHaveLength(2); + const published = vi.mocked(publishLiveEvent).mock.calls.map(([event]) => event).filter(event => event.type === "heartbeat.run.event"); + expect(published).toHaveLength(2); + for (const row of rows) { + const text = JSON.stringify({ message: row.message, payload: row.payload }); + expect(text).not.toContain("alice"); + expect(text).not.toContain("bob"); + expect(text).not.toContain("test-credential-material"); + expect(published.find(event => event.payload?.seq === row.seq)?.payload).toMatchObject({ message: row.message, payload: row.payload }); + } + }); +}); diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts new file mode 100644 index 0000000000..55215e6301 --- /dev/null +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -0,0 +1,2982 @@ +import { + DEFERRED_WAKE_CONTEXT_KEY, + EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + shouldRequireIssueCommentForWake, +} from "./queue.js"; +import { + HEARTBEAT_RUN_TERMINAL_STATUSES, + DETACHED_PROCESS_ERROR_CODE, +} from "./recovery.js"; +import { normalizeAgentNameKey } from "./retries.js"; +import { + type RunSessionOutcome, + deriveTaskKeyWithHeartbeatFallback, + type UsageTotals, +} from "./run-state.js"; +import { CONFIGURATION_INCOMPLETE_FAILURE_CODE } from "./run-preparation.js"; +import { WORKSPACE_VALIDATION_FAILURE_CODE } from "./workspaces.js"; +import { boundHeartbeatRunEventPayloadForStorage } from "./run-log.js"; +import { preserveWorkspaceRestoreRecoveryMetadata } from "../workspace-restore-recovery-state.js"; +import { activeIssueInteractionCondition } from "../issue-question-context.js"; +import { + nativeRetryCancellationCommitCondition, + rethrowNativeCancellationLockConflict, +} from "../native-runtime/native-cancellation-request.js"; +import { cancellationResultJson } from "../run-cancellation.js"; +import { accountRunCost } from "../run-cost-accounting.js"; +import { isWaitingConversation } from "../agent-conversations.js"; +import { PROCESS_IDENTITY_RECORDED } from "../native-local-process-stop.js"; +import { + hasAcknowledgedNativeReassignmentStopIntent, + hasAcknowledgedNativeStopIntent, +} from "../acknowledged-native-stop.js"; +import { connectionIntentService } from "../connection-intents.js"; +import { + CONVERSATION_CONTINUATION_POLICY, + isConversationAdapter, +} from "../conversation-continuation.js"; +import { terminalizeLegacyExecution } from "../legacy-execution-recovery.js"; +import { buildHeartbeatRunStatusLiveEventPayload } from "../heartbeat-run-status-payload.js"; +import { randomUUID } from "node:crypto"; +import { + and, + desc, + eq, + inArray, + isNull, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, + type HeartbeatRunStatusPhase, + type RequestConfirmationResult, + type RunLivenessState, +} from "@paperclipai/shared"; +import { + agents, + agentRuntimeState, + agentWakeupRequests, + activityLog, + approvals, + companySkillTestRuns, + documentRevisions, + issueDocuments, + heartbeatRunEvents, + heartbeatRuns, + issueApprovals, + issueComments, + issueRecoveryActions, + issueRelations, + issueThreadInteractions, + issues, + issueWorkProducts, + routines, + workspaceOperations, +} from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import { publishLiveEvent } from "../live-events.js"; +import { + appendHeartbeatRunEvent, + type AppendHeartbeatRunEventInput, +} from "../heartbeat-run-events.js"; +import type { AdapterExecutionResult } from "../../adapters/index.js"; +import { + parseObject, + asBoolean, + asNumber, +} from "../../adapters/utils.js"; +import { + isNativeRunnerOwnershipHeld, + nativeRunnerOwnershipNotHeldCondition, +} from "../native-runtime/native-runner-ownership.js"; +import { + type RunFailureReportOptions, +} from "../run-failure-diagnostics.js"; +import { + findHeartbeatRunCompletionComment, + hasAcceptedSemanticResult, + readCompletedAssistantMessageCandidate, + selectHeartbeatRunFinalAgentMessage, + type RunPresentationDecision, +} from "../heartbeat-run-summary.js"; +import { + buildHeartbeatRunStopMetadata, + mergeHeartbeatRunStopMetadata, +} from "../heartbeat-stop-metadata.js"; +import { CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js"; +import { + classifyRunLiveness, + type RunLivenessClassificationInput, +} from "../run-liveness.js"; +import { + logActivity, + publishPluginDomainEvent, +} from "../activity-log.js"; +import { runnerGoalService } from "../runner-goals.js"; +import { visibleIssueCondition } from "../issue-visibility.js"; +import { + getIssueContinuationSummaryDocument, + refreshIssueContinuationSummary, +} from "../issue-continuation-summary.js"; +import { + RECOVERY_ORIGIN_KINDS, + FINISH_SUCCESSFUL_RUN_HANDOFF_REASON, + SUCCESSFUL_RUN_MISSING_STATE_REASON, + RUN_LIVENESS_CONTINUATION_REASON, + buildRunLivenessContinuationIdempotencyKey, + buildFinishSuccessfulRunHandoffIdempotencyKey, + buildSuccessfulRunHandoffRequiredNotice, + decideRunLivenessContinuation, + decideSuccessfulRunHandoff, + findExistingFinishSuccessfulRunHandoffWake, + findExistingRunLivenessContinuationWake, + isSuccessfulRunHandoffValidPathSkip, + SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, + readContinuationAttempt, +} from "../recovery/index.js"; +import { withRecoveryContext } from "../recovery/status-only-context.js"; +import { deriveCommentId } from "../../modules/run-dispatch/index.js"; +import { + buildIssueReviewPathLostIdempotencyKey, + decideIssueReviewPathRecovery, + ISSUE_REVIEW_PATH_LOST_WAKE_REASON, + isReviewPathRecoveryIdempotencyConflict, + reviewPathConsumedRefFromRun, +} from "../recovery/review-path-recovery.js"; +import { resolveRequiredSuccessfulRunHandoffOnValidPath } from "../successful-run-handoff-state.js"; +import { + redactCurrentUserText, + redactCurrentUserValue, + type CurrentUserRedactionOptions, +} from "../../log-redaction.js"; +import { + redactEventPayload, + redactSensitiveText, +} from "../../redaction.js"; +import { + type RuntimeStatusUpdate, +} from "@paperclipai/adapter-utils"; +import { + UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + UNMANAGED_BACKGROUND_TASK_STOP_REASON, +} from "@paperclipai/adapter-utils/server-utils"; +import { + clearHeartbeatRunRuntimeStatus, + getHeartbeatRunRuntimeStatus, + MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, + MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, + setHeartbeatRunRuntimeStatus, +} from "../heartbeat-run-runtime-status.js"; +import { readProcessStartedAt } from "../hot-restart.js"; + +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatRunPreparation } from "./run-preparation.js"; +import type { createHeartbeatQueue } from "./queue.js"; +import type { HeartbeatRetryDependencies } from "./retries.js"; +import type { issueService } from "../issues.js"; +import type { recoveryService } from "../recovery/index.js"; +import type { companySkillService } from "../company-skills.js"; +import type { budgetService, BudgetServiceHooks } from "../budgets.js"; +import type { issueTreeControlService } from "../issue-tree-control.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; + +/** Lifecycle owns run writes; the service supplies reporting and admission effects. */ +export interface HeartbeatLifecycleDependencies extends Pick, + "getRun" | "resolveSessionBeforeForWakeup" | "ensureRuntimeState">, + Pick, + "getIssueExecutionContext" | "resolveResponsibleUserIdForRunContext">, + Pick { + issuesSvc: Pick, "addComment" | "listReviewAttention">; + recovery: Pick, "escalateStrandedAssignedIssue">; + companySkills: Pick, "completeTestRunForIssue">; + budgets: Pick, "getInvocationBlock">; + treeControlSvc: Pick, "getActivePauseHoldGate">; + enqueueWakeup: ReturnType["enqueueWakeup"]; + getCurrentUserRedactionOptions: () => Promise; + budgetHooks: BudgetServiceHooks; + emitTerminalAgentTaskRun: (run: HeartbeatRun, previousStatus: string | null, failureReport?: RunFailureReportOptions) => void; +} + +export function redactDetectedSuccessfulRunProgressSummaryForBoard( + summary: string, + currentUserRedactionOptions?: CurrentUserRedactionOptions, +) { + const normalized = summary.replace(/\s+/g, " ").trim(); + const redacted = redactSensitiveText( + redactCurrentUserText(normalized, currentUserRedactionOptions), + ); + return redacted.length <= 280 ? redacted : `${redacted.slice(0, 277)}...`; +} + +export function redactSuccessfulRunHandoffEvidence( + value: string | null, + currentUserRedactionOptions?: CurrentUserRedactionOptions, +) { + if (!value) return null; + return redactSensitiveText( + redactCurrentUserText(value, currentUserRedactionOptions), + ); +} + +const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [ + "environment.lease_acquired", + "environment.lease_released", + "cost.reported", +]; + +export const NON_RETRYABLE_PREFLIGHT_FAILURE_CODES = new Set([ + "low_trust_isolation_unavailable", + "low_trust_requires_isolated_workspace", + "low_trust_boundary_mismatch", + "low_trust_requires_sandbox_environment", + "low_trust_runtime_services_denied", + "chat_failed_run_retry_not_authorized", + CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, +]); + +// Error codes that mark a pre-dispatch setup failure. The adapter process never +// started, so no agent could post an issue comment. The setup catch writes one +// of these codes when a failure happens before `adapter.execute`. +const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set([ + "setup_failed", + CONFIGURATION_INCOMPLETE_FAILURE_CODE, + WORKSPACE_VALIDATION_FAILURE_CODE, + ...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, +]); + +const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2; + +const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10; + +const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000; + +const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000; + +interface MaxTurnContinuationPolicy { + enabled: boolean; + maxAttempts: number; + delayMs: number; +} + +export function isHeartbeatRunTerminalStatus( + status: string | null | undefined, +): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] { + return HEARTBEAT_RUN_TERMINAL_STATUSES.includes( + status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number], + ); +} + +export function isHeartbeatRunRuntimeStatusActive( + status: string | null | undefined, +): boolean { + return status === "queued" || status === "running"; +} + +type HeartbeatRunRuntimeStatusRunLike = { + id: string; + status?: string | null; + companyId?: string | null; + agentId?: string | null; + issueId?: string | null; + contextSnapshot?: Record | null; +}; + +function readRuntimeStatusIssueIdCandidate( + run: HeartbeatRunRuntimeStatusRunLike, +): string | null | undefined { + if ("issueId" in run) return readNonEmptyString(run.issueId) ?? null; + if ("contextSnapshot" in run) { + return readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? null; + } + return undefined; +} + +export function decorateHeartbeatRunRuntimeStatus< + T extends HeartbeatRunRuntimeStatusRunLike, +>( + run: T, + expected: { + companyId?: string | null; + issueId?: string | null; + agentId?: string | null; + } = {}, +): T & { + currentStatusMessage: string | null; + currentStatusUpdatedAt: Date | null; + currentToolName: string | null; + lastAssistantSnippet: string | null; + lastEventAt: Date | null; +} { + if (isHeartbeatRunTerminalStatus(run.status)) { + clearHeartbeatRunRuntimeStatus(run.id); + } + + const companyId = expected.companyId ?? run.companyId ?? null; + const agentId = expected.agentId ?? run.agentId ?? null; + const issueId = + expected.issueId !== undefined + ? expected.issueId + : readRuntimeStatusIssueIdCandidate(run); + const currentStatus = + isHeartbeatRunRuntimeStatusActive(run.status) && companyId && agentId + ? getHeartbeatRunRuntimeStatus(run.id, { + companyId, + agentId, + ...(issueId !== undefined ? { issueId } : {}), + }) + : null; + + return { + ...run, + currentStatusMessage: currentStatus?.message ?? null, + currentStatusUpdatedAt: currentStatus?.updatedAt ?? null, + currentToolName: currentStatus?.currentToolName ?? null, + lastAssistantSnippet: currentStatus?.lastAssistantSnippet ?? null, + lastEventAt: currentStatus?.lastEventAt ?? null, + }; +} + +export function publishHeartbeatRunRuntimeProgress(status: { + companyId: string; + runId: string; + agentId: string; + issueId: string | null; + phase: HeartbeatRunStatusPhase; + message: string; + updatedAt: Date; + currentToolName?: string | null; + lastAssistantSnippet?: string | null; + lastEventAt?: Date | null; +}) { + publishLiveEvent({ + companyId: status.companyId, + type: "heartbeat.run.progress", + payload: { + runId: status.runId, + agentId: status.agentId, + issueId: status.issueId, + phase: status.phase, + message: status.message, + currentToolName: status.currentToolName ?? null, + lastAssistantSnippet: status.lastAssistantSnippet ?? null, + lastEventAt: (status.lastEventAt ?? status.updatedAt).toISOString(), + updatedAt: status.updatedAt.toISOString(), + }, + }); +} + +function recordHeartbeatRunRuntimeProgress( + run: Pick< + typeof heartbeatRuns.$inferSelect, + "id" | "companyId" | "agentId" | "status" | "contextSnapshot" + >, + update: RuntimeStatusUpdate, + issueId: string | null, +) { + if (!isHeartbeatRunRuntimeStatusActive(run.status)) return null; + const status = setHeartbeatRunRuntimeStatus({ + companyId: run.companyId, + issueId, + agentId: run.agentId, + runId: run.id, + phase: update.phase as HeartbeatRunStatusPhase, + message: update.message, + currentToolName: readNonEmptyString(update.currentToolName) ?? null, + lastAssistantSnippet: + readNonEmptyString(update.lastAssistantSnippet) ?? null, + lastEventAt: update.lastEventAt ? new Date(update.lastEventAt) : new Date(), + }); + if (!status) return null; + + publishHeartbeatRunRuntimeProgress(status); + return status; +} + +function sanitizeLiveRunProgressText( + value: string, + maxChars: number, +): string | null { + const normalized = value.replace(/\s+/g, " ").trim(); + if (!normalized) return null; + const redacted = redactSensitiveText(normalized); + if (redacted.length <= maxChars) return redacted; + return `${redacted.slice(0, maxChars - 3)}...`; +} + +function readLiveRunProgressString( + value: unknown, + maxChars: number, +): string | null { + return typeof value === "string" + ? sanitizeLiveRunProgressText(value, maxChars) + : null; +} + +function readFirstLiveRunProgressString( + maxChars: number, + values: unknown[], +): string | null { + for (const value of values) { + const text = readLiveRunProgressString(value, maxChars); + if (text) return text; + } + return null; +} + +function readLiveRunToolName( + payload: Record | null, + eventType: string, +): string | null { + const toolCall = + parseObject(payload?.tool_call) ?? parseObject(payload?.toolCall); + const message = parseObject(payload?.message); + const direct = readFirstLiveRunProgressString( + MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, + [ + payload?.toolName, + payload?.tool_name, + payload?.tool, + payload?.name, + payload?.title, + toolCall?.name, + toolCall?.toolName, + message?.name, + message?.toolName, + ], + ); + if (direct) return direct; + + const normalizedEventType = eventType.toLowerCase(); + if (!normalizedEventType.includes("tool")) return null; + return readLiveRunProgressString( + eventType.replace(/[._-]+/g, " "), + MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, + ); +} + +function readLiveRunAssistantSnippet( + payload: Record | null, + eventType: string, + message: string | null, +): string | null { + const normalizedEventType = eventType.toLowerCase(); + const messagePayload = parseObject(payload?.message); + const direct = readFirstLiveRunProgressString( + MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, + [ + payload?.text, + payload?.delta, + payload?.text_delta, + payload?.content, + payload?.summary, + messagePayload?.text, + messagePayload?.content, + ], + ); + if (direct) return direct; + + if ( + normalizedEventType.includes("assistant") || + normalizedEventType.includes("text_delta") || + normalizedEventType.includes("message.delta") || + normalizedEventType.includes("message_delta") + ) { + return message + ? sanitizeLiveRunProgressText( + message, + MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, + ) + : null; + } + + return null; +} + +function buildRunEventRuntimeProgress(input: { + eventType: string; + message: string | null; + payload: Record | null; + at: Date; +}) { + const normalizedEventType = input.eventType.toLowerCase(); + if ( + normalizedEventType === "lifecycle" || + normalizedEventType === "adapter.invoke" + ) { + return null; + } + + const currentToolName = readLiveRunToolName(input.payload, input.eventType); + const lastAssistantSnippet = readLiveRunAssistantSnippet( + input.payload, + input.eventType, + input.message, + ); + const fallbackMessage = + readLiveRunProgressString( + input.message, + MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, + ) ?? + sanitizeLiveRunProgressText( + input.eventType.replace(/[._-]+/g, " "), + MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, + ); + const message = currentToolName + ? `Using ${currentToolName}` + : (lastAssistantSnippet ?? fallbackMessage); + + if (!message) return null; + return { + phase: "run_activity" as const, + message, + currentToolName, + lastAssistantSnippet, + lastEventAt: input.at, + }; +} + +export async function persistHeartbeatRunProcessMetadata( + db: Db, + runId: string, + meta: { pid: number; processGroupId: number | null; startedAt: string }, +) { + const observedStartedAt = await readProcessStartedAt(meta.pid).catch( + () => null, + ); + const startedAt = new Date(observedStartedAt ?? meta.startedAt); + return db.transaction(async tx => { + const run = await tx + .update(heartbeatRuns) + .set({ + processPid: meta.pid, + processGroupId: meta.processGroupId, + processStartedAt: Number.isNaN(startedAt.getTime()) + ? new Date() + : startedAt, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, runId)) + .returning() + .then((rows) => rows[0] ?? null); + if (run?.runtimeMode === "native") await appendHeartbeatRunEvent(tx as unknown as Db, { + companyId: run.companyId, runId, agentId: run.agentId, + eventType: PROCESS_IDENTITY_RECORDED, stream: "system", level: "info", + message: "Process identity recorded; prior stop evidence no longer applies.", + }); + return run; + }); +} + +type SkillTestHeartbeatCompletion = { + outcome: "failed" | "cancelled"; + error: string | null; + heartbeatOutcome: RunSessionOutcome; +}; + +export function resolveSkillTestRunCompletionForHeartbeatOutcome( + outcome: RunSessionOutcome, + error: string | null | undefined, +): SkillTestHeartbeatCompletion | null { + if (outcome === "cancelled") { + return { + outcome: "cancelled", + error: error ?? "Harness run was cancelled", + heartbeatOutcome: outcome, + }; + } + if (outcome === "timed_out") { + return { + outcome: "failed", + error: error ?? "Timed out", + heartbeatOutcome: outcome, + }; + } + if (outcome === "failed") { + return { + outcome: "failed", + error: error ?? "Adapter failed", + heartbeatOutcome: outcome, + }; + } + return null; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +/** Run lifecycle binds status, progress, liveness, and completion operations. */ +export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycleDependencies) { + const { + issuesSvc, + recovery, + companySkills, + getRun, + emitTerminalAgentTaskRun, + getIssueExecutionContext, + budgets, + enqueueWakeup, + getCurrentUserRedactionOptions, + treeControlSvc, + getAgentInvokability, + resolveSessionBeforeForWakeup, + resolveResponsibleUserIdForRunContext, + ensureRuntimeState, + budgetHooks, + } = dependencies; + + function isPlanApprovalConfirmationPayload(payload: unknown) { + const target = parseObject(parseObject(payload).target); + return ( + readNonEmptyString(target.type) === "issue_document" && + readNonEmptyString(target.key) === "plan" + ); + } + + async function getAcceptedPlanApprovalInteractionForRun( + run: typeof heartbeatRuns.$inferSelect, + issueId: string | null, + ) { + const context = parseObject(run.contextSnapshot); + const interactionId = readNonEmptyString(context.interactionId); + if (!issueId || !interactionId) return null; + + const interaction = await db + .select({ + id: issueThreadInteractions.id, + kind: issueThreadInteractions.kind, + status: issueThreadInteractions.status, + payload: issueThreadInteractions.payload, + result: issueThreadInteractions.result, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, run.companyId), + eq(issueThreadInteractions.issueId, issueId), + eq(issueThreadInteractions.id, interactionId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!interaction) return null; + if ( + interaction.kind !== "request_confirmation" || + interaction.status !== "accepted" + ) + return null; + return isPlanApprovalConfirmationPayload(interaction.payload) + ? interaction + : null; + } + + function planApprovalResumeFailureErrorCode( + run: typeof heartbeatRuns.$inferSelect, + ) { + return readNonEmptyString(run.errorCode) ?? "unknown_error"; + } + + function buildPlanApprovalResumeFailureComment(input: { + run: typeof heartbeatRuns.$inferSelect; + status: "retrying" | "needs_attention"; + attempt: number; + maxAttempts: number; + }) { + const errorCode = planApprovalResumeFailureErrorCode(input.run); + if (input.status === "retrying") { + return `Agent failed to resume after approval: \`${errorCode}\` — retrying (attempt ${input.attempt}/${input.maxAttempts})`; + } + return `Agent failed to resume after approval: \`${errorCode}\` — needs attention`; + } + + function buildPlanApprovalResumeFailureResult(input: { + run: typeof heartbeatRuns.$inferSelect; + status: "retrying" | "needs_attention"; + attempt: number; + maxAttempts: number; + retryRunId?: string | null; + recoveryActionId?: string | null; + }): NonNullable { + return { + status: input.status, + errorCode: planApprovalResumeFailureErrorCode(input.run), + attempt: input.attempt, + maxAttempts: input.maxAttempts, + runId: input.run.id, + retryRunId: input.retryRunId ?? null, + recoveryActionId: input.recoveryActionId ?? null, + updatedAt: new Date().toISOString(), + }; + } + + async function updatePlanApprovalInteractionResumeFailure(input: { + interaction: NonNullable< + Awaited> + >; + failure: NonNullable; + }) { + const result = parseObject(input.interaction.result); + const nextResult = { + ...result, + version: 1 as const, + outcome: "accepted" as const, + resumeFailure: input.failure, + } satisfies RequestConfirmationResult; + + await db + .update(issueThreadInteractions) + .set({ + result: nextResult, + updatedAt: new Date(), + }) + .where(eq(issueThreadInteractions.id, input.interaction.id)); + } + + async function addPlanApprovalResumeFailureCommentOnce(input: { + issueId: string; + run: typeof heartbeatRuns.$inferSelect; + body: string; + }) { + const existing = await db + .select({ id: issueComments.id }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, input.run.companyId), + eq(issueComments.issueId, input.issueId), + or( + eq(issueComments.body, input.body), + sql`${issueComments.body} like ${`${input.body}\n%`}`, + ), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (existing) return null; + return issuesSvc.addComment( + input.issueId, + input.body, + { runId: input.run.id }, + { authorType: "system" }, + ); + } + + async function getActiveRecoveryActionId( + companyId: string, + sourceIssueId: string, + ) { + return db + .select({ id: issueRecoveryActions.id }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + eq(issueRecoveryActions.sourceIssueId, sourceIssueId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + ), + ) + .orderBy(desc(issueRecoveryActions.updatedAt)) + .limit(1) + .then((rows) => rows[0]?.id ?? null); + } + + async function recordPlanApprovalResumeFailureRetry(input: { + run: typeof heartbeatRuns.$inferSelect; + issueId: string | null; + retryRunId: string | null; + attempt: number; + maxAttempts: number; + }) { + const interaction = await getAcceptedPlanApprovalInteractionForRun( + input.run, + input.issueId, + ); + if (!interaction || !input.issueId) return null; + + const body = buildPlanApprovalResumeFailureComment({ + run: input.run, + status: "retrying", + attempt: input.attempt, + maxAttempts: input.maxAttempts, + }); + await addPlanApprovalResumeFailureCommentOnce({ + issueId: input.issueId, + run: input.run, + body, + }); + await updatePlanApprovalInteractionResumeFailure({ + interaction, + failure: buildPlanApprovalResumeFailureResult({ + run: input.run, + status: "retrying", + attempt: input.attempt, + maxAttempts: input.maxAttempts, + retryRunId: input.retryRunId, + }), + }); + return interaction.id; + } + + async function escalatePlanApprovalResumeFailureNeedsAttention(input: { + run: typeof heartbeatRuns.$inferSelect; + issueId: string | null; + attempt: number; + maxAttempts: number; + }) { + const interaction = await getAcceptedPlanApprovalInteractionForRun( + input.run, + input.issueId, + ); + if (!interaction || !input.issueId) return null; + + const issue = await db + .select() + .from(issues) + .where( + and( + eq(issues.companyId, input.run.companyId), + eq(issues.id, input.issueId), + ), + ) + .then((rows) => rows[0] ?? null); + if (!issue) return null; + if ( + issue.status !== "todo" && + issue.status !== "in_progress" && + issue.status !== "in_review" + ) + return null; + + const body = buildPlanApprovalResumeFailureComment({ + run: input.run, + status: "needs_attention", + attempt: input.attempt, + maxAttempts: input.maxAttempts, + }); + await recovery.escalateStrandedAssignedIssue({ + issue, + previousStatus: issue.status, + latestRun: input.run, + comment: body, + }); + await addPlanApprovalResumeFailureCommentOnce({ + issueId: issue.id, + run: input.run, + body, + }); + + const recoveryActionId = await getActiveRecoveryActionId( + issue.companyId, + issue.id, + ); + await updatePlanApprovalInteractionResumeFailure({ + interaction, + failure: buildPlanApprovalResumeFailureResult({ + run: input.run, + status: "needs_attention", + attempt: input.attempt, + maxAttempts: input.maxAttempts, + recoveryActionId, + }), + }); + return interaction.id; + } + + async function completeSkillTestRunForHeartbeatOutcome(input: { + run: typeof heartbeatRuns.$inferSelect; + issueId: string | null; + issueWorkMode?: string | null; + outcome: RunSessionOutcome; + error: string | null; + }) { + const completion = resolveSkillTestRunCompletionForHeartbeatOutcome( + input.outcome, + input.error, + ); + if (!completion || !input.issueId) return null; + + let isSkillTestIssue = input.issueWorkMode === "skill_test"; + if (!isSkillTestIssue && input.issueWorkMode === undefined) { + const issueRow = await db + .select({ + workMode: issues.workMode, + harnessKind: issues.harnessKind, + }) + .from(issues) + .where( + and( + eq(issues.companyId, input.run.companyId), + eq(issues.id, input.issueId), + ), + ) + .then((rows) => rows[0] ?? null); + isSkillTestIssue = + issueRow?.workMode === "skill_test" || + issueRow?.harnessKind === "skill_test"; + } + if (!isSkillTestIssue) return null; + + const existingRun = await db + .select({ + id: companySkillTestRuns.id, + status: companySkillTestRuns.status, + }) + .from(companySkillTestRuns) + .where( + and( + eq(companySkillTestRuns.companyId, input.run.companyId), + eq(companySkillTestRuns.issueId, input.issueId), + ), + ) + .then((rows) => rows[0] ?? null); + if ( + !existingRun || + ["succeeded", "failed", "cancelled"].includes(existingRun.status) + ) + return null; + + const completedRun = await companySkills.completeTestRunForIssue({ + companyId: input.run.companyId, + issueId: input.issueId, + outcome: completion.outcome, + error: completion.error, + }); + if (!completedRun) return null; + + await logActivity(db, { + companyId: input.run.companyId, + actorType: "system", + actorId: "heartbeat_finalize", + agentId: input.run.agentId, + runId: input.run.id, + action: "company.skill_test_run_completed", + entityType: "company_skill_test_run", + entityId: completedRun.id, + issueId: input.issueId, + details: { + issueId: input.issueId, + status: completedRun.status, + outputDocumentKey: completedRun.outputDocumentKey, + heartbeatOutcome: completion.heartbeatOutcome, + source: "heartbeat.run_finalized", + }, + }); + + return completedRun; + } + + async function recordCurrentHeartbeatRunRuntimeProgress( + run: Pick< + typeof heartbeatRuns.$inferSelect, + "id" | "companyId" | "agentId" | "status" | "contextSnapshot" + >, + update: RuntimeStatusUpdate, + issueId: string | null, + ) { + if (!isHeartbeatRunRuntimeStatusActive(run.status)) { + clearHeartbeatRunRuntimeStatus(run.id); + return null; + } + + const currentRun = await getRun(run.id); + if (!currentRun || !isHeartbeatRunRuntimeStatusActive(currentRun.status)) { + clearHeartbeatRunRuntimeStatus(run.id); + return null; + } + + return recordHeartbeatRunRuntimeProgress(currentRun, update, issueId); + } + + async function setRunStatus( + runId: string, + status: string, + patch?: Partial, + ) { + const previousStatus = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + + if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch, + resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson), + }; + + // Preserve the receipt-source fence when finalization enriches usage. A + // late spool replay must still be able to complete an unfinished receipt. + if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } }; + + // Cancelling a queued run that never acquired provider execution is + // positive bootstrap evidence. It must not hold unrelated queued messages. + if (previousStatus && (status === "cancelled" || status === "interrupted")) { + patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) }; + } + if ( + status === "cancelled" && + previousStatus?.status === "queued" && + previousStatus.runtimeMode !== "native" && + !previousStatus.startedAt && + !previousStatus.processPid + ) { + patch = { + ...patch, + resultJson: { + ...previousStatus.resultJson, + ...patch?.resultJson, + executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + }, + }; + } + const updated = + previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status) + ? await terminalizeLegacyExecution({ + db, + run: previousStatus, + status, + patch, + reconcileIfNeeded: true, + }) + : await db + .update(heartbeatRuns) + .set({ + status, + ...patch, + executionStatusDeliveryId: randomUUID(), + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, runId)) + .returning() + .then((rows) => rows[0] ?? null); + + if (updated) { + publishLiveEvent({ + companyId: updated.companyId, + type: "heartbeat.run.status", + payload: buildHeartbeatRunStatusLiveEventPayload(updated), + }); + publishRunLifecyclePluginEvent(updated); + emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null); + } + + return updated; + } + + async function setRunStatusIfRunning( + runId: string, + status: string, + patch?: Partial, + failureReport?: RunFailureReportOptions, + ) { + return setRunStatusFromLive(runId, status, ["running"], patch, failureReport); + } + + // Move a run to a new status only when its current status is one of + // `fromStatuses`. The compare-and-set is a single conditional update, so a + // concurrent path can win the race. When this update matches nothing, the + // function reads the current row and reports updated=false, so the caller can + // keep the terminal outcome that another path already wrote. + async function setRunStatusFromLive( + runId: string, + status: string, + fromStatuses: string[], + patch?: Partial, + failureReport?: RunFailureReportOptions, + cancellationCondition?: ReturnType, + ) { + // fromStatuses can name a terminal status as its own source (for example, + // an idempotent "still failed" patch), so the write below is not always a + // genuine transition. Read the pre-write status to tell the two apart. + const previousStatus = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + + if (previousStatus && (status === "cancelled" || status === "interrupted")) { + patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) }; + } + + if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch, + resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson), + }; + + // Preserve the receipt-source fence when finalization enriches usage. A + // late spool replay must still be able to complete an unfinished receipt. + if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } }; + + // Cancelling a queued run that never acquired provider execution is + // positive bootstrap evidence. It must not hold unrelated queued messages. + if ( + status === "cancelled" && + previousStatus?.status === "queued" && + previousStatus.runtimeMode !== "native" && + !previousStatus.startedAt && + !previousStatus.processPid + ) { + patch = { + ...patch, + resultJson: { + ...previousStatus.resultJson, + ...patch?.resultJson, + executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + }, + }; + } + const updated = + previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status) + ? await terminalizeLegacyExecution({ + db, + run: previousStatus, + status, + patch, + fromStatuses, + reconcileIfNeeded: true, + writeConditions: [nativeRunnerOwnershipNotHeldCondition(), ...(cancellationCondition ? [cancellationCondition] : [])], + }) + : await db + .update(heartbeatRuns) + .set({ + status, + ...patch, + executionStatusDeliveryId: randomUUID(), + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, runId), + inArray(heartbeatRuns.status, fromStatuses), + ...(cancellationCondition ? [cancellationCondition] : []), + ...(isHeartbeatRunTerminalStatus(status) + ? [nativeRunnerOwnershipNotHeldCondition()] + : []), + ), + ) + .returning() + .then((rows) => rows[0] ?? null) + .catch((error: unknown) => { + if (cancellationCondition) rethrowNativeCancellationLockConflict(error); + throw error; + }); + + if (updated) { + publishLiveEvent({ + companyId: updated.companyId, + type: "heartbeat.run.status", + payload: buildHeartbeatRunStatusLiveEventPayload(updated), + }); + publishRunLifecyclePluginEvent(updated); + emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport); + return { run: updated, updated: true as const }; + } + + const current = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + + return { run: current, updated: false as const }; + } + + // Invariant: when a run releases its environment lease, the run row must be + // terminal. The finalizer writes the terminal status in a step that is + // separate from the agent status=done PATCH. If the sandbox or the run + // process stops between the two steps, heartbeat_runs.status stays "running". + // The UI reads liveness from that row, so a finished task shows "Live" + // forever. This function closes the gap in the run teardown: when the run is + // still running or queued, it forces a terminal status before the lease is + // released. It never overwrites a status that another path already made + // terminal. + async function terminalizeRunOnLeaseRelease( + run: typeof heartbeatRuns.$inferSelect, + ): Promise { + if (isNativeRunnerOwnershipHeld(run)) return run; + if (isHeartbeatRunTerminalStatus(run.status)) return run; + if (run.status !== "running" && run.status !== "queued") return run; + + // Choose the terminal status that reflects the true outcome. When the issue + // already reached a terminal status, the run reached its goal, so use the + // matching terminal run status. Otherwise the teardown cut the run short, + // so use "interrupted". + const issueId = readNonEmptyString( + parseObject(run.contextSnapshot).issueId, + ); + let terminalStatus: "succeeded" | "cancelled" | "interrupted" = + "interrupted"; + if (issueId) { + const issueStatus = await db + .select({ status: issues.status }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0]?.status ?? null); + if (issueStatus === "done") terminalStatus = "succeeded"; + else if (issueStatus === "cancelled") terminalStatus = "cancelled"; + } + + // Teardown can beat the cancellation finalizer. Preserve the acknowledged + // user intent instead of reporting an infrastructure interruption. + if (hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run)) terminalStatus = "cancelled"; + + const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`; + // Match both "running" and "queued". A queued run has released its lease but + // never reached "running", so a running-only update would miss it and leave + // a phantom live run behind. + const write = await setRunStatusFromLive( + run.id, + terminalStatus, + ["running", "queued"], + { + finishedAt: run.finishedAt ?? new Date(), + error: run.error ?? (terminalStatus === "interrupted" ? message : null), + errorCode: + run.errorCode ?? + (terminalStatus === "interrupted" + ? "lease_released_before_terminal" + : null), + }, + ); + if (!write.updated) { + // Another path already finalized the run. Keep that terminal outcome. + return write.run ?? run; + } + + const terminalRun = write.run; + if (terminalRun) { + await appendRunEvent(terminalRun, { + eventType: "lifecycle", + stream: "system", + level: terminalStatus === "interrupted" ? "warn" : "info", + message, + payload: { + previousStatus: run.status, + terminalStatus, + reason: "environment_lease_release", + ...(issueId ? { issueId } : {}), + }, + }).catch((eventErr) => { + logger.warn( + { err: eventErr, runId: run.id }, + "failed to append run event for lease-release terminalization", + ); + }); + } + return terminalRun ?? run; + } + + function publishRunLifecyclePluginEvent( + run: typeof heartbeatRuns.$inferSelect, + ) { + publishRunLifecyclePluginEventData({ + companyId: run.companyId, + runId: run.id, + agentId: run.agentId, + status: run.status, + invocationSource: run.invocationSource, + triggerDetail: run.triggerDetail, + error: run.error, + errorCode: run.errorCode, + issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId), + startedAt: run.startedAt, + finishedAt: run.finishedAt, + }); + } + + function publishRunLifecyclePluginEventData(run: { + companyId: string; + runId: string; + agentId: string; + status: string; + invocationSource: string; + triggerDetail: string | null; + error: string | null; + errorCode: string | null; + issueId: string | null; + startedAt: Date | null; + finishedAt: Date | null; + }) { + const eventType = + run.status === "running" + ? "agent.run.started" + : run.status === "succeeded" + ? "agent.run.finished" + : run.status === "failed" || run.status === "timed_out" + ? "agent.run.failed" + : run.status === "cancelled" + ? "agent.run.cancelled" + : null; + if (!eventType) return; + publishPluginDomainEvent({ + eventId: randomUUID(), + eventType, + occurredAt: new Date().toISOString(), + actorId: run.agentId, + actorType: "agent", + entityId: run.runId, + entityType: "heartbeat_run", + companyId: run.companyId, + payload: { + runId: run.runId, + agentId: run.agentId, + status: run.status, + invocationSource: run.invocationSource, + triggerDetail: run.triggerDetail, + error: run.error ?? null, + errorCode: run.errorCode ?? null, + issueId: run.issueId, + startedAt: run.startedAt ? new Date(run.startedAt).toISOString() : null, + finishedAt: run.finishedAt + ? new Date(run.finishedAt).toISOString() + : null, + }, + }); + } + + async function setWakeupStatus( + wakeupRequestId: string | null | undefined, + status: string, + patch?: Partial, + ) { + if (!wakeupRequestId) return; + await db + .update(agentWakeupRequests) + .set({ status, ...patch, updatedAt: new Date() }) + // Reassignment revokes requests transactionally. Late execution settlement + // must not turn a revoked request back into authority for a later retry. + .where(and(eq(agentWakeupRequests.id, wakeupRequestId), ne(agentWakeupRequests.status, "cancelled"))); + } + + async function addContinuationExhaustedCommentOnce(input: { + run: typeof heartbeatRuns.$inferSelect; + issueId: string; + comment: string; + }) { + const existing = await db + .select({ id: issueComments.id }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, input.run.companyId), + eq(issueComments.issueId, input.issueId), + eq(issueComments.createdByRunId, input.run.id), + sql`${issueComments.body} like 'Bounded liveness continuation exhausted%'`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (existing) return; + await issuesSvc.addComment(input.issueId, input.comment, { + agentId: input.run.agentId, + runId: input.run.id, + }); + } + + async function handleRunLivenessContinuation( + run: typeof heartbeatRuns.$inferSelect, + ) { + const context = parseObject(run.contextSnapshot); + if ( + readNonEmptyString(context.goalControlRequestId) || + context.resumeSessionGoalHeartbeat === true + ) + return; + const livenessState = run.livenessState as RunLivenessState | null; + if (livenessState !== "plan_only" && livenessState !== "empty_response") + return; + + const issueId = readNonEmptyString(context.issueId); + if (!issueId) return; + const waitingContext = await getIssueExecutionContext(run.companyId, issueId); + if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return; + + const [issue, agent] = await Promise.all([ + db + .select({ + id: issues.id, + companyId: issues.companyId, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + executionState: issues.executionState, + projectId: issues.projectId, + }) + .from(issues) + .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) + .then((rows) => rows[0] ?? null), + db + .select({ + id: agents.id, + companyId: agents.companyId, + status: agents.status, + }) + .from(agents) + .where(eq(agents.id, run.agentId)) + .then((rows) => rows[0] ?? null), + ]); + + const budgetBlock = + issue && agent + ? await budgets.getInvocationBlock(issue.companyId, agent.id, { + issueId: issue.id, + projectId: issue.projectId, + }) + : null; + + const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1; + const idempotencyKey = issue + ? buildRunLivenessContinuationIdempotencyKey({ + issueId: issue.id, + sourceRunId: run.id, + livenessState, + nextAttempt, + }) + : null; + const existingWake = idempotencyKey + ? await findExistingRunLivenessContinuationWake(db, { + companyId: run.companyId, + idempotencyKey, + }) + : null; + + const decision = decideRunLivenessContinuation({ + run, + issue, + agent, + livenessState, + livenessReason: run.livenessReason, + nextAction: run.nextAction, + budgetBlocked: Boolean(budgetBlock), + idempotentWakeExists: Boolean(existingWake), + }); + + if (decision.kind === "exhausted") { + await setRunStatus(run.id, run.status, { + livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation attempts exhausted`, + }); + await addContinuationExhaustedCommentOnce({ + run, + issueId, + comment: decision.comment, + }); + return; + } + + if (decision.kind !== "enqueue") return; + + const continuationRun = await enqueueWakeup(run.agentId, { + source: "automation", + triggerDetail: "system", + reason: RUN_LIVENESS_CONTINUATION_REASON, + payload: decision.payload, + contextSnapshot: { + ...decision.contextSnapshot, + originIdentityContextId: null, + parentRunId: run.id, + }, + idempotencyKey: decision.idempotencyKey, + requestedByActorType: "system", + requestedByActorId: "heartbeat", + }); + + if (continuationRun) { + await db + .update(heartbeatRuns) + .set({ + continuationAttempt: decision.nextAttempt, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + } + } + + function issueUiLink( + issue: Pick, + ) { + const label = issue.identifier ?? issue.id; + const prefix = issue.identifier?.split("-")[0] || "PAP"; + return `[${label}](/${prefix}/issues/${label})`; + } + + function hasUnmanagedBackgroundTaskEvidence( + resultJson: Record | null | undefined, + ) { + const evidence = parseObject(resultJson?.unmanagedBackgroundTask); + return ( + evidence.stopped === true && + (evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON || + evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON) + ); + } + + function withUnmanagedBackgroundTaskStopReason( + resultJson: Record | null | undefined, + ) { + return { + ...(resultJson ?? {}), + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + }; + } + + function buildDetectedSuccessfulRunProgressSummary( + run: typeof heartbeatRuns.$inferSelect, + currentUserRedactionOptions: CurrentUserRedactionOptions, + ) { + const resultJson = parseObject(run.resultJson); + const candidates = [ + hasUnmanagedBackgroundTaskEvidence(resultJson) + ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON + : null, + readNonEmptyString(run.nextAction) + ? `Next action noted: ${readNonEmptyString(run.nextAction)}` + : null, + readNonEmptyString(run.livenessReason), + readNonEmptyString(resultJson.summary), + readNonEmptyString(resultJson.result), + readNonEmptyString(resultJson.message), + ].filter((value): value is string => Boolean(value)); + const summary = candidates[0]; + if (!summary) return null; + return redactDetectedSuccessfulRunProgressSummaryForBoard( + summary, + currentUserRedactionOptions, + ); + } + + async function addSuccessfulRunHandoffCommentOnce(input: { + issue: Pick< + typeof issues.$inferSelect, + "id" | "identifier" | "title" | "status" + >; + run: typeof heartbeatRuns.$inferSelect; + agent: Pick; + detectedProgressSummary: string; + }) { + const existing = await db + .select({ id: issueComments.id }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, input.run.companyId), + eq(issueComments.issueId, input.issue.id), + eq(issueComments.createdByRunId, input.run.id), + sql`(${issueComments.body} = ${SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY} or ${issueComments.body} like '## This issue still needs a next step%' or ${issueComments.body} like '## Successful run missing issue disposition%')`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (existing) return null; + const notice = buildSuccessfulRunHandoffRequiredNotice(input); + return issuesSvc.addComment( + input.issue.id, + notice.body, + { runId: input.run.id }, + { + authorType: "system", + presentation: notice.presentation, + metadata: notice.metadata, + }, + ); + } + + async function handleSuccessfulRunHandoff( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + ) { + if (run.status !== "succeeded") return; + const context = parseObject(run.contextSnapshot); + const issueId = + readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); + if (!issueId) return; + const waitingContext = await getIssueExecutionContext(run.companyId, issueId); + if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return; + if ( + readNonEmptyString(context.goalControlRequestId) || + context.resumeSessionGoalHeartbeat === true + ) { + const goalProjection = await runnerGoalService(db).projection( + run.companyId, + issueId, + run.agentId, + ); + if (goalProjection?.goal?.status !== "complete") return; + } + + const issue = await db + .select({ + id: issues.id, + companyId: issues.companyId, + identifier: issues.identifier, + title: issues.title, + description: issues.description, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + executionState: issues.executionState, + monitorNextCheckAt: issues.monitorNextCheckAt, + projectId: issues.projectId, + originKind: issues.originKind, + }) + .from(issues) + .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) + .then((rows) => rows[0] ?? null); + const idempotencyKey = issue + ? buildFinishSuccessfulRunHandoffIdempotencyKey({ + issueId: issue.id, + sourceRunId: run.id, + }) + : null; + const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null); + const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); + const detectedProgressSummary = buildDetectedSuccessfulRunProgressSummary( + run, + currentUserRedactionOptions, + ); + const resultJson = parseObject(run.resultJson); + const finalReport = redactSuccessfulRunHandoffEvidence( + [ + readNonEmptyString(resultJson.summary), + readNonEmptyString(resultJson.result), + readNonEmptyString(resultJson.message), + ].find((value): value is string => Boolean(value)) ?? null, + currentUserRedactionOptions, + ); + const nextAction = redactSuccessfulRunHandoffEvidence( + readNonEmptyString(run.nextAction), + currentUserRedactionOptions, + ); + + const [ + activeExecutionPath, + queuedWake, + pendingInteraction, + pendingApproval, + explicitBlocker, + openRecoveryIssue, + existingWake, + budgetBlock, + pauseHold, + activeRoutineContinuation, + ] = await Promise.all([ + issue + ? db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, issue.companyId), + eq(heartbeatRuns.agentId, run.agentId), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), + sql`( + ${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id} + or ${heartbeatRuns.contextSnapshot} ->> 'taskId' = ${issue.id} + )`, + sql`${heartbeatRuns.id} <> ${run.id}`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : Promise.resolve(null), + issue + ? db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + eq(agentWakeupRequests.agentId, run.agentId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + "claimed", + ]), + sql`( + ${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id} + or ${agentWakeupRequests.payload} ->> 'taskId' = ${issue.id} + or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issue.id} + or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issue.id} + )`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : Promise.resolve(null), + issue + ? db + .select({ id: issueThreadInteractions.id }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, issue.companyId), + eq(issueThreadInteractions.issueId, issue.id), + eq(issueThreadInteractions.status, "pending"), + activeIssueInteractionCondition(), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : Promise.resolve(null), + issue + ? db + .select({ id: issueApprovals.approvalId }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where( + and( + eq(issueApprovals.companyId, issue.companyId), + eq(issueApprovals.issueId, issue.id), + inArray(approvals.status, ["pending", "revision_requested"]), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : Promise.resolve(null), + issue + ? db + .select({ id: issueRelations.issueId }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, issue.companyId), + eq(issueRelations.relatedIssueId, issue.id), + eq(issueRelations.type, "blocks"), + sql`exists ( + select 1 + from issues blocker + where blocker.id = ${issueRelations.issueId} + and blocker.company_id = ${issue.companyId} + and blocker.status not in ('done', 'cancelled') + and blocker.hidden_at is null + )`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : Promise.resolve(null), + issue + ? db + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.companyId, issue.companyId), + inArray(issues.originKind, [ + RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, + RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + ]), + eq(issues.originId, issue.id), + visibleIssueCondition(), + notInArray(issues.status, ["done", "cancelled"]), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : Promise.resolve(null), + idempotencyKey + ? findExistingFinishSuccessfulRunHandoffWake(db, { + companyId: run.companyId, + idempotencyKey, + }) + : Promise.resolve(null), + issue + ? budgets.getInvocationBlock(issue.companyId, run.agentId, { + issueId: issue.id, + projectId: issue.projectId, + }) + : Promise.resolve(null), + issue + ? treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id) + : Promise.resolve(null), + issue + ? db + .select({ id: routines.id }) + .from(routines) + .where( + and( + eq(routines.companyId, issue.companyId), + eq(routines.parentIssueId, issue.id), + eq(routines.status, "active"), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : Promise.resolve(null), + ]); + + const decision = decideSuccessfulRunHandoff({ + run, + issue, + agent, + livenessState: run.livenessState as RunLivenessState | null, + detectedProgressSummary, + finalReport, + nextAction, + taskKey, + hasActiveExecutionPath: Boolean(activeExecutionPath), + hasQueuedWake: Boolean(queuedWake), + hasPendingInteractionOrApproval: Boolean( + pendingInteraction || pendingApproval, + ), + hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt), + hasExplicitBlockerPath: Boolean(explicitBlocker), + hasOpenRecoveryIssue: Boolean(openRecoveryIssue), + hasPauseHold: Boolean(pauseHold), + hasActiveRoutineContinuation: Boolean(activeRoutineContinuation), + budgetBlocked: Boolean(budgetBlock), + idempotentWakeExists: Boolean(existingWake), + }); + + if (isSuccessfulRunHandoffValidPathSkip(decision) && issue) { + await resolveRequiredSuccessfulRunHandoffOnValidPath(db, { + companyId: issue.companyId, + issueId: issue.id, + issueIdentifier: issue.identifier, + agentId: run.agentId, + runId: run.id, + skipReason: decision.reason, + }); + } + + if (decision.kind !== "enqueue" || !issue) return; + + if (hasUnmanagedBackgroundTaskEvidence(parseObject(run.resultJson))) { + await db + .update(heartbeatRuns) + .set({ + livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || + ${JSON.stringify({ stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON })}::jsonb`, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + } + + const handoffRun = await enqueueWakeup(decision.targetAgentId, { + source: "automation", + triggerDetail: "system", + reason: FINISH_SUCCESSFUL_RUN_HANDOFF_REASON, + payload: decision.payload, + contextSnapshot: { + ...decision.contextSnapshot, + originIdentityContextId: null, + parentRunId: run.id, + }, + idempotencyKey: decision.idempotencyKey, + requestedByActorType: "system", + requestedByActorId: "heartbeat", + }); + if (!handoffRun) return; + + await addSuccessfulRunHandoffCommentOnce({ + issue, + run, + agent, + detectedProgressSummary: + detectedProgressSummary ?? + "The run reported progress, but did not choose a next step.", + }); + await logActivity(db, { + companyId: issue.companyId, + actorType: "system", + actorId: "heartbeat", + agentId: run.agentId, + runId: run.id, + action: "issue.successful_run_handoff_required", + entityType: "issue", + entityId: issue.id, + details: { + label: "Successful run missing issue disposition", + sourceRunId: run.id, + correctiveRunId: handoffRun.id, + handoffReason: SUCCESSFUL_RUN_MISSING_STATE_REASON, + missingDisposition: "clear_next_step", + detectedProgressSummary, + issue: issueUiLink(issue), + }, + }); + } + + async function handleIssueReviewPathDisposition( + run: typeof heartbeatRuns.$inferSelect, + ) { + const contextSnapshot = parseObject(run.contextSnapshot); + if (readNonEmptyString(contextSnapshot.goalControlRequestId)) return; + const issueId = + readNonEmptyString(contextSnapshot.issueId) ?? + readNonEmptyString(contextSnapshot.taskId); + if (!issueId) return; + const waitingContext = await getIssueExecutionContext(run.companyId, issueId); + if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return; + + const issue = await db + .select({ + id: issues.id, + companyId: issues.companyId, + identifier: issues.identifier, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + }) + .from(issues) + .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) + .then((rows) => rows[0] ?? null); + if (!issue || issue.status !== "in_review" || !issue.assigneeAgentId) + return; + + const reviewAttention = await issuesSvc + .listReviewAttention(issue.companyId, [issue]) + .then( + (map) => + map.get(issue.id) ?? { + state: "none" as const, + paths: [], + reason: null, + }, + ); + if (reviewAttention.state !== "stalled") return; + + const consumedPathRef = reviewPathConsumedRefFromRun({ + runId: run.id, + issueId: issue.id, + contextSnapshot, + }); + const idempotencyKey = buildIssueReviewPathLostIdempotencyKey({ + issueId: issue.id, + consumedPathRef, + }); + const existingWake = await db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + eq(agentWakeupRequests.idempotencyKey, idempotencyKey), + notInArray(agentWakeupRequests.status, ["skipped"]), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + + const decision = decideIssueReviewPathRecovery({ + issueId: issue.id, + sourceRunId: run.id, + assigneeAgentId: issue.assigneeAgentId, + contextSnapshot, + reviewAttention, + existingWake: Boolean(existingWake), + }); + if (decision.kind !== "enqueue") return; + + const recoveryRun = await enqueueWakeup(issue.assigneeAgentId, { + source: "automation", + triggerDetail: "system", + reason: ISSUE_REVIEW_PATH_LOST_WAKE_REASON, + idempotencyKey: decision.idempotencyKey, + payload: decision.payload, + contextSnapshot: decision.contextSnapshot, + requestedByActorType: "system", + requestedByActorId: "heartbeat", + }).catch((error: unknown) => { + if (isReviewPathRecoveryIdempotencyConflict(error)) return null; + throw error; + }); + if (!recoveryRun) return; + + await logActivity(db, { + companyId: issue.companyId, + actorType: "system", + actorId: "heartbeat", + agentId: issue.assigneeAgentId, + runId: run.id, + action: "issue.review_path_recovery_queued", + entityType: "issue", + entityId: issue.id, + details: { + sourceRunId: run.id, + recoveryRunId: recoveryRun.id, + consumedPathRef, + recoveryAttempt: 1, + maxRecoveryAttempts: 1, + }, + }); + } + + async function appendRunEvent( + run: typeof heartbeatRuns.$inferSelect, + event: { + eventType: string; + stream?: "system" | "stdout" | "stderr"; + level?: "info" | "warn" | "error"; + color?: string; + message?: string; + payload?: Record; + retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"]; + }, + ) { + const eventAt = new Date(); + const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); + const sanitizedMessage = event.message + ? redactSensitiveText( + redactCurrentUserText(event.message, currentUserRedactionOptions), + ) + : event.message; + const boundedPayload = event.payload + ? boundHeartbeatRunEventPayloadForStorage(event.payload) + : event.payload; + const secretSanitizedPayload = boundedPayload + ? redactEventPayload(boundedPayload) + : boundedPayload; + const sanitizedPayload = secretSanitizedPayload + ? redactCurrentUserValue( + secretSanitizedPayload, + currentUserRedactionOptions, + ) + : secretSanitizedPayload; + const issueId = readRuntimeStatusIssueIdCandidate(run) ?? null; + const progress = buildRunEventRuntimeProgress({ + eventType: event.eventType, + message: sanitizedMessage ?? null, + payload: sanitizedPayload ?? null, + at: eventAt, + }); + const persistedEvent = await appendHeartbeatRunEvent(db, { + companyId: run.companyId, + runId: run.id, + agentId: run.agentId, + eventType: event.eventType, + stream: event.stream, + level: event.level, + color: event.color, + message: sanitizedMessage, + payload: sanitizedPayload, + retryExhaustion: event.retryExhaustion, + }); + if (persistedEvent.disposition === "duplicate") return; + const seq = persistedEvent.row.seq; + + publishLiveEvent({ + companyId: run.companyId, + type: "heartbeat.run.event", + payload: { + runId: run.id, + agentId: run.agentId, + issueId, + seq, + eventType: event.eventType, + stream: event.stream ?? null, + level: event.level ?? null, + color: event.color ?? null, + message: sanitizedMessage ?? null, + currentToolName: progress?.currentToolName ?? null, + lastAssistantSnippet: progress?.lastAssistantSnippet ?? null, + lastEventAt: (progress?.lastEventAt ?? eventAt).toISOString(), + payload: sanitizedPayload ?? null, + }, + }); + if (progress && isHeartbeatRunRuntimeStatusActive(run.status)) { + const status = setHeartbeatRunRuntimeStatus({ + companyId: run.companyId, + issueId, + agentId: run.agentId, + runId: run.id, + phase: progress.phase, + message: progress.message, + updatedAt: eventAt, + currentToolName: progress.currentToolName, + lastAssistantSnippet: progress.lastAssistantSnippet, + lastEventAt: progress.lastEventAt, + }); + if (status) publishHeartbeatRunRuntimeProgress(status); + } + } + + async function persistRunProcessMetadata( + runId: string, + meta: { pid: number; processGroupId: number | null; startedAt: string }, + ) { + return persistHeartbeatRunProcessMetadata(db, runId, meta); + } + + async function clearDetachedRunWarning(runId: string) { + const updated = await db + .update(heartbeatRuns) + .set({ + error: null, + errorCode: null, + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.status, "running"), + eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!updated) return null; + + await appendRunEvent(updated, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: + "Detached child process reported activity; cleared detached warning", + }); + return updated; + } + + async function patchRunIssueCommentStatus( + runId: string, + patch: Partial< + Pick< + typeof heartbeatRuns.$inferInsert, + | "issueCommentStatus" + | "issueCommentSatisfiedByCommentId" + | "issueCommentRetryQueuedAt" + > + >, + ) { + return db + .update(heartbeatRuns) + .set({ ...patch, updatedAt: new Date() }) + .where(eq(heartbeatRuns.id, runId)) + .returning() + .then((rows) => rows[0] ?? null); + } + + async function findRunIssueComment( + runId: string, + companyId: string, + issueId: string, + resultJson?: Record | null, + ) { + const comments = await db + .select({ + id: issueComments.id, + body: issueComments.body, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + eq(issueComments.issueId, issueId), + eq(issueComments.createdByRunId, runId), + ), + ) + .orderBy(desc(issueComments.createdAt), desc(issueComments.id)); + return findHeartbeatRunCompletionComment(comments, resultJson); + } + + async function findLatestCompletedFinalAgentMessage( + runId: string, + companyId: string, + ) { + const rows = await db + .select({ + seq: heartbeatRunEvents.seq, + payload: heartbeatRunEvents.payload, + }) + .from(heartbeatRunEvents) + .where( + and( + eq(heartbeatRunEvents.companyId, companyId), + eq(heartbeatRunEvents.runId, runId), + eq(heartbeatRunEvents.eventType, "item.completed"), + ), + ) + .orderBy(desc(heartbeatRunEvents.seq)) + .limit(200); + const candidates: Array<{ + seq: number; + text: string; + sourceEventId: string | null; + channel: "final" | "unknown"; + }> = []; + for (const row of rows) { + const prpEvent = parseObject(parseObject(row.payload).prpEvent); + const candidate = readCompletedAssistantMessageCandidate({ + seq: row.seq, + prpEvent, + }); + if (candidate) candidates.push(candidate); + } + const recoveryBoundary = await db + .select({ + seq: heartbeatRunEvents.seq, + payload: heartbeatRunEvents.payload, + }) + .from(heartbeatRunEvents) + .where( + and( + eq(heartbeatRunEvents.companyId, companyId), + eq(heartbeatRunEvents.runId, runId), + eq(heartbeatRunEvents.eventType, "lifecycle"), + ), + ) + .orderBy(heartbeatRunEvents.seq) + .limit(200) + .then( + (lifecycleRows) => + lifecycleRows.find( + (row) => + parseObject(row.payload).retryReasonCode === + "semantic_result_missing", + )?.seq ?? null, + ); + return selectHeartbeatRunFinalAgentMessage({ + candidates, + semanticResultRecoveryAfterSeq: recoveryBoundary, + }); + } + + async function refreshContinuationSummaryForRun( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + ) { + const contextSnapshot = parseObject(run.contextSnapshot); + const issueId = readNonEmptyString(contextSnapshot.issueId); + if (!issueId) return null; + try { + return await refreshIssueContinuationSummary({ + db, + issueId, + run: { + id: run.id, + status: run.status, + error: run.error, + errorCode: run.errorCode, + resultJson: run.resultJson as Record | null, + stdoutExcerpt: run.stdoutExcerpt, + stderrExcerpt: run.stderrExcerpt, + finishedAt: run.finishedAt, + }, + agent: { + id: agent.id, + name: agent.name, + adapterType: agent.adapterType, + }, + }); + } catch (err) { + logger.warn( + { + err, + runId: run.id, + issueId, + agentId: agent.id, + }, + "failed to refresh issue continuation summary", + ); + return null; + } + } + + async function enqueueMissingIssueCommentRetry( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + issueId: string, + ) { + const invokability = await getAgentInvokability(agent); + if (!invokability.invokable) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + "Missing-comment retry suppressed because the agent is not invokable", + payload: { + reason: invokability.reason, + invalidOrgChain: invokability.invalidOrgChain, + ...invokability.details, + }, + }); + return null; + } + + const contextSnapshot = parseObject(run.contextSnapshot); + const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); + const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); + const retryContextSnapshot = withRecoveryContext( + { + ...contextSnapshot, + retryOfRunId: run.id, + wakeReason: "missing_issue_comment", + retryReason: "missing_issue_comment", + missingIssueCommentForRunId: run.id, + }, + "status_only", + ); + const responsibleUserId = await resolveResponsibleUserIdForRunContext( + run, + retryContextSnapshot, + ); + const now = new Date(); + + const retryRun = await db.transaction(async (tx) => { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`, + ); + + const issue = await tx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ) + .then((rows) => rows[0] ?? null); + if (!issue) return null; + + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId: run.companyId, + agentId: run.agentId, + source: "automation", + triggerDetail: "system", + reason: "missing_issue_comment", + payload: withRecoveryContext( + { + issueId, + retryOfRunId: run.id, + retryReason: "missing_issue_comment", + }, + "status_only", + ), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + const queuedRun = await tx + .insert(heartbeatRuns) + .values({ + companyId: run.companyId, + agentId: run.agentId, + scopeKind: "issue", + issueId, + invocationSource: "automation", + triggerDetail: "system", + status: "queued", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: retryContextSnapshot, + responsibleUserId, + sessionIdBefore: sessionBefore, + retryOfRunId: run.id, + issueCommentStatus: "not_applicable", + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ + runId: queuedRun.id, + updatedAt: now, + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + await tx + .update(issues) + .set({ + executionRunId: queuedRun.id, + executionAgentNameKey: normalizeAgentNameKey(agent.name), + executionLockedAt: now, + updatedAt: now, + }) + .where(eq(issues.id, issue.id)); + + await tx + .update(heartbeatRuns) + .set({ + issueCommentStatus: "retry_queued", + issueCommentRetryQueuedAt: now, + updatedAt: now, + }) + .where(eq(heartbeatRuns.id, run.id)); + + return queuedRun; + }); + + if (!retryRun) return null; + + publishLiveEvent({ + companyId: retryRun.companyId, + type: "heartbeat.run.queued", + payload: { + runId: retryRun.id, + agentId: retryRun.agentId, + invocationSource: retryRun.invocationSource, + triggerDetail: retryRun.triggerDetail, + wakeupRequestId: retryRun.wakeupRequestId, + }, + }); + + return retryRun; + } + + async function hasDeferredIssueCommentWake( + companyId: string, + issueId: string, + agentId: string, + ) { + const deferredPayloads = await db + .select({ payload: agentWakeupRequests.payload }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, + ), + ); + + return deferredPayloads.some(({ payload }) => { + const parsedPayload = parseObject(payload); + const deferredContext = parseObject( + parsedPayload[DEFERRED_WAKE_CONTEXT_KEY], + ); + return Boolean(deriveCommentId(deferredContext, parsedPayload)); + }); + } + + async function finalizeIssueCommentPolicy( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + presentationDecision?: RunPresentationDecision | null, + ) { + const contextSnapshot = parseObject(run.contextSnapshot); + // The explicit receipt admitted one turn. A prose-only follow-up cannot + // reuse it or renew it under the separate transient-failure retry policy. + if (readNonEmptyString(contextSnapshot.goalControlRequestId) || contextSnapshot.explicitUserContinuation) { + if (run.issueCommentStatus !== "not_applicable") { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + } + return { outcome: "not_applicable" as const, queuedRun: null }; + } + const issueId = readNonEmptyString(contextSnapshot.issueId); + if (!issueId) { + if (run.issueCommentStatus !== "not_applicable") { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + } + return { outcome: "not_applicable" as const, queuedRun: null }; + } + + // A failed/timed-out/cancelled run may immediately enqueue the normal + // assignment or continuation recovery below. Do not let the lower-priority + // status-only missing-comment wake claim the issue first. The dedicated + // missing-comment retry still records exhaustion if that retry itself + // fails, but an original failed execution remains on the established + // direct-adapter recovery path. + if ( + run.status !== "succeeded" && + readNonEmptyString(contextSnapshot.retryReason) !== + "missing_issue_comment" + ) { + if (run.issueCommentStatus !== "not_applicable") { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + } + return { outcome: "not_applicable" as const, queuedRun: null }; + } + + // A settled run may legitimately have no user-facing prose. The response + // resolver owns that decision; do not wake the agent again merely to force + // an artificial comment into the issue thread. + if ( + presentationDecision?.chosenSource === "none" && + (hasAcceptedSemanticResult(parseObject(run.resultJson)) || + presentationDecision.reasonCodes.includes( + "legacy_adapter_summary_ambiguous", + )) + ) { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + return { outcome: "not_applicable" as const, queuedRun: null }; + } + + // A pre-dispatch setup failure means the adapter process never started (for + // example an unresolved workspace base ref). No agent could run, so no agent + // could post an issue comment. A missing-comment retry cannot help and would + // loop the identical pre-adapter failure, so mark the policy not_applicable + // and queue nothing. + if ( + run.errorCode != null && + PRE_ADAPTER_SETUP_FAILURE_CODES.has(run.errorCode) + ) { + if (run.issueCommentStatus !== "not_applicable") { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + } + return { outcome: "not_applicable" as const, queuedRun: null }; + } + + const postedComment = await findRunIssueComment( + run.id, + run.companyId, + issueId, + parseObject(run.resultJson), + ); + if (postedComment) { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "satisfied", + issueCommentSatisfiedByCommentId: postedComment.id, + issueCommentRetryQueuedAt: null, + }); + return { outcome: "satisfied" as const, queuedRun: null }; + } + + // Missing-comment recovery is a legacy compatibility path for otherwise + // successful runs. A failed, timed-out, or cancelled run is already owned + // by lifecycle recovery and its terminal system presentation. Queuing a + // prose-only retry here can seize the issue execution lock before the + // authoritative continuation is materialized, replacing real recovery + // with a cheap status-only turn. + if (run.status !== "succeeded") { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + return { outcome: "not_applicable" as const, queuedRun: null }; + } + + if ( + readNonEmptyString(contextSnapshot.retryReason) === + "missing_issue_comment" + ) { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "retry_exhausted", + issueCommentSatisfiedByCommentId: null, + }); + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + "Run ended without an issue comment after one retry; no further comment wake will be queued", + }); + return { outcome: "retry_exhausted" as const, queuedRun: null }; + } + + if (!shouldRequireIssueCommentForWake(contextSnapshot)) { + if (run.issueCommentStatus !== "not_applicable") { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + } + return { outcome: "not_applicable" as const, queuedRun: null }; + } + + if ( + await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId) + ) { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: + "Run ended without an issue comment; a deferred comment wake already exists for this issue", + }); + return { outcome: "not_applicable" as const, queuedRun: null }; + } + + const queuedRun = await enqueueMissingIssueCommentRetry( + run, + agent, + issueId, + ); + if (queuedRun) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + "Run ended without an issue comment; queued one follow-up wake to require a comment", + }); + return { outcome: "retry_queued" as const, queuedRun }; + } + + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "retry_exhausted", + issueCommentSatisfiedByCommentId: null, + }); + return { outcome: "retry_exhausted" as const, queuedRun: null }; + } + + function parseMaxTurnContinuationPolicy( + agent: typeof agents.$inferSelect, + ): MaxTurnContinuationPolicy { + const runtimeConfig = parseObject(agent.runtimeConfig); + const heartbeat = parseObject(runtimeConfig.heartbeat); + const configured = parseObject(heartbeat.maxTurnContinuation); + const rawMaxAttempts = Math.floor( + asNumber( + configured.maxAttempts, + MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS, + ), + ); + const rawDelayMs = Math.floor( + asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS), + ); + + return { + enabled: asBoolean(configured.enabled, true), + maxAttempts: Math.max( + 0, + Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts), + ), + delayMs: Math.max( + 0, + Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs), + ), + }; + } + + function mergeRunStopMetadataForAgent( + agent: Pick, + outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out", + options?: { + resultJson?: Record | null; + conversationContinuationEligible?: boolean; + errorCode?: string | null; + errorMessage?: string | null; + }, + ) { + const stopMetadata = buildHeartbeatRunStopMetadata({ + adapterType: agent.adapterType, + adapterConfig: parseObject(agent.adapterConfig), + outcome, + errorCode: options?.errorCode ?? null, + errorMessage: options?.errorMessage ?? null, + }); + const result = mergeHeartbeatRunStopMetadata( + options?.resultJson ?? null, + stopMetadata, + ); + const cancellationAcknowledged = + parseObject(result?.executionCancellation).state === "acknowledged"; + return options?.conversationContinuationEligible !== false && outcome !== "succeeded" && + (outcome !== "cancelled" || cancellationAcknowledged) && + isConversationAdapter(agent.adapterType) + ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY } + : result; + } + + function countValue(value: unknown) { + const parsed = Number(value ?? 0); + return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0; + } + + function dateValue(value: unknown) { + if (value instanceof Date) + return Number.isNaN(value.getTime()) ? null : value; + if (typeof value === "string" || typeof value === "number") { + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; + } + return null; + } + + function latestDate(...values: unknown[]) { + let latest: Date | null = null; + for (const value of values) { + const parsed = dateValue(value); + if (!parsed) continue; + if (!latest || parsed.getTime() > latest.getTime()) latest = parsed; + } + return latest; + } + + async function buildRunLivenessInput( + run: typeof heartbeatRuns.$inferSelect, + resultJson: Record | null | undefined, + ): Promise { + const context = parseObject(run.contextSnapshot); + const contextIssueId = readNonEmptyString(context.issueId); + const continuationAttempt = asNumber( + context.continuationAttempt, + run.continuationAttempt ?? 0, + ); + + const issue = contextIssueId + ? await db + .select({ + status: issues.status, + title: issues.title, + description: issues.description, + workMode: issues.workMode, + }) + .from(issues) + .where( + and( + eq(issues.companyId, run.companyId), + eq(issues.id, contextIssueId), + ), + ) + .then((rows) => rows[0] ?? null) + : null; + + const [commentStats] = contextIssueId + ? await db + .select({ + count: sql`count(*)::int`, + latestAt: sql`max(${issueComments.createdAt})`, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, run.companyId), + eq(issueComments.issueId, contextIssueId), + eq(issueComments.createdByRunId, run.id), + isNull(issueComments.deletedAt), + ), + ) + : [{ count: 0, latestAt: null }]; + + const issueCommentBodies = contextIssueId + ? await db + .select({ body: issueComments.body }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, run.companyId), + eq(issueComments.issueId, contextIssueId), + eq(issueComments.createdByRunId, run.id), + ), + ) + .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) + .limit(5) + .then((rows) => rows.reverse().map((row) => row.body)) + : []; + + const continuationSummary = contextIssueId + ? await getIssueContinuationSummaryDocument(db, contextIssueId) + : null; + + const [documentStats] = contextIssueId + ? await db + .select({ + count: sql`count(*)::int`, + planCount: sql`count(*) filter (where ${issueDocuments.key} = 'plan')::int`, + latestAt: sql`max(${documentRevisions.createdAt})`, + }) + .from(documentRevisions) + .innerJoin( + issueDocuments, + eq(documentRevisions.documentId, issueDocuments.documentId), + ) + .where( + and( + eq(documentRevisions.companyId, run.companyId), + eq(documentRevisions.createdByRunId, run.id), + eq(issueDocuments.companyId, run.companyId), + eq(issueDocuments.issueId, contextIssueId), + sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`, + ), + ) + : [{ count: 0, planCount: 0, latestAt: null }]; + + const [workProductStats] = contextIssueId + ? await db + .select({ + count: sql`count(*)::int`, + latestAt: sql`max(${issueWorkProducts.createdAt})`, + }) + .from(issueWorkProducts) + .where( + and( + eq(issueWorkProducts.companyId, run.companyId), + eq(issueWorkProducts.issueId, contextIssueId), + eq(issueWorkProducts.createdByRunId, run.id), + ), + ) + : [{ count: 0, latestAt: null }]; + + const [workspaceOperationStats] = await db + .select({ + count: sql`count(*)::int`, + latestAt: sql`max(${workspaceOperations.startedAt})`, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, run.companyId), + eq(workspaceOperations.heartbeatRunId, run.id), + ), + ); + + const [activityStats] = await db + .select({ + count: sql`count(*)::int`, + latestAt: sql`max(${activityLog.createdAt})`, + }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, run.companyId), + eq(activityLog.runId, run.id), + notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS), + ), + ); + + const [eventStats] = await db + .select({ + count: sql`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`, + latestAt: sql`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`, + }) + .from(heartbeatRunEvents) + .where( + and( + eq(heartbeatRunEvents.companyId, run.companyId), + eq(heartbeatRunEvents.runId, run.id), + ), + ); + + return { + runStatus: run.status, + issue, + resultJson: resultJson ?? run.resultJson ?? null, + issueCommentBodies, + continuationSummaryBody: continuationSummary?.body ?? null, + stdoutExcerpt: run.stdoutExcerpt ?? null, + stderrExcerpt: run.stderrExcerpt ?? null, + error: run.error ?? null, + errorCode: run.errorCode ?? null, + continuationAttempt, + evidence: { + issueCommentsCreated: countValue(commentStats?.count), + documentRevisionsCreated: countValue(documentStats?.count), + planDocumentRevisionsCreated: countValue(documentStats?.planCount), + workProductsCreated: countValue(workProductStats?.count), + workspaceOperationsCreated: countValue(workspaceOperationStats?.count), + activityEventsCreated: countValue(activityStats?.count), + toolOrActionEventsCreated: countValue(eventStats?.count), + latestEvidenceAt: latestDate( + commentStats?.latestAt, + documentStats?.latestAt, + workProductStats?.latestAt, + workspaceOperationStats?.latestAt, + activityStats?.latestAt, + eventStats?.latestAt, + ), + }, + }; + } + + async function classifyAndPersistRunLiveness( + run: typeof heartbeatRuns.$inferSelect, + resultJson?: Record | null, + ) { + const authRepair = run.status === "failed" + ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => { + logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available"); + return null; + }) + : null; + const classification = classifyRunLiveness({ + ...await buildRunLivenessInput(run, resultJson), + authenticationRepairRequested: Boolean(authRepair?.interactionId), + }); + return db + .update(heartbeatRuns) + .set({ + livenessState: classification.livenessState, + livenessReason: classification.livenessReason, + continuationAttempt: classification.continuationAttempt, + lastUsefulActionAt: classification.lastUsefulActionAt, + nextAction: classification.nextAction, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)) + .returning() + .then((rows) => rows[0] ?? null); + } + + async function updateRuntimeState( + agent: typeof agents.$inferSelect, + run: typeof heartbeatRuns.$inferSelect, + result: AdapterExecutionResult, + session: { legacySessionId: string | null }, + normalizedUsage?: UsageTotals | null, + ) { + await ensureRuntimeState(agent); + await accountRunCost(db, run.id, budgetHooks); + await db + .update(agentRuntimeState) + .set({ + adapterType: agent.adapterType, + sessionId: session.legacySessionId, + lastRunId: run.id, + lastRunStatus: run.status, + lastError: run.error ?? null, + updatedAt: new Date(), + }) + .where(eq(agentRuntimeState.agentId, agent.id)); + + } + + return { + appendRunEvent, + escalatePlanApprovalResumeFailureNeedsAttention, + recordPlanApprovalResumeFailureRetry, + setRunStatusIfRunning, + setWakeupStatus, + mergeRunStopMetadataForAgent, + classifyAndPersistRunLiveness, + setRunStatusFromLive, + setRunStatus, + publishRunLifecyclePluginEventData, + publishRunLifecyclePluginEvent, + persistRunProcessMetadata, + recordCurrentHeartbeatRunRuntimeProgress, + completeSkillTestRunForHeartbeatOutcome, + refreshContinuationSummaryForRun, + findRunIssueComment, + findLatestCompletedFinalAgentMessage, + parseMaxTurnContinuationPolicy, + finalizeIssueCommentPolicy, + handleIssueReviewPathDisposition, + handleRunLivenessContinuation, + handleSuccessfulRunHandoff, + updateRuntimeState, + terminalizeRunOnLeaseRelease, + clearDetachedRunWarning, + }; +} diff --git a/server/src/services/invite-grants.ts b/server/src/services/invite-grants.ts index b2141e0930..2f41ea9003 100644 --- a/server/src/services/invite-grants.ts +++ b/server/src/services/invite-grants.ts @@ -1,6 +1,6 @@ import { PERMISSION_KEYS } from "@paperclipai/shared"; import type { HumanCompanyMembershipRole } from "@paperclipai/shared"; -import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope } from "./agent-permissions.js"; +import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope } from "../lib/agent-permissions.js"; import { grantsForHumanRole } from "./company-member-roles.js"; export function grantsFromDefaults( diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index d28d6a4ec0..58555ccb48 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,4 +1,5 @@ import { normalizeEscapedLineBreaks } from "@paperclipai/shared/validators/text"; +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { activeIssueInteractionCondition, historicalQuestionCondition } from "./issue-question-context.js"; import { currentContinuationOrigins, @@ -2829,6 +2830,7 @@ export function issueThreadInteractionService( .returning(); if (!row) throw interactionAlreadyResolvedError(); if (status === "accepted" || status === "rejected") { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); await tx.insert(connectionIntentDeliveries).values({ interactionId, companyId: issue.companyId }).onConflictDoNothing(); } return row; @@ -5026,6 +5028,7 @@ export function issueThreadInteractionService( // This answer updates conversation history only. It must not resume // the completed source run or enqueue new work for the closed task. if (!historicalAnswer) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.question); await tx .insert(issueQuestionResponseDeliveries) .values(questionResponseDeliveryValues(answered)); diff --git a/server/src/services/onboarding-seed.ts b/server/src/services/onboarding-seed.ts index 92399bef34..9aff67f47f 100644 --- a/server/src/services/onboarding-seed.ts +++ b/server/src/services/onboarding-seed.ts @@ -5,11 +5,13 @@ import type { ApplyOnboardingSeed } from "@paperclipai/shared"; import { writePaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/server-utils"; import { findActiveServerAdapter } from "../adapters/registry.js"; import { agentService } from "./agents.js"; +import { createAgentLifecycle, scheduleAgentLifecycle } from "./agent-lifecycle.js"; +import { withDedicatedDbConnection } from "@paperclipai/db"; import { PAPERCLIP_CORE_SKILL_KEYS } from "./company-skills.js"; import { goalService } from "./goals.js"; import { projectService } from "./projects.js"; import { issueService } from "./issues.js"; -import { readBuiltInAgentMarker } from "./built-in-agent-metadata.js"; +import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { logActivity, publishActivity, type ActivityPublication } from "./activity-log.js"; /** @@ -179,19 +181,45 @@ export function onboardingSeedService(db: Db) { return created.id; } - /** - * The seed application proper, run inside the per-company transaction the - * public `apply` opens. Every read and write goes through `dbx` — the locked - * transaction — so it is serialized against a concurrent push for the same - * company. Services are reconstructed on `dbx` for the same reason. - */ + async function prepareAgent(dbx: Db, companyId: string, seed: ApplyOnboardingSeed, audit?: OnboardingSeedAuditActor) { + const existing = await readRecord(dbx, companyId); + if (existing?.revision === seed.revision) return existing.agentId; + const agentSvc = agentService(dbx); + const agentName = seed.agent?.name.trim() || null; + const agentRole = seed.agent?.role?.trim() || null; + // 2. Agent → the customer's first hire, the lead the first task is + // assigned to. + let agentId = await resolveTargetAgentId(dbx, companyId, existing?.agentId ?? null); + if (agentName) { + if (agentId) { + await agentSvc.update(agentId, { name: agentName, title: agentRole }); + } else { + const adapterType = seededAgentAdapterType(); + const created = await createAgentLifecycle(dbx).requestHire(companyId, { + name: agentName, + role: SEEDED_AGENT_ROLE, + title: agentRole, + adapterType, + adapterConfig: seededAgentAdapterConfig(adapterType), + runtimeConfig: {}, + permissions: {}, + status: "idle", + spentMonthlyCents: 0, + lastHeartbeatAt: null, + }, { createdByUserId: audit?.actorType === "user" ? audit.actorId : null }); + agentId = created.id; + } + } + + return agentId; + } + async function applyWithin( dbx: Db, companyId: string, seed: ApplyOnboardingSeed, - audit?: OnboardingSeedAuditActor, + preparedAgentId: string | null, ): Promise { - const agentSvc = agentService(dbx); const goalSvc = goalService(dbx); const projectSvc = projectService(dbx); const issueSvc = issueService(dbx); @@ -237,48 +265,9 @@ export function onboardingSeedService(db: Db) { } } - // 2. Agent → the customer's first hire, the lead the first task is - // assigned to. - let agentId = await resolveTargetAgentId(dbx, companyId, existing?.agentId ?? null); - if (agentName) { - if (agentId) { - await agentSvc.update(agentId, { name: agentName, title: agentRole }); - } else { - const adapterType = seededAgentAdapterType(); - const created = await agentSvc.create(companyId, { - name: agentName, - role: SEEDED_AGENT_ROLE, - title: agentRole, - adapterType, - adapterConfig: seededAgentAdapterConfig(adapterType), - runtimeConfig: {}, - permissions: {}, - status: "idle", - spentMonthlyCents: 0, - lastHeartbeatAt: null, - }, { createdByUserId: audit?.actorType === "user" ? audit.actorId : null }); - agentId = created.id; - } - } + const agentId = preparedAgentId; - // 3. First task → an issue in the Onboarding project, assigned to the - // lead so the dashboard opens with work on it. - // - // No-first-task contract (PAP-67 r17.4): on the Cloud walk this branch - // never runs. The seed Cloud sends is mission-only — `agent` and - // `firstTask` are unpopulated by the signup wizard and a paperclip-cloud - // `node:test` in `src/onboarding/` pins that — so `firstTaskTitle` is - // null here and the first task stays owned by the tenant's own - // server-owned onboarding path (`POST /issues` with - // `onboardingFirstTask: true`). That path is the only one that stamps - // `ONBOARDING_FIRST_TASK_ORIGIN_KIND` and races safely on the partial - // unique index `issues_onboarding_first_task_uq`. If this receiver ever - // created the first task on the cloud walk it would produce a *second*, - // unstamped one: no agent-authored greeting, the brief rendered as a - // right-aligned user bubble, and two onboarding tasks — silently, - // because the uq index only guards origin-stamped rows. The branch is - // retained for the endpoint's documented body contract, but the - // mission-only seed is what keeps it inert on the cloud path. + // A mission-only seed leaves the first task to the onboarding route. let issueId = existing?.issueId ?? null; if (firstTaskTitle) { if (await issueStillExists(dbx, companyId, issueId)) { @@ -357,38 +346,8 @@ export function onboardingSeedService(db: Db) { return { revision: seed.revision, changed: true, goalId, agentId, issueId }; } - /** - * Apply an onboarding seed to a company. - * - * Idempotent per `revision`: a replay of the revision already stored is a - * no-op that still reports success, because Cloud reads any 2xx as "the - * tenant holds this content" and retries otherwise. A *different* revision - * (the customer edited their answers in Cloud) updates the goal, agent and - * task this seed previously created rather than creating a second set. - * - * Every write happens before the caller responds — Cloud records the applied - * revision only on a 2xx, and the redirect into the tenant dashboard is - * gated on it, so a partially-applied seed must surface as a failure rather - * than as an acknowledged one. - * - * Concurrency: Cloud's reconcile runs off portfolio fetches, which can - * overlap, so two pushes for the same company can arrive at once. Both would - * otherwise pass the revision check before either wrote the seed record and - * each create a company goal, a lead agent and an Onboarding project. A - * per-company advisory lock held for the transaction serializes them — the - * same idiom `folders` and `decision-queues` use — so the second push sees - * the first push's writes (the record, the reused goal/agent/project) and - * updates in place instead of duplicating. - * - * Auditing: when `audit` is supplied and the push changed anything, the - * `company.onboarding_seed_applied` entry is written *inside* this same - * transaction. That is the only arrangement in which the entry cannot go - * permanently missing. Logging after the commit forces a choice between two - * broken outcomes — answer 500 and the retry returns `changed: false` and - * never logs, or answer 200 and Cloud stops retrying while the entry stays - * absent. Writing it transactionally removes the choice: either both land, or - * neither does and the retry re-applies from a clean slate. - */ + // Hire first. If the content transaction fails, a retry reuses the committed agent. + // The dedicated transaction holds only the workflow lock, not the hire writes. async function apply( companyId: string, seed: ApplyOnboardingSeed, @@ -400,42 +359,46 @@ export function onboardingSeedService(db: Db) { // late. const publications: ActivityPublication[] = []; - const result = await db.transaction(async (tx) => { - await tx.execute( - sql`select pg_advisory_xact_lock(hashtextextended(${`paperclip:onboarding-seed:${companyId}`}, 0))`, - ); - const dbx = tx as unknown as Db; - const applied = await applyWithin(dbx, companyId, seed, audit); + const database = db; + const lockKey = `paperclip:onboarding-seed:${companyId}`; + return withDedicatedDbConnection(db, lockDb => lockDb.transaction(async lock => { + await lock.execute(sql`select pg_advisory_xact_lock(hashtextextended(${lockKey}, 0))`); + const preparedAgentId = await prepareAgent(database, companyId, seed, audit); + const result = await database.transaction(async (tx) => { + const dbx = tx as unknown as Db; + const applied = await applyWithin(dbx, companyId, seed, preparedAgentId); - if (applied.changed && audit) { - await logActivity( - dbx, - { - companyId, - actorType: audit.actorType, - actorId: audit.actorId, - agentId: audit.agentId, - runId: audit.runId, - agentApiKeyId: audit.agentApiKeyId, - action: "company.onboarding_seed_applied", - entityType: "company", - entityId: companyId, - details: { - revision: applied.revision, - goalId: applied.goalId, - agentId: applied.agentId, - issueId: applied.issueId, + if (applied.changed && audit) { + await logActivity( + dbx, + { + companyId, + actorType: audit.actorType, + actorId: audit.actorId, + agentId: audit.agentId, + runId: audit.runId, + agentApiKeyId: audit.agentApiKeyId, + action: "company.onboarding_seed_applied", + entityType: "company", + entityId: companyId, + details: { + revision: applied.revision, + goalId: applied.goalId, + agentId: applied.agentId, + issueId: applied.issueId, + }, }, - }, - publications, - ); - } + publications, + ); + } - return applied; - }); + return applied; + }); - for (const publication of publications) publishActivity(publication); - return result; + for (const publication of publications) publishActivity(publication); + if (result.agentId) scheduleAgentLifecycle(db, result.agentId); + return result; + })); } return { apply, get: (companyId: string) => readRecord(db, companyId) }; diff --git a/server/src/services/plugin-host-services.ts b/server/src/services/plugin-host-services.ts index d6213fb8b3..1c592fb4cb 100644 --- a/server/src/services/plugin-host-services.ts +++ b/server/src/services/plugin-host-services.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "./agent-lifecycle.js"; import type { Db } from "@paperclipai/db"; import { activityLog, @@ -719,6 +720,7 @@ export function buildHostServices( const secretsHandler = createPluginSecretsHandler({ db, pluginId }); const companies = companyService(db); const agents = agentService(db); + const agentsLifecycle = createAgentLifecycle(db); const managedAgents = pluginManagedAgentService(db, { pluginId, pluginKey, @@ -3013,14 +3015,14 @@ export function buildHostServices( await ensurePluginAvailableForCompany(companyId); const agent = await agents.getById(params.agentId); requireInCompany("Agent", agent, companyId); - return (await agents.pause(params.agentId)) as Agent; + return (await agentsLifecycle.pauseAgent(params.agentId)) as Agent; }, async resume(params) { const companyId = ensureCompanyId(params.companyId); await ensurePluginAvailableForCompany(companyId); const agent = await agents.getById(params.agentId); requireInCompany("Agent", agent, companyId); - return (await agents.resume(params.agentId)) as Agent; + return (await agentsLifecycle.resumeAgent(params.agentId)) as Agent; }, async invoke(params) { const companyId = ensureCompanyId(params.companyId); diff --git a/server/src/services/plugin-managed-agents.ts b/server/src/services/plugin-managed-agents.ts index 4c4e2ee1ae..39ca9fd3b0 100644 --- a/server/src/services/plugin-managed-agents.ts +++ b/server/src/services/plugin-managed-agents.ts @@ -1,3 +1,4 @@ +import { createAgentLifecycle } from "./agent-lifecycle.js"; import { and, eq, isNull, ne } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { @@ -183,6 +184,7 @@ export function pluginManagedAgentService( options: PluginManagedAgentServiceOptions, ) { const agentSvc = agentService(db); + const agentSvcLifecycle = createAgentLifecycle(db); const approvalSvc = approvalService(db); const instructions = agentInstructionsService(db); @@ -486,7 +488,7 @@ export function pluginManagedAgentService( const requiresApproval = company.requireBoardApprovalForNewAgents; const adapterType = await resolveManagedAdapterType(companyId, declaration); const initialStatus = requiresApproval ? "pending_approval" : declaration.status ?? "idle"; - let created = await agentSvc.create(companyId, { + let created = await agentSvcLifecycle.requestHire(companyId, { ...declarationPatch(declaration, { adapterType }), status: initialStatus, pauseReason: initialStatus === "paused" ? managedAgentPauseReason(options.pluginKey) : null, @@ -569,26 +571,13 @@ export function pluginManagedAgentService( ) { if ( declaration.status !== "paused" - || agent.status !== "paused" + || agent.lifecycleState !== "paused" || agent.pauseReason !== null ) { return agent; } - const updated = await db - .update(agents) - .set({ - pauseReason: managedAgentPauseReason(options.pluginKey), - updatedAt: new Date(), - }) - .where(and( - eq(agents.id, agent.id), - eq(agents.companyId, companyId), - eq(agents.status, "paused"), - isNull(agents.pauseReason), - )) - .returning({ id: agents.id }) - .then((rows) => rows[0] ?? null); + const updated = await agentSvcLifecycle.pauseAgent(agent.id, managedAgentPauseReason(options.pluginKey)); if (!updated) { const current = await agentSvc.getById(agent.id) as Agent | null; diff --git a/server/src/services/public-mcp/oauth.ts b/server/src/services/public-mcp/oauth.ts index ec878f35a4..16d5e88181 100644 --- a/server/src/services/public-mcp/oauth.ts +++ b/server/src/services/public-mcp/oauth.ts @@ -1,5 +1,7 @@ +import { updateAgentConfigurationInTransaction } from "../agent-configuration-transaction.js"; import { instanceSettingsService } from "../instance-settings.js"; import { createHash, randomBytes } from "node:crypto"; +import { canConfigureAgentConnection } from "../../modules/agent-lifecycle/index.js"; import { and, eq, gt, inArray, isNull, lt, lte, notExists, sql } from "drizzle-orm"; import { z } from "zod"; import type { Request } from "express"; @@ -7,7 +9,6 @@ import { type Db, activityLog, agents, authUsers, companies, companyLogos, dotAgentBindings, mcpOauthClients, mcpOauthGrants, mcpOauthRequests, mcpOauthTokens, mcpOauthDeviceRequests, mcpOauthMetadataAdmissions, } from "@paperclipai/db"; import { DOT_RUNNER_MCP_PATH, DOT_RUNNER_MCP_SCOPES, PUBLIC_MCP_PATH, PUBLIC_MCP_SCOPES, type McpConnectionRequest, type McpDotPairingPreview } from "@paperclipai/shared"; -import { agentService } from "../agents.js"; import { boardAuthService } from "../board-auth.js"; import { logActivity } from "../activity-log.js"; import { createClientMetadataResolver, mcpRedirectMatches, validMcpRedirect as validRedirect, type MetadataFetch } from "./client-metadata.js"; @@ -274,7 +275,7 @@ export function createPublicMcpOAuth(db: Db, config: PublicMcpConfig, options: { if (!binding || binding.status !== "pairing" || binding.grantId || !binding.pairingExpiresAt || binding.pairingExpiresAt <= new Date()) throw invalidGrant(); if (request.requestedCompanyId && request.requestedCompanyId !== binding.companyId) throw invalidGrant(); const [agent] = await queryDb.select().from(agents).where(and(eq(agents.id, binding.agentId), eq(agents.companyId, binding.companyId))).for("update"); - if (!agent || agent.adapterType !== "paperclip_runner" || ["paused", "terminated", "pending_approval"].includes(agent.status)) throw invalidGrant(); + if (!agent || agent.adapterType !== "paperclip_runner" || !canConfigureAgentConnection(agent)) throw invalidGrant(); return { request, binding, agent }; } @@ -459,7 +460,7 @@ export function createPublicMcpOAuth(db: Db, config: PublicMcpConfig, options: { await tx.update(mcpOauthGrants).set({ agentId: binding.agentId }).where(eq(mcpOauthGrants.id, grant.id)); await tx.update(dotAgentBindings).set({ grantId: grant.id, status: "connected", pairingCodeHash: null, pairingExpiresAt: null, updatedAt: new Date() }).where(eq(dotAgentBindings.id, binding.id)); - await agentService(tx as unknown as Db).update(agent.id, { adapterConfig: { ...agent.adapterConfig, dotBindingId: binding.id } }, + await updateAgentConfigurationInTransaction(tx as unknown as Db, agent.id, { adapterConfig: { ...agent.adapterConfig, dotBindingId: binding.id } }, { recordRevision: { createdByUserId: binding.operatorId, source: "dot-pairing" } }); await logActivity(tx as unknown as Db, { companyId: binding.companyId, actorType: "user", actorId: binding.operatorId, action: "dot.paired", entityType: "agent", entityId: binding.agentId, diff --git a/server/src/services/question-response-delivery.ts b/server/src/services/question-response-delivery.ts index 5db1e05150..dad35c77d7 100644 --- a/server/src/services/question-response-delivery.ts +++ b/server/src/services/question-response-delivery.ts @@ -1340,5 +1340,9 @@ export function questionResponseDeliveryService( return counts; } - return { deliver, sweepPending }; + async function hasPending() { + return (await db.select({ id: issueQuestionResponseDeliveries.id }).from(issueQuestionResponseDeliveries) + .where(inArray(issueQuestionResponseDeliveries.status, ["pending", "delivering"])).limit(1)).length > 0; + } + return { deliver, sweepPending, hasPending }; } diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index ff2de0c336..d27f3e2717 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1,3 +1,4 @@ +import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; import { isAiAuthenticationBlocked } from "../ai-auth-failure.js"; import { hasCommittedNativePlanWait } from "../native-runtime/native-plan-wait.js"; import { isNativeWorkspaceExportRepairCause } from "@paperclipai/shared"; @@ -4494,6 +4495,10 @@ export function recoveryService( } const agent = await getAgent(agentId); + if (agent?.companyId === issue.companyId && isAgentAwaitingSetup(agent)) { + result.skipped += 1; + continue; + } const agentInvokable = agent && agent.companyId === issue.companyId ? await isAgentInvokable(agent) diff --git a/server/src/services/secret-proposals.ts b/server/src/services/secret-proposals.ts index f3b29ae862..fbefd53d5f 100644 --- a/server/src/services/secret-proposals.ts +++ b/server/src/services/secret-proposals.ts @@ -1,3 +1,4 @@ +import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js"; import { withAgentAppearance } from "@paperclipai/shared"; import { and, count, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -699,7 +700,7 @@ export function createSecretProposalsService(db: Db) { } adapterConfig[proposal.configPath] = binding; } - const updated = await agentSvc.update(target.id, { adapterConfig }, { + const updated = await updateAgentConfigurationInTransaction(txDb, target.id, { adapterConfig }, { recordRevision: { createdByUserId: resolvedByUserId, source: "patch" }, }); if (!updated) throw notFound("Target agent not found"); @@ -720,18 +721,22 @@ export function createSecretProposalsService(db: Db) { overrides?: { name?: string; description?: string | null; providerConfigId?: string | null }; assertCanResolve?: (proposal: Proposal, txDb: Db) => Promise; }) { - return db.transaction(async (tx) => { - const txDb = tx as unknown as Db; + const loadForApproval = async (txDb: Db) => { const proposal = await requirePending(companyId, proposalId, txDb, true); assertNotExpired(proposal); await input.assertCanResolve?.(proposal, txDb); await assertBindingSnapshotCurrent(proposal, txDb, true); + return proposal; + }; + const saved = await db.transaction(async (tx) => { + const txDb = tx as unknown as Db; + const proposal = await loadForApproval(txDb); if (proposal.kind === "secret") { const created = await applySecretApproval(txDb, proposal, input); - return markApproved(txDb, proposal, { + return { kind: "secret" as const, proposal: await markApproved(txDb, proposal, { resolvedByUserId: input.resolvedByUserId, createdSecretId: created.id, - }); + }) }; } let secretId = proposal.secretId; @@ -757,7 +762,15 @@ export function createSecretProposalsService(db: Db) { } } if (!secretId) throw conflict("Binding proposal has no approved secret"); - const liveSecret = await secretService(txDb).getById(secretId); + return { kind: "binding" as const, secretId }; + }); + if (saved.kind === "secret") return saved.proposal; + + // Keep the approved secret if applying its reference fails. + return db.transaction(async tx => { + const txDb = tx as unknown as Db; + const proposal = await loadForApproval(txDb); + const liveSecret = await secretService(txDb).getById(saved.secretId); if (!liveSecret || liveSecret.companyId !== companyId || liveSecret.status !== "active") { throw conflict("Binding proposal secret is not active"); } diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 5a4c8ae6c1..4a7439b6aa 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -21,7 +21,7 @@ import type { import { companySearchQuerySchema, STATUS_CARD_AGENT_MAX_CARDS } from "@paperclipai/shared"; import { conflict, forbidden, notFound, unprocessable } from "../errors.js"; import { logger } from "../middleware/logger.js"; -import { readBuiltInAgentMarker } from "./built-in-agent-metadata.js"; +import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService } from "./built-in-agents.js"; import { companySearchService } from "./company-search.js"; import { issueService } from "./issues.js"; diff --git a/server/src/services/summary-slots.ts b/server/src/services/summary-slots.ts index 6f07b83872..bb4ca9e952 100644 --- a/server/src/services/summary-slots.ts +++ b/server/src/services/summary-slots.ts @@ -24,7 +24,7 @@ import { type WriteSummarySlotResponse, } from "@paperclipai/shared"; import { conflict, forbidden, notFound, unprocessable } from "../errors.js"; -import { readBuiltInAgentMarker } from "./built-in-agent-metadata.js"; +import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService } from "./built-in-agents.js"; import { agentService } from "./agents.js"; import { issueService } from "./issues.js"; diff --git a/server/src/services/tool-action-delivery.ts b/server/src/services/tool-action-delivery.ts index 63f217de9a..0718ed2d44 100644 --- a/server/src/services/tool-action-delivery.ts +++ b/server/src/services/tool-action-delivery.ts @@ -345,6 +345,12 @@ export function toolActionDeliveryService( } return { deliver, + async hasPending() { + // Include receipts awaiting review/execution: their later terminal write + // must remain recoverable even if its optional fast-path callback fails. + return (await db.select({ id: toolActionDeliveries.actionRequestId }).from(toolActionDeliveries) + .where(isNull(toolActionDeliveries.deliveredAt)).limit(1)).length > 0; + }, async deliverForRun(input: { companyId: string; runId: string }) { // A review can settle before its source run yields. Retry that run's // receipts as soon as execution cleanup finishes; the periodic sweep diff --git a/server/src/services/tool-action-review.ts b/server/src/services/tool-action-review.ts index b429b35ce6..3d0effa9f0 100644 --- a/server/src/services/tool-action-review.ts +++ b/server/src/services/tool-action-review.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { and, eq } from "drizzle-orm"; import { issues, @@ -206,6 +207,7 @@ export async function commitToolActionReview( : { version: 1, outcome: "rejected", reason: input.reason }, }) .where(eq(issueThreadInteractions.id, interaction.id)); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.toolAction); await tx .insert(toolActionDeliveries) .values({ diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index 86d79cd2f9..4ef4f1092e 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { composeConnectionInstructions } from "./connection-instructions.js"; import { isInsufficientConnectionScope, INSUFFICIENT_CONNECTION_SCOPE_MESSAGE } from "./connection-permission-errors.js"; import { boundedMcpToolName } from "./mcp-tool-names.js"; @@ -2615,15 +2616,18 @@ export function createToolGatewayService( ); } - await db - .insert(toolActionDeliveries) - .values({ - companyId: input.session.companyId, - actionRequestId: actionRequest.id, - issueId: input.session.issueId, - interactionId: interaction.id, - }) - .onConflictDoNothing(); + await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.toolAction); + await tx + .insert(toolActionDeliveries) + .values({ + companyId: input.session.companyId, + actionRequestId: actionRequest.id, + issueId: input.session.issueId!, + interactionId: interaction.id, + }) + .onConflictDoNothing(); + }); await writeToolCallEvent({ invocationId: input.invocation.id, diff --git a/server/src/services/work-scheduler.ts b/server/src/services/work-scheduler.ts new file mode 100644 index 0000000000..1d2cdadfe3 --- /dev/null +++ b/server/src/services/work-scheduler.ts @@ -0,0 +1,45 @@ +/** In-process deadlines. Owners restore durable tasks at startup. */ +export function createWorkScheduler() { + const tasks = new Map void }>(); + let timer: ReturnType | null = null; + let stopped = false; + const nextWakeAt = () => tasks.size ? Math.min(...Array.from(tasks.values(), task => task.at)) : null; + + function arm() { + if (timer) clearTimeout(timer); + timer = null; + const next = nextWakeAt(); + if (stopped || next === null) return; + timer = setTimeout(() => { + timer = null; + const now = Date.now(); + const due = Array.from(tasks.entries()).filter(([, task]) => task.at <= now); + for (const [key, task] of due) { + // Another due callback can cancel or replace this task. + if (tasks.get(key) !== task) continue; + tasks.delete(key); + try { task.run(); } + catch (error) { process.emitWarning(`Scheduled work failed: ${String(error)}`); } + } + arm(); + }, Math.min(2_147_483_647, Math.max(0, next - Date.now()))); + timer.unref?.(); + } + + return { + nextWakeAt, + schedule(key: string, at: number, run: () => void) { + if (stopped) return; + if (!Number.isFinite(at)) throw new Error("Work deadline must be finite"); + tasks.set(key, { at, run }); + arm(); + }, + cancel(key: string) { tasks.delete(key); arm(); }, + stop() { + stopped = true; + tasks.clear(); + if (timer) clearTimeout(timer); + timer = null; + }, + }; +} diff --git a/server/src/types/activity-publication.ts b/server/src/types/activity-publication.ts new file mode 100644 index 0000000000..611a167afa --- /dev/null +++ b/server/src/types/activity-publication.ts @@ -0,0 +1,7 @@ +import type { PluginEvent } from "@paperclipai/plugin-sdk"; + +export interface ActivityPublication { + companyId: string; + payload: Record; + pluginEvent: PluginEvent | null; +} diff --git a/tests/e2e/acp-stop-continuation.spec.ts b/tests/e2e/acp-stop-continuation.spec.ts index 318ddf65f6..8e03550841 100644 --- a/tests/e2e/acp-stop-continuation.spec.ts +++ b/tests/e2e/acp-stop-continuation.spec.ts @@ -21,9 +21,12 @@ for (const { unfinishedWrite, stopResponse } of [{ unfinishedWrite: false, stopR name: "ACP Stop fixture", role: "engineer", adapterType: "claude_local", adapterConfig: { engine: "acp", cwd: root, stateDir: path.join(root, "state"), agentCommand: `${JSON.stringify(process.execPath)} ${JSON.stringify(path.resolve("scripts/mcp-fixtures/servers/acp-stop-agent.mjs"))}`, - env: { PAPERCLIP_STOP_FIXTURE_ROOT: root, PAPERCLIP_STOP_FIXTURE_FINISH_TASK: "1", ...(unfinishedWrite ? { PAPERCLIP_STOP_FIXTURE_TOOL: "write" } : {}) }, + // The ACP fixture has no provider login. Use a fixture key to keep setup offline. + env: { ANTHROPIC_API_KEY: "acp-stop-fixture-key", PAPERCLIP_STOP_FIXTURE_ROOT: root, PAPERCLIP_STOP_FIXTURE_FINISH_TASK: "1", ...(unfinishedWrite ? { PAPERCLIP_STOP_FIXTURE_TOOL: "write" } : {}) }, }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } }, } })); + await expect.poll(async () => (await json(await request.get(`/api/agents/${owner.id}`))).lifecycleState, + { timeout: 45_000 }).toBe("ready"); const issue = await json(await request.post(`/api/companies/${company.id}/issues`, { data: { title: "ACP Stop continuation", status: "backlog", assigneeAgentId: owner.id, } })); diff --git a/tests/e2e/ai-personal-connection-recovery.spec.ts b/tests/e2e/ai-personal-connection-recovery.spec.ts index 3da4fb68ba..22f9cacd42 100644 --- a/tests/e2e/ai-personal-connection-recovery.spec.ts +++ b/tests/e2e/ai-personal-connection-recovery.spec.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { expect, test } from "@playwright/test"; import { json } from "./agent-chat.shared"; -test("missing onboarding key offers personal Claude setup in chat and resumes once", async ({ page, request }) => { +test("a ready agent with a missing personal key offers Claude setup in chat and resumes once", async ({ page, request }) => { test.setTimeout(120_000); const root = await mkdtemp(path.join(os.tmpdir(), "personal-ai-recovery-")); const original = await json(await request.get("/api/instance/settings/experimental")); @@ -16,12 +16,17 @@ test("missing onboarding key offers personal Claude setup in chat and resumes on })); await writeFile(path.join(root, "continued"), "ready"); const agent = await json(await request.post(`/api/companies/${company.id}/agents`, { data: { - name: "Chief of Staff", role: "general", adapterType: "claude_local", + name: "Chief of Staff", role: "general", adapterType: "process", + adapterConfig: { command: process.execPath }, + runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } }, + } })); + await expect.poll(async () => (await json(await request.get(`/api/agents/${agent.id}`))).lifecycleState).toBe("ready"); + await json(await request.patch(`/api/agents/${agent.id}`, { data: { + adapterType: "claude_local", adapterConfig: { engine: "acp", cwd: root, stateDir: path.join(root, "state"), agentCommand: `${JSON.stringify(process.execPath)} ${JSON.stringify(path.resolve("scripts/mcp-fixtures/servers/acp-stop-agent.mjs"))}`, env: { ANTHROPIC_API_KEY: { type: "user_secret_ref", key: definition.key, version: "latest", required: true }, PAPERCLIP_STOP_FIXTURE_ROOT: root } }, - runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } }, } })); const chatPath = `/api/companies/${company.id}/chats/${agent.id}`; await page.goto(`/${company.issuePrefix}/chats/${agent.id}`); diff --git a/tests/e2e/legacy-failure-continuation.spec.ts b/tests/e2e/legacy-failure-continuation.spec.ts index e57369bf24..dc7875e862 100644 --- a/tests/e2e/legacy-failure-continuation.spec.ts +++ b/tests/e2e/legacy-failure-continuation.spec.ts @@ -27,9 +27,12 @@ for (const action of ["task_retry", "thread_retry", "inbox_retry", "message", "q name: "Recovery fixture", role: "engineer", adapterType: "claude_local", adapterConfig: { engine: "acp", cwd: root, stateDir: path.join(root, "state"), agentCommand: `${JSON.stringify(process.execPath)} ${JSON.stringify(path.resolve("scripts/mcp-fixtures/servers/acp-stop-agent.mjs"))}`, - env: { PAPERCLIP_STOP_FIXTURE_ROOT: root, PAPERCLIP_STOP_FIXTURE_FINISH_TASK: "1" } }, + // The ACP fixture has no provider login. Use a fixture key to keep setup offline. + env: { ANTHROPIC_API_KEY: "acp-stop-fixture-key", PAPERCLIP_STOP_FIXTURE_ROOT: root, PAPERCLIP_STOP_FIXTURE_FINISH_TASK: "1" } }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } }, } })); + await expect.poll(async () => (await json(await request.get(`/api/agents/${agent.id}`))).lifecycleState, + { timeout: 45_000 }).toBe("ready"); const issue = await json(await request.post(`/api/companies/${company.id}/issues`, { data: { title: "Continue after startup failure", description: "Answer the pending follow-up once.", status: "backlog", assigneeAgentId: agent.id,