diff --git a/doc/connections/CONNECTOR-PLAYBOOK.md b/doc/connections/CONNECTOR-PLAYBOOK.md index f924f9aa9e4..8f4b2566b33 100644 --- a/doc/connections/CONNECTOR-PLAYBOOK.md +++ b/doc/connections/CONNECTOR-PLAYBOOK.md @@ -721,6 +721,12 @@ as `agentmail_send` rather than extending core tools with provider-specific branches. Existing MCP connectors continue to use their normal MCP tool catalog; they do not need a duplicate native wrapper just to supply a skill. +For a gated catalog connector, declare `experimentalFeatureKey` in its +`AppDefinition`. Shared availability checks resolve the saved application key +and evaluate that feature before setup, discovery, and execution. Keep the +feature's settings visibility separate from this enforcement. Do not add +provider-name checks to the gateway or setup screens. + **Resolve eligibility from current assignments and access.** An AgentMail account credential alone does not give an agent email capabilities. An active inbox assigned to that agent does, provided both the inbox connection and saved diff --git a/doc/connections/PAPERCLIP-NOTEBOOK.md b/doc/connections/PAPERCLIP-NOTEBOOK.md new file mode 100644 index 00000000000..aa970293e88 --- /dev/null +++ b/doc/connections/PAPERCLIP-NOTEBOOK.md @@ -0,0 +1,101 @@ +# Paperclip Notebook + +Paperclip Notebook uses the existing remote HTTP MCP connection, authorization-code +OAuth with PKCE, Dynamic Client Registration, and instance credential vault. +Hosted and self-hosted Paperclip use the same connection protocol; Cloud broker +enrollment is unnecessary. Cloud authenticates the person, while Notebook owns +pages, reader policies, deployments, and its resource-specific OAuth grants. + +## Connection and rollout + +The curated connection uses `https://notebook.paperclip.app/mcp`. It is hidden +behind the default-off `enablePaperclipNotebook` flag, with no Settings toggle, +including developer mode. Operators can enable it through managed configuration +(`features.enablePaperclipNotebook: true`); self-hosted operators can use the +existing instance experimental-settings API. Cloud tenants cannot self-enable +this hidden flag, including through staging tenant opt-ins. + +Catalog metadata and the saved application identity govern availability. While +disabled, setup, discovery, reconnect, OAuth completion, and curated Notebook +calls are denied. Local removal and grant revocation remain available. This +rollout gate does not blacklist independent generic remote MCP endpoints. + +The personal connection requests `notebook:read`, `notebook:write`, +`notebook:share`, and `notebook:delete`. Sharing includes publishing a page +publicly in Explore. Notebook enforces ownership and reader policy; Paperclip +continues to enforce current agent assignment and Allowed / Ask first / Off +controls. Connection labels use the normal catalog name. Reconnect preserves +assignments and action restrictions. + +## Publishing local files + +A remote MCP server cannot read an agent-local filesystem path. Ordinary MCP +clients can use `notebook_publish` for inline UTF-8 or base64 content up to 1 MiB +decoded. Clients that upload directly can continue to use the existing scoped +S3-ticket protocol; upload URLs are temporary authority and must not be logged. + +For a local file or built directory, Notebook provides a standalone Node.js +script. It is owned and served by Notebook; Paperclip adds no publishing CLI +command, native runtime tool, private transfer channel, or credential flow. +The script requires a Notebook release advertising `uploadLimits` through +`notebook_get_account` and supporting `notebook_begin_upload` with +`transport: "mcp"` plus `notebook_upload_chunk`. The script and these tools must +be deployed together; this source addition does not establish that deployment. + +Run it only in an approved agent shell that already has `PAPERCLIP_API_URL`, +`PAPERCLIP_API_KEY`, and `PAPERCLIP_RUN_ID`. These are the existing Paperclip run +credentials, not Notebook credentials. It creates no login or API key. Download +and inspect the trusted script before running it: + +```sh +curl --fail --silent --show-error --proto '=https' \ + https://notebook.paperclip.app/client/publish.mjs \ + -o /tmp/paperclip-notebook-publish.mjs +node /tmp/paperclip-notebook-publish.mjs ./dist \ + --connection CONNECTION_ID --title "Weekly report" \ + --idempotency-key weekly-report-build +``` + +Build client applications before publishing. Use `--entry-file` and `--spa` +when needed. Updates require `--page-id` and `--expected-version`. The script +reads only the selected local source under the shell's existing filesystem +authority. It rejects symbolic links, nonregular files, noncanonical paths, and advertised +size/file limits before publication mutations, then verifies declared content +again before sending chunks. +Agents without those shell credentials or filesystem access use the ordinary +MCP tools available to their runtime; the script grants neither authority. + +The script opens an ordinary run-bound tool gateway session and sends every +operation through `/api/tool-gateway/tools/call` on the selected connection. +Notebook OAuth credentials remain in Paperclip's vault. File bytes travel as +bounded base64 tool arguments under normal gateway handling, not through signed +upload URLs or a private data channel. Notebook verifies chunk identity, +checksums, ownership, scopes, and the original upload window before publication. + +If an action requires approval, stop and wait for the normal task approval. +Rerun with the same publication key and unchanged files after approval or a lost +response; Notebook reports the missing chunk indexes so completed chunks are +not repeated. Off actions, removed assignments, disabled connections, and the +rollout gate continue to prevent dispatch. Throttles are honored within a +bounded attempt. Uploads expire after 15 minutes; an expired upload requires a +new publication key. A validating operation is not a working link: use +`notebook_get_operation` until Notebook reports success and the effective audience. + +## Verification and limits + +The connection fixtures exercise personal OAuth, registration, PKCE/resource +binding, tool discovery, assignment, refresh, reconnect policy preservation, +and local removal. Uploader tests use real local files and controlled HTTP +services; final acceptance must additionally exercise the ordinary authenticated +Paperclip gateway against Notebook. These checks do not establish deployed +hosted/self-hosted browser acceptance or maximum-path AWS publication. Notebook +advertises accepted upload limits; clients do not infer its S3 key layout. + +Removing a connection disables local use and clears saved credentials through +Paperclip's existing removal flow. It does not automatically revoke Notebook's +remote authorization; revoke that separately in Notebook's Connected apps. +Automatic upstream revocation, account-email labels, and custom rotating-refresh +response recovery remain deferred. A lost refresh response may require reconnecting. + +Notebook artwork uses the repository's first-party Paperclip mark, with fixed +light and dark variants for the connector icon renderer. diff --git a/doc/connections/tool-method-permission-reviews.json b/doc/connections/tool-method-permission-reviews.json index b22e6554de4..b81e206d17f 100644 --- a/doc/connections/tool-method-permission-reviews.json +++ b/doc/connections/tool-method-permission-reviews.json @@ -2746,6 +2746,26 @@ "reviewedAt": "2026-10-15", "liveProof": "not-run", "keyPermissions": "Create an organization key with the read, write, and delete permissions for the resources you intend to manage. Daytona toolbox access is available to every valid organization key, even without selected scopes. Some user or administration endpoints require provider user authentication and will reject an API key." + }, + { + "app": "paperclip-notebook", + "method": "mcp-oauth", + "auth": "oauth", + "policy": "explicit", + "requestedScopes": [ + "notebook:read", + "notebook:write", + "notebook:share", + "notebook:delete" + ], + "capability": "write", + "supportedActions": "Read available pages; create and update owned pages; change sharing including public discovery; manage versions; delete and restore owned pages.", + "evidence": [ + "doc/connections/PAPERCLIP-NOTEBOOK.md" + ], + "reviewedAt": "2026-10-20", + "liveProof": "not-run", + "evidenceLimit": "First-party implementation contract; canonical service availability and live OAuth qualification are deployment prerequisites. Uses the existing personal OAuth flow with default connection labels; automatic remote revocation and custom keyed refresh recovery are deferred." } ] } diff --git a/doc/idle-execution-control.md b/doc/idle-execution-control.md new file mode 100644 index 00000000000..a18600124ec --- /dev/null +++ b/doc/idle-execution-control.md @@ -0,0 +1,71 @@ +# Execution-control workers and idle instances + +Execution-control maintenance uses the shared delivery coordinator instead of an +unconditional 15-second interval. This changes when recovery is inspected; it +does not change the authority, evidence, locking, idempotency, or cleanup rules +inside the six owning services. No table or migration is added. + +## Duties and durable state + +| Duty | Trigger and durable source | Remaining scheduled work | Quiet condition | +| --- | --- | --- | --- | +| Abandoned finalization | Committed control-deadline writes on `heartbeat_runs` and `native_run_finalizations` | Earliest non-null control deadline; a due row still present after recovery retries after 15 seconds | No control deadlines | +| Safe native replacement | Committed run transitions and reconciliation decisions; existing failed run/coordinator rows | Retry while the existing replacement predicate selects a candidate, including one awaiting provider termination or sandbox release | No replacement candidate, or a recorded successor/denial | +| Reconciled continuation | Committed recovery-action marking/resolution; `evidence.continuationDelivery` | Retry pending delivery after 15 seconds | No resolved action awaiting continuation delivery | +| Execution status | Committed run status writes; `execution_status_delivery_id` | Retry while a delivery marker remains | All markers acknowledged | +| Automatic disposition | Committed run/recovery-action changes; existing eligible recovery-action queries | Retry while a selected action still needs disposition or an obsolete conversation hold remains | No eligible action/hold; resolved history alone is quiet | +| Local login cleanup | Committed local login creation; existing `adapter_auth_sessions.expires_at` | Earliest local login expiry; retry overdue failed cleanup | No local login expiry remains | + +Each worker runs once on startup to restore durable obligations. Passes are +coalesced at the former 15-second resolution: a tenant's repeated writes cannot +force continuous global scans. This is a minimum spacing between requested +passes, not a recurring poll; an empty worker never arms that spacing by itself. +A newly discovered deadline within the spacing window runs at its end, retaining +the former maximum 15-second scheduling resolution. Future deadlines outside +that window retain their exact due time. Workers remain +independent: a slow replacement proof cannot stall status delivery. Batches and +failed obligations retain the existing 15-second retry budget; empty workers do +not repeatedly inspect the database. Login authorization still checks expiry on +use even before physical cleanup completes. + +## Producer contract + +`notifyExecutionRunWork(transaction)` signals the run transition's status, +finalization, replacement, and disposition consumers before the write. Existing +transaction owners call it directly. Standalone deadline and recovery-action +writes use `writeExecutionControl` to give the write a transaction boundary. +Nested transactions retain the existing outer-commit notification semantics. + +Producer coverage includes heartbeat status transitions, legacy terminalization, +native finalization and restart/failure settlement, replacement successors and +denials, recovery-action upserts/resolution, operator reconciliation, control +deadline creation/cancellation, and local login creation. Workers acknowledge +existing records rather than creating a second queue. The replacement and +disposition pending checks reuse the executing services' eligibility queries. + +This is the existing single live server / shared `createDb` owner signal model. +Independent SQL writers or separate replicas must integrate notifications; these +workers do not continuously discover unannounced external writes. Startup still +recovers committed rows after process loss. Uncertain commit handling remains in +`packages/db/src/work-signals.ts`. + +## Lifecycle and verification + +A worker consumes a requested generation only after its sweep and deadline +inspection succeed. Writes during a sweep, startup during drain, and a failure +before drain remain pending for resume. Passive waiting during drain performs no +worker SQL. Shutdown uses the coordinator to unsubscribe and await in-flight work. + +`server/src/__tests__/execution-control-work.test.ts` covers empty and retained +history with the real sweep implementations, a real producer inside an outer +commit/rollback, saved deadlines, deadline replacement/cancellation, durable +status retries, recovery continuations, independent duties, and drain/resume. +Database tests use embedded PostgreSQL by default. An isolated external test +instance can be supplied through `PAPERCLIP_EXECUTION_SLEEP_TEST_DATABASE_URL`; +the database must have a dedicated name ending in `_test` and is cleared by the +suite. Never use a tenant database. + +Worker quiet is not host sleep authorization. The existing idle-safety blockers +remain, and Cloud's external wake contract does not gain coverage for execution +or login deadlines through this change. Provider suspension and wake must be +qualified separately before relaxing those blockers. diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md index 35db2511e4a..e77a8719c71 100644 --- a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md +++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md @@ -536,13 +536,13 @@ Longer polling intervals alone do not complete an item. retain subscription scope, expiry, and replay safety. Source: [events.ts](../../server/src/services/public-mcp/events.ts). -- [ ] **JOB-06 — Execution-control reconciliation.** Current: independent - **15s** recovery/delivery lanes, including when the main heartbeat is disabled. - Change: activate lanes from durable pending work and deadlines. **Complete - when:** empty lanes are quiet and abandoned finalizations, replacements, - continuations, status/disposition delivery, and login expiry still recover. - Sources: [index.ts](../../server/src/index.ts), - [execution-control-deadline.ts](../../server/src/services/execution-control-deadline.ts). +- [x] **JOB-06 — Execution-control reconciliation.** Implemented in + [PR #17094](https://github.com/paperclipai/paperclip/pull/17094): six independent + workers wake from committed writes, saved control/login deadlines, or concrete + unfinished obligations. Empty queues and completed history no longer cause + recurring SQL. Existing 15s retries remain for pending work; Cloud sleep + eligibility and external alarm coverage are unchanged. See the + [worker and producer contract](../idle-execution-control.md). - [ ] **JOB-07 — Plugin job scheduling.** Current: **30s** SQL even without jobs. Change: register nearest `nextRunAt` and signal registration/update/removal. diff --git a/doc/plugins/PLUGIN_AUTHORING_GUIDE.md b/doc/plugins/PLUGIN_AUTHORING_GUIDE.md index e6913a203df..94ca672bd0d 100644 --- a/doc/plugins/PLUGIN_AUTHORING_GUIDE.md +++ b/doc/plugins/PLUGIN_AUTHORING_GUIDE.md @@ -798,3 +798,14 @@ A plugin can declare `supportsTasks` on an environment driver and implement `onEnvironmentTask`. See the [task API](PLUGIN_ENVIRONMENT_TASKS.md) for the operations, validation, and lease rules. This hook does not add a shell or file transfer API. Native agent execution is a separate integration. + +## Plugin-provided Runner execution + +Plugins can [provide Runner execution through environment drivers](PLUGIN_RUNNER_EXECUTION.md). +Core can select this path for `paperclip_runner` agents assigned to a saved plugin +environment. The plugin owns project volumes and starts Runner from structured +task launch inputs. Core connects PRP and verifies cancellation and durable cleanup. +Resource transfer and credential synchronization use a separate workflow. +Persistent computers keep their agent home and session files between runs. No remote shell, file-command, +or workspace hook is required. The guide defines the task operations, lease +hooks, artifact qualification, and supported per-turn lifecycle. diff --git a/doc/plugins/PLUGIN_ENVIRONMENT_TASKS.md b/doc/plugins/PLUGIN_ENVIRONMENT_TASKS.md index 505e2f579cc..6a4affe8a90 100644 --- a/doc/plugins/PLUGIN_ENVIRONMENT_TASKS.md +++ b/doc/plugins/PLUGIN_ENVIRONMENT_TASKS.md @@ -3,8 +3,8 @@ An environment driver can provide a Runner task through a worker hook. The task API has no shell command or file transfer operation. -This API is a server and worker contract. The agent environment test can verify -a task-capable provider. Native heartbeat dispatch is a separate integration. +This API is a server and worker contract. The [native execution integration](PLUGIN_RUNNER_EXECUTION.md) +connects a task-capable environment to a Runner run. ## Declare the hook diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md new file mode 100644 index 00000000000..12a68da5a0b --- /dev/null +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -0,0 +1,139 @@ +# Plugin Runner execution + +Core can run `paperclip_runner` tasks in a plugin environment. +The plugin owns the execution host and its files. +Core owns run identity, selected policy, assigned tools, and the PRP journal. + +## Required hooks + +Set `supportsTasks: true` on the environment driver. +Declare the `environment.drivers.register` capability. +Implement these hooks: + +- `onEnvironmentAcquireLease` +- `onEnvironmentTask` +- `onEnvironmentReleaseLease` +- `onEnvironmentDestroyLease` + +Task execution does not require `onEnvironmentExecute` or +`onEnvironmentRealizeWorkspace`. The task API has no shell or file transfer +operation. Use a separate workflow to install and save agent files. + +Core supports per-turn workspace sessions with Codex, OpenCode, and ACPX. +OpenCode and ACPX require the existing qualified provider profiles. +Workspace-free providers and warm process retention are not supported here. +The plugin owns project workspace preparation. Core does not prepare or reuse +a Core Git worktree for a task-only environment. +For plugin-backed project storage, Core uses the workspace provider contract. +The storage plugin must also provide the selected execution environment. +Its declared workspace modes and strategies remain in effect. +Core passes the pinned workspace binding with each task operation. +Core rejects configured workspace services for task-only environments. +Fresh retries that import retained files are not supported for plugin storage. +An agent-owned workspace cannot move to another agent or environment. +Core also rejects reuse of a local workspace in a task-only environment. +Keep its original owner, or use the workspace storage provider contract. + +## Prepare and launch + +1. Return an opaque task ID from lease acquisition as `providerLeaseId`. +2. Return the installed Runner version and SHA-256 digest from `describe`. +3. Return `launchPreparationVersion: 1` from `describe`. +4. Handle `prepare` with the [launch preparation contract](../runner-launch-preparation.md). +5. Save the result under its configuration ID. +6. Handle `submit` with that ID in `preparedLaunchId`. +7. Return an authenticated WSS endpoint from `connection`. + +Preparation must not start a process. It must preserve Core's provider, model, +policy, instructions, and resource assignment identities. Resolve assigned +resources to execution-host paths. Use `bindRunnerLaunchResources` and +`createRunnerProviderLaunch` when the built-in descriptor builder is suitable. + +Core checks the returned requirements digest and descriptor before it registers +PRP or submits the task. A failed check does not enable a fallback launch. +Recovery within the same run must return the exact saved configuration. + +The plugin supplies executable paths and the provider workspace in the +descriptor. It owns the process environment and account installation. +For example, a persistent computer can use these locations: + +```text +HOME=/home/agent +AGENT_HOME=/home/agent/.agents +CODEX_HOME=/home/agent/.codex +instructions=/home/agent/.agents/AGENTS.md +skills=/home/agent/.agents/skills//SKILL.md +``` + +These are execution-host locations. Core does not copy the home for each run. +The plugin must declare `projectAuthCheckVersion: 1` for all native tasks. +Core requires `check_project_auth` to allow the exact prepared configuration +before it sends credentials. The plugin must reject conflicting project or +parent authentication settings. It must keep the checked configuration fixed +until submission. See [managed account validation](PLUGIN_ENVIRONMENT_TASKS.md#managed-account-validation). + +Core sends authorized project IDs, the PRP identity, launch limits, and the +bootstrap ticket through `submit`. It does not send a process command. +The plugin must submit each task at most once. A lost reply requires status +reconciliation for the same task. + +## Credentials and tools + +Core can transfer a selected Codex account or a direct Claude credential. +Other managed routes fail closed until they have a typed credential contract. +Do not put credentials in task metadata, journals, logs, or errors. + +Assigned MCP tools run through the Core gateway over PRP. +The gateway credentials stay on Core. +The plugin must preserve the MCP binding identity in the prepared context. + +Core records credential writeback authority before dispatch. +After process termination, Core saves refreshed credentials before it completes +the task. A failed save retains the task grants for cleanup. + +## Stop and recovery + +A terminal phase does not prove process termination. +Return `executionStopped: true` only after the task and its descendants stop. +Return `sessionStateAvailable: true` only when the suspended state remains usable. +Keep retained state when Core completes the task. +Core checks managed plugin workspaces through their storage provider before +completion, including after a restart. Core does not inspect these folders locally. +If this check fails, Core defers the run and continues recovery for other runs. +Core keeps known branch and owner error codes with the repair instructions. + +The task contract exposes two checkpoint operations: + +- `read_state` returns the exact Runner durable state for the supplied identity. +- `archive_state` moves a suspended Runner authority checkpoint under the supplied + archive key. It returns the same state on retry. It keeps provider session files. + +These operations do not accept paths. Core checks the active lease and retained +source task before it calls the plugin. The existing Runner transport validates +state identity and suspension before it rotates authority. + +Core checks company, agent, environment, workspace, driver, and termination +receipts before it supplies `resumeFrom`. For same-run controller recovery, Core +first stops and releases the old task. A failed stop prevents replacement. +A new run prepares its current assignments. Provider compatibility checks remain +in the existing backends. Missing or incompatible retained state fails closed. + +## Setup verification + +Core creates a private probe lease with an expiry. +It runs the same describe, prepare, submit, and WSS/PRP path. +The probe checks provider authentication and the selected model with tools denied. +Core bounds Runner lifetime to the remaining probe time. +It then requires suspension, confirmed termination, and task cleanup. +For a new agent-owned computer, the saved-agent lifecycle runs verification +after it saves the agent. Core does not require a computer before this save. + +## Verification + +- [Lease and SDK RPC integration](../../server/src/__tests__/plugin-runner-task.integration.test.ts) +- [Real Runner setup and recovery](../../server/src/services/native-runtime/plugin-runner-probe.test.ts) +- [Task ownership and cancellation](../../server/src/services/native-runtime/plugin-runner-task.test.ts) +- [Credential transfer](../../server/src/services/native-runtime/plugin-runner-credentials.test.ts) + +The Runner fixture uses a synthetic Codex provider and local sockets. +It does not qualify a deployed plugin, live account, or production WSS proxy. diff --git a/doc/runner-launch-preparation.md b/doc/runner-launch-preparation.md index 4495d364bcc..653edb25e47 100644 --- a/doc/runner-launch-preparation.md +++ b/doc/runner-launch-preparation.md @@ -48,6 +48,6 @@ unsettled checkpoint and prevents reuse. ## Plugin task protocol The [environment task API](plugins/PLUGIN_ENVIRONMENT_TASKS.md#launch-preparation) -defines the plugin preparation exchange. This callback does not implement a -plugin task process owner or a Launcher endpoint. Those integrations are -separate changes. +defines the plugin preparation exchange. The [native plugin integration](plugins/PLUGIN_RUNNER_EXECUTION.md) +connects the callback to task execution. A Launcher implementation remains the +execution owner's responsibility. diff --git a/maps/agent-access-map/connection-setup.md b/maps/agent-access-map/connection-setup.md index ee64513a67a..0d3c322f2a8 100644 --- a/maps/agent-access-map/connection-setup.md +++ b/maps/agent-access-map/connection-setup.md @@ -43,6 +43,24 @@ remaining operations before qualifying the whole capability. Gaps: Runtime tool broker searches available services without granting connection administration. +Notebook is internally gated by `enablePaperclipNotebook` (default off, no settings toggle). Disabled instances hide setup/discovery and deny curated Notebook dispatch and publishing; local removal and grant revocation remain available. + +The connector catalog declares the feature key. Shared availability checks use +the saved application identity. Notebook owns publication and file uploads; +Paperclip uses its existing MCP tools and OAuth flow. + +Notebook source addition: `paperclip-notebook` is discoverable through +`connections_search` and requestable through `connection_request`. Its standard +personal authorization requests read, write, share, and delete scopes. Notebook +page ownership remains enforced by the remote service; a client-supplied agent +label grants no authority. The direct MCP connection requires no Cloud instance +enrollment. Notebook fixture evidence: `verified` against disposable PostgreSQL 18 +with controlled MCP responses — [Notebook integration](../../doc/connections/PAPERCLIP-NOTEBOOK.md) +and [generic OAuth fixture](../../server/src/__tests__/generic-mcp-connection.test.ts). +Ordinary MCP calls retain assignment and Allowed / Ask first / Off policy. +Live deployed publication and hosted/self-hosted OAuth remain release checks. + + ## `identity-and-access` Capability ID: `connection-setup/identity-and-access`. diff --git a/maps/agent-access-map/execution-environments.md b/maps/agent-access-map/execution-environments.md index 69bd94a2d62..210291cf46b 100644 --- a/maps/agent-access-map/execution-environments.md +++ b/maps/agent-access-map/execution-environments.md @@ -165,6 +165,17 @@ remaining operations before qualifying the whole capability. Gaps: Choosing among permitted targets differs from creating/configuring host targets; test authorized and denied selection. +Scoped plugin execution update: `paperclip_runner` supports a saved plugin +environment through the server-selected lease and the SDK `environmentTask` RPC. +This is an execution path, not a new agent-callable management endpoint. Host +company/project checks and native PRP authorization remain required; declaring a +plugin capability does not authorize the agent to change its environment. +The [integration fixture](../../server/src/__tests__/plugin-runner-task.integration.test.ts) +exercises the host and SDK with synthetic tasks and a mocked WebSocket. +The task API does not transfer files or reveal guest filesystem paths. +This source update does not qualify live agent access, warm retention, or a provider. +See the [execution contract](../../doc/plugins/PLUGIN_RUNNER_EXECUTION.md). + ## `images` Capability ID: `execution-environments/images`. @@ -189,3 +200,16 @@ own policy and access review. Gaps: No default agent custom-image administration/terminal path; this is a policy decision before adding a tool. + +The task-only setup path uses a Core-issued expiring probe identity and a real, +read-only Runner hello. The plugin checks project authentication before provider startup. +[Setup probe coverage](../../server/src/services/native-runtime/plugin-runner-probe.test.ts) +uses the real Runner and a synthetic Codex provider through loopback PRP; TLS +termination is replaced by the fixture. It does not qualify a live plugin provider. + +The plugin prepares execution-host resources and the provider descriptor. Core validates the result, then supplies task identity and PRP launch limits. + +Managed account transfer requires `projectAuthCheckVersion: 1` and a successful +`check_project_auth` result for the exact prepared configuration, provider, and +project set. New preparation removes old validation. Core checks the saved +validation again in the dispatch update. diff --git a/maps/feature-map/connection-setup.md b/maps/feature-map/connection-setup.md index b3017e5db66..3efb0d3713b 100644 --- a/maps/feature-map/connection-setup.md +++ b/maps/feature-map/connection-setup.md @@ -99,6 +99,23 @@ saved connection. Run a permitted read-only action with an eligible agent and verify its useful result. Try an ineligible agent and confirm denial. Repeat with a user who cannot install company-wide; the UI must not imply broad access. +Notebook is internally gated by `enablePaperclipNotebook` (default off, no settings toggle). Disabled instances hide setup/discovery and deny curated Notebook dispatch and publishing; local removal and grant revocation remain available. + +Availability is catalog metadata evaluated by shared connector checks. Notebook +publication and file uploads stay in the Notebook service. + +Paperclip Notebook uses the existing personal-account MCP OAuth setup at +`https://notebook.paperclip.app/mcp`. Sign in through Cloud, assign agents, and +verify tool discovery. The connection keeps its default catalog name. +Reconnect must preserve agent assignments and Allowed / Ask first / Off choices. +Removal disables local use and clears saved credentials. Remote authorization +can be revoked separately in Notebook's Connected apps screen; automatic remote +cleanup and custom keyed refresh recovery are deferred. A lost rotating-token +response may require reconnecting. Notebook tools use the selected connection's +ordinary action gateway. See [Notebook](../../doc/connections/PAPERCLIP-NOTEBOOK.md) +for the Notebook-owned uploader and fixture coverage. Live Notebook qualification +still requires the deployed service and both hosted and self-hosted clients. + Daytona follows this same key/access wizard. Its native method exposes the published platform and Toolbox HTTP actions using provider sandbox IDs, without a Paperclip task or lease requirement. [Daytona browser setup](../../tests/e2e/daytona-connector.spec.ts) diff --git a/maps/feature-map/execution-environments.md b/maps/feature-map/execution-environments.md index 61d0022cbe7..cbe6739bf34 100644 --- a/maps/feature-map/execution-environments.md +++ b/maps/feature-map/execution-environments.md @@ -80,6 +80,17 @@ execution. Remote-driver probes and a real bounded run need their own evidence. ### `remote-execution` +For `paperclip_runner`, a saved plugin environment can provide per-turn Runner +execution when its manifest and worker expose the Runner task API. The task +contract starts a provider-owned Runner without shell or workspace hooks. Core +resolves the saved lease, connects through PRP, and verifies stop before lease +cleanup. The provider owns the installed agent home and retained session files; +data transfer is outside task execution. See the +[plugin contract](../../doc/plugins/PLUGIN_RUNNER_EXECUTION.md). +The [fake-provider integration](../../server/src/__tests__/plugin-runner-task.integration.test.ts) +checks acquisition, SDK dispatch, scope denial, stop proof and deletion recovery. +Its WebSocket is mocked; it does not qualify a live provider or a real Runner handshake. + Automated: [execution target](../../server/src/__tests__/environment-execution-target.test.ts) and [custom images](../../server/src/__tests__/environment-custom-images-service.test.ts) cover contracts; [SSH live test](../../server/src/__tests__/environment-live-ssh.test.ts) requires its own external setup. The [environment runtime regression](../../server/src/__tests__/environment-runtime.test.ts) @@ -98,6 +109,14 @@ Manual: Run a bounded command in the intended target, inspect its output and wor - The local option may appear only when editing an existing local environment; availability follows current UI/provider capabilities. - Workspace service reachability and environment reachability are separate checks. +The task-only setup path uses a Core-issued expiring probe identity and a real, +read-only Runner hello. The plugin checks project authentication before provider startup. +[Setup probe coverage](../../server/src/services/native-runtime/plugin-runner-probe.test.ts) +uses the real Runner and a synthetic Codex provider through loopback PRP; TLS +termination is replaced by the fixture. It does not qualify a live plugin provider. + +The plugin prepares execution-host resources and the provider descriptor. Core validates the result, then supplies task identity and PRP launch limits. + ### Storybook verification #### `configuration` @@ -135,3 +154,8 @@ Scope: ui Storybook: Custom-image setup/capture/progress, terminal connect/disconnect/failure, unsupported capability and active-image drift. Production: Execute owned provider setup/capture/probe/terminal transport with correct saved image identity, authority, cleanup and shutdown; terminal screenshot cannot prove operation. Sources: Source: [CompanyEnvironments.tsx](../../ui/src/pages/CompanyEnvironments.tsx), [environment-custom-images.ts](../../server/src/services/environment-custom-images.ts). Tests: [CompanyEnvironments.test.tsx](../../ui/src/pages/CompanyEnvironments.test.tsx), [environment-custom-images-service.test.ts](../../server/src/__tests__/environment-custom-images-service.test.ts) + +Managed account transfer requires `projectAuthCheckVersion: 1` and a successful +`check_project_auth` result for the exact prepared configuration, provider, and +project set. New preparation removes old validation. Core checks the saved +validation again in the dispatch update. diff --git a/packages/plugins/sdk/src/types.ts b/packages/plugins/sdk/src/types.ts index a81b7b56d99..7fd9ce0cbbc 100644 --- a/packages/plugins/sdk/src/types.ts +++ b/packages/plugins/sdk/src/types.ts @@ -1720,6 +1720,8 @@ export interface PluginAgentRuntimeState { agent: Agent; instructionsRootPath: string | null; instructionsEntryFile: string; + /** Default entry content when the agent has no instruction directory. */ + instructionsEntryContent?: string; instanceSecrets: Record; } diff --git a/packages/shared/src/app-definitions-url.test.ts b/packages/shared/src/app-definitions-url.test.ts index 7a8e327ceae..4070f908512 100644 --- a/packages/shared/src/app-definitions-url.test.ts +++ b/packages/shared/src/app-definitions-url.test.ts @@ -53,6 +53,13 @@ describe("tool app gallery URL matching", () => { } }); + it("uses Notebook's canonical endpoint without per-instance configuration", () => { + const notebook = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "paperclip-notebook"); + expect(notebook?.availability?.available).toBe(true); + expect(notebook?.methods[0]?.defaults?.serverUrl).toBe("https://notebook.paperclip.app/mcp"); + expect(getAppDefinitionForUrl("https://notebook.paperclip.app/p/page-id")?.slug).toBe("paperclip-notebook"); + }); + it("offers region and custom model connections without claiming arbitrary URLs", () => { for (const slug of ["bedrock", "responses-api", "messages-api", "chat-completions-api", "local"]) { const app = CONNECTABLE_APP_DEFINITIONS.find(app => app.slug === slug); diff --git a/packages/shared/src/app-definitions.generated.ts b/packages/shared/src/app-definitions.generated.ts index 467412e6ea1..cc57bf2b612 100644 --- a/packages/shared/src/app-definitions.generated.ts +++ b/packages/shared/src/app-definitions.generated.ts @@ -102,6 +102,7 @@ import a100 from "./app-definitions/chat-completions-api.json" with { type: "jso import a101 from "./app-definitions/local.json" with { type: "json" }; import a102 from "./app-definitions/daytona.json" with { type: "json" }; import a103 from "./app-definitions/monday.json" with { type: "json" }; -import a104 from "./app-definitions/speko.json" with { type: "json" }; +import a104 from "./app-definitions/paperclip-notebook.json" with { type: "json" }; +import a105 from "./app-definitions/speko.json" with { type: "json" }; import type { AppDefinition } from "./types/app-definition.js"; -export const APP_DEFINITIONS=[a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19,a20,a21,a22,a23,a24,a25,a26,a27,a28,a29,a30,a31,a32,a33,a34,a35,a36,a37,a38,a39,a40,a41,a42,a43,a44,a45,a46,a47,a48,a49,a50,a51,a52,a53,a54,a55,a56,a57,a58,a59,a60,a61,a62,a63,a64,a65,a66,a67,a68,a69,a70,a71,a72,a73,a74,a75,a76,a77,a78,a79,a80,a81,a82,a83,a84,a85,a86,a87,a88,a89,a90,a91,a92,a93,a94,a95,a96,a97,a98,a99,a100,a101,a102,a103,a104] as AppDefinition[]; +export const APP_DEFINITIONS=[a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19,a20,a21,a22,a23,a24,a25,a26,a27,a28,a29,a30,a31,a32,a33,a34,a35,a36,a37,a38,a39,a40,a41,a42,a43,a44,a45,a46,a47,a48,a49,a50,a51,a52,a53,a54,a55,a56,a57,a58,a59,a60,a61,a62,a63,a64,a65,a66,a67,a68,a69,a70,a71,a72,a73,a74,a75,a76,a77,a78,a79,a80,a81,a82,a83,a84,a85,a86,a87,a88,a89,a90,a91,a92,a93,a94,a95,a96,a97,a98,a99,a100,a101,a102,a103,a104,a105] as AppDefinition[]; diff --git a/packages/shared/src/app-definitions.test.ts b/packages/shared/src/app-definitions.test.ts index a7c6bf0cc7d..7629aadb6e0 100644 --- a/packages/shared/src/app-definitions.test.ts +++ b/packages/shared/src/app-definitions.test.ts @@ -32,6 +32,7 @@ import { appDefinitionSchema, appDefinitionsSchema } from "./validators/app-defi const googleScope = (scope: string) => `https://www.googleapis.com/auth/${scope}`; + const GOOGLE_WORKSPACE_PROFILE_EXPECTATIONS = [ { profile: "gmail.read", @@ -885,7 +886,7 @@ describe("AppDefinition catalog", () => { "ticktick", "xero", ]); - expect(APP_STORE_DEFINITIONS).toHaveLength(82); + expect(APP_STORE_DEFINITIONS).toHaveLength(83); const connectableSlugs = new Set( CONNECTABLE_APP_DEFINITIONS.map((entry) => entry.slug), ); diff --git a/packages/shared/src/app-definitions.ts b/packages/shared/src/app-definitions.ts index d88f8509157..4f325b202e3 100644 --- a/packages/shared/src/app-definitions.ts +++ b/packages/shared/src/app-definitions.ts @@ -18,6 +18,7 @@ export const CONNECTABLE_APP_SLUGS = new Set([ "executor", "slack", "notion", + "paperclip-notebook", "railway", "posthog", "linear", diff --git a/packages/shared/src/app-definitions/paperclip-notebook.json b/packages/shared/src/app-definitions/paperclip-notebook.json new file mode 100644 index 00000000000..4bdf194d6ba --- /dev/null +++ b/packages/shared/src/app-definitions/paperclip-notebook.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 1, + "slug": "paperclip-notebook", + "experimentalFeatureKey": "enablePaperclipNotebook", + "name": "Paperclip Notebook", + "description": "Publish, find, and share reports, files, and interactive pages with your Paperclip Cloud account.", + "categories": [ + "content", + "productivity" + ], + "featured": true, + "branding": { + "logoUrl": "/brands/apps/paperclip-notebook.svg", + "darkLogoUrl": "/brands/apps/paperclip-notebook-dark.svg" + }, + "urlPatterns": [ + "https://notebook.paperclip.app/*" + ], + "availability": { + "available": true + }, + "methods": [ + { + "key": "mcp-oauth", + "transport": "mcp_remote", + "auth": "oauth", + "ownershipModes": [ + "dcr" + ], + "oauthClientRegistration": "dcr", + "grantKinds": [ + "user" + ], + "whenToUse": "Sign in with Paperclip Cloud and let your assigned agents publish and manage your Notebook pages.", + "defaults": { + "serverUrl": "https://notebook.paperclip.app/mcp", + "scopesHint": [ + "notebook:read", + "notebook:write", + "notebook:share", + "notebook:delete" + ], + "mcpSessionRequired": true + }, + "guidanceMd": "Connect your Paperclip Cloud account. Assigned agents can read available pages, publish and update your pages, change sharing including making pages public, manage versions, and delete or restore your pages. Manage individual actions in Permissions.", + "riskTier": "S3" + } + ], + "redirectConstraints": "https-or-loopback-http" +} diff --git a/packages/shared/src/connector-availability.test.ts b/packages/shared/src/connector-availability.test.ts new file mode 100644 index 00000000000..a1e17eb4072 --- /dev/null +++ b/packages/shared/src/connector-availability.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { getConnectableAppDefinition } from "./app-definitions.js"; +import { isConnectorFeatureEnabled } from "./connector-availability.js"; +import { appDefinitionSchema } from "./validators/app-definition.js"; + +describe("connector availability", () => { + const ordinary = getConnectableAppDefinition("notion")!; + const gated = { ...ordinary, slug: "private-research", name: "Private Research", experimentalFeatureKey: "enableCases" as const }; + + it("honors feature metadata for an arbitrary connector and leaves ungated connectors alone", () => { + for (const settings of [undefined, null, {}, { enableCases: false }]) { + expect(isConnectorFeatureEnabled(gated, settings)).toBe(false); + expect(isConnectorFeatureEnabled(ordinary, settings)).toBe(true); + } + expect(isConnectorFeatureEnabled(gated, { enablePaperclipNotebook: true })).toBe(false); + expect(isConnectorFeatureEnabled(gated, { enableCases: true })).toBe(true); + }); + + it("validates the feature key and declares Notebook's hidden rollout in its manifest", () => { + expect(appDefinitionSchema.parse(gated).experimentalFeatureKey).toBe("enableCases"); + expect(appDefinitionSchema.safeParse({ ...gated, experimentalFeatureKey: "typoFeature" }).success).toBe(false); + const notebook = getConnectableAppDefinition("paperclip-notebook")!; + expect(notebook.experimentalFeatureKey).toBe("enablePaperclipNotebook"); + expect(isConnectorFeatureEnabled(notebook, {})).toBe(false); + expect(isConnectorFeatureEnabled(notebook, { enablePaperclipNotebook: true })).toBe(true); + }); +}); diff --git a/packages/shared/src/connector-availability.ts b/packages/shared/src/connector-availability.ts new file mode 100644 index 00000000000..1e1b5c0a340 --- /dev/null +++ b/packages/shared/src/connector-availability.ts @@ -0,0 +1,9 @@ +import type { AppDefinition } from "./types/app-definition.js"; +import type { InstanceExperimentalSettings } from "./types/instance.js"; + +export function isConnectorFeatureEnabled( + app: Pick | null | undefined, + settings: Partial | null | undefined, +): boolean { + return !app?.experimentalFeatureKey || settings?.[app.experimentalFeatureKey] === true; +} diff --git a/packages/shared/src/connector-identity.test.ts b/packages/shared/src/connector-identity.test.ts new file mode 100644 index 00000000000..d58357707fa --- /dev/null +++ b/packages/shared/src/connector-identity.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { getAppDefinitionForApplicationKey } from "./connector-identity.js"; + +describe("saved connector identity", () => { + it("resolves known catalog keys and company-scoped gallery keys", () => { + expect(getAppDefinitionForApplicationKey("fireflies")?.slug).toBe("fireflies"); + expect(getAppDefinitionForApplicationKey("app-gallery:fireflies:44444444-4444-4444-8444-444444444444")?.slug).toBe("fireflies"); + }); + + it.each([null, undefined, {}, "custom-mcp", "app-gallery:unknown:44444444-4444-4444-8444-444444444444", "app-gallery:fireflies:anything", "app-gallery:fireflies:44444444-4444-4444-8444-444444444444:extra", "fireflies.example.test"])("does not infer a catalog identity from %j", value => { + expect(getAppDefinitionForApplicationKey(value)).toBeNull(); + }); +}); diff --git a/packages/shared/src/connector-identity.ts b/packages/shared/src/connector-identity.ts new file mode 100644 index 00000000000..5ede83c9c52 --- /dev/null +++ b/packages/shared/src/connector-identity.ts @@ -0,0 +1,8 @@ +import { getConnectableAppDefinition } from "./app-definitions.js"; +import type { AppDefinition } from "./types/app-definition.js"; + +export function getAppDefinitionForApplicationKey(key: unknown): AppDefinition | null { + if (typeof key !== "string") return null; + const gallery = /^app-gallery:([a-z0-9]+(?:-[a-z0-9]+)*):[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.exec(key); + return getConnectableAppDefinition(gallery ? gallery[1].toLowerCase() : key); +} diff --git a/packages/shared/src/environment-support.test.ts b/packages/shared/src/environment-support.test.ts index 0b462267ba9..f0bc2520cd5 100644 --- a/packages/shared/src/environment-support.test.ts +++ b/packages/shared/src/environment-support.test.ts @@ -13,6 +13,7 @@ describe("isSandboxProviderSupportedForAdapter", () => { "local", "ssh", "sandbox", + "plugin", ]); }); diff --git a/packages/shared/src/environment-support.ts b/packages/shared/src/environment-support.ts index ab10a20e172..3a61f86a51b 100644 --- a/packages/shared/src/environment-support.ts +++ b/packages/shared/src/environment-support.ts @@ -81,6 +81,7 @@ export function adapterSupportsRemoteManagedEnvironments(adapterType: string): b } export function supportedEnvironmentDriversForAdapter(adapterType: string): EnvironmentDriver[] { + if (adapterType === "paperclip_runner") return ["local", "ssh", "sandbox", "plugin"]; return adapterSupportsRemoteManagedEnvironments(adapterType) ? ["local", "ssh", "sandbox"] : ["local"]; @@ -194,7 +195,7 @@ export function getEnvironmentCapabilities( local: "supported", ssh: "supported", sandbox: "supported", - plugin: "unsupported", + plugin: adapterTypes.includes("paperclip_runner") ? "supported" : "unsupported", }, sandboxProviders, }; diff --git a/packages/shared/src/feature-catalog.test.ts b/packages/shared/src/feature-catalog.test.ts index 5b63525b553..d3e000c0394 100644 --- a/packages/shared/src/feature-catalog.test.ts +++ b/packages/shared/src/feature-catalog.test.ts @@ -30,6 +30,12 @@ function schemaBooleanFlagKeys(): string[] { } describe("INSTANCE_FEATURE_CATALOG", () => { + it("keeps Notebook a managed rollout disabled for Cloud and self-hosted instances", () => { + expect(INSTANCE_FEATURE_CATALOG.enablePaperclipNotebook).toMatchObject({ + tier: "managed", cloudDefault: false, selfHostedDefault: false, + }); + expect(buildFeatureCatalogArtifact("2026.1020.0").features.enablePaperclipNotebook).toEqual({ tier: "managed" }); + }); it("covers exactly the boolean flag keys of the experimental settings schema", () => { expect([...INSTANCE_FEATURE_KEYS]).toEqual(schemaBooleanFlagKeys()); }); diff --git a/packages/shared/src/feature-catalog.ts b/packages/shared/src/feature-catalog.ts index 583aae340d1..d66bbd9cacc 100644 --- a/packages/shared/src/feature-catalog.ts +++ b/packages/shared/src/feature-catalog.ts @@ -161,6 +161,13 @@ export const INSTANCE_FEATURE_CATALOG: Record; transport?:{location:"query"|"header";name:string;format?:"string"|"csv"|"boolean";omitFalse?:boolean} } export interface ConnectionMethodDef { ai?: import("../ai-connections.js").AiConnectionMetadata; key:string; label?:string; purpose?:ToolConnectionPurpose; provider?:"slack"|"github"|"discord"|"microsoft-teams"|"telegram"|"agentmail"|"imessage-photon"|"speko"; transport:ToolConnectionTransport; auth:"oauth"|"api_key"|"none"; oauthStrategy?:"paperclip_cloud_connector"|"paperclip_id_connector"; connectorProfile?:string; oauthClientSecretRequired?:boolean; oauthClientRegistration?:"dcr"; capabilityProfile?:{key:string;label:string;description?:string}; grantKinds?:ConnectionGrantKind[]; ownershipModes:ToolConnectionOwnership[]; whenToUse:string; defaults?:{serverUrl?:string;serverUrlTemplate?:string;discoveryUrl?:string|null;serviceHost?:string;templateKey?:string;authorizationEndpoint?:string;tokenEndpoint?:string;metadataUrl?:string;scopesHint?:string[];oauthAuthorizationParams?:{access_type?:"offline";prompt?:"consent"};toolArgumentDefaults?:Record;mcpSessionRequired?:boolean}; tenantFields?:FieldDef[]; extensionFields?:FieldDef[]; configRequirements?:{atLeastOneOf?:string[]}; credentialFields?:FieldDef[]; keyPlacement?:{location:"header"|"query"|"body_json"|"env";name:string;prefix?:string|null}; credentialSources?:{vercelConnect?:{services:string[];principalModes:VercelConnectPrincipalMode[];scopes:string[];header:{name:string;prefix?:string|null}}}; guidanceMd:string; consoleLinks?:{register?:string;keys?:string;settings?:string;docs?:string}; warnings?:string[]; variants?:Array<{key:string;label:string;whenToUse:string;tenantFields?:FieldDef[]}>; riskTier:"S1"|"S2"|"S3"|"S4"; requiredResourceFilters?:string[] } -export interface AppDefinition { aiConnectionRouter?: { pluginKey: string }; agentInstructions?: import("../connection-instructions.js").ConnectionInstructionTemplate; schemaVersion:1; slug:string; name:string; description:string; categories:AppCategory[]; tags?:string[]; featured?:boolean; branding:{logoUrl:string;darkLogoUrl?:string;backgroundColor?:string;accentColor?:string}; urlPatterns:string[]; docsUrl?:string; setupPrerequisite?:{title:string;description:string;steps?:string[];actionLabel:string;actionUrl:string}; redirectConstraints?:OAuthRedirectConstraints; methods:ConnectionMethodDef[]; suggestable?:boolean; availability?:{available:boolean;reason?:string;robotEmail?:string}; ownershipAvailability?:Partial> } +export interface AppDefinition { experimentalFeatureKey?: import("../feature-catalog.js").InstanceFeatureKey; aiConnectionRouter?: { pluginKey: string }; agentInstructions?: import("../connection-instructions.js").ConnectionInstructionTemplate; schemaVersion:1; slug:string; name:string; description:string; categories:AppCategory[]; tags?:string[]; featured?:boolean; branding:{logoUrl:string;darkLogoUrl?:string;backgroundColor?:string;accentColor?:string}; urlPatterns:string[]; docsUrl?:string; setupPrerequisite?:{title:string;description:string;steps?:string[];actionLabel:string;actionUrl:string}; redirectConstraints?:OAuthRedirectConstraints; methods:ConnectionMethodDef[]; suggestable?:boolean; availability?:{available:boolean;reason?:string;robotEmail?:string}; ownershipAvailability?:Partial> } export type SelfServeMcpAuthMode = | "dcr" diff --git a/packages/shared/src/types/instance.ts b/packages/shared/src/types/instance.ts index c41e1e26f85..916ff72b8eb 100644 --- a/packages/shared/src/types/instance.ts +++ b/packages/shared/src/types/instance.ts @@ -84,6 +84,7 @@ export interface InstanceExperimentalSettings { enableMcpAggregators: boolean; /** Show experimental memory connection setup. Existing connections remain usable. */ enableMemoryConnectors: boolean; + enablePaperclipNotebook: boolean; enableCases: boolean; enableAgentChat: boolean; /** Reorganized left nav, Inbox-as-Tasks-views, and the chat agent rail + task cards. */ diff --git a/packages/shared/src/validators/app-definition.ts b/packages/shared/src/validators/app-definition.ts index cbb0879cbbc..9fb660249ea 100644 --- a/packages/shared/src/validators/app-definition.ts +++ b/packages/shared/src/validators/app-definition.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { INSTANCE_FEATURE_KEYS } from "../feature-catalog.js"; import { connectionInstructionTemplateSchema } from "../connection-instructions.js"; import { AI_CONNECTION_CAPABILITIES, aiConnectionMetadataSchema } from "../ai-connections.js"; import { connectionGrantKindSchema, toolConnectionOwnershipSchema, toolConnectionPurposeSchema, toolConnectionTransportSchema } from "./tool-access.js"; @@ -8,7 +9,7 @@ const appBrandAssetUrlSchema=z.string().refine((value)=>{ },{message:"Brand assets must be HTTPS URLs or local /brands/apps SVG/PNG paths"}); const field=z.object({key:z.string().min(1),label:z.string().min(1),type:z.enum(["text","password","textarea","datetime","select","checkbox"]),required:z.boolean().optional(),advanced:z.boolean().optional(),hidden:z.boolean().optional(),placeholder:z.string().optional(),helperMd:z.string().optional(),secret:z.boolean().optional(),prefix:z.string().optional(),defaultValue:z.union([z.string(),z.boolean()]).optional(),validation:z.object({pattern:z.string().optional(),maxLength:z.number().int().positive().optional()}).optional(),options:z.array(z.object({value:z.string(),label:z.string()})).optional(),transport:z.object({location:z.enum(["query","header"]),name:z.string().min(1),format:z.enum(["string","csv","boolean"]).optional(),omitFalse:z.boolean().optional()}).optional()}).superRefine((v,c)=>{if(v.required&&v.type!=="checkbox"&&!v.placeholder)c.addIssue({code:"custom",message:"Required fields need placeholders",path:["placeholder"]});if(v.type==="select"&&(!v.options||v.options.length===0))c.addIssue({code:"custom",message:"Select fields need options",path:["options"]});if(v.hidden&&v.defaultValue===undefined)c.addIssue({code:"custom",message:"Hidden fields need defaults",path:["defaultValue"]})}); export const connectionMethodDefSchema=z.object({key:z.string().min(1),label:z.string().min(1).optional(),purpose:z.union([toolConnectionPurposeSchema,z.literal("ai")]).optional(),provider:z.enum(["slack","github","discord","microsoft-teams","telegram","speko","agentmail","imessage-photon"]).optional(),transport:z.union([toolConnectionTransportSchema,z.literal("runtime_auth")]),ai:aiConnectionMetadataSchema.optional(),auth:z.enum(["oauth","api_key","none"]),oauthStrategy:z.enum(["paperclip_cloud_connector","paperclip_id_connector"]).optional(),connectorProfile:z.string().regex(/^[a-z0-9]+(?:[.-][a-z0-9]+)*$/).optional(),oauthClientSecretRequired:z.boolean().optional(),oauthClientRegistration:z.literal("dcr").optional(),capabilityProfile:z.object({key:z.string().min(1),label:z.string().min(1),description:z.string().min(1).optional()}).optional(),grantKinds:z.array(connectionGrantKindSchema).min(1).optional(),ownershipModes:z.array(toolConnectionOwnershipSchema).min(1),whenToUse:z.string().min(1),defaults:z.object({serverUrl:z.string().url().optional(),serverUrlTemplate:z.string().regex(/^https:\/\//).optional(),discoveryUrl:z.string().url().nullable().optional(),serviceHost:z.string().optional(),templateKey:z.string().optional(),authorizationEndpoint:z.string().url().optional(),tokenEndpoint:z.string().url().optional(),metadataUrl:z.string().url().optional(),scopesHint:z.array(z.string()).optional(),oauthAuthorizationParams:z.object({access_type:z.literal("offline").optional(),prompt:z.literal("consent").optional()}).optional(),toolArgumentDefaults:z.record(z.string(),z.unknown()).optional(),mcpSessionRequired:z.boolean().optional()}).optional(),tenantFields:z.array(field).optional(),extensionFields:z.array(field).optional(),configRequirements:z.object({atLeastOneOf:z.array(z.string().min(1)).min(1).optional()}).optional(),credentialFields:z.array(field).optional(),keyPlacement:z.object({location:z.enum(["header","query","body_json","env"]),name:z.string().min(1),prefix:z.string().nullable().optional()}).optional(),credentialSources:z.object({vercelConnect:z.object({services:z.array(z.string().min(1)).min(1),principalModes:z.array(z.enum(["app","user"])).min(1),scopes:z.array(z.string().min(1)).min(1),header:z.object({name:z.string().min(1),prefix:z.string().nullable().optional()})}).optional()}).optional(),guidanceMd:z.string().min(1),consoleLinks:z.object({register:z.string().url().optional(),keys:z.string().url().optional(),settings:z.string().url().optional(),docs:z.string().url().optional()}).optional(),warnings:z.array(z.string()).optional(),variants:z.array(z.object({key:z.string(),label:z.string(),whenToUse:z.string(),tenantFields:z.array(field).optional()})).optional(),riskTier:z.enum(["S1","S2","S3","S4"]),requiredResourceFilters:z.array(z.string()).optional()}).superRefine((v,c)=>{if((v.transport==="runtime_auth")!==Boolean(v.ai))c.addIssue({code:"custom",message:"Runtime authentication requires AI metadata and AI metadata requires runtime_auth",path:["ai"]});if(v.ai&&!AI_CONNECTION_CAPABILITIES[v.ai.provider].methods[v.ai.method])c.addIssue({code:"custom",message:"Unsupported AI sign-in method",path:["ai","method"]});if(v.ai&&v.auth!==(v.ai.method==="subscription"?"oauth":"api_key"))c.addIssue({code:"custom",message:"AI sign-in method must match authentication",path:["auth"]});if(v.oauthClientRegistration&&(v.auth!=="oauth"||v.transport!=="mcp_remote"||!v.ownershipModes.includes("dcr")))c.addIssue({code:"custom",message:"DCR client registration requires a remote MCP OAuth method with DCR ownership",path:["oauthClientRegistration"]});if(v.defaults?.mcpSessionRequired&&v.transport!=="mcp_remote")c.addIssue({code:"custom",message:"MCP session initialization preferences require remote MCP transport",path:["defaults","mcpSessionRequired"]});const purpose=v.purpose??"tool";if((purpose==="ai")!==(v.transport==="runtime_auth"))c.addIssue({code:"custom",message:"AI methods require runtime_auth and runtime_auth requires AI purpose",path:["purpose"]});if(["chat_sdk","voice"].includes(v.transport)&&purpose!=="channel")c.addIssue({code:"custom",message:"Chat SDK methods must be channel connections",path:["purpose"]});if(purpose==="channel"&&!["chat_sdk","voice"].includes(v.transport)&&!(v.provider==="agentmail"&&v.transport==="rest_api"))c.addIssue({code:"custom",message:"Channel connections must use the Chat SDK transport",path:["transport"]});if(purpose==="channel"&&((v.provider==="speko")!==(v.transport==="voice")))c.addIssue({code:"custom",message:"Speko channels require the voice transport",path:["transport"]});if(purpose==="channel"&&!v.provider)c.addIssue({code:"custom",message:"Channel connections require a chat provider",path:["provider"]});if(v.auth==="api_key"&&!v.keyPlacement&&purpose!=="channel")c.addIssue({code:"custom",message:"API-key tool methods require keyPlacement",path:["keyPlacement"]});if(v.oauthStrategy&&v.auth!=="oauth")c.addIssue({code:"custom",message:"OAuth strategies require OAuth auth",path:["oauthStrategy"]});if(v.oauthStrategy&&!v.connectorProfile)c.addIssue({code:"custom",message:"Paperclip Cloud connector methods require connectorProfile",path:["connectorProfile"]});if(v.connectorProfile&&!v.oauthStrategy)c.addIssue({code:"custom",message:"connectorProfile requires a Paperclip Cloud OAuth strategy",path:["connectorProfile"]});if(v.credentialSources?.vercelConnect&&(v.transport!=="mcp_remote"||v.auth==="none"))c.addIssue({code:"custom",message:"Vercel Connect requires an authenticated remote MCP method",path:["credentialSources","vercelConnect"]});const keys=new Set([...(v.tenantFields??[]),...(v.extensionFields??[])].map((entry)=>entry.key));for(const key of v.configRequirements?.atLeastOneOf??[])if(!keys.has(key))c.addIssue({code:"custom",message:"Config requirement references an unknown field",path:["configRequirements","atLeastOneOf"]});if(v.defaults?.serverUrl&&v.defaults.serverUrlTemplate)c.addIssue({code:"custom",message:"Use either serverUrl or serverUrlTemplate",path:["defaults"]});for(const placeholder of v.defaults?.serverUrlTemplate?.matchAll(/\{([a-zA-Z0-9_-]+)\}/g)??[])if(!keys.has(placeholder[1]))c.addIssue({code:"custom",message:"Server URL template references an unknown field",path:["defaults","serverUrlTemplate"]})}); -export const appDefinitionSchema=z.object({agentInstructions:connectionInstructionTemplateSchema.optional(),aiConnectionRouter:z.object({pluginKey:z.string().min(1)}).optional(),schemaVersion:z.literal(1),slug:z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/),name:z.string().min(1),description:z.string().min(1),categories:z.array(z.enum(["ai","analytics","commerce","communication","content","data","developer","productivity","other"])).min(1),tags:z.array(z.string().min(1)).optional(),featured:z.boolean().optional(),branding:z.object({logoUrl:appBrandAssetUrlSchema,darkLogoUrl:appBrandAssetUrlSchema.optional(),backgroundColor:z.string().optional(),accentColor:z.string().optional()}),urlPatterns:z.array(z.string()),docsUrl:z.string().url().optional(),setupPrerequisite:z.object({title:z.string().min(1),description:z.string().min(1),steps:z.array(z.string().min(1)).min(1).optional(),actionLabel:z.string().min(1),actionUrl:z.string().url()} ).optional(),redirectConstraints:z.enum(["https-or-loopback-http"]).optional(),methods:z.array(connectionMethodDefSchema),suggestable:z.boolean().optional(),availability:z.object({available:z.boolean(),reason:z.string().optional(),robotEmail:z.string().optional()}).optional(),ownershipAvailability:z.object({platform_shared:z.boolean().optional(),platform_provisioned:z.boolean().optional(),customer:z.boolean().optional(),dcr:z.boolean().optional()}).optional()}).superRefine((value, ctx) => { +export const appDefinitionSchema=z.object({experimentalFeatureKey:z.enum(INSTANCE_FEATURE_KEYS).optional(),agentInstructions:connectionInstructionTemplateSchema.optional(),aiConnectionRouter:z.object({pluginKey:z.string().min(1)}).optional(),schemaVersion:z.literal(1),slug:z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/),name:z.string().min(1),description:z.string().min(1),categories:z.array(z.enum(["ai","analytics","commerce","communication","content","data","developer","productivity","other"])).min(1),tags:z.array(z.string().min(1)).optional(),featured:z.boolean().optional(),branding:z.object({logoUrl:appBrandAssetUrlSchema,darkLogoUrl:appBrandAssetUrlSchema.optional(),backgroundColor:z.string().optional(),accentColor:z.string().optional()}),urlPatterns:z.array(z.string()),docsUrl:z.string().url().optional(),setupPrerequisite:z.object({title:z.string().min(1),description:z.string().min(1),steps:z.array(z.string().min(1)).min(1).optional(),actionLabel:z.string().min(1),actionUrl:z.string().url()} ).optional(),redirectConstraints:z.enum(["https-or-loopback-http"]).optional(),methods:z.array(connectionMethodDefSchema),suggestable:z.boolean().optional(),availability:z.object({available:z.boolean(),reason:z.string().optional(),robotEmail:z.string().optional()}).optional(),ownershipAvailability:z.object({platform_shared:z.boolean().optional(),platform_provisioned:z.boolean().optional(),customer:z.boolean().optional(),dcr:z.boolean().optional()}).optional()}).superRefine((value, ctx) => { if (value.aiConnectionRouter) { if (value.methods.length !== 0) ctx.addIssue({ code: "custom", message: "Pooled connectors use existing connections, not authentication methods", path: ["methods"] }); if (!value.categories.includes("ai")) ctx.addIssue({ code: "custom", message: "Pooled connectors must use the AI category", path: ["categories"] }); diff --git a/packages/shared/src/validators/instance.test.ts b/packages/shared/src/validators/instance.test.ts index 1a8eca32c6c..f1db216ab7b 100644 --- a/packages/shared/src/validators/instance.test.ts +++ b/packages/shared/src/validators/instance.test.ts @@ -2,9 +2,21 @@ import { describe, expect, it } from "vitest"; import { instanceExperimentalSettingsSchema, patchInstanceExperimentalSettingsSchema, + instanceExperimentalSettingsWithManagedSchema, } from "./instance.js"; describe("instance experimental settings validators", () => { + it("keeps Notebook disabled by default and accepts only explicit boolean rollout values", () => { + expect(instanceExperimentalSettingsSchema.parse({ enableApps: true, enableMemoryConnectors: true }).enablePaperclipNotebook).toBe(false); + expect(patchInstanceExperimentalSettingsSchema.parse({})).toEqual({}); + for (const enabled of [true, false]) { + expect(patchInstanceExperimentalSettingsSchema.parse({ enablePaperclipNotebook: enabled })).toEqual({ enablePaperclipNotebook: enabled }); + expect(instanceExperimentalSettingsWithManagedSchema.parse({ enablePaperclipNotebook: enabled, + managedKeys: { enablePaperclipNotebook: { managed: true, managedBy: "paperclip-cloud" } }, + }).enablePaperclipNotebook).toBe(enabled); + } + expect(patchInstanceExperimentalSettingsSchema.safeParse({ enablePaperclipNotebook: "true" }).success).toBe(false); + }); it("defaults GitHub review bots off independently of chat and accepts explicit changes", () => { expect(instanceExperimentalSettingsSchema.parse({ enableChatConnectors: true }).enableGitHubReviewBots).toBe(false); expect(patchInstanceExperimentalSettingsSchema.parse({ enableGitHubReviewBots: true })).toEqual({ enableGitHubReviewBots: true }); diff --git a/packages/shared/src/validators/instance.ts b/packages/shared/src/validators/instance.ts index 1512366e6bb..66298470985 100644 --- a/packages/shared/src/validators/instance.ts +++ b/packages/shared/src/validators/instance.ts @@ -60,6 +60,7 @@ export const instanceExperimentalSettingsSchema = z.object({ // Compatibility only: old stored and managed values must still parse. enableMcpAggregators: z.boolean().default(true), enableMemoryConnectors: z.boolean().default(false), + enablePaperclipNotebook: z.boolean().default(false), enableCases: z.boolean().default(false), enableAgentChat: z.boolean().default(false), enableCombinedInboxTasks: z.boolean().default(false), diff --git a/scripts/app-definition-overrides/paperclip-notebook.json b/scripts/app-definition-overrides/paperclip-notebook.json new file mode 100644 index 00000000000..4bdf194d6ba --- /dev/null +++ b/scripts/app-definition-overrides/paperclip-notebook.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 1, + "slug": "paperclip-notebook", + "experimentalFeatureKey": "enablePaperclipNotebook", + "name": "Paperclip Notebook", + "description": "Publish, find, and share reports, files, and interactive pages with your Paperclip Cloud account.", + "categories": [ + "content", + "productivity" + ], + "featured": true, + "branding": { + "logoUrl": "/brands/apps/paperclip-notebook.svg", + "darkLogoUrl": "/brands/apps/paperclip-notebook-dark.svg" + }, + "urlPatterns": [ + "https://notebook.paperclip.app/*" + ], + "availability": { + "available": true + }, + "methods": [ + { + "key": "mcp-oauth", + "transport": "mcp_remote", + "auth": "oauth", + "ownershipModes": [ + "dcr" + ], + "oauthClientRegistration": "dcr", + "grantKinds": [ + "user" + ], + "whenToUse": "Sign in with Paperclip Cloud and let your assigned agents publish and manage your Notebook pages.", + "defaults": { + "serverUrl": "https://notebook.paperclip.app/mcp", + "scopesHint": [ + "notebook:read", + "notebook:write", + "notebook:share", + "notebook:delete" + ], + "mcpSessionRequired": true + }, + "guidanceMd": "Connect your Paperclip Cloud account. Assigned agents can read available pages, publish and update your pages, change sharing including making pages public, manage versions, and delete or restore your pages. Manage individual actions in Permissions.", + "riskTier": "S3" + } + ], + "redirectConstraints": "https-or-loopback-http" +} diff --git a/server/src/__tests__/agent-runtime-hooks.test.ts b/server/src/__tests__/agent-runtime-hooks.test.ts index 10a93c5ae64..69fb20f5df8 100644 --- a/server/src/__tests__/agent-runtime-hooks.test.ts +++ b/server/src/__tests__/agent-runtime-hooks.test.ts @@ -155,6 +155,13 @@ describeDatabase("agent runtime synchronization hooks", () => { expect(await fs.readFile(path.join(state!.instructionsRootPath!, state!.instructionsEntryFile), "utf8")).toBe("restored instructions"); }); + it("supplies default instructions when the agent has no instruction directory", async () => { + await db.update(agents).set({ adapterConfig: { promptTemplate: "Fallback task instructions" } }).where(eq(agents.id, agentId)); + expect(await pluginAgentRuntimeState(db, pluginId, companyId, agentId)).toMatchObject({ + instructionsRootPath: null, instructionsEntryFile: "AGENTS.md", instructionsEntryContent: "Fallback task instructions", + }); + }); + it("resolves instructions without reading or hashing bundle contents", async () => { const [agent] = await db.select().from(agents).where(eq(agents.id, agentId)); const root = resolveManagedInstructionsRoot(agent); diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index 69550ec9f3a..183052c5105 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -15,7 +15,7 @@ import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:f import os from "node:os"; import path from "node:path"; import { and, eq, sql } from "drizzle-orm"; -import { createDb, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db"; +import { createDb, companies, agents, agentTaskSessions, heartbeatRuns, companyMemberships, connectionGrants, toolApplications, connectionGrantDelegations, connectionGrantMembers, toolConnections, toolConnectionInstalls, aiConnectionDefaults, aiProviderDefaults, adapterAuthSessions, environments, plugins, issues, issueThreadInteractions, issueRecoveryActions, connectionIntentDeliveries, agentWakeupRequests, companySecrets, companySecretVersions, userSecretDefinitions, principalPermissionGrants } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres"; import { fetchCompanyQuotaWindows } from "../services/quota-windows.js"; import { quotaCredentialRecoveryPath, quotaCredentialRecovery, quotaCredentialHash } from "../services/quota-credential-recovery.js"; @@ -72,6 +72,36 @@ beforeAll(async () => { afterAll(async () => { await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); }); describe("managed AI connections", () => { + it("defers a new plugin agent's Copilot check until its saved-agent lifecycle can acquire the computer", async () => { + const { errorHandler } = await import("../middleware/index.js"); + const owner = "copilot-plugin-bootstrap", pluginId = randomUUID(); + const pluginKey = `test.bootstrap.${pluginId}`; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + await db.insert(plugins).values({ id: pluginId, pluginKey, packageName: "@test/bootstrap", version: "1.0.0", apiVersion: 1, + categories: ["automation"], status: "ready", manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", + displayName: "Bootstrap", description: "Bootstrap fixture", author: "Test", categories: ["automation"], + capabilities: ["environment.drivers.register"], entrypoints: { worker: "worker.js" }, + environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" } }] } }); + const [environment] = await db.insert(environments).values({ companyId, name: "Agent computer", driver: "plugin", + config: { pluginKey, driverKey: "tasks", driverConfig: {} } }).returning(); + const token = "github_pat_fixture_plugin_bootstrap"; + const account = await service.save(companyId, owner, { provider: "github", method: "api_key", ownership: "personal", name: "Bootstrap", apiKey: token, agentIds: [], allAgents: false }, token); + await service.setDefault(companyId, owner, account.grantId); + const probe = vi.spyOn(copilotProbe, "probeCopilotExecutionTarget").mockRejectedValue(new Error("No computer exists before the agent is saved")); + const app = express(); app.use(express.json()); + app.use((req, _res, next) => { req.actor = { type: "board", source: "local_implicit", userId: owner, companyIds: [companyId], isInstanceAdmin: true }; next(); }); + app.use("/api", agentRoutes(db)); app.use(errorHandler); + try { + const response = await request(app).post(`/api/companies/${companyId}/agents`).send({ name: "Plugin bootstrap", role: "general", + adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "copilot", model: "gpt-5.6-luna" }, + defaultEnvironmentId: environment.id, runtimeConfig: { aiConnection: { provider: "github", method: "api_key", mode: "responsible_user" } } }); + expect(response.status, JSON.stringify(response.body)).toBe(201); + expect(response.body.defaultEnvironmentId).toBe(environment.id); + expect(response.body.lifecycleState).toBe("preparing"); + expect(probe).not.toHaveBeenCalled(); + } finally { probe.mockRestore(); } + }); + it.each(["create", "hire", "edit"])("refuses Copilot %s before probing or persisting when forced Kubernetes is missing", async action => { const { errorHandler } = await import("../middleware/index.js"); const owner = `copilot-kubernetes-${action}`, id = randomUUID(); diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index 4807b47d65f..92c241f3970 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -391,6 +391,21 @@ describe("environmentRunOrchestrator — realizeForRun", () => { expect(result.executionTarget).toBeNull(); }); + it("forwards authorized project IDs to task-only providers without staging directories", async () => { + const environment = makeEnvironment("plugin" as Environment["driver"]); + mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "remote", transport: "sandbox", + providerKey: "plugin", remoteCwd: "provider/project", runnerTask: vi.fn() }); + const runtime = makeMockRuntime({ supportsTasks: vi.fn().mockResolvedValue(true) }); + const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime }); + const input = { ...makeRealizeInput({ environment }), adapterType: "paperclip_runner", + runnerTaskProjectIds: ["anchor", "authorized-reference", "anchor"] }; + input.executionWorkspace.projectId = "anchor"; + const result = await orchestrator.realizeForRun(input); + expect(result.executionTarget).toMatchObject({ runnerTaskProjectIds: ["anchor", "authorized-reference"] }); + expect(runtime.realizeWorkspace).not.toHaveBeenCalled(); + expect(runtime.execute).not.toHaveBeenCalled(); + }); + it("persisted metadata is updated on lease and execution workspace after realization", async () => { const persistedExecutionWorkspace = makePersistedExecutionWorkspace(); const updatedLease = makeLease({ diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts new file mode 100644 index 00000000000..9d107b75dd7 --- /dev/null +++ b/server/src/__tests__/execution-control-work.test.ts @@ -0,0 +1,289 @@ +import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { reconcileSafeNativeReplacements } from "../services/native-runtime/native-safe-replacement.js"; +import { localAiLoginService } from "../services/local-ai-login.js"; +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, applyPendingMigrations } from "@paperclipai/db"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js"; +import { executionControlSchedule, registerExecutionControlWork } from "../services/execution-control-work.js"; +import { deliverExecutionStatuses } from "../services/execution-status-delivery.js"; +import { terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; +import { reconcileAbandonedExecutionControl } from "../services/execution-control-reconciliation.js"; +import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; +import { markExecutionReconciliation, deliverReconciledExecutions, settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const coordinators: ReturnType[] = []; +afterEach(async () => { for (const worker of coordinators.splice(0)) await worker.stop(); vi.useRealTimers(); }); +function coordinator(owner: object, canRun = () => true, onError: (error: unknown) => void = vi.fn()) { + const c = createDeliveryWorkCoordinator({ owner, canRun, canReconcile: () => true, onError }); + coordinators.push(c); return c; +} +function fakeOwner() { + const execute = vi.fn(async () => [{ xid: "42" }]); + const owner = installDatabaseWorkSignals({ execute, async transaction(fn: (tx: any) => Promise): Promise { + return fn({ execute, transaction: owner.transaction }); + } }); + return owner; +} + +describe("execution-control lifecycle", () => { + it.each(["startup", "commit", "failure"])("retains %s work through drain without querying while paused", async kind => { + vi.useFakeTimers(); + const owner = fakeOwner(); let draining = kind === "startup"; + const errors = vi.fn(); const c = coordinator(owner, () => !draining, errors); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionStatus, run, next }]).ready; + if (kind !== "startup") { + if (kind === "failure") run.mockRejectedValueOnce(new Error("delivery unavailable")); + else draining = true; + await owner.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.executionStatus)); + await vi.advanceTimersByTimeAsync(15_000); + draining = true; + } + const reads = next.mock.calls.length, runs = run.mock.calls.length; + await vi.advanceTimersByTimeAsync(60_000); + expect(next).toHaveBeenCalledTimes(reads); expect(run).toHaveBeenCalledTimes(runs); + draining = false; + await vi.advanceTimersByTimeAsync(15_000); + expect(run).toHaveBeenCalledTimes(runs + 1); + expect(c.nextWakeAt()).toBeNull(); + }); + it("keeps a change arriving during a sweep and does not overlap it", async () => { + vi.useFakeTimers(); const owner = fakeOwner(); const c = coordinator(owner); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const run = vi.fn(async () => {}); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionStatus, run, next: async () => null }]).ready; + run.mockImplementationOnce(() => gate); + await owner.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.executionStatus)); + await vi.advanceTimersByTimeAsync(15_000); + await owner.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.executionStatus)); + await vi.advanceTimersByTimeAsync(1); + expect(run).toHaveBeenCalledTimes(2); + release(); await vi.advanceTimersByTimeAsync(15_000); + expect(run).toHaveBeenCalledTimes(3); expect(c.nextWakeAt()).toBeNull(); + }); + it("coalesces tenant write bursts without repeatedly scanning global queues", async () => { + vi.useFakeTimers(); const owner = fakeOwner(); const c = coordinator(owner); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionStatus, run, next }]).ready; + for (let i = 0; i < 100; i++) { + await owner.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.executionStatus, `company-${i % 2}`)); + await vi.advanceTimersByTimeAsync(10); + } + expect(run).toHaveBeenCalledTimes(1); expect(next).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(14_000); + expect(run).toHaveBeenCalledTimes(2); expect(next).toHaveBeenCalledTimes(2); + expect(c.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(60_000); + expect(run).toHaveBeenCalledTimes(2); + }); + it.each([false, true])("does not start sweeps or inspect deadlines once server shutdown begins (notification=%s)", async notification => { + vi.useFakeTimers(); const owner = fakeOwner(); const c = coordinator(owner); let stopping = false; + const run = vi.fn(async () => {}), next = vi.fn(async () => Date.now() + 1000); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionStatus, run, next }], () => !stopping).ready; + stopping = true; + if (notification) await owner.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.executionStatus)); + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(1); expect(next).toHaveBeenCalledTimes(1); + await c.stop(); expect(c.nextWakeAt()).toBeNull(); + }); + it("keeps other obligations responsive while one is waiting, and awaits shutdown", async () => { + vi.useFakeTimers(); const owner = fakeOwner(); const c = coordinator(owner); + let release!: () => void; const gate = new Promise(resolve => { release = resolve; }); + const status = vi.fn(async () => {}); + const started = registerExecutionControlWork(c, [ + { queue: DELIVERY_QUEUES.executionReplacement, run: () => gate, next: async () => null }, + { queue: DELIVERY_QUEUES.executionStatus, run: status, next: async () => null }, + ]); + await vi.advanceTimersByTimeAsync(1); expect(status).toHaveBeenCalledTimes(1); + let stopped = false; const stopping = c.stop().then(() => { stopped = true; }); + await vi.advanceTimersByTimeAsync(1); expect(stopped).toBe(false); + release(); await started.ready; await stopping; expect(stopped).toBe(true); + }); +}); + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_EXECUTION_SLEEP_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("execution-control durable work", () => { + let db: ReturnType, observed: typeof db; + let temporary: Awaited> | undefined; + let calls: number, companyId: string, agentId: string, issueId: string; + const errors: unknown[] = []; + beforeAll(async () => { + if (external) await applyPendingMigrations(external); + else temporary = await startEmbeddedPostgresTestDatabase("execution-sleep-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 }); + }, 90000); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + beforeEach(async () => { + await db.execute(sql`TRUNCATE TABLE companies CASCADE`); + companyId = randomUUID(); agentId = randomUUID(); issueId = randomUUID(); calls = 0; errors.length = 0; + await db.insert(companies).values({ id: companyId, name: "Idle execution", issuePrefix: "EXE" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "codex_local" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Execution", status: "in_progress", assigneeAgentId: agentId }); + observed = new Proxy(db, { get(target, key) { + if (!["select", "update", "insert", "delete", "execute", "transaction"].includes(String(key))) return Reflect.get(target, key); + return (...args: unknown[]) => { calls++; return Reflect.apply(Reflect.get(target, key), target, args); }; + } }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); + }); + async function settle() { + await vi.advanceTimersByTimeAsync(15_000); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0), { timeout: 5000 }); + await vi.advanceTimersByTimeAsync(1); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0), { timeout: 5000 }); + expect(errors).toEqual([]); + } + async function quiet(c: ReturnType) { + await settle(); expect(c.nextWakeAt()).toBeNull(); const before = calls; + await vi.advanceTimersByTimeAsync(90_000); expect(calls).toBe(before); + } + async function runRow(status = "running") { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status, runtimeMode: "legacy", contextSnapshot: { issueId } }).returning(); + return run!; + } + it("stops all six pending/deadline scans with empty queues and completed history", async () => { + await runRow("succeeded"); + const schedule = executionControlSchedule(observed); + const c = coordinator(db, undefined, error => { errors.push(error); }); + const topics = [DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionContinuation, + DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.localLoginCleanup]; + const runs = [() => reconcileAbandonedExecutionControl(observed), () => reconcileSafeNativeReplacements(observed), + () => deliverReconciledExecutions(observed, vi.fn(async () => null)), () => deliverExecutionStatuses(observed), + () => settleUnrecoverableExecutions(observed), () => localAiLoginService(observed).reapExpired()]; + await registerExecutionControlWork(c, Object.values(schedule).map((next, i) => ({ queue: topics[i]!, run: runs[i]!, next }))).ready; + await quiet(c); + }); + it("bounds real producer bursts with another company's completed history present", async () => { + const otherCompany = randomUUID(), otherAgent = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Other tenant", issuePrefix: "OTH" }); + await db.insert(agents).values({ id: otherAgent, companyId: otherCompany, name: "Other worker", adapterType: "codex_local" }); + await db.insert(heartbeatRuns).values({ companyId: otherCompany, agentId: otherAgent, status: "succeeded" }); + const run = await runRow(); const publish = vi.fn(); const c = coordinator(db, undefined, error => { errors.push(error); }); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionStatus, + run: () => deliverExecutionStatuses(observed, { publish }), next: executionControlSchedule(observed).status }]).ready; + const before = calls; + for (let i = 0; i < 20; i++) { + await terminalizeLegacyExecution({ db, run, status: "succeeded", fromStatuses: ["running", "succeeded"] }); + await vi.advanceTimersByTimeAsync(10); + } + expect(calls).toBe(before); expect(publish).not.toHaveBeenCalled(); + await settle(); expect(publish).toHaveBeenCalledTimes(1); + expect(publish.mock.calls[0]?.[0]).toMatchObject({ companyId, payload: { runId: run.id } }); + await quiet(c); + }); + it("wakes status delivery through the real producer only after outer commit, then becomes quiet", async () => { + const run = await runRow(); const publish = vi.fn(); const c = coordinator(db, undefined, error => { errors.push(error); }); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionStatus, + run: () => deliverExecutionStatuses(observed, { publish }), next: executionControlSchedule(observed).status }]).ready; + await quiet(c); + await db.transaction(async tx => { + await terminalizeLegacyExecution({ db: tx as unknown as typeof db, run, status: "succeeded", patch: { finishedAt: new Date() } }); + await vi.advanceTimersByTimeAsync(1); expect(publish).not.toHaveBeenCalled(); + }); + await settle(); expect(publish).toHaveBeenCalledTimes(1); + await quiet(c); + const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(saved?.status).toBe("succeeded"); expect(saved?.executionStatusDeliveryId).toBeNull(); + }); + it("rolls back a nested producer without delivering or leaving periodic work", async () => { + const run = await runRow(), publish = vi.fn(); const c = coordinator(db, undefined, error => { errors.push(error); }); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionStatus, + run: () => deliverExecutionStatuses(observed, { publish }), next: executionControlSchedule(observed).status }]).ready; + await expect(db.transaction(async tx => { + await terminalizeLegacyExecution({ db: tx as unknown as typeof db, run, status: "succeeded" }); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(c); expect(publish).not.toHaveBeenCalled(); + }); + it("restores an exact control deadline on restart without querying before it", async () => { + const run = await runRow("succeeded"), due = new Date(Date.now() + 60_000); + await db.update(heartbeatRuns).set({ executionControlDeadlineAt: due }).where(eq(heartbeatRuns.id, run.id)); + const c = coordinator(db, undefined, error => { errors.push(error); }); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionFinalization, + run: () => reconcileAbandonedExecutionControl(observed), next: executionControlSchedule(observed).finalization }]).ready; + expect(c.nextWakeAt()).toBe(due.getTime()); const before = calls; + await vi.advanceTimersByTimeAsync(55_000); expect(calls).toBe(before); + await vi.advanceTimersByTimeAsync(5000); await settle(); await quiet(c); + }); + it("retains replacement retries for an unresolved failed execution, excludes completed/denied history", async () => { + const run = await runRow("failed"); + await db.update(heartbeatRuns).set({ nativeIssueId: issueId }).where(eq(heartbeatRuns.id, run.id)); + await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId, issueId, phase: "terminal_failure", failureCode: "provider_transport_failed" }); + const schedule = executionControlSchedule(observed); + expect(await schedule.replacement()).toBeGreaterThan(Date.now()); + await db.update(nativeRunFinalizations).set({ failureDetail: { replacementDenied: "operator_reconciled" } }).where(eq(nativeRunFinalizations.runId, run.id)); + expect(await schedule.replacement()).toBeNull(); + }); + it("a recovery-action producer wakes automatic disposition and leaves resolved history quiet", async () => { + const run = await runRow("failed"); const c = coordinator(db, undefined, error => { errors.push(error); }); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionDisposition, + run: () => settleUnrecoverableExecutions(observed), next: executionControlSchedule(observed).disposition }]).ready; + await quiet(c); + const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", cause: "execution_finalization_deadline_exceeded", fingerprint: run.id, + returnOwnerAgentId: agentId, evidence: { runId: run.id }, nextAction: "Recover execution" }); + await settle(); + expect(await issueRecoveryActionService(db).getActiveForIssue(companyId, issueId)).toBeNull(); + expect(action.id).toBeTruthy(); await quiet(c); + }); + it("retries a failed broadcast only while its durable marker remains, including after restart", async () => { + const run = await runRow(); + const publish = vi.fn(() => { throw new Error("receiver unavailable"); }); + let c = coordinator(db, undefined, error => { errors.push(error); }); + const duty = { queue: DELIVERY_QUEUES.executionStatus, + run: () => deliverExecutionStatuses(observed, { publish }), next: executionControlSchedule(observed).status }; + await registerExecutionControlWork(c, [duty]).ready; + await terminalizeLegacyExecution({ db, run, status: "succeeded" }); + await settle(); expect(publish).toHaveBeenCalledTimes(1); + const before = calls; + await vi.advanceTimersByTimeAsync(10_000); expect(calls).toBe(before); + await c.stop(); + publish.mockImplementation(() => undefined); + c = coordinator(db, undefined, error => { errors.push(error); }); + await registerExecutionControlWork(c, [duty]).ready; + expect(publish).toHaveBeenCalledTimes(2); await quiet(c); + }); + it("reschedules an earlier deadline and disarms a cancelled deadline", async () => { + const run = await runRow("succeeded"); const c = coordinator(db, undefined, error => { errors.push(error); }); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionFinalization, + run: () => reconcileAbandonedExecutionControl(observed), next: executionControlSchedule(observed).finalization }]).ready; + const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); + for (const at of [due, earlier]) { + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); + } + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + await quiet(c); + }); + it("delivers a reconciled continuation when marking and resolving commit together", async () => { + const run = await runRow("failed"), successor = await runRow("queued"); + const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", cause: "execution_finalization_deadline_exceeded", fingerprint: run.id, + returnOwnerAgentId: agentId, evidence: { runId: run.id }, nextAction: "Recover execution" }); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId, recoveryActionId: action.id, previousRunId: run.id } }) + .where(eq(heartbeatRuns.id, successor.id)); + const wake = vi.fn(async () => successor), c = coordinator(db, undefined, error => { errors.push(error); }); + await registerExecutionControlWork(c, [{ queue: DELIVERY_QUEUES.executionContinuation, + run: () => deliverReconciledExecutions(observed, wake), next: executionControlSchedule(observed).continuation }]).ready; + await db.transaction(async tx => { + await markExecutionReconciliation(tx as unknown as typeof db, action, + { runId: run.id, providerStopped: true, actionOutcome: "not_performed", outcomeEvidence: "Verified the provider stopped without performing an external action." }, "operator"); + await issueRecoveryActionService(tx).resolveActiveForIssue({ companyId, sourceIssueId: issueId, actionId: action.id, status: "resolved", outcome: "restored" }); + await vi.advanceTimersByTimeAsync(1); expect(wake).not.toHaveBeenCalled(); + }); + await settle(); expect(wake).toHaveBeenCalledTimes(1); await quiet(c); + const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, successor.id)); + expect(saved?.retryOfRunId).toBe(run.id); + }); + +}); diff --git a/server/src/__tests__/generic-mcp-connection.test.ts b/server/src/__tests__/generic-mcp-connection.test.ts index fb166deabd0..4e8075c9a1d 100644 --- a/server/src/__tests__/generic-mcp-connection.test.ts +++ b/server/src/__tests__/generic-mcp-connection.test.ts @@ -7,6 +7,9 @@ import pino from "pino"; import request from "supertest"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + applyPendingMigrations, + closeRegisteredClients, activityLog, agents, authUsers, @@ -17,6 +20,7 @@ import { companySecrets, companySecretVersions, createDb, + heartbeatRuns, issueThreadInteractions, issues, principalPermissionGrants, @@ -24,6 +28,9 @@ import { toolAccessAuditEvents, toolCallEvents, toolInvocations, + toolGatewaySessions, + toolMcpGateways, + toolMcpGatewayTokens, toolActionRequests, toolApplications, toolCatalogEntries, @@ -35,13 +42,16 @@ import { toolProfiles, toolRuntimeSlots, } from "@paperclipai/db"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { APP_DEFINITIONS, MCP_CONFIG_HELP_PROMPT } from "@paperclipai/shared"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { toolAccessService } from "../services/tool-access.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import * as instanceSettingsModule from "../services/instance-settings.js"; +import { connectionIntentService } from "../services/connection-intents.js"; import { ComposioApiError, type ComposioClient } from "../services/composio.js"; import { createComposioSessionManager } from "../services/composio-session-manager.js"; import { createToolGatewayService } from "../services/tool-gateway.js"; @@ -52,7 +62,8 @@ import { createHttpLogger } from "../middleware/logger.js"; import { HTTP_LOG_REDACT_PATHS } from "../middleware/http-log-redaction.js"; import { startOAuthRefreshFixture } from "../../../scripts/mcp-fixtures/servers/oauth-refresh-fixture.mjs"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim(); +const embeddedPostgresSupport = externalTestDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; /** @@ -81,6 +92,11 @@ const FIXTURE_TOOLS = [ ]; type FixtureOptions = { + notebookAccount?: { id: string; email: string; emailVerified: boolean }; + contextResults?: Record; + scopes?: string[]; + rotateRefreshTokens?: boolean; + dropFirstRefreshResponse?: boolean; /** How the MCP endpoint authenticates. */ auth?: "public" | "oauth" | "header"; /** For `auth: "header"`, the header the endpoint requires and its value. */ @@ -169,18 +185,23 @@ function installMcpOAuthFixture(options: FixtureOptions = {}) { const requests: FixtureRequest[] = []; const issuedCodes = new Map(); let accessToken: string | null = null; + let refreshToken = "fixture-refresh"; + let refreshResponseDropped = false; + const tokenResponses: { access_token: string; refresh_token: string }[] = []; const tools = options.tools ?? FIXTURE_TOOLS; + const scopes = options.scopes ?? (options.notebookAccount ? ["notebook:read", "notebook:write", "notebook:share", "notebook:delete"] : null); const resourceMetadataUrl = `${MCP_ORIGIN}/.well-known/oauth-protected-resource/mcp`; const authorizationServerMetadata = () => ({ issuer: ISSUER, authorization_endpoint: options.authorizationEndpoint ?? `${ISSUER}/authorize`, token_endpoint: options.tokenEndpoint ?? `${ISSUER}/token`, + ...(options.notebookAccount ? { revocation_endpoint: `${MCP_ORIGIN}/oauth/revoke` } : {}), ...(options.dcr === false ? {} : { registration_endpoint: `${ISSUER}/register` }), ...(options.cimd ? { client_id_metadata_document_supported: true } : {}), code_challenge_methods_supported: ["S256"], token_endpoint_auth_methods_supported: options.tokenEndpointAuthMethods ?? ["none"], - scopes_supported: ["mcp:read", "mcp:write"], + scopes_supported: scopes ?? ["mcp:read", "mcp:write"], }); const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url, init) => { @@ -204,6 +225,16 @@ function installMcpOAuthFixture(options: FixtureOptions = {}) { if (supplied !== options.requiredHeader.value) return unauthorizedMcpResponse(resourceMetadataUrl); } const rpc = parsedBody as Record; + if (options.notebookAccount && rpc.method === "initialize") { + return jsonResponse({ jsonrpc: "2.0", id: rpc.id, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "Notebook fixture", version: "1" } } }); + } + if (options.notebookAccount && rpc.method === "notifications/initialized") return jsonResponse({}, 202); + if (options.notebookAccount && rpc.method === "tools/call" && (rpc.params as Record)?.name === "notebook_get_account") { + return jsonResponse({ jsonrpc: "2.0", id: rpc.id, result: { structuredContent: options.notebookAccount, content: [] } }); + } + if (typeof rpc.method === "string" && options.contextResults && Object.hasOwn(options.contextResults, rpc.method)) { + return jsonResponse({ jsonrpc: "2.0", id: rpc.id, result: options.contextResults[rpc.method] }); + } if (rpc.method === "tools/call") { return jsonResponse({ jsonrpc: "2.0", id: rpc.id, result: { content: [{ type: "text", text: "Fixture meeting data" }], @@ -254,14 +285,24 @@ function installMcpOAuthFixture(options: FixtureOptions = {}) { const issued = issuedCodes.get(body.get("code") ?? ""); if (!issued) return jsonResponse({ error: "invalid_grant" }, 400); } + if (grantType === "refresh_token" && options.rotateRefreshTokens && body.get("refresh_token") !== refreshToken) { + return jsonResponse({ error: "invalid_grant" }, 400); + } accessToken = `fixture-access-${randomUUID()}`; - return jsonResponse({ + if (options.rotateRefreshTokens) refreshToken = `fixture-refresh-${randomUUID()}`; + const response = { access_token: accessToken, - refresh_token: "fixture-refresh", + refresh_token: refreshToken, expires_in: 3600, token_type: "Bearer", - scope: "mcp:read", - }); + scope: scopes?.join(" ") ?? "mcp:read", + }; + tokenResponses.push(response); + if (grantType === "refresh_token" && options.dropFirstRefreshResponse && !refreshResponseDropped) { + refreshResponseDropped = true; + throw new TypeError("Fixture lost the committed rotating refresh response"); + } + return jsonResponse(response); } // 404 rather than throw: discovery legitimately probes several well-known @@ -272,6 +313,7 @@ function installMcpOAuthFixture(options: FixtureOptions = {}) { return { fetchMock, requests, + tokenResponses, /** Pretend the operator approved the consent screen and got a code back. */ issueAuthorizationCode(authorizationUrl: string) { const parsed = new URL(authorizationUrl); @@ -339,15 +381,23 @@ function createRouteApp( describeEmbeddedPostgres("generic remote MCP connections", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; + const restoreCatalogEndpoints: (() => void)[] = []; beforeAll(async () => { + if (externalTestDatabaseUrl) { + db = createDb(externalTestDatabaseUrl); + await applyPendingMigrations(externalTestDatabaseUrl); + return; + } tempDb = await startEmbeddedPostgresTestDatabase("paperclip-generic-mcp-"); db = createDb(tempDb.connectionString); - }, 20_000); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { + for (const restore of restoreCatalogEndpoints.splice(0)) restore(); vi.restoreAllMocks(); vi.unstubAllEnvs(); + await instanceSettingsService(db).updateExperimental({ enablePaperclipNotebook: false }); await db.delete(toolCallEvents); await db.delete(toolInvocations); await db.delete(toolActionRequests); @@ -358,6 +408,9 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { await db.delete(companySecrets); await db.delete(activityLog); await db.delete(toolAccessAuditEvents); + await db.delete(toolGatewaySessions); + await db.delete(toolMcpGatewayTokens); + await db.delete(toolMcpGateways); await db.delete(toolRuntimeSlots); await db.delete(toolConnectionInstalls); await db.delete(toolProfileBindings); @@ -367,6 +420,7 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { await db.delete(toolConnections); await db.delete(toolApplications); await db.delete(issueThreadInteractions); + await db.delete(heartbeatRuns); await db.delete(issues); await db.delete(principalPermissionGrants); await db.delete(companyMemberships); @@ -376,6 +430,7 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { }); afterAll(async () => { + if (externalTestDatabaseUrl) await closeRegisteredClients(externalTestDatabaseUrl); await tempDb?.cleanup(); }); @@ -397,6 +452,292 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { return false; } + function useNotebookFixtureEndpoint() { + const method = APP_DEFINITIONS.find(app => app.slug === "paperclip-notebook")!.methods.find(method => method.key === "mcp-oauth")!; + const originalUrl = method.defaults!.serverUrl; + method.defaults!.serverUrl = MCP_URL; + restoreCatalogEndpoints.push(() => { method.defaults!.serverUrl = originalUrl; }); + } + + async function notebookContextFixture() { + const settings = instanceSettingsService(db); + await settings.updateExperimental({ enablePaperclipNotebook: true }); + useNotebookFixtureEndpoint(); + const fixture = installMcpOAuthFixture({ auth: "oauth", notebookAccount: { id: "cloud-owner", email: "owner@example.test", emailVerified: true }, + tools: [{ name: "notebook_get_account", annotations: { readOnlyHint: true } }], + contextResults: { + "resources/list": { resources: [{ uri: "notes://one", name: "Note one" }] }, + "resources/read": { contents: [{ uri: "notes://one", text: "resource body" }] }, + "prompts/list": { prompts: [{ name: "summarize", title: "Summarize note" }] }, + "prompts/get": { messages: [{ role: "user", content: { type: "text", text: "Summarize it" } }] }, + } }); + const company = await createCompany(db), service = toolAccessService(db, { paperclipCloudConnector: null }); + const actor = { actorType: "user" as const, actorId: "board-user" }; + const connected = await service.connectGalleryApp(company.id, { galleryKey: "paperclip-notebook" }, actor); + const start = await service.startOAuth(company.id, connected.connectionId, { redirectUri: REDIRECT_URI, actor }); + const completed = await service.completeOAuthCallback({ state: new URL(start.authorizationUrl).searchParams.get("state")!, + code: fixture.issueAuthorizationCode(start.authorizationUrl), iss: ISSUER, redirectUri: REDIRECT_URI, actor }); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Context reader", role: "engineer", status: "active", adapterType: "process", adapterConfig: {}, runtimeConfig: {} }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Read Notebook context", status: "in_progress", assigneeAgentId: agent!.id }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent!.id, invocationSource: "assignment", status: "running", responsibleUserId: "board-user", contextSnapshot: { issueId: issue!.id } }).returning(); + await service.finishGalleryAppConnection(company.id, connected.connectionId, { enabledCatalogEntryIds: completed.catalog.map(entry => entry.id), askFirstCatalogEntryIds: [], access: { agentIds: [agent!.id] } }, actor); + const [profile] = await db.select().from(toolProfiles).where(eq(toolProfiles.profileKey, `app:${connected.connectionId}`)); + await db.insert(toolProfileEntries).values({ companyId: company.id, profileId: profile!.id, selectorType: "connection", effect: "include", connectionId: connected.connectionId }); + const gateway = createToolGatewayService(db); + const named = await gateway.createNamedGateway({ companyId: company.id, body: { name: "Notebook context", profileId: profile!.id, agentId: agent!.id, issueId: issue!.id } }); + const token = await gateway.createNamedGatewayToken({ companyId: company.id, gatewayId: named.id, body: { name: "Context run", subjectType: "heartbeat_run", subjectId: run!.id } }); + const context = (method: "resources/list" | "resources/read" | "prompts/list" | "prompts/get", params?: Record) => + gateway.executeContextForNamedGateway({ gatewayPublicId: named.gatewayPublicId, bearerToken: token.token, method, params }); + const resources = await context("resources/list"), prompts = await context("prompts/list"); + const uri = (resources.resources as { uri: string }[])[0]!.uri, name = (prompts.prompts as { name: string }[])[0]!.name; + expect(await context("resources/read", { uri })).toMatchObject({ contents: [{ uri, text: "resource body" }] }); + expect(await context("prompts/get", { name })).toMatchObject({ messages: [{ role: "user" }] }); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.connectionId, connected.connectionId)); + await db.update(connectionGrants).set({ providerTenant: { ...grant!.providerTenant, oauth: { + ...grant!.providerTenant?.oauth, accessTokenExpiresAt: new Date(0).toISOString(), + } } }).where(eq(connectionGrants.id, grant!.id)); + return { settings, fixture, service, connected, context, uri, name }; + } + + it.each(["missing", "relabelled"])("omits disabled Notebook resources and prompts and denies retained handles with %s markers", async marker => { + const f = await notebookContextFixture(), saved = await f.service.getConnection(f.connected.connectionId); + const config = { ...saved.config }; + if (marker === "missing") delete config.sourceTemplateKey; + else config.sourceTemplateKey = "fireflies"; + await f.service.updateConnection(f.connected.connectionId, { config }); + await f.settings.updateExperimental({ enablePaperclipNotebook: false }); + const calls = f.fixture.requests.length, secretAccess = await db.select().from(secretAccessEvents); + await expect(f.context("resources/list")).resolves.toEqual({ resources: [] }); + await expect(f.context("prompts/list")).resolves.toEqual({ prompts: [] }); + await expect(f.context("resources/read", { uri: f.uri })).rejects.toMatchObject({ reasonCode: "mcp_resource_not_found" }); + await expect(f.context("prompts/get", { name: f.name })).rejects.toMatchObject({ reasonCode: "mcp_prompt_not_found" }); + expect(f.fixture.requests).toHaveLength(calls); + expect(await db.select().from(secretAccessEvents)).toEqual(secretAccess); + await db.update(toolApplications).set({ applicationKey: "fireflies" }).where(eq(toolApplications.id, f.connected.application.id)); + await expect(f.context("resources/list")).resolves.toMatchObject({ resources: [{ uri: f.uri }] }); + await expect(f.context("prompts/list")).resolves.toMatchObject({ prompts: [{ name: f.name }] }); + await expect(f.context("resources/read", { uri: f.uri })).resolves.toMatchObject({ contents: [{ uri: f.uri }] }); + await expect(f.context("prompts/get", { name: f.name })).resolves.toMatchObject({ messages: [{ role: "user" }] }); + }); + + it.each(["resources/read", "prompts/get"] as const)("rechecks saved availability before credentials when %s eligibility becomes stale", async method => { + const f = await notebookContextFixture(), actualService = instanceSettingsModule.instanceSettingsService; + let settingsReads = 0; + vi.spyOn(instanceSettingsModule, "instanceSettingsService").mockImplementation((...args) => { + const service = actualService(...args); + return { ...service, async getExperimental() { + const settings = await service.getExperimental(); + if (++settingsReads === 1) await f.settings.updateExperimental({ enablePaperclipNotebook: false }); + return settings; + } }; + }); + const calls = f.fixture.requests.length, secretAccess = await db.select().from(secretAccessEvents); + await expect(f.context(method, method === "resources/read" ? { uri: f.uri } : { name: f.name })).rejects.toMatchObject({ reasonCode: "connector_unavailable" }); + expect(settingsReads).toBe(2); + expect(f.fixture.requests).toHaveLength(calls); + expect(await db.select().from(secretAccessEvents)).toEqual(secretAccess); + }); + + it.each(["missing", "relabelled"])("keeps Notebook setup internal after %s markers, denying pending OAuth and finalization while allowing removal", async marker => { + const company = await createCompany(db), settings = instanceSettingsService(db); + await settings.updateExperimental({ enablePaperclipNotebook: false }); + const service = toolAccessService(db, { paperclipCloudConnector: null }); + const actor = { actorType: "user" as const, actorId: "board-user" }; + const app = createRouteApp(db); + const gallery = await request(app).get(`/api/companies/${company.id}/tools/gallery`).expect(200); + expect(gallery.body.apps.some((item: { slug: string }) => item.slug === "paperclip-notebook")).toBe(false); + await expect(service.connectGalleryApp(company.id, { galleryKey: "paperclip-notebook", grantKind: "user" }, actor)) + .rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + await request(app).get(`/api/companies/${company.id}/tools/apps/paperclip-notebook/preflight`).expect(403); + await expect(service.createConnection(company.id, { name: "Notebook bypass", transport: "mcp_remote", config: { + sourceTemplateKey: "paperclip-notebook", url: MCP_URL, + } }, actor)).rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + expect(await db.select().from(toolConnections).where(eq(toolConnections.companyId, company.id))).toEqual([]); + expect(await db.select().from(companySecrets).where(eq(companySecrets.companyId, company.id))).toEqual([]); + await settings.updateExperimental({ enablePaperclipNotebook: true }); + useNotebookFixtureEndpoint(); + const fixture = installMcpOAuthFixture({ auth: "oauth", notebookAccount: { id: "cloud-owner", email: "owner@example.test", emailVerified: true } }); + const enabledGallery = await request(app).get(`/api/companies/${company.id}/tools/gallery`).expect(200); + expect(enabledGallery.body.apps.some((item: { slug: string }) => item.slug === "paperclip-notebook")).toBe(true); + const connected = await service.connectGalleryApp(company.id, { galleryKey: "paperclip-notebook", grantKind: "user" }, actor); + const start = await service.startOAuth(company.id, connected.connectionId, { redirectUri: REDIRECT_URI, actor }); + const saved = await service.getConnection(connected.connectionId, company.id); + const config = { ...saved.config }; + delete config.sourceTemplateKey; + if (marker === "relabelled") config.sourceTemplateKey = "custom-mcp"; + await service.updateConnection(connected.connectionId, { config }); + await settings.updateExperimental({ enablePaperclipNotebook: false }); + await expect(service.completeOAuthCallback({ state: new URL(start.authorizationUrl).searchParams.get("state")!, + code: fixture.issueAuthorizationCode(start.authorizationUrl), iss: ISSUER, redirectUri: REDIRECT_URI, actor })) + .rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + expect(fixture.requestsTo("/token")).toHaveLength(0); + await expect(service.startOAuth(company.id, connected.connectionId, { redirectUri: REDIRECT_URI, actor })) + .rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + await expect(service.reconnectGalleryApp(connected.connectionId, company.id, { credentialValues: {} }, actor)) + .rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + await expect(service.connectGalleryApp(company.id, { resumeConnectionId: connected.connectionId, grantKind: "user" }, actor)) + .rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + await expect(service.finishGalleryAppConnection(company.id, connected.connectionId, { enabledCatalogEntryIds: [], askFirstCatalogEntryIds: [], access: { agentIds: [] } }, actor)) + .rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + await expect(service.archiveConnection(connected.connectionId, company.id, actor)).resolves.toMatchObject({ connection: { status: "archived" } }); + }); + + it("connects Notebook with personal OAuth and preserves policies across ordinary rotation, reconnect, and local removal", async () => { + await instanceSettingsService(db).updateExperimental({ enablePaperclipNotebook: true }); + useNotebookFixtureEndpoint(); + const scopes = ["notebook:read", "notebook:write", "notebook:share", "notebook:delete"]; + const fixture = installMcpOAuthFixture({ auth: "oauth", scopes, rotateRefreshTokens: true, + notebookAccount: { id: "cloud-owner", email: "owner@example.test", emailVerified: true }, + tools: [ + { name: "notebook_get_account", annotations: { readOnlyHint: true } }, + { name: "notebook_list_pages", annotations: { readOnlyHint: true } }, + { name: "notebook_publish", annotations: { readOnlyHint: false } }, + { name: "notebook_set_sharing", annotations: { readOnlyHint: false } }, + { name: "notebook_delete_page", annotations: { readOnlyHint: false, destructiveHint: true } }, + ] }); + const company = await createCompany(db); + const serviceDb = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); + const service = toolAccessService(serviceDb, { paperclipCloudConnector: null }); + const actor = { actorType: "user" as const, actorId: "board-user" }; + const connected = await service.connectGalleryApp(company.id, { galleryKey: "paperclip-notebook" }, actor); + const authorize = async () => { + const start = await service.startOAuth(company.id, connected.connectionId, { redirectUri: REDIRECT_URI, actor }); + expect(start.registrationSource).toBe("dcr"); + const authorization = new URL(start.authorizationUrl); + expect(authorization.searchParams.get("resource")).toBe(MCP_URL); + expect(authorization.searchParams.get("code_challenge_method")).toBe("S256"); + expect(authorization.searchParams.get("scope")?.split(" ")).toEqual(scopes); + return service.completeOAuthCallback({ state: authorization.searchParams.get("state")!, code: fixture.issueAuthorizationCode(start.authorizationUrl), + iss: ISSUER, redirectUri: REDIRECT_URI, actor }); + }; + const completed = await authorize(); + expect(completed.connection).toMatchObject({ name: "Paperclip Notebook", credentialPolicy: "per_user" }); + expect(completed.catalog).toHaveLength(5); + expect(completed.catalog.every(entry => entry.status === "active")).toBe(true); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.connectionId, connected.connectionId)); + expect(grant).toMatchObject({ kind: "user", subjectUserId: "board-user" }); + expect(grant!.providerTenant).not.toHaveProperty("name"); + expect(grant!.providerTenant).not.toHaveProperty("externalId"); + expect(fixture.requestsTo("/mcp").filter(call => (call.body as Record)?.method === "tools/call")).toEqual([]); + const stored = await db.select().from(companySecrets).where(inArray(companySecrets.id, grant!.credentialSecretRefs.map(ref => ref.secretId))); + expect(stored).toHaveLength(2); + expect(stored.every(secret => secret.scope === "user" && secret.ownerUserId === "board-user")).toBe(true); + expect(JSON.stringify(completed)).not.toContain("fixture-access-"); + for (let rotation = 0; rotation < 2; rotation += 1) { + await service.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: connected.connectionId, grantId: grant!.id, forceRefresh: true, actor }); + } + const refreshRequests = fixture.requestsTo("/token").filter(call => (call.body as URLSearchParams).get("grant_type") === "refresh_token"); + expect(refreshRequests).toHaveLength(2); + expect((refreshRequests[0]!.body as URLSearchParams).get("refresh_token")).toBe(fixture.tokenResponses[0]!.refresh_token); + expect((refreshRequests[1]!.body as URLSearchParams).get("refresh_token")).toBe(fixture.tokenResponses[1]!.refresh_token); + expect(fixture.tokenResponses[0]!.refresh_token).not.toBe(fixture.tokenResponses[1]!.refresh_token); + expect(fixture.requests.every(call => call.headers["idempotency-key"] === undefined)).toBe(true); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Notebook author", role: "engineer", status: "active", adapterType: "process", adapterConfig: {}, runtimeConfig: {} }).returning(); + const [unassigned] = await db.insert(agents).values({ companyId: company.id, name: "Unassigned", role: "engineer", status: "active", adapterType: "process", adapterConfig: {}, runtimeConfig: {} }).returning(); + await service.finishGalleryAppConnection(company.id, connected.connectionId, { + enabledCatalogEntryIds: completed.catalog.filter(entry => entry.toolName !== "notebook_delete_page").map(entry => entry.id), + askFirstCatalogEntryIds: completed.catalog.filter(entry => entry.toolName === "notebook_set_sharing").map(entry => entry.id), access: { agentIds: [agent!.id] }, + }, actor); + await authorize(); + expect(await db.select({ id: connectionGrants.id }).from(connectionGrants).where(eq(connectionGrants.connectionId, connected.connectionId))).toEqual([{ id: grant!.id }]); + const policy = toolAccessPolicyService(db); + const decision = (name: string, agentId = agent!.id) => { + const entry = completed.catalog.find(entry => entry.toolName === name)!; + return policy.decide({ companyId: company.id, actor: { actorType: "agent", actorId: agentId, agentId }, + request: { connectionId: connected.connectionId, catalogEntryId: entry.id, toolName: name } }); + }; + await expect(decision("notebook_publish")).resolves.toMatchObject({ allowed: true }); + await expect(decision("notebook_set_sharing")).resolves.toMatchObject({ allowed: false, decision: "require_approval" }); + await expect(decision("notebook_delete_page")).resolves.toMatchObject({ allowed: false, decision: "deny" }); + await expect(decision("notebook_publish", unassigned!.id)).resolves.toMatchObject({ allowed: false }); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Notebook MCP publication", status: "in_progress", assigneeAgentId: agent!.id }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent!.id, invocationSource: "assignment", status: "running", + responsibleUserId: "board-user", contextSnapshot: { issueId: issue!.id } }).returning(); + const [profile] = await db.select().from(toolProfiles).where(eq(toolProfiles.profileKey, `app:${connected.connectionId}`)); + const gateway = createToolGatewayService(db, { toolActionSigningSecret: "notebook-mcp-fixture" }); + const named = await gateway.createNamedGateway({ companyId: company.id, body: { name: "Notebook MCP", profileId: profile!.id, agentId: agent!.id, issueId: issue!.id } }); + const token = await gateway.createNamedGatewayToken({ companyId: company.id, gatewayId: named.id, body: { name: "Notebook run", subjectType: "heartbeat_run", subjectId: run!.id } }); + const descriptors = await gateway.listToolsForNamedGateway({ gatewayPublicId: named.gatewayPublicId, bearerToken: token.token }); + const publish = descriptors.find(tool => tool.upstreamToolName === "notebook_publish")!; + const share = descriptors.find(tool => tool.upstreamToolName === "notebook_set_sharing")!; + expect(descriptors.some(tool => tool.upstreamToolName === "notebook_delete_page")).toBe(false); + await expect(gateway.executeTool({ gatewayPublicId: named.gatewayPublicId, sessionToken: token.token, tool: publish.name, parameters: {} })).resolves.toMatchObject({ status: "completed" }); + const callsBeforeApproval = fixture.requestsTo("/mcp").length; + await expect(gateway.executeTool({ gatewayPublicId: named.gatewayPublicId, sessionToken: token.token, tool: share.name, parameters: {} })).rejects.toMatchObject({ reasonCode: "approval_required" }); + expect(fixture.requestsTo("/mcp")).toHaveLength(callsBeforeApproval); + const [action] = await db.select().from(toolActionRequests).where(and(eq(toolActionRequests.companyId, company.id), eq(toolActionRequests.status, "pending"))); + expect(action).toBeDefined(); + const saved = await service.getConnection(connected.connectionId, company.id); + const config = { ...saved.config }; + delete config.sourceTemplateKey; + await service.updateConnection(connected.connectionId, { config }); + const [currentGrant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); + await db.update(connectionGrants).set({ providerTenant: { ...currentGrant!.providerTenant, oauth: { + ...currentGrant!.providerTenant?.oauth, accessTokenExpiresAt: new Date(0).toISOString(), + } } }).where(eq(connectionGrants.id, grant!.id)); + await instanceSettingsService(db).updateExperimental({ enablePaperclipNotebook: false }); + const callsBeforeDisabledDispatch = fixture.requestsTo("/mcp").length; + expect(await gateway.listToolsForNamedGateway({ gatewayPublicId: named.gatewayPublicId, bearerToken: token.token })).toEqual([]); + await expect(gateway.executeTool({ gatewayPublicId: named.gatewayPublicId, sessionToken: token.token, tool: publish.name, parameters: {} })).rejects.toMatchObject({ reasonCode: "tool_not_found" }); + await expect(gateway.approveActionRequest({ companyId: company.id, actionRequestId: action!.id, actor: { userId: "board-user" } })).rejects.toMatchObject({ reasonCode: "tool_not_found" }); + const intents = connectionIntentService(db); + const claims = { sub: agent!.id, company_id: company.id, run_id: run!.id, responsible_user_id: "board-user" }; + expect((await intents.search(claims, "Paperclip Notebook")).results.some(item => item.service === "paperclip-notebook")).toBe(false); + await expect(intents.request(claims, "paperclip-notebook")).rejects.toMatchObject({ status: 404 }); + expect(fixture.requestsTo("/mcp")).toHaveLength(callsBeforeDisabledDispatch); + const tokenRequestsWhileOff = fixture.requestsTo("/token").length; + const mcpRequestsWhileOff = fixture.requestsTo("/mcp").length; + const secretAccessBeforeOff = await db.select({ id: secretAccessEvents.id }).from(secretAccessEvents); + await expect(service.checkHealth(connected.connectionId, actor)).rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + await expect(service.refreshCatalog(connected.connectionId, actor)).rejects.toMatchObject({ status: 403, details: { code: "connector_disabled" } }); + expect(fixture.requestsTo("/token")).toHaveLength(tokenRequestsWhileOff); + expect(fixture.requestsTo("/mcp")).toHaveLength(mcpRequestsWhileOff); + expect(await db.select({ id: secretAccessEvents.id }).from(secretAccessEvents)).toEqual(secretAccessBeforeOff); + const removed = await service.archiveConnection(connected.connectionId, company.id, actor); + expect(removed.connection.status).toBe("archived"); + expect(removed.removal).not.toHaveProperty("oauthRevocationPending"); + await expect(decision("notebook_publish")).resolves.toMatchObject({ allowed: false }); + const [removedGrant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); + expect(removedGrant).toMatchObject({ status: "revoked", credentialSecretRefs: [] }); + expect(await db.select().from(companySecrets).where(inArray(companySecrets.id, currentGrant!.credentialSecretRefs.map(ref => ref.secretId)))).toEqual([]); + expect(fixture.requestsTo("/oauth/revoke")).toEqual([]); + await instanceSettingsService(db).updateExperimental({ enablePaperclipNotebook: true }); + await service.connectGalleryApp(company.id, { galleryKey: "paperclip-notebook", applicationId: connected.application.id }, actor); + await authorize(); + expect(fixture.requestsTo("/oauth/revoke")).toEqual([]); + expect(fixture.requests.every(call => call.headers["idempotency-key"] === undefined)).toBe(true); + }); + + it("reconnects Notebook after a lost rotating refresh response requires new authorization", async () => { + await instanceSettingsService(db).updateExperimental({ enablePaperclipNotebook: true }); + useNotebookFixtureEndpoint(); + const fixture = installMcpOAuthFixture({ auth: "oauth", rotateRefreshTokens: true, dropFirstRefreshResponse: true, + notebookAccount: { id: "cloud-owner", email: "owner@example.test", emailVerified: true } }); + const company = await createCompany(db), service = toolAccessService(db, { paperclipCloudConnector: null }); + const actor = { actorType: "user" as const, actorId: "board-user" }; + const connected = await service.connectGalleryApp(company.id, { galleryKey: "paperclip-notebook" }, actor); + const authorize = async () => { + const start = await service.startOAuth(company.id, connected.connectionId, { redirectUri: REDIRECT_URI, actor }); + return service.completeOAuthCallback({ state: new URL(start.authorizationUrl).searchParams.get("state")!, code: fixture.issueAuthorizationCode(start.authorizationUrl), + iss: ISSUER, redirectUri: REDIRECT_URI, actor }); + }; + await authorize(); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.connectionId, connected.connectionId)); + const refresh = () => service.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: connected.connectionId, grantId: grant!.id, forceRefresh: true, actor }); + await expect(refresh()).rejects.toThrow("Fixture lost the committed rotating refresh response"); + expect(fixture.requestsTo("/token")).toHaveLength(2); + await expect(refresh()).rejects.toMatchObject({ status: 422, details: { code: "oauth_reauthorization_required" } }); + const [expired] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); + expect(expired!.status).toBe("needs_reauthorization"); + await authorize(); + const [renewed] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); + expect(renewed).toMatchObject({ status: "active", credentialSecretRefs: expect.any(Array) }); + expect(renewed!.credentialSecretRefs).toHaveLength(2); + await expect(refresh()).resolves.toMatchObject({ status: "active" }); + expect(fixture.requestsTo("/oauth/revoke")).toEqual([]); + expect(fixture.requests.every(call => call.headers["idempotency-key"] === undefined)).toBe(true); + }); + it.each(["mcp-oauth", "mcp-api-key"])("connects Fireflies through %s with vaulted credentials and its stable meeting tools", async (methodKey) => { // Keep the real catalog method and governance identity; redirect only its // transport URL to the deterministic protocol fixture. @@ -1332,7 +1673,7 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { it("completes organization OAuth with a single database connection", async () => { const fixture = installMcpOAuthFixture({ auth: "oauth" }); const company = await createCompany(db); - const callbackDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); + const callbackDb = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); const service = toolAccessService(callbackDb); let deadline: ReturnType | null = null; @@ -1416,8 +1757,8 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { it("serializes organization OAuth completion behind membership revocation", async () => { const fixture = installMcpOAuthFixture({ auth: "oauth" }); const company = await createCompany(db); - const callbackDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); - const removalDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); + const callbackDb = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); + const removalDb = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); const service = toolAccessService(callbackDb); const actor = { actorType: "user" as const, actorId: "board-user" }; let releaseRemoval!: () => void; @@ -2187,7 +2528,7 @@ describeEmbeddedPostgres("generic remote MCP connections", () => { // serialize the two callbacks and prove nothing. With two connections both // callbacks can read a live state row before either deletes it, so the // atomic `DELETE … RETURNING` is what picks the winner. - const otherDb = createDb(tempDb!.connectionString); + const otherDb = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString); const otherService = toolAccessService(otherDb); let settled: PromiseSettledResult[]; try { diff --git a/server/src/__tests__/heartbeat-plugin-environment.test.ts b/server/src/__tests__/heartbeat-plugin-environment.test.ts index 3965b0ef721..99c5abfa58b 100644 --- a/server/src/__tests__/heartbeat-plugin-environment.test.ts +++ b/server/src/__tests__/heartbeat-plugin-environment.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, rm, stat } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -93,6 +93,59 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { await stopDb?.(); }); + it("dispatches a plugin-only project without Core workspace files through the native heartbeat boundary", async () => { + const companyId = randomUUID(), projectId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sourceId = randomUUID(); + const environmentId = randomUUID(), pluginId = randomUUID(), pluginKey = `test.tasks.${pluginId}`; + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-task-heartbeat-")); tempRoots.push(root); + await db.insert(companies).values({ id: companyId, name: "Task fixture", issuePrefix: "TASK", defaultResponsibleUserId: "task-user" }); + await grantResponsibleUser(companyId, "task-user"); + await db.insert(projects).values({ id: projectId, companyId, name: "Task fixture", status: "active", + executionWorkspacePolicy: { enabled: true, storageProviderPluginId: pluginId } }); + const pluginOnlyCwd = path.join(root, "plugin-only", "project"); + await db.insert(projectWorkspaces).values([ + { id: sourceId, companyId, projectId, name: "Primary", cwd: pluginOnlyCwd, isPrimary: true }, + { companyId, projectId, name: "Extra", cwd: null, repoUrl: "/unavailable-controller-repository.git", isPrimary: false }, + ]); + await db.insert(plugins).values({ id: pluginId, pluginKey, packageName: "@test/tasks", version: "1.0.0", apiVersion: 1, + categories: ["automation"], status: "ready", manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", + displayName: "Tasks", description: "Task fixture", author: "Paperclip", categories: ["automation"], + capabilities: ["environment.drivers.register", "workspaces.manage", "events.subscribe"], resourceLifecycle: ["project"], + workspaceProvider: { modes: ["shared_workspace", "isolated_workspace", "operator_branch"], strategies: ["project_primary", "git_worktree"] }, + entrypoints: { worker: "worker.js" }, + environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" } }] } }); + await db.insert(environments).values({ id: environmentId, companyId, name: "Tasks", driver: "plugin", status: "active", + config: { pluginKey, driverKey: "tasks", driverConfig: {} } }); + await db.insert(agents).values({ id: agentId, companyId, name: "Task agent", role: "engineer", status: "idle", + adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol" }, runtimeConfig: {}, + defaultEnvironmentId: environmentId, permissions: {} }); + await db.insert(issues).values({ id: issueId, companyId, projectId, projectWorkspaceId: sourceId, title: "Task-only dispatch", status: "in_progress", + assigneeAgentId: agentId, issueNumber: 1, identifier: "TASK-1" }); + const call = vi.fn(async (_id: string, method: string, params?: any) => { + if (method === "workspaceOperation") return { kind: "prepared", binding: { pluginId, projectId, + executionWorkspaceId: params.executionWorkspaceId, projectWorkspaceId: params.operation.source.id, + sourceRevision: params.operation.source.revision, workspaceRef: "provider:heartbeat-workspace" }, + branchName: params.operation.branchName, baseSha: "a".repeat(40), created: true, branchCreatedByRuntime: false }; + if (method === "environmentAcquireLease") return { providerLeaseId: "heartbeat-task-attempt", metadata: { remoteCwd: "/project" } }; + if (method === "environmentReleaseLease" || method === "environmentDestroyLease") return undefined; + throw new Error(`Unexpected task provider call: ${method}`); + }); + const workerManager = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["workspaceOperation", "environmentTask", "environmentAcquireLease", "environmentReleaseLease", "environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; + const entered = vi.fn(() => { throw new Error("synthetic_native_boundary_reached"); }); + const heartbeat = heartbeatService(db, { pluginWorkerManager: workerManager, nativeSessionBackendFactory: entered }); + heartbeats.push(heartbeat); + const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", contextSnapshot: { issueId, projectId } }); + expect(run).not.toBeNull(); + await vi.waitFor(async () => { expect((await heartbeat.getRun(run!.id))?.status).toBe("failed"); }, { timeout: 10_000 }); + expect(entered).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({ runId: run!.id, companyId, issueId }) })); + expect((await heartbeat.getRun(run!.id))?.error).toContain("synthetic_native_boundary_reached"); + await heartbeat.drainActiveRunExecutions(); + expect(call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: "heartbeat-task-attempt" })); + expect(call.mock.calls.every(([, method]) => ["workspaceOperation", "environmentAcquireLease", "environmentDestroyLease"].includes(method))).toBe(true); + expect(call.mock.calls.filter(([, method]) => method === "workspaceOperation")).toHaveLength(1); + expect(adapterExecute).not.toHaveBeenCalled(); + await expect(stat(pluginOnlyCwd)).rejects.toMatchObject({ code: "ENOENT" }); + }, 20_000); + it("acquires plugin environment leases through the heartbeat execution path", async () => { const companyId = randomUUID(); const projectId = randomUUID(); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index b4faf3b74d1..d0d98e6a52c 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -8297,19 +8297,28 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const lease = originalWatch(...args); return { ...lease, stop() { stopped(); lease.stop(); } }; }); - const originalUpdate = db.update.bind(db); + const originalTransaction = db.transaction.bind(db); const cleanupFailure = vi.fn(() => { throw new Error("fixture_deadline_cleanup_failure"); }); - const update = vi.spyOn(db, "update").mockImplementation(((table: typeof heartbeatRuns) => { - const builder = originalUpdate(table); - const originalSet = builder.set.bind(builder); - builder.set = ((values: Record) => { - if (table === heartbeatRuns && Object.keys(values).length === 1 && values.executionControlDeadlineAt === null) { - return { where: async () => cleanupFailure() }; + const transaction = vi.spyOn(db, "transaction").mockImplementation((operation, config) => + originalTransaction(async tx => { + const originalUpdate = tx.update.bind(tx); + const update = vi.spyOn(tx, "update").mockImplementation(((table: typeof heartbeatRuns) => { + const builder = originalUpdate(table); + const originalSet = builder.set.bind(builder); + builder.set = ((values: Record) => { + if (table === heartbeatRuns && Object.keys(values).length === 1 && values.executionControlDeadlineAt === null) { + return { where: async () => cleanupFailure() }; + } + return originalSet(values); + }) as typeof builder.set; + return builder; + }) as typeof tx.update); + try { + return await operation(tx); + } finally { + update.mockRestore(); } - return originalSet(values); - }) as typeof builder.set; - return builder; - }) as typeof db.update); + }, config)); const heartbeat = heartbeatService(db); try { await heartbeat.resumeQueuedRuns(); @@ -8319,7 +8328,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(adapterExecutionControls.has(runId)).toBe(false); expect((await heartbeat.getRun(runId))?.status).not.toBe("running"); } finally { - update.mockRestore(); + transaction.mockRestore(); watcher.mockRestore(); } }); diff --git a/server/src/__tests__/instance-settings-managed-overlay.test.ts b/server/src/__tests__/instance-settings-managed-overlay.test.ts index 5285f5b4694..60f9964fb64 100644 --- a/server/src/__tests__/instance-settings-managed-overlay.test.ts +++ b/server/src/__tests__/instance-settings-managed-overlay.test.ts @@ -85,6 +85,43 @@ describe("applyManagedExperimentalOverlay", () => { }); describe("instanceSettingsService managed overlay", () => { + it.each(["team.staging.paperclip.app", "team.paperclip.app"])( + "enables private Notebook through managed config without changing stored opt-out on %s", + async (primaryHost) => { + const row = settingsRow({ enablePaperclipNotebook: false }); + const { db, persistedSets } = stubDb(row); + const service = instanceSettingsService(db, { runtimeEnv: { + PAPERCLIP_PRIMARY_HOST: primaryHost, + ...managedEnv(JSON.stringify({ + v: 1, mode: "cloud", catalogVersion: "test", + features: { enablePaperclipNotebook: true }, plugins: { autoInstall: [] }, + })), + } }); + expect(await service.getExperimental()).toMatchObject({ + enablePaperclipNotebook: true, + managedKeys: { enablePaperclipNotebook: { managed: true, managedBy: "paperclip-cloud" } }, + }); + expect((await service.get()).experimental.enablePaperclipNotebook).toBe(true); + expect(persistedSets).toHaveLength(0); + expect(row.experimental).toEqual({ enablePaperclipNotebook: false }); + expect((await instanceSettingsService(db, { runtimeEnv: {} }).getExperimental()).enablePaperclipNotebook).toBe(false); + }, + ); + + it("persists an explicit self-hosted Notebook opt-in and disable across service recreation", async () => { + const row = settingsRow({}); + const { db, persistedSets } = stubDb(row); + expect((await instanceSettingsService(db, { runtimeEnv: {} }).getExperimental()).enablePaperclipNotebook).toBe(false); + for (const enabled of [true, false]) { + const service = instanceSettingsService(db, { runtimeEnv: {} }); + const updated = await service.updateExperimental({ enablePaperclipNotebook: enabled }); + Object.assign(row, persistedSets.at(-1)); + expect(updated.experimental.enablePaperclipNotebook).toBe(enabled); + expect((row.experimental as Record).enablePaperclipNotebook).toBe(enabled); + expect((await instanceSettingsService(db, { runtimeEnv: {} }).getExperimental()).enablePaperclipNotebook).toBe(enabled); + } + }); + it.each(["team.staging.paperclip.app", "team.paperclip.app", undefined])( "enforces every Cloud-enabled flag over old tenant values on %s", async (primaryHost) => { diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.ts index 4078d9ad6f8..2ddd8f31552 100644 --- a/server/src/__tests__/instance-settings-routes.test.ts +++ b/server/src/__tests__/instance-settings-routes.test.ts @@ -912,6 +912,38 @@ describe("instance settings routes", () => { }); describe("Cloud staging experiments", () => { + it.each([ + ["team.staging.paperclip.app", 200], + ["team.paperclip.app", 403], + ] as const)( + "keeps private Notebook operator-only under the real hidden wildcard on %s", + async (primaryHost, visibleExperimentStatus) => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "tenant-token"); + vi.stubEnv("PAPERCLIP_PRIMARY_HOST", primaryHost); + vi.stubEnv("PAPERCLIP_HIDDEN_SETTINGS", "instance.experimental.*"); + mockInstanceSettingsService.getExperimental.mockResolvedValue({ + enablePaperclipNotebook: false, enableChatConnectors: false, + }); + const app = createApp({ + type: "board", userId: "stack-admin-1", source: "cloud_tenant", + companyIds: ["company-1"], isInstanceAdmin: false, canManageExperimentalSettings: true, + }); + const denied = await request(app).patch("/api/instance/settings/experimental") + .send({ enablePaperclipNotebook: true }).expect(403); + expect(denied.body.details).toMatchObject({ code: "settings_operator_managed" }); + expect(mockInstanceSettingsService.updateExperimental).not.toHaveBeenCalled(); + expect(mockLogActivity).not.toHaveBeenCalled(); + + await request(app).patch("/api/instance/settings/experimental") + .send({ enableChatConnectors: true }).expect(visibleExperimentStatus); + if (visibleExperimentStatus === 200) { + expect(mockInstanceSettingsService.updateExperimental).toHaveBeenCalledExactlyOnceWith({ enableChatConnectors: true }); + } else { + expect(mockInstanceSettingsService.updateExperimental).not.toHaveBeenCalled(); + } + }, + ); + it.each(["team.staging.paperclip.app", "team.paperclip.app"])( "reads Cloud-enabled isolation and rejects disabling it through the real service on %s", async (primaryHost) => { diff --git a/server/src/__tests__/instance-settings-service.test.ts b/server/src/__tests__/instance-settings-service.test.ts index 2502015867c..631c1389844 100644 --- a/server/src/__tests__/instance-settings-service.test.ts +++ b/server/src/__tests__/instance-settings-service.test.ts @@ -7,6 +7,18 @@ import { } from "../services/instance-settings.js"; describe("instance settings service", () => { + it("keeps Notebook off for legacy or invalid settings and round-trips explicit operator choices", () => { + for (const stored of [undefined, null, {}, { enableApps: true }, + { enablePaperclipNotebook: "true" }, { enablePaperclipNotebook: 1 }, + { enablePaperclipNotebook: true, enableCases: "invalid" }]) { + expect(normalizeExperimentalSettings(stored).enablePaperclipNotebook).toBe(false); + } + const enabled = applyExperimentalSettingsPatch({}, { enablePaperclipNotebook: true }); + const restored = normalizeExperimentalSettings(JSON.parse(JSON.stringify(enabled))); + expect(restored).toMatchObject({ enablePaperclipNotebook: true, enableMemoryConnectors: false }); + const disabled = applyExperimentalSettingsPatch(restored, { enablePaperclipNotebook: false }); + expect(normalizeExperimentalSettings(JSON.parse(JSON.stringify(disabled))).enablePaperclipNotebook).toBe(false); + }); it("keeps OpenAI Dot opt-in and persists its setting independently of its prerequisites", () => { expect(normalizeExperimentalSettings({}).enableOpenAiDot).toBe(false); const enabled = applyExperimentalSettingsPatch({}, { enableOpenAiDot: true }); @@ -67,6 +79,7 @@ describe("instance settings service", () => { enableChatConnectors: false, enableGitHubReviewBots: false, enableMemoryConnectors: false, + enablePaperclipNotebook: false, enableConferenceRoomChat: false, enableClassicTaskInterface: false, enableExternalObjects: false, diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 0246d509ff3..e1151c9c899 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -106,6 +106,7 @@ describe("issueRecoveryActionService", () => { }); const fakeDb = { + async transaction(fn: (tx: unknown) => Promise): Promise { return fn(fakeDb); }, select: vi.fn(() => makeSelectQuery(selectResults.shift() ?? [])), update: vi.fn(() => ({ set: vi.fn(() => ({ diff --git a/server/src/__tests__/plugin-runner-task.integration.test.ts b/server/src/__tests__/plugin-runner-task.integration.test.ts index 644d332a7f5..45f347726b8 100644 --- a/server/src/__tests__/plugin-runner-task.integration.test.ts +++ b/server/src/__tests__/plugin-runner-task.integration.test.ts @@ -1,18 +1,28 @@ import * as pluginProbe from "../services/native-runtime/plugin-runner-probe.js"; import { randomUUID } from "node:crypto"; import { PassThrough } from "node:stream"; +import { setTimeout as delay } from "node:timers/promises"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { agents, companies, createDb, environmentLeases, environments, plugins } from "@paperclipai/db"; +import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins, projects } from "@paperclipai/db"; import { definePlugin } from "../../../packages/plugins/sdk/src/define-plugin.js"; import { startWorkerRpcHost } from "../../../packages/plugins/sdk/src/worker-rpc-host.js"; import { createRequest, parseMessage, serializeMessage, isJsonRpcSuccessResponse, isJsonRpcErrorResponse } from "../../../packages/plugins/sdk/src/protocol.js"; -import { type PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk"; +import { environmentTaskOperationSchema, type PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import { agentEnvironmentTestService } from "../services/agent-environment-test.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { remoteExecutionHasStopped } from "../services/remote-execution-termination.js"; +import { resolvePluginTaskResume } from "../services/environment-task-runtime.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; +import { environmentRunOrchestrator } from "../services/environment-run-orchestrator.js"; +import { createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type RunnerProviderLaunchRequirements } from "@paperclipai/paperclip-runner"; +import { createPluginRunnerTaskOwner } from "../services/native-runtime/plugin-runner-task.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +const ingress = vi.hoisted(() => ({ connect: vi.fn() })); +vi.mock("../realtime/runner-prp-outbound.js", () => ({ connectRunnerPrpIngress: ingress.connect })); const support = await getEmbeddedPostgresTestSupport(); const databaseSuite = support.supported ? describe : describe.skip; @@ -54,9 +64,9 @@ databaseSuite("Core plugin Runner task lifecycle", () => { }); afterAll(async () => { await stopDb?.(); }); - it("uses a bounded setup lease through SDK RPC and releases it after stop proof", async () => { - const companyId = randomUUID(), agentId = randomUUID(); - const environmentId = randomUUID(), pluginId = randomUUID(); + it("uses an acquired lease through SDK RPC, native dispatch, connection, stop proof and release", async () => { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); + const environmentId = randomUUID(), pluginId = randomUUID(), projectId = randomUUID(); const manifest: PaperclipPluginManifestV1 = { id: "test.runner-tasks", apiVersion: 1, version: "1.0.0", displayName: "Runner tasks", description: "Synthetic task provider", author: "Paperclip", categories: ["automation"], @@ -66,6 +76,8 @@ databaseSuite("Core plugin Runner task lifecycle", () => { await db.insert(companies).values({ id: companyId, name: "Fixture", status: "active" }); await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", role: "engineer", status: "active", adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {} }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" }); + await db.insert(projects).values({ id: projectId, companyId, name: "Fixture", status: "active" }); const [environment] = await db.insert(environments).values({ id: environmentId, name: "Fixture", driver: "plugin", status: "active", config: { pluginKey: manifest.id, driverKey: "tasks", driverConfig: {} } }).returning(); await db.insert(plugins).values({ id: pluginId, pluginKey: manifest.id, packageName: "@test/runner-tasks", @@ -82,8 +94,18 @@ databaseSuite("Core plugin Runner task lifecycle", () => { switch (params.operation.kind) { case "describe": return { kind: "description", launchPreparationVersion: 1, projectAuthCheckVersion: 1, taskId: params.taskId, runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; + case "prepare": return { kind: "prepared", taskId: params.taskId, launch: { + configurationId: `config-${params.taskId}`, requirementsDigest: runnerLaunchRequirementsDigest(params.operation.requirements as RunnerProviderLaunchRequirements), + provider: createRunnerProviderLaunch(params.operation.requirements as RunnerProviderLaunchRequirements, { + options: { codexArgs: [] }, root: "/runner", cwd: "/workspace", codexHome: "/account", + codexCommand: "/bin/codex", providerNodeCommand: "/bin/node", opencodeProxyPath: "", acpxSidecarPath: "", + }), + } }; case "check_project_auth": return { kind: "project_auth", taskId: params.taskId, configurationId: params.operation.configurationId, provider: params.operation.provider, allowed: true }; + case "read_state": return { kind: "state", taskId: params.taskId, state: { + schema: "paperclip.runner.durable.state.v1", ...params.operation.identity, lifecycle: "suspended", + } }; case "submit": stopRequested = false; executionStopped = false; return { kind: "accepted", taskId: params.taskId }; case "status": return { kind: "status", taskId: params.taskId, phase: stopRequested ? "cancelled" : "running", executionStopped, sessionStateAvailable: executionStopped }; case "connection": return { kind: "connection", taskId: params.taskId, endpoint: { @@ -99,17 +121,17 @@ databaseSuite("Core plugin Runner task lifecycle", () => { const { supportedMethods } = await harness.initialize(); const workers = { isRunning: () => true, getWorker: () => ({ supportedMethods }), call: (_id: string, method: string, params: unknown) => harness.call(method, params) } as unknown as PluginWorkerManager; - try { - const setup = agentEnvironmentTestService(db, workers); - await setup.assertAdapterTestEnvironmentForCompany(companyId, environmentId, "paperclip_runner"); - const probe = await setup.resolveAdapterTestExecutionContext({ companyId, agentId, adapterType: "paperclip_runner", environmentId }); - expect(probe.fallbackChecks).toEqual([]); - expect(probe.executionTarget).toMatchObject({ runnerTaskProbe: { runId: expect.any(String), sessionId: expect.any(String) } }); - if (probe.executionTarget?.kind !== "remote" || probe.executionTarget.transport !== "sandbox" || !probe.executionTarget.runnerTaskProbe) throw new Error("Probe task target unavailable"); - await expect(probe.executionTarget.runnerTask!({ kind: "describe" })) - .resolves.toMatchObject({ kind: "description" }); - const [probeLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, probe.executionTarget.leaseId!)); - expect(probeLease.heartbeatRunId).toBeNull(); + const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers }); + const setup = agentEnvironmentTestService(db, workers); + await setup.assertAdapterTestEnvironmentForCompany(companyId, environmentId, "paperclip_runner"); + const probe = await setup.resolveAdapterTestExecutionContext({ companyId, agentId, adapterType: "paperclip_runner", environmentId }); + expect(probe.fallbackChecks).toEqual([]); + expect(probe.executionTarget).toMatchObject({ runnerTaskProbe: { runId: expect.any(String), sessionId: expect.any(String) } }); + if (probe.executionTarget?.kind !== "remote" || probe.executionTarget.transport !== "sandbox" || !probe.executionTarget.runnerTaskProbe) throw new Error("Probe task target unavailable"); + await expect(probe.executionTarget.runnerTask!({ kind: "describe" })) + .resolves.toMatchObject({ kind: "description" }); + const [probeLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, probe.executionTarget.leaseId!)); + expect(probeLease.heartbeatRunId).toBeNull(); const selected = { identity: "core-selected-revision", sessionIdentity: "core-selected-account", home: "/core-owned-home" }; const verify = vi.spyOn(pluginProbe, "probePluginRunnerEnvironment").mockResolvedValue({ adapterType: "paperclip_runner", status: "pass", testedAt: new Date().toISOString(), checks: [] }); try { @@ -117,21 +139,174 @@ databaseSuite("Core plugin Runner task lifecycle", () => { { provider: "openai" } as any, selected as any); expect(verify).toHaveBeenCalledWith(expect.anything(), { identity: selected.identity, accountIdentity: selected.sessionIdentity, home: "/core-owned-home/provider" }); } finally { verify.mockRestore(); } + await probe.executionTarget.runnerTask!({ kind: "check_project_auth", configurationId: "probe-config", provider: "openai", projectIds: [] }); + await probe.executionTarget.runnerTask!({ kind: "submit", preparedLaunchId: "probe-config", bootstrapTicket: "synthetic-probe-ticket", projectIds: [], + runner: { protocolMin: 1, protocolMax: 3, harness: "codex", runnerId: "probe-runner", leaseId: probeLease.id, + runId: probe.executionTarget.runnerTaskProbe.runId, sessionId: probe.executionTarget.runnerTaskProbe.sessionId, turnId: "probe-turn", itemId: "probe-item" }, + launch: { runnerVersion: "fixture", runnerDigest: `sha256:${"a".repeat(64)}`, + maxOutboxBytes: 262144, p0ReserveBytes: 65536, maxRuntimeMs: 0, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + }); + const probeSubmit = calls.find((op): op is Extract => op.kind === "submit")!; + expect(probeSubmit.launch?.maxRuntimeMs).toBeGreaterThan(0); + expect(probeSubmit.launch?.maxRuntimeMs).toBeLessThanOrEqual(120_000); + executionStopped = true; + await probe.release(); released.mockClear(); complete.mockClear(); calls.length = 0; + const orchestrator = environmentRunOrchestrator(db, { environmentRuntime: runtime }); + const acquired = await orchestrator.acquireForRun({ companyId, selectedEnvironmentId: environmentId, localEnvironmentId: randomUUID(), + agentId, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, + executionWorkspaceSettings: null, adapterType: "paperclip_runner" }); + const realization = await orchestrator.realizeForRun({ companyId, environment: acquired.environment, lease: acquired.lease, + adapterType: "paperclip_runner", issueId: null, heartbeatRunId: runId, + executionWorkspace: { baseCwd: "/local/project", cwd: "/local/project", source: "project_primary", projectId, + workspaceId: null, repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null, + warnings: [], created: false, branchCreatedByRuntime: false }, + effectiveExecutionWorkspaceMode: null, persistedExecutionWorkspace: null }); + const target = realization.executionTarget; + expect(target).toMatchObject({ kind: "remote", transport: "sandbox", remoteCwd: "/local/project", leaseId: acquired.lease.id }); + if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask) throw new Error("Missing plugin target"); + expect(target.runner).toBeUndefined(); + await expect(target.runnerTask({ kind: "describe" })) + .resolves.toMatchObject({ kind: "description", runner: { version: "fixture" } }); + const onDispatched = vi.fn(), checkpoint = vi.fn(async () => {}), close = vi.fn(async () => {}); + ingress.connect.mockReturnValue({ ready: Promise.resolve(), close }); + const owner = createPluginRunnerTaskOwner({ task: target.runnerTask, projectAuthCheckVersion: 1, + credentialIdentity: "revision", accountIdentity: "account", credential: { type: "codex-auth-json", authJson: '{"OPENAI_API_KEY":"synthetic-account-key"}' }, projectIds: target.runnerTaskProjectIds ?? [], harness: "codex", + onSessionRetained: checkpoint, onDispatched, pollMs: 1 }); + const identity = { runnerInstanceId: randomUUID(), environmentLeaseId: acquired.lease.id, runId, + normalizedSessionId: randomUUID(), turnId: "turn", itemId: "item" }; + const requirements: RunnerProviderLaunchRequirements = { schema: "paperclip.runner.launch-requirements.v1", identity, + provider: "codex", model: "fixture", instructions: "Core policy", runtimeContext: null, settings: {}, + approvalPolicy: "never", readOnly: false, preparedConversation: false, includeCollaborationModeInstructions: false }; + const prepared = await owner.driver.prepareLaunch!(requirements); + // Re-preparation invalidates the old credential validation before the worker reply. + await runtime.task({ companyId, leaseId: acquired.lease.id, operation: environmentTaskOperationSchema.parse({ + kind: "prepare", requirements, projectIds: [projectId], + }) }); + await expect(runtime.task({ companyId, leaseId: acquired.lease.id, operation: { + kind: "submit", projectIds: [projectId], preparedLaunchId: prepared.configurationId, bootstrapTicket: "synthetic-ticket", + accountIdentity: "account", credential: { type: "codex-auth-json", authJson: '{"OPENAI_API_KEY":"never-transfer-stale-proof"}' }, + runner: { protocolMin: 1, protocolMax: 3, harness: "codex", runnerId: identity.runnerInstanceId, + leaseId: identity.environmentLeaseId, runId: identity.runId, sessionId: identity.normalizedSessionId, + turnId: identity.turnId, itemId: identity.itemId }, + } })).rejects.toThrow("validation unavailable"); + expect(calls.some(op => op.kind === "submit")).toBe(false); + await owner.driver.prepareLaunch!(requirements); + const registration = await owner.registration({} as never, runId); + const handle = owner.launcher({ command: "ignored", args: [], cwd: "/ignored", environment: { OPENAI_API_KEY: "synthetic-launch-key" }, + nativeRunner: { identity, ticket: "synthetic-bootstrap-ticket", + connection: registration.connection, stateDirectory: "/runner/state", runnerVersion: "fixture", runnerDigest: `sha256:${"a".repeat(64)}`, + maxOutboxBytes: 262144, p0ReserveBytes: 65536, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } } }); + let settled = false; + void handle.completion.then(() => { settled = true; }); + try { + registration.activate(); await registration.ready(); + expect(onDispatched).toHaveBeenCalledOnce(); + expect(ingress.connect).toHaveBeenCalledWith(expect.objectContaining({ endpoint: expect.objectContaining({ websocketUrl: "wss://runner.example.test/connect" }) })); + const submit = calls.find(operation => operation.kind === "submit"); + expect(submit).toMatchObject({ kind: "submit", projectIds: [projectId], runner: { leaseId: acquired.lease.id, runId }, + launch: { runnerVersion: "fixture" } }); + const [saved] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, acquired.lease.id)); + expect(saved.metadata?.runnerTaskDispatched).toBe(true); + expect(JSON.stringify(saved.metadata)).not.toMatch(/synthetic-account-key|synthetic-launch-key|synthetic-bootstrap-ticket|synthetic-fixture/); + await expect(runtime.task({ companyId: randomUUID(), leaseId: acquired.lease.id, operation: { kind: "stop" } })).rejects.toThrow("lease unavailable"); + const before = calls.filter(value => value.kind === "submit").length; + await expect(runtime.task({ companyId, leaseId: acquired.lease.id, operation: { ...submit!, projectIds: [randomUUID()] } as never })).rejects.toThrow("project unavailable"); + expect(calls.filter(value => value.kind === "submit")).toHaveLength(before); + handle.child.kill("SIGTERM"); + await vi.waitFor(() => expect(stopRequested).toBe(true)); + await delay(10); expect(settled).toBe(false); + executionStopped = true; await handle.completion; + await registration.release(); expect(complete).not.toHaveBeenCalled(); + await registration.checkpoint("settled"); + expect(checkpoint).toHaveBeenCalledOnce(); expect(complete).toHaveBeenCalledOnce(); + await runtime.releaseRunLeases(runId, "released"); + expect(released).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); + expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true); - await probe.executionTarget.runnerTask!({ kind: "check_project_auth", configurationId: "probe-config", provider: "openai", projectIds: [] }); - await probe.executionTarget.runnerTask!({ kind: "submit", preparedLaunchId: "probe-config", bootstrapTicket: "synthetic-probe-ticket", projectIds: [], - runner: { protocolMin: 1, protocolMax: 3, harness: "codex", runnerId: "probe-runner", leaseId: probeLease.id, - runId: probe.executionTarget.runnerTaskProbe.runId, sessionId: probe.executionTarget.runnerTaskProbe.sessionId, turnId: "probe-turn", itemId: "probe-item" }, - launch: { runnerVersion: "fixture", runnerDigest: `sha256:${"a".repeat(64)}`, - maxOutboxBytes: 262144, p0ReserveBytes: 65536, maxRuntimeMs: 0, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, - }); - const probeSubmit = calls.find((op): op is Extract => op.kind === "submit")!; - expect(probeSubmit.launch?.maxRuntimeMs).toBeGreaterThan(0); - expect(probeSubmit.launch?.maxRuntimeMs).toBeLessThanOrEqual(120_000); + // A controller that dies after dispatch leaves only the lease marker. + const second = await runtime.acquireRunLease({ companyId, environment: environment as never, agentId, + issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, adapterType: "paperclip_runner" }); + if (submit?.kind !== "submit") throw new Error("Missing submit fixture"); + await runtime.task({ companyId, leaseId: second.lease.id, operation: { kind: "check_project_auth", configurationId: "second-config", provider: "openai", projectIds: [projectId] } }); + await runtime.task({ companyId, leaseId: second.lease.id, + operation: { ...submit, credential: undefined, accountIdentity: undefined, credentialIdentity: undefined, + preparedLaunchId: "second-config", runner: { ...submit.runner, leaseId: second.lease.id } } }); + await runtime.task({ companyId, leaseId: second.lease.id, operation: { kind: "complete" } }); + const errors = vi.fn(); + await runtime.releaseRunLeases(runId, "released", errors); + expect(errors).toHaveBeenCalledOnce(); + expect(released).toHaveBeenCalledOnce(); // No release while termination is unknown. + const [pending] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, second.lease.id)); + expect(pending).toMatchObject({ status: "pending_cleanup", metadata: { runnerTaskDispatched: true } }); + await db.delete(environments).where(eq(environments.id, environmentId)); executionStopped = true; - await probe.release(); - expect(released).toHaveBeenCalledOnce(); - } finally { harness.stop(); } + releaseFailure = true; + const recovery = heartbeatService(db, { environmentRuntime: runtime, pluginWorkerManager: workers }); + expect((await recovery.sweepPendingCleanupLeases()).destroyed).toBe(0); + const [retryable] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, second.lease.id)); + expect(retryable.status).toBe("pending_cleanup"); + releaseFailure = false; + await db.update(environmentLeases).set({ metadata: { ...retryable.metadata, pendingCleanupRetryAfterMs: 0 } }).where(eq(environmentLeases.id, second.lease.id)); + const sweep = await recovery.sweepPendingCleanupLeases(); + expect(released).toHaveBeenCalledTimes(2); + expect(sweep.destroyed).toBe(1); + const [cleaned] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, second.lease.id)); + expect(cleaned).toMatchObject({ status: "expired", cleanupStatus: "success", environmentId: null, + metadata: { runnerTaskDispatched: false, remoteExecutionTermination: { state: "stopped" } } }); + } finally { executionStopped = true; handle.child.kill(); await handle.completion.catch(() => {}); harness.stop(); } }); + + it.each([false, true])("replaces a same-run task only after the provider confirms termination (%s)", async stopped => { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), pluginId = randomUUID(), environmentId = randomUUID(), oldLeaseId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Task recovery", issuePrefix: `R${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" }); + await db.insert(environments).values({ id: environmentId, name: `Recovery ${environmentId}`, driver: "plugin", status: "active", config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-recovery", version: "1.0.0", status: "ready", manifestJson: { + capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }], + } as never }); + const priorIdentity = { runnerInstanceId: "runner", environmentLeaseId: oldLeaseId, runId, normalizedSessionId: "session" }; + const runner = { runnerId: "runner", leaseId: oldLeaseId, runId, sessionId: "session", turnId: "turn", itemId: "item", harness: "codex", protocolMin: 1, protocolMax: 3 }; + await db.insert(environmentLeases).values({ id: oldLeaseId, companyId, environmentId, heartbeatRunId: runId, status: "active", provider: `plugin:${pluginId}:tasks`, providerLeaseId: "old-task", + metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", agentId, runnerTaskLease: true, runnerTaskDispatched: true, runnerTaskIdentity: runner } }); + const operations: string[] = []; + const workers = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentTask", "environmentReleaseLease", "environmentAcquireLease"] }), + call: async (_id: string, method: string, params: any) => { + operations.push(method === "environmentTask" ? params.operation.kind : method); + if (method === "environmentAcquireLease") { + expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true); + return { providerLeaseId: "new-task", metadata: {} }; + } + if (method === "environmentReleaseLease") return; + if (params.operation.kind === "check_project_auth") return { kind: "project_auth", taskId: params.taskId, configurationId: params.operation.configurationId, provider: params.operation.provider, allowed: true }; + if (params.operation.kind === "status") return { kind: "status", taskId: "old-task", phase: "cancelled", executionStopped: stopped, sessionStateAvailable: stopped }; + return { kind: "accepted", taskId: params.taskId }; + } } as unknown as PluginWorkerManager; + const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers }); + const acquire = environmentRunOrchestrator(db, { environmentRuntime: runtime }).acquireForRun({ + companyId, selectedEnvironmentId: environmentId, localEnvironmentId: randomUUID(), adapterType: "paperclip_runner", agentId, + issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, executionWorkspaceSettings: null, recoverPluginTask: true, + }); + if (!stopped) { + await expect(acquire).rejects.toThrow("termination_unconfirmed"); + expect(operations).not.toContain("environmentAcquireLease"); + expect(operations).not.toContain("complete"); + return; + } + const next = await acquire; + expect(operations).toEqual(["stop", "status", "complete", "environmentReleaseLease", "environmentAcquireLease"]); + expect(next.lease.id).not.toBe(oldLeaseId); + await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).resolves.toBe("old-task"); + await expect(runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "submit", projectIds: [], bootstrapTicket: "fixture", runner } })).rejects.toThrow("identity mismatch"); + await runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "check_project_auth", configurationId: "recovery-config", provider: "openai", projectIds: [] } }); + await expect(runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "submit", preparedLaunchId: "recovery-config", projectIds: [], bootstrapTicket: "fixture", runner, resumeFrom: "old-task" } })).resolves.toMatchObject({ kind: "accepted" }); + const competingId = randomUUID(); + await db.insert(environmentLeases).values({ id: competingId, companyId, environmentId, heartbeatRunId: runId, status: "active", provider: `plugin:${pluginId}:tasks`, providerLeaseId: "competing-task" }); + await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).rejects.toThrow("not authorized"); + await db.delete(environmentLeases).where(eq(environmentLeases.id, competingId)); + await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, oldLeaseId)); + await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).rejects.toThrow("not authorized"); + }); + }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 96d94ef248a..db22ce7b2c2 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -790,16 +790,9 @@ describe("startServer feedback export wiring", () => { } }); - it("reconciles native replacements at startup and on the execution-control interval", async () => { + it("registers native replacement recovery without an execution-control interval", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); - let executionControlTick: (() => void) | undefined; - const setIntervalSpy = vi.spyOn(globalThis, "setInterval").mockImplementation((( - callback: () => void, - interval: number, - ) => { - if (interval === EXECUTION_RECONCILIATION_INTERVAL_MS) executionControlTick = callback; - return 1 as unknown as ReturnType; - }) as typeof setInterval); + const setIntervalSpy = vi.spyOn(globalThis, "setInterval"); try { await startServer(); await new Promise((resolve) => setImmediate(resolve)); @@ -808,11 +801,8 @@ describe("startServer feedback export wiring", () => { expect.any(Date), { verifyStoppedSession: expect.any(Function) }, ); - - expect(executionControlTick).toBeDefined(); - executionControlTick?.(); - await new Promise((resolve) => setImmediate(resolve)); - expect(reconcileSafeNativeReplacements).toHaveBeenCalledTimes(2); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.executionReplacement)).toBe(true); + expect(setIntervalSpy.mock.calls.some(([, interval]) => interval === EXECUTION_RECONCILIATION_INTERVAL_MS)).toBe(false); } finally { setIntervalSpy.mockRestore(); } diff --git a/server/src/__tests__/task-workspace-api.test.ts b/server/src/__tests__/task-workspace-api.test.ts index 07dbc05d74d..52d6b9be602 100644 --- a/server/src/__tests__/task-workspace-api.test.ts +++ b/server/src/__tests__/task-workspace-api.test.ts @@ -87,6 +87,17 @@ const support = await getEmbeddedPostgresTestSupport(); expect(runs).toHaveLength(1); expect(runs[0]).toMatchObject({ id: f.runId, status: "running" }); }); + it("rejects repository preparation on an environment-owned workspace before queuing a receipt", async () => { + const f = await server.fixture({ conversation: true, disableWakeOnDemand: true }); + const [workspace] = await server.db.insert(executionWorkspaces).values({ companyId: f.companyId, sourceIssueId: f.issueId, + name: "Plugin workspace", cwd: ".", mode: "adapter_managed", strategyType: "adapter_managed", providerType: "adapter_managed" }).returning(); + await server.db.update(issues).set({ projectId: null, projectWorkspaceId: null, executionWorkspaceId: workspace.id }).where(eq(issues.id, f.issueId)); + const call = (tool: string, args: Record) => f.authority.execute({ tool, arguments: args, callId: randomUUID() }); + await expect(call("get_workspace", {})).resolves.toMatchObject({ capabilities: { prepareRepository: "unsupported" } }); + await expect(call("prepare_repository", { repository: { kind: "url", url: "https://github.com/public/example" }, requestKey: "unsupported-plugin" })) + .rejects.toThrow("environment-owned workspace"); + expect(await server.db.select().from(executionWorkspaceRepositories).where(eq(executionWorkspaceRepositories.executionWorkspaceId, workspace.id))).toEqual([]); + }); it("allows a projectless task to use a directory also configured by an unrelated protected project", async () => { const f = await server.fixture({ conversation: true, disableWakeOnDemand: true }); const [project] = await server.db.insert(projects).values({ companyId: f.companyId, name: "Protected physical source", diff --git a/server/src/__tests__/tool-access-service.test.ts b/server/src/__tests__/tool-access-service.test.ts index f2213242972..20df4507b80 100644 --- a/server/src/__tests__/tool-access-service.test.ts +++ b/server/src/__tests__/tool-access-service.test.ts @@ -94,6 +94,7 @@ import { HttpError } from "../errors.js"; import type { VercelConnectClient } from "../services/vercel-connect.js"; import { invalidatePaperclipCloudConnectorCapabilities, type PaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; +const externalDatabaseUrl = process.env.PAPERCLIP_TOOL_ACCESS_TEST_DATABASE_URL?.trim(); const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe @@ -909,8 +910,6 @@ describeEmbeddedPostgres("tool access service", () => { > | null = null; beforeAll(async () => { - const externalDatabaseUrl = - process.env.PAPERCLIP_TOOL_ACCESS_TEST_DATABASE_URL?.trim(); if (externalDatabaseUrl) { db = createDb(externalDatabaseUrl); await instanceSettingsService(db).updateExperimental({ enableMemoryConnectors: true }); @@ -1104,8 +1103,8 @@ describeEmbeddedPostgres("tool access service", () => { const { run } = await createIssueAndRun(db, company.id, agent.id); const { connection } = await createBrokerConnection(db, company.id); await allowConnectionForAgent(db, company.id, agent.id, connection.id); - const mintDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); - const revocationDb = createDb(tempDb!.connectionString, { + const mintDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); + const revocationDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1, }); const service = createTestToolAccessService(mintDb); @@ -1691,9 +1690,9 @@ describeEmbeddedPostgres("tool access service", () => { .values(personalGrantRow(company, connection)) .returning() .then((rows) => rows[0]!); - const serviceDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); + const serviceDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); const service = createTestToolAccessService(serviceDb); - const removalDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); + const removalDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); let releaseRemoval!: () => void; const removalMayCommit = new Promise((resolve) => { releaseRemoval = resolve; @@ -1973,13 +1972,13 @@ describeEmbeddedPostgres("tool access service", () => { subjectId: "previous-user", }); - const firstReplacementDb = createDb(tempDb!.connectionString, { + const firstReplacementDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1, }); - const replacementDb = createDb(tempDb!.connectionString, { + const replacementDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1, }); - const cleanupDb = createDb(tempDb!.connectionString, { + const cleanupDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1, }); const replacementService = createTestToolAccessService(replacementDb); @@ -7579,7 +7578,7 @@ describeEmbeddedPostgres("tool access service", () => { const company = await createCompany(db); const userId = `gmail-member-${randomUUID()}`; await grantBoardUser(db, company.id, userId, []); - const callbackDb = createDb(tempDb!.connectionString, { + const callbackDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1, }); const connector = fakeGmailConnector(company.id, userId); @@ -9383,7 +9382,7 @@ describeEmbeddedPostgres("tool access service", () => { await grantBoardUser(db, company.id, userId, [], "owner"); const actor = { actorType: "user" as const, actorId: userId }; const connector = fakeGmailConnector(company.id, userId); - const refreshDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); + const refreshDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); const service = createTestToolAccessService(db, { paperclipCloudConnector: connector }); const definition = getConnectableAppDefinition("gmail")!; const availability = definition.ownershipAvailability; @@ -9523,10 +9522,10 @@ describeEmbeddedPostgres("tool access service", () => { const company = await createCompany(db); const userId = `gmail-member-${randomUUID()}`; await grantBoardUser(db, company.id, userId, []); - const callbackDb = createDb(tempDb!.connectionString, { + const callbackDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1, }); - const removalDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); + const removalDb = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); const service = createTestToolAccessService(callbackDb, { paperclipCloudConnector: fakeGmailConnector(company.id, userId), }); diff --git a/server/src/index.ts b/server/src/index.ts index 1645b0d7bcb..a7b0d17fabe 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -39,7 +39,7 @@ import { deliverExecutionStatuses } from "./services/execution-status-delivery.j import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./services/execution-recovery-resolution.js"; import { reconcileSafeNativeReplacements } from "./services/native-runtime/native-safe-replacement.js"; import { reconcileAbandonedExecutionControl } from "./services/execution-control-reconciliation.js"; -import { EXECUTION_RECONCILIATION_INTERVAL_MS } from "./services/execution-control-deadline.js"; +import { executionControlSchedule, registerExecutionControlWork } from "./services/execution-control-work.js"; import { connectionIntentDeliveryService } from "./services/connection-intent-delivery.js"; import { existsSync, readFileSync, rmSync } from "node:fs"; import type { Server, RequestListener } from "node:http"; @@ -1205,30 +1205,6 @@ async function startServerWithDatabaseTeardown( await Promise.allSettled([...heartbeatSchedulerInFlight]); } }; - const executionControlSweepsInFlight = new Set(); - const executionControlSweeps = [ - ["finalization", () => reconcileAbandonedExecutionControl(db)], - ["replacement", () => heartbeat ? reconcileSafeNativeReplacements(db, new Date(), { verifyStoppedSession: run => verifyStoppedNativeSessionForReplacement(db, run) }) : undefined], - ["reconciliation_delivery", () => heartbeat ? deliverReconciledExecutions(db, heartbeat.wakeup) : undefined], - ["status_delivery", () => deliverExecutionStatuses(db)], - ["automatic_disposition", () => settleUnrecoverableExecutions(db)], - ["local_ai_login_cleanup", () => localAiLoginService(db).reapExpired()], - ] as const; - const sweepExecutionControl = () => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped || isWarmStandby()) return; - // Independent durable queues must not block one another. Each queue remains - // single-flight; a later sweep observes committed transitions from its peers. - for (const [queue, work] of executionControlSweeps) { - if (executionControlSweepsInFlight.has(queue)) continue; - executionControlSweepsInFlight.add(queue); - trackHeartbeatSchedulerWork(Promise.resolve().then(async () => { await work(); }) - .catch(err => logger.error({ err, queue }, "execution control reconciliation failed")) - .finally(() => { executionControlSweepsInFlight.delete(queue); })); - } - }; - const executionControlInterval = setInterval(sweepExecutionControl, EXECUTION_RECONCILIATION_INTERVAL_MS); - executionControlInterval.unref?.(); - sweepExecutionControl(); const startHeartbeatSchedulerInterval = (callback: () => void) => { heartbeatSchedulerInterval = setInterval(() => { if (!isWarmStandby() && !isIdleTaskDrainActive()) callback(); @@ -1346,6 +1322,18 @@ async function startServerWithDatabaseTeardown( resolveNativeQuestion: (interaction) => deliverNativeQuestionResponse(db as any, interaction), }); const deliveryWork = app.locals.deliveryWork as ReturnType; + const executionSchedule = executionControlSchedule(db); + registerExecutionControlWork(deliveryWork, [ + { queue: DELIVERY_QUEUES.executionFinalization, run: () => reconcileAbandonedExecutionControl(db), next: executionSchedule.finalization }, + { queue: DELIVERY_QUEUES.executionStatus, run: () => deliverExecutionStatuses(db), next: executionSchedule.status }, + { queue: DELIVERY_QUEUES.executionDisposition, run: () => settleUnrecoverableExecutions(db), next: executionSchedule.disposition }, + { queue: DELIVERY_QUEUES.localLoginCleanup, run: () => localAiLoginService(db).reapExpired(), next: executionSchedule.localLogin }, + ...(heartbeat ? [ + { queue: DELIVERY_QUEUES.executionReplacement, run: () => reconcileSafeNativeReplacements(db, new Date(), { verifyStoppedSession: run => verifyStoppedNativeSessionForReplacement(db, run) }), next: executionSchedule.replacement }, + { queue: DELIVERY_QUEUES.executionContinuation, run: () => deliverReconciledExecutions(db, heartbeat.wakeup), next: executionSchedule.continuation }, + ] : []), + ], () => !heartbeatSchedulerStopped); + const budgetEnforcementWorker = registerBudgetEnforcementWork(db, deliveryWork, { cancelWorkForScope: accountingHeartbeat.cancelBudgetScopeWork, }); @@ -2092,7 +2080,6 @@ async function startServerWithDatabaseTeardown( unsubscribeChatCompletions(); await deliveryWork.stop(); unregisterTerminalWorkspaceProbe?.(); - clearInterval(executionControlInterval); if (heartbeatSchedulerInterval) { clearInterval(heartbeatSchedulerInterval); heartbeatSchedulerInterval = null; diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 372531d971d..fefd2afed44 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -3139,7 +3139,13 @@ export function agentRoutes( if (test || binding.provider === "github") { const testEnvironmentId = await resolveAdapterTestEnvironmentId(companyId, environmentId, binding.provider === "github"); if (testEnvironmentId) await assertAdapterTestEnvironmentForCompany(companyId, testEnvironmentId, adapterType); - const target = await resolveAdapterTestExecutionContext({ companyId, adapterType, agentId: newAgent ? undefined : agentId, environmentId: testEnvironmentId }); + const hasSavedAgent = !newAgent || (isUuidLike(agentId) && Boolean(await svc.getById(agentId))); + if (!hasSavedAgent && !test && testEnvironmentId && (await environmentsSvc.getById(testEnvironmentId))?.driver === "plugin") { + // The lifecycle verifies the saved agent after its plugin can create the agent-owned computer. + return { connectionId: selection.connection.id }; + } + const target = await resolveAdapterTestExecutionContext({ companyId, adapterType, + agentId: hasSavedAgent ? agentId : undefined, environmentId: testEnvironmentId }); try { if (!target.executionTarget && target.fallbackChecks.length > 0) throw unprocessable("The agent environment is not available for adoption"); await withManagedAiProbe(db, { companyId, agentId, responsibleUserId: userId, adapterType, binding, config, allowUninstalledPersonal: newAgent, allowUninstalledShared, allowLegacyValidation: true }, async managed => { diff --git a/server/src/routes/task-workspaces.ts b/server/src/routes/task-workspaces.ts index 93677c7631e..64b71897a14 100644 --- a/server/src/routes/task-workspaces.ts +++ b/server/src/routes/task-workspaces.ts @@ -33,7 +33,7 @@ export function taskWorkspaceRoutes(db: Db) { router.get("/issues/:id/workspace", async (req, res) => { const issue = await task(req); const view = await workspaces.inspectTaskWorkspace(issue.companyId, issue.id, req.actor); - res.json({ ...view, repositories: issue.executionWorkspaceId ? await workspaces.listTaskRepositories(issue.companyId, issue.executionWorkspaceId) : [], capabilities: { prepareRepository: "next_normal_admission", selectWorkspace: "next_normal_admission" }, fileLocations: { taskFiles: "workspace", personalInstructionsAndMemory: "AGENT_HOME", providerHome: "runtime_managed" } }); + res.json({ ...view, repositories: issue.executionWorkspaceId ? await workspaces.listTaskRepositories(issue.companyId, issue.executionWorkspaceId) : [], capabilities: { prepareRepository: view.workspace?.providerType === "adapter_managed" ? "unsupported" : "next_normal_admission", selectWorkspace: "next_normal_admission" }, fileLocations: { taskFiles: "workspace", personalInstructionsAndMemory: "AGENT_HOME", providerHome: "runtime_managed" } }); }); router.put("/issues/:id/workspace", validate(selectTaskWorkspaceSchema), async (req, res) => { const issue = await task(req, true); diff --git a/server/src/routes/tool-access.ts b/server/src/routes/tool-access.ts index 48a590a5bc1..9ffebb7b9ed 100644 --- a/server/src/routes/tool-access.ts +++ b/server/src/routes/tool-access.ts @@ -9,6 +9,7 @@ import { agents, companies, connectionGrants, issueThreadInteractions, toolConne import { and, eq, or } from "drizzle-orm"; import { APP_STORE_DEFINITIONS, + isConnectorFeatureEnabled, isMemoryConnectorId, GITHUB_CONNECTOR_PROFILES, GOOGLE_WORKSPACE_CONNECTOR_PROFILES, @@ -819,7 +820,7 @@ function connectorEnrollmentPrincipal(req: Request): string { ? await options.paperclipCloudConnector.getCapabilities() : []; const vercelConnect = vercelConnectIntegrationStatus(); - const { enableMemoryConnectors } = await instanceSettingsService(db).getExperimental(); + const settings = await instanceSettingsService(db).getExperimental(); let oauthCallbackUrl: string | undefined; try { // Setup must display the callback used by its initiating POST. Provider @@ -850,7 +851,8 @@ function connectorEnrollmentPrincipal(req: Request): string { : "Vercel Connect setup is disabled on this Paperclip instance.", }, }, - apps: [...APP_STORE_DEFINITIONS.filter((app) => (enableMemoryConnectors || !isMemoryConnectorId(app.slug))).map((app) => + apps: [...APP_STORE_DEFINITIONS.filter((app) => (settings.enableMemoryConnectors || !isMemoryConnectorId(app.slug)) + && isConnectorFeatureEnabled(app, settings)).map((app) => appWithPaperclipCloudConnectorAvailability(app, advertisedProfiles) ), ...await aiConnectionRouterService(db).catalog()], }); diff --git a/server/src/services/connection-intents.ts b/server/src/services/connection-intents.ts index 90381467751..41cf08ceb90 100644 --- a/server/src/services/connection-intents.ts +++ b/server/src/services/connection-intents.ts @@ -29,6 +29,8 @@ import { } from "@paperclipai/db"; import { APP_STORE_DEFINITIONS, + isConnectorFeatureEnabled, + getAppDefinitionForApplicationKey, AGGREGATOR_PRIORITY, AGGREGATOR_NAMES, AGGREGATOR_CATALOG_SOURCES, findAggregatorService, searchAggregatorServices, prepareConnectionSearch, scoreConnectionSearch, explicitAggregatorQuery, normalizeConnectionQuery, parseAggregatorRoute, aggregatorProviderQuestion, aggregatorProviderQuestionSet, aggregatorContinuationInstruction, isRemoteMcpConnectorId, askUserQuestionsPayloadSchema, askUserQuestionsResultSchema, @@ -364,6 +366,9 @@ export function connectionIntentService(db: Db) { if (!service.startsWith("connection:")) { const app = getAppStoreDefinition(service); if (!app) throw notFound("Connection service was not found"); + if (app.experimentalFeatureKey && !isConnectorFeatureEnabled(app, await instanceSettingsService(db).getExperimental())) { + throw notFound("Connection service was not found"); + } if (purpose === "channel" && service !== "agentmail") { throw unprocessable("This email connection is not available"); } @@ -389,6 +394,14 @@ export function connectionIntentService(db: Db) { || (grant.kind === "agent" && grant.subjectAgentId === agentId) ))) throw notFound("Configured connection was not found"); const application = await access.getApplication(connection.applicationId, companyId); + const definition = application.applicationKey != null + ? getAppDefinitionForApplicationKey(application.applicationKey) + : typeof connection.config?.sourceTemplateKey === "string" + ? getAppStoreDefinition(connection.config?.sourceTemplateKey) : null; + if (definition?.experimentalFeatureKey && !isConnectorFeatureEnabled(definition, + await instanceSettingsService(db).getExperimental())) { + throw notFound("Configured connection was not found"); + } return { slug: service, name: connection.name, description: application.description, searchCapabilities: "", branding: { logoUrl: undefined, darkLogoUrl: undefined }, @@ -413,7 +426,8 @@ export function connectionIntentService(db: Db) { const discoveryMethods = (app: (typeof APP_STORE_DEFINITIONS)[number]) => getAvailableConnectionMethods(app) .filter(method => method.purpose !== "channel" || app.slug === "agentmail" || (method.provider === "github" ? settings.enableGitHubReviewBots : settings.enableChatConnectors)); - const services = [...APP_STORE_DEFINITIONS.filter(app => discoveryMethods(app).length).map((app) => app.slug), + const services = [...APP_STORE_DEFINITIONS.filter(app => isConnectorFeatureEnabled(app, settings) + && discoveryMethods(app).length).map((app) => app.slug), ...inventory.connections.filter((connection) => sourceSlugForConnection(connection, inventory.applicationsById)?.startsWith("connection:") && connection.status !== "archived").map((connection) => `connection:${connection.id}`)]; diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index b1c7c62062a..26f737b39cd 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,12 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + executionFinalization: "execution-finalization", + executionReplacement: "execution-replacement", + executionContinuation: "execution-continuation", + executionStatus: "execution-status", + executionDisposition: "execution-disposition", + localLoginCleanup: "local-login-cleanup", budgetEnforcement: "budget-enforcement", accounting: "cost-accounting", agentLifecycle: "agent-lifecycle", diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 8016bb567a5..8757efe6cf9 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -15,7 +15,9 @@ * and transport logic. */ -import type { Db } from "@paperclipai/db"; +import { environmentLeases, type Db } from "@paperclipai/db"; +import { and, eq } from "drizzle-orm"; +import { remoteExecutionHasStopped } from "./remote-execution-termination.js"; import type { Environment, EnvironmentLease, @@ -276,6 +278,8 @@ export function environmentRunOrchestrator( executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; /** Existing live runner recovery must use its original active lease, including ephemeral leases. */ reattachRemoteLease?: { leaseId: string; providerLeaseId: string; remoteCwd: string }; + /** A new controller must stop the previous plugin task before allocating a replacement. */ + recoverPluginTask?: boolean; }): Promise { // Step 1: Resolve environment const selectedEnvironment = await resolveEnvironment({ @@ -288,6 +292,20 @@ export function environmentRunOrchestrator( selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, ); + if (input.recoverPluginTask && environment.driver === "plugin" && input.adapterType === "paperclip_runner") { + const prior = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.heartbeatRunId, input.heartbeatRunId), + )); + if (prior.some(lease => lease.metadata?.runnerTaskLease !== true || lease.environmentId !== environment.id || + lease.issueId !== input.issueId || lease.executionWorkspaceId !== (input.persistedExecutionWorkspace?.id ?? null) || + lease.metadata?.agentId !== input.agentId)) throw new Error("native_plugin_recovery_lease_mismatch"); + const errors: unknown[] = []; + await environmentRuntime.releaseRunLeases(input.heartbeatRunId, "released", (_id, error) => { errors.push(error); }); + if (errors.length || (prior.length && !await remoteExecutionHasStopped(db, input.companyId, input.heartbeatRunId))) { + throw new Error("native_plugin_recovery_termination_unconfirmed", { cause: errors[0] }); + } + } + // Step 2: Acquire lease const acquisitionInput = { companyId: input.companyId, @@ -346,7 +364,7 @@ export function environmentRunOrchestrator( }); // Step 4: Resolve execution transport - const executionTransport = await resolveTransport({ + const executionTransport = environment.driver === "plugin" && input.adapterType === "paperclip_runner" ? null : await resolveTransport({ companyId: input.companyId, adapterType: input.adapterType, environment, @@ -383,6 +401,7 @@ export function environmentRunOrchestrator( executionWorkspace: RealizedExecutionWorkspace; effectiveExecutionWorkspaceMode: string | null; persistedExecutionWorkspace: ExecutionWorkspace | null; + runnerTaskProjectIds?: readonly string[]; /** * The host duplex observability recorder for this run. The orchestrator threads * it to `resolveEnvironmentExecutionTarget`, which stamps it on the sandbox @@ -426,6 +445,10 @@ export function environmentRunOrchestrator( // Step 2: Realize workspace in the environment via the runtime driver let workspaceRealization: Record = {}; let realizedWorkspaceCwd: string | null = null; + const pluginTaskWorkspace = environment.driver === "plugin" && adapterType === "paperclip_runner"; + if (pluginTaskWorkspace && !await environmentRuntime.supportsTasks({ lease })) { + throw new EnvironmentRunError("unsupported_adapter_environment", "Plugin environment does not provide native Runner task execution"); + } if (ENVIRONMENT_DRIVER_TRAITS[environment.driver].realizesWorkspace) { try { const remoteCwd = @@ -478,7 +501,7 @@ export function environmentRunOrchestrator( // for `local`. Keep the step for `ssh`, which runs the command on the // remote host that shares the workspace path. const driverSkipsHostProvision = - environment.driver === "local" || environment.driver === "sandbox"; + environment.driver === "local" || environment.driver === "sandbox" || pluginTaskWorkspace; if (provisionCommand && environment.driver === "sandbox") { logger.info( { @@ -569,7 +592,7 @@ export function environmentRunOrchestrator( !("runnerTask" in executionTarget) || typeof executionTarget.runnerTask !== "function")) { throw new Error("Managed workspace execution requires the native plugin task bridge"); } - const realizationMode = workspaceRealization.mode === "in_place" ? "in_place" : "copy"; + const realizationMode = pluginTaskWorkspace || workspaceRealization.mode === "in_place" ? "in_place" : "copy"; const authoritativeRoot = typeof workspaceRealization.authoritativeRoot === "string" && workspaceRealization.authoritativeRoot.trim().length > 0 ? workspaceRealization.authoritativeRoot.trim() @@ -592,6 +615,11 @@ export function environmentRunOrchestrator( if (executionTarget) { executionTarget = { ...executionTarget, + ...(executionTarget.kind === "remote" && executionTarget.transport === "sandbox" && executionTarget.runnerTask + ? { runnerTaskProjectIds: [...new Set([executionWorkspace.projectId, + ...input.runnerTaskProjectIds ?? [], + ...(executionWorkspace.additionalWorkspaces ?? []).map(workspace => workspace.projectId)] + .filter((id): id is string => Boolean(id)))] } : {}), ...(executionTarget.kind === "remote" && realizationMode === "in_place" ? { remoteCwd: authoritativeRoot } : {}), diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts new file mode 100644 index 00000000000..fb59deafbe8 --- /dev/null +++ b/server/src/services/execution-control-notifications.ts @@ -0,0 +1,20 @@ +import type { Db } from "@paperclipai/db"; +import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; + +export async function notifyExecutionRunWork(transaction: object) { + for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, + DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition]) { + await notifyDeliveryWork(transaction, queue); + } +} + +export function writeExecutionControl( + db: Db | Parameters[0]>[0], + queue: DeliveryQueue | readonly DeliveryQueue[], + write: (tx: Db) => PromiseLike, +): Promise { + return db.transaction(async transaction => { + for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); + return write(transaction as unknown as Db); + }); +} diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 72266731dff..5b3a7aa3735 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -1,3 +1,4 @@ +import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { randomUUID } from "node:crypto"; import { logger } from "../middleware/logger.js"; import { and, eq, isNotNull, lte, sql } from "drizzle-orm"; @@ -181,6 +182,7 @@ export async function reconcileAbandonedExecutionControl( updatedAt: now, }) .where(eq(nativeRunFinalizations.runId, run.id)); + await notifyExecutionRunWork(tx); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-control-work.ts b/server/src/services/execution-control-work.ts new file mode 100644 index 00000000000..1c7a1a6d376 --- /dev/null +++ b/server/src/services/execution-control-work.ts @@ -0,0 +1,98 @@ +import { and, asc, eq, isNotNull, sql } from "drizzle-orm"; +import { adapterAuthSessions, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { type DeliveryQueue } from "./delivery-work-notifications.js"; +import { EXECUTION_RECONCILIATION_INTERVAL_MS } from "./execution-control-deadline.js"; +import { obsoleteExecutionConversationHold, unresolvedExecutionDispositions } from "./execution-recovery-resolution.js"; +import { nativeReplacementPending } from "./native-runtime/native-safe-replacement.js"; + +export function registerExecutionControlWork( + coordinator: ReturnType, + duties: ReadonlyArray<{ + queue: DeliveryQueue; + run: () => Promise; + next: () => Promise; + }>, + canRun: () => boolean = () => true, +) { + const workers = duties.map(duty => { + let requested = 1; + let reconciled = 0; + let next: number | null = null; + let earliestRunAt = 0; + return coordinator.register(duty.queue, { + retryMs: EXECUTION_RECONCILIATION_INTERVAL_MS, + pausePassiveMaintenance: true, + onSettled: () => { requested += 1; }, + run: async () => { + if (!canRun()) { requested += 1; next = null; return; } + const now = Date.now(); + if (now < earliestRunAt) { + if (reconciled !== requested || (next !== null && next <= now)) next = earliestRunAt; + return; + } + const generation = ++requested; + earliestRunAt = now + EXECUTION_RECONCILIATION_INTERVAL_MS; + await duty.run(); + const deadline = await duty.next(); + next = deadline === null ? null : Math.max(deadline, earliestRunAt); + reconciled = generation; + }, + hasPending: async () => reconciled !== requested, + nextRunAt: () => next, + }); + }); + return { ready: Promise.all(workers.map(worker => worker.ready)) }; +} + +export function executionControlSchedule(db: Db) { + const retry = () => Date.now() + EXECUTION_RECONCILIATION_INTERVAL_MS; + const deadline = (dates: Array) => { + const at = dates.flatMap(date => date ? [date.getTime()] : []); + if (!at.length) return null; + const earliest = Math.min(...at); + return earliest > Date.now() ? earliest : retry(); + }; + return { + async finalization() { + const [native, legacy] = await Promise.all([ + db.select({ at: nativeRunFinalizations.controlDeadlineAt }).from(nativeRunFinalizations) + .where(isNotNull(nativeRunFinalizations.controlDeadlineAt)).orderBy(asc(nativeRunFinalizations.controlDeadlineAt)).limit(1), + db.select({ at: heartbeatRuns.executionControlDeadlineAt }).from(heartbeatRuns) + .where(isNotNull(heartbeatRuns.executionControlDeadlineAt)).orderBy(asc(heartbeatRuns.executionControlDeadlineAt)).limit(1), + ]); + return deadline([native[0]?.at, legacy[0]?.at]); + }, + async replacement() { + const rows = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)) + .where(nativeReplacementPending()).limit(1); + return rows.length ? retry() : null; + }, + async continuation() { + const rows = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + )).limit(1); + return rows.length ? retry() : null; + }, + async status() { + const rows = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(isNotNull(heartbeatRuns.executionStatusDeliveryId)).limit(1); + return rows.length ? retry() : null; + }, + async disposition() { + const [conversation, execution] = await Promise.all([ + db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(obsoleteExecutionConversationHold()).limit(1), + unresolvedExecutionDispositions(db, 1), + ]); + return conversation.length || execution.length ? retry() : null; + }, + async localLogin() { + const rows = await db.select({ at: adapterAuthSessions.expiresAt }).from(adapterAuthSessions).where(and( + eq(adapterAuthSessions.connectionMethod, "local_subscription"), isNotNull(adapterAuthSessions.expiresAt), + )).orderBy(asc(adapterAuthSessions.expiresAt)).limit(1); + return deadline([rows[0]?.at]); + }, + }; +} diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index 3323bceb93e..e56141565b9 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -1,3 +1,5 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { randomUUID } from "node:crypto"; import { canonicalExecutionUuid } from "./execution-identity.js"; @@ -201,38 +203,43 @@ export async function markExecutionReconciliation( throw conflict("The authorized chat retry owner is no longer valid."); } } - await db - .update(nativeRunFinalizations) - .set({ - failureDetail: sql`coalesce(${nativeRunFinalizations.failureDetail}, '{}'::jsonb) || ${JSON.stringify({ replacementDenied: "operator_reconciled" })}::jsonb`, - }) - .where( - and( - eq(nativeRunFinalizations.companyId, action.companyId), - eq(nativeRunFinalizations.runId, decision.runId), - ), - ); - await db - .update(issueRecoveryActions) - .set({ - evidence: { - ...action.evidence, - automaticRecovery: undefined, - executionReconciliation: { - ...decision, - actorId, - recordedAt: new Date().toISOString(), + await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionReplacement); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); + await tx + .update(nativeRunFinalizations) + .set({ + failureDetail: sql`coalesce(${nativeRunFinalizations.failureDetail}, '{}'::jsonb) || ${JSON.stringify({ replacementDenied: "operator_reconciled" })}::jsonb`, + }) + .where( + and( + eq(nativeRunFinalizations.companyId, action.companyId), + eq(nativeRunFinalizations.runId, decision.runId), + ), + ); + await tx + .update(issueRecoveryActions) + .set({ + evidence: { + ...action.evidence, + automaticRecovery: undefined, + executionReconciliation: { + ...decision, + actorId, + recordedAt: new Date().toISOString(), + }, + continuationDelivery: options?.workspaceRepairOnly ? "not_requested" : deliveryOwner ? "delegated" : "pending", + ...(deliveryOwner ? { continuationDeliveryOwner: deliveryOwner } : {}), }, - continuationDelivery: options?.workspaceRepairOnly ? "not_requested" : deliveryOwner ? "delegated" : "pending", - ...(deliveryOwner ? { continuationDeliveryOwner: deliveryOwner } : {}), - }, - }) - .where( - and( - eq(issueRecoveryActions.companyId, action.companyId), - eq(issueRecoveryActions.id, action.id), - ), - ); + }) + .where( + and( + eq(issueRecoveryActions.companyId, action.companyId), + eq(issueRecoveryActions.id, action.id), + ), + ); + }); } export async function deliverReconciledExecutions( @@ -337,60 +344,8 @@ export async function deliverReconciledExecutions( } } -/** - * Failed execution is a system responsibility, not a user questionnaire. After - * automatic recovery is ruled out, preserve evidence and stop without replay. - * This is NOT evidence that an external action succeeded or never happened. - * The resolved record retains a dispatch hold until actual evidence clears it. - */ -export async function settleUnrecoverableExecutions( - db: Db, - now = new Date(), - options: { failpoint?: (phase: "persisted") => void } = {}, -) { - // Fold obsolete conversation holds without waking historical work on upgrade. - // Keep their evidence and record the policy change in the task's activity log. - const obsoleteConversationHold = and( - conversationRecoveryActionPredicate(), - sql`not (${issueRecoveryActions.evidence} ? 'systemRecovery')`, - or( - inArray(issueRecoveryActions.status, ["active", "escalated"]), - sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, - ), - ); - await db.transaction(async tx => { - const foldable = await tx.select().from(issueRecoveryActions).where(obsoleteConversationHold) - .limit(25).for("update", { skipLocked: true }); - for (const candidate of foldable) { - if (await getConversationOwnershipBlocker(tx as unknown as Db, candidate.companyId, candidate.sourceIssueId)) continue; - const [action] = await tx.update(issueRecoveryActions).set({ - status: "resolved", - outcome: "cancelled", - resolvedAt: now, - updatedAt: now, - nextAction: "Automatic attempts stopped. Send a new message to continue the conversation.", - resolutionNote: "Conversation continuation does not replay prior tool calls.", - wakePolicy: null, - monitorPolicy: null, - evidence: sql`case when ${issueRecoveryActions.evidence} ? 'automaticRecovery' - then jsonb_set(${issueRecoveryActions.evidence}, '{automaticRecovery,replay}', '"conversation_continuation"'::jsonb) - else ${issueRecoveryActions.evidence} end`, - }).where(and(obsoleteConversationHold, eq(issueRecoveryActions.id, candidate.id))).returning(); - if (!action) continue; - await persistActivity(tx as unknown as Db, { - companyId: action.companyId, - actorType: "system", - actorId: "execution-recovery", - action: "issue.execution_recovery_settled", - entityType: "issue", - entityId: action.sourceIssueId, - details: { recoveryActionId: action.id, outcome: "cancelled", continuation: "conversation" }, - }); - } - }); - // Filter eligibility before applying the batch limit. A queue of sessions - // awaiting replacement must not starve settled incidents behind it. - const candidates = await db +export function unresolvedExecutionDispositions(db: Db, limit: number) { + return db .select({ action: issueRecoveryActions }) .from(issueRecoveryActions) .innerJoin( @@ -437,7 +392,67 @@ export async function settleUnrecoverableExecutions( or coalesce(${nativeRunFinalizations.failureDetail}->>'replacementDenied', '') <> '')`, ), ) - .limit(25); + .limit(limit); +} + +export function obsoleteExecutionConversationHold() { + return and( + conversationRecoveryActionPredicate(), + sql`not (${issueRecoveryActions.evidence} ? 'systemRecovery')`, + or( + inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, + ), + ); +} + +/** + * Failed execution is a system responsibility, not a user questionnaire. After + * automatic recovery is ruled out, preserve evidence and stop without replay. + * This is NOT evidence that an external action succeeded or never happened. + * The resolved record retains a dispatch hold until actual evidence clears it. + */ +export async function settleUnrecoverableExecutions( + db: Db, + now = new Date(), + options: { failpoint?: (phase: "persisted") => void } = {}, +) { + // Fold obsolete conversation holds without waking historical work on upgrade. + // Keep their evidence and record the policy change in the task's activity log. + const obsoleteConversationHold = obsoleteExecutionConversationHold(); + await db.transaction(async tx => { + const foldable = await tx.select().from(issueRecoveryActions).where(obsoleteConversationHold) + .limit(25).for("update", { skipLocked: true }); + for (const candidate of foldable) { + if (await getConversationOwnershipBlocker(tx as unknown as Db, candidate.companyId, candidate.sourceIssueId)) continue; + const [action] = await tx.update(issueRecoveryActions).set({ + status: "resolved", + outcome: "cancelled", + resolvedAt: now, + updatedAt: now, + nextAction: "Automatic attempts stopped. Send a new message to continue the conversation.", + resolutionNote: "Conversation continuation does not replay prior tool calls.", + wakePolicy: null, + monitorPolicy: null, + evidence: sql`case when ${issueRecoveryActions.evidence} ? 'automaticRecovery' + then jsonb_set(${issueRecoveryActions.evidence}, '{automaticRecovery,replay}', '"conversation_continuation"'::jsonb) + else ${issueRecoveryActions.evidence} end`, + }).where(and(obsoleteConversationHold, eq(issueRecoveryActions.id, candidate.id))).returning(); + if (!action) continue; + await persistActivity(tx as unknown as Db, { + companyId: action.companyId, + actorType: "system", + actorId: "execution-recovery", + action: "issue.execution_recovery_settled", + entityType: "issue", + entityId: action.sourceIssueId, + details: { recoveryActionId: action.id, outcome: "cancelled", continuation: "conversation" }, + }); + } + }); + // Filter eligibility before applying the batch limit. A queue of sessions + // awaiting replacement must not starve settled incidents behind it. + const candidates = await unresolvedExecutionDispositions(db, 25); for (const { action: candidate } of candidates) { const runId = candidate.evidence.runId; if (typeof runId !== "string") continue; @@ -586,6 +601,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); + await notifyExecutionRunWork(tx); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/execution-workspace-repositories.ts b/server/src/services/execution-workspace-repositories.ts index 2186d5638b0..fae172296ff 100644 --- a/server/src/services/execution-workspace-repositories.ts +++ b/server/src/services/execution-workspace-repositories.ts @@ -153,6 +153,7 @@ export function executionWorkspaceRepositoryService(db: Db) { const [workspace] = await measureRunPhase("workspace.sources.inventory", async () => tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, issue.executionWorkspaceId!), eq(executionWorkspaces.companyId, input.companyId), taskWorkspaceSelectableCondition())).for("no key update")); if (!workspace) throw conflict("Execution workspace is unavailable"); await assertTaskWorkspaceExecutionAccess(tx, input.actor, input.companyId, { issueId: input.issueId, workspaceId: workspace.id }); + if (workspace.providerType === "adapter_managed") throw conflict("Repository preparation is unavailable for this environment-owned workspace"); const anonymousOnly = await requiresAnonymousAcquisition(tx, { issue, workspace, agentId: input.actor.agentId, runId: input.actor.runId }); const source = resolveAcquisitionSource(parsed.repository, available, anonymousOnly); diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index c449167dd00..a33d421a24f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -1,3 +1,4 @@ +import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { canSelectFreshWorkspace, type ContinuationReadiness } from "./fresh-workspace-continuation.js"; import { readTaskRetryIntent } from "./task-retry.js"; import { readSystemRecovery } from "./system-recovery-state.js"; @@ -356,16 +357,19 @@ export async function admitExplicitNativeContinuation(input: { ...(queuedInterrupt ? { queuedCommentInterruptId: input.queuedCommentInterruptId } : {}), ...(queuedRequest ? { queuedCommentRequestId: input.queuedCommentRequestId } : {}), runId: input.successorRunId, previousRunId: previous.id, recordedAt: new Date().toISOString() }; - for (const runId of cancelledStartupIds) { - await db.update(nativeRunFinalizations).set({ - phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, - controlDeadlineAt: null, updatedAt: new Date(), - }).where(and(eq(nativeRunFinalizations.companyId, companyId), eq(nativeRunFinalizations.runId, runId))); - await db.update(heartbeatRuns).set({ - ...(nativeSources.some(run => run.id === runId) ? { nativePhase: "terminal_failure", nativePhaseUpdatedAt: new Date() } : {}), - executionControlDeadlineAt: null, - }).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))); - } + if (cancelledStartupIds.size) await db.transaction(async tx => { + await notifyExecutionRunWork(tx); + for (const runId of cancelledStartupIds) { + await tx.update(nativeRunFinalizations).set({ + phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, + controlDeadlineAt: null, updatedAt: new Date(), + }).where(and(eq(nativeRunFinalizations.companyId, companyId), eq(nativeRunFinalizations.runId, runId))); + await tx.update(heartbeatRuns).set({ + ...(nativeSources.some(run => run.id === runId) ? { nativePhase: "terminal_failure", nativePhaseUpdatedAt: new Date() } : {}), + executionControlDeadlineAt: null, + }).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))); + } + }); if (nativeSources.length) await db.update(nativeRunFinalizations).set({ failureDetail: sql`coalesce(${nativeRunFinalizations.failureDetail}, '{}'::jsonb) || ${JSON.stringify({ replacementDenied: "explicit_user_continuation" })}::jsonb`, updatedAt: new Date(), diff --git a/server/src/services/heartbeat-github-launchers.test.ts b/server/src/services/heartbeat-github-launchers.test.ts index 5bb4038bacb..10626f369be 100644 --- a/server/src/services/heartbeat-github-launchers.test.ts +++ b/server/src/services/heartbeat-github-launchers.test.ts @@ -11,6 +11,16 @@ import { prepareHeartbeatGitHubLaunchers } from "./heartbeat-github-launchers.js const target = { kind: "remote" as const, transport: "sandbox" as const, providerKey: "daytona", remoteCwd: "/workspace" }; describe("heartbeat GitHub launcher lifetime", () => { + it("leaves a task provider's runtime untouched", async () => { + const prepare = vi.fn(), mint = vi.fn(); + const env = { OPENAI_API_KEY: "selected-credential" }; + const result = await prepareHeartbeatGitHubLaunchers({ native: true, githubConfigured: true, agentId: "agent", + runId: "run", target: { ...target, runnerTask: async () => ({}) }, cwd: "/core/workspace", + env, brokerUrl: "https://paperclip.test", createBrokerToken: mint }, prepare); + expect(result).toEqual({ env, cleanupLocation: null }); + expect(prepare).not.toHaveBeenCalled(); + expect(mint).not.toHaveBeenCalled(); + }); it.each([target, null])("defers native managed authorization/staging to the session owner (%j)", async (target) => { const prepare = vi.fn(); const mint = vi.fn(); diff --git a/server/src/services/heartbeat-github-launchers.ts b/server/src/services/heartbeat-github-launchers.ts index 83786a48eb2..df24082f126 100644 --- a/server/src/services/heartbeat-github-launchers.ts +++ b/server/src/services/heartbeat-github-launchers.ts @@ -15,6 +15,10 @@ export async function prepareHeartbeatGitHubLaunchers( prepareLaunchers = prepareGitHubOperationLaunchers, cleanupLaunchers = cleanupGitHubOperationLaunchers, ) { + if (input.target?.kind === "remote" && input.target.transport === "sandbox" && input.target.runnerTask) { + // Provider-owned tasks decide their own runtime layout and command environment. + return { env: input.env, cleanupLocation: null }; + } // Native configured access is owned by the provider session supervisor. // Defer staging until it has acquired that session; never mutate a live // process's authorization from heartbeat preparation. diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index b571364d22d..0d2d33e61d8 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,5 @@ +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { writeExecutionControl } from "./execution-control-notifications.js"; import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js"; import { promptInjectionPreventionService } from "./prompt-injection-prevention.js"; import { createHeartbeatAgentStatus } from "./heartbeat/agent-status.js"; @@ -1001,6 +1003,7 @@ export function heartbeatService( sweepOrphanedActiveLeases, drainRunningRunsForShutdown, } = createHeartbeatRecovery(db, { + pluginWorkerManager: options.pluginWorkerManager, enterShutdown: () => { shutdownInProgress = true; }, scheduleBoundedRetryForRun, appendRunEvent, @@ -3009,7 +3012,7 @@ export function heartbeatService( // and the per-failure reason for a partial sync. const referencedProjectObservability = buildReferencedProjectRunObservability({ - syncedProjectIds: resolvedWorkspace.additionalWorkspaces.map( + syncedProjectIds: resolvedWorkspace.runnerTaskProjectIds?.filter(id => id !== resolvedWorkspace.projectId) ?? resolvedWorkspace.additionalWorkspaces.map( (additional) => additional.projectId, ), failures: resolvedWorkspace.referencedProjectFailures, @@ -3657,10 +3660,10 @@ export function heartbeatService( let latestRun = await getRun(run.id).catch(() => null); try { if (latestRun && isHeartbeatRunTerminalStatus(latestRun.status)) { - await db + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id)); + .where(eq(heartbeatRuns.id, run.id))); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/instruction-working-copy.test.ts b/server/src/services/heartbeat/instruction-working-copy.test.ts index a3f57cce7c2..b29d226b49e 100644 --- a/server/src/services/heartbeat/instruction-working-copy.test.ts +++ b/server/src/services/heartbeat/instruction-working-copy.test.ts @@ -49,6 +49,18 @@ function fixture(executionTarget: Input["workspace"]["executionTarget"] = null, beforeEach(() => { vi.clearAllMocks(); native.claim.mockResolvedValue(null); }); describe("instruction working-copy extraction boundary", () => { + it("leaves the persistent agent home with the task provider", async () => { + const task = vi.fn(); + const f = fixture({ kind: "remote", transport: "sandbox", providerKey: "plugin", remoteCwd: "/unused", leaseId: "task-lease", + runnerTask: task, runnerLifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }); + await f.controller.prepare(f.prepare); + await f.controller.collectStoppedInstructions(); + expect(f.controller.nativeInstructionWorkingCopy()).toBeUndefined(); + expect(f.copies.prepare).not.toHaveBeenCalled(); + expect(f.copies.collectStopped).not.toHaveBeenCalled(); + expect(task).not.toHaveBeenCalled(); + }); + it.each([ ["copilot", false, true], ["copilot", true, true], ["cursor", false, true], ["cursor", true, true], diff --git a/server/src/services/heartbeat/instruction-working-copy.ts b/server/src/services/heartbeat/instruction-working-copy.ts index 558b7b0a583..449dcbd53cf 100644 --- a/server/src/services/heartbeat/instruction-working-copy.ts +++ b/server/src/services/heartbeat/instruction-working-copy.ts @@ -79,6 +79,7 @@ export function createHeartbeatInstructionWorkingCopy(input: InstructionWorkingC let nativeInstructionReservation: Awaited> = null; let instructionCopy: Awaited> = null; let instructionSave: Record | null = null; + const task = executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" ? executionTarget.runnerTask : undefined; const instructionPreparationKey = createHash("sha256").update(JSON.stringify({ adapterType: agent.adapterType, adapterConfig: agent.adapterConfig, runtimeConfig: agent.runtimeConfig, sessionConfigMetadata, duplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(executionTarget), @@ -127,7 +128,7 @@ export function createHeartbeatInstructionWorkingCopy(input: InstructionWorkingC nativeRuntimeResolution: Selection["nativeRuntimeResolution"]; hasInstructionFilesystem: boolean; }) { - if (hasInstructionFilesystem) { + if (hasInstructionFilesystem && !task) { try { // Missing contract fields on a restored session mean the deployed // legacy format. New sessions opt into whole-directory persistence. diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index dad4a0ea53b..8964c628b94 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -1,3 +1,4 @@ +import type { WorkspaceWorkers } from "../plugin-workspaces.js"; import { createHeartbeatAgentStatus } from "./agent-status.js"; import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js"; import { preserveCancelledNativeWorkspaces } from "../native-runtime/native-failed-workspace-recovery.js"; @@ -99,6 +100,7 @@ export interface HeartbeatRecoveryDependencies extends Pick["getRun"]>> | null >; environmentRuntime: ReturnType; + pluginWorkerManager?: WorkspaceWorkers; environmentsSvc: Pick, "getById" | "getLeaseById" | "releaseLease">; instructionCopies: Pick, "recoverStopped" | "recoverCaptured">; runtimeEnv: Record; @@ -919,6 +921,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD // whether any provider authority needs to be reopened. await reconcileNativeFinalizations(db, undefined, { environmentRuntime, + pluginWorkerManager: dependencies.pluginWorkerManager, onWorkspaceSettled: settleRecoveredNativeWorkspace, }); await preserveCancelledNativeWorkspaces(db); @@ -1960,6 +1963,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD // mode, never the current feature flag. await reconcileNativeFinalizations(db, undefined, { environmentRuntime, + pluginWorkerManager: dependencies.pluginWorkerManager, onWorkspaceSettled: settleRecoveredNativeWorkspace, }).catch((error) => { logger.warn( diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index fbebed7f69f..0ee088a4d51 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import os from "node:os"; import { DEFERRED_WAKE_CONTEXT_KEY, @@ -1011,12 +1012,12 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl } async function writeRunStatus(status: string, companyId: string | undefined, write: (tx: Db) => Promise) { - if (!isHeartbeatRunTerminalStatus(status)) return write(db); // Register on the outer transaction before the terminal write. An unknown // commit cannot strand newly-actionable accounting after the live event. return db.transaction(async transaction => { const tx = transaction as unknown as Db; - await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); + if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); + await notifyExecutionRunWork(tx); return write(tx); }); } diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index 361fe422b69..2b4e91c57cc 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -2,13 +2,14 @@ import { randomUUID } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db"; +import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { trace } from "@opentelemetry/api"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { createRunPreparationTiming, measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import * as pluginWorkspaces from "../plugin-workspaces.js"; import * as legacyRecovery from "../legacy-execution-recovery.js"; import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; import { createAdapterExecutionControl } from "../adapter-execution-control.js"; @@ -202,6 +203,28 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = expectCleanup(input); }); + it.each(["expected", "wrong"])("finalizes a plugin worktree through its owner (branch: %s)", async branch => { + const { input } = await fixture(true); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId: input.agent.companyId, + name: "Plugin worktree", providerType: "plugin", providerRef: "provider:opaque-worktree", cwd: "/workspaces/plugin-only", + mode: "isolated_workspace", strategyType: "git_worktree", branchName: "task-branch" }).returning(); + input.workspace.persistedExecutionWorkspace = workspace; + const inspect = vi.spyOn(pluginWorkspaces, "inspectPluginWorkspace").mockResolvedValue({ kind: "inspection", + branchName: branch === "expected" ? "task-branch" : "other-branch", headSha: "a".repeat(40), + hasDirtyTrackedFiles: true, hasUntrackedFiles: false, hasLocalOnlyData: true, isMergedIntoBase: false, remotelyRecoverable: false }); + try { + const result = executeHeartbeatRuntime(db, input); + if (branch === "expected") { + await expect(result).resolves.toMatchObject({ dispatched: true }); + expect(await finalizations()).toMatchObject([{ status: "succeeded" }]); + } else { + await expect(result).rejects.toThrow("plugin_workspace_branch_mismatch"); + expect((await finalizations()).some(row => row.status === "succeeded")).toBe(false); + } + expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id, providerRef: workspace.providerRef })); + } finally { inspect.mockRestore(); } + }); + it("retains the same-run retry signal without recording a failed workspace", async () => { const { input, run } = await fixture(true); await db.update(heartbeatRuns).set({ nativeIssueId: input.task.issueId }).where(eq(heartbeatRuns.id, run.id)); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index bba22d1442f..039dbe69348 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -1,3 +1,6 @@ +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import { writeExecutionControl } from "../execution-control-notifications.js"; +import { inspectPluginWorkspaceForFinalization } from "../native-runtime/native-plugin-workspace-finalization.js"; import { AsyncLocalStorage } from "node:async_hooks"; import { measureRunPhase } from "../execution-preparation-timing.js"; import { parseNativeSessionGoalControl } from "./scheduling.js"; @@ -272,6 +275,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe } let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; let preserveRunGatewayTokens = false; + let pluginWorkspaceFinalization: Record | null = null; const inspectFinalizeWorkspaceBranch = async () => { const workspaceRecord = persistedExecutionWorkspace?.id ? await executionWorkspacesSvc.getStoredById( @@ -279,7 +283,10 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe persistedExecutionWorkspace.id, ) : persistedExecutionWorkspace; - if (workspaceRecord?.strategyType !== "git_worktree") return null; + pluginWorkspaceFinalization = workspaceRecord + ? await inspectPluginWorkspaceForFinalization(db, options.pluginWorkerManager, workspaceRecord, agent.id) + : null; + if (pluginWorkspaceFinalization || workspaceRecord?.strategyType !== "git_worktree") return null; const worktreePath = readNonEmptyString(workspaceRecord.providerRef) ?? @@ -476,6 +483,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe } } return { status, metadata: { + ...(pluginWorkspaceFinalization ? { pluginWorkspace: pluginWorkspaceFinalization } : {}), ...(finalizeBranchMetadata ? { managedGitWorktreeBranch: finalizeBranchMetadata } : {}), ...(finalizeBranchRepairMetadata ? { managedGitWorktreeBranchRepair: finalizeBranchRepairMetadata } : {}), } }; @@ -631,6 +639,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe managedGitHub: !useHostGitHub && githubSelection.configured, billingIdentity: managedAiRuntime ? { provider: managedAiRuntime.attribution.provider, biller: managedAiRuntime.attribution.provider === "openai" ? resolveManagedOpenAiBilling(managedAiRuntime.config.managedAiRouting)?.biller ?? managedAiRuntime.attribution.provider : managedAiRuntime.attribution.provider, billingType: managedAiRuntime.attribution.method === "subscription" ? "subscription_included" : "metered_api" } : undefined, managedAiCredentialIdentity: managedAiRuntime?.identity, + managedAiAccountIdentity: managedAiRuntime?.sessionIdentity, + managedAiCredentialWriteback: managedAiRuntime?.credentialWriteback, managedAiCredentialHome: managedAiRuntime ? String((managedAiRuntime.config.env as Record).CODEX_HOME) : undefined, dotWorkspaceRoot: nativeExecution.provider.kind === "openai_dot" && resolvedConfig.dotWorkspaceAccess === true ? executionWorkspace.cwd : undefined, runnerEnvironment: { @@ -659,6 +669,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe } : {}), }, + onTaskDispatched: markDispatchStarted, runnerExecutionTarget: executionTarget, runnerIngressAuthorized: isRunnerIngressAuthorized( nativeRuntimeResolution, @@ -933,7 +944,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe })); }); } await ownership?.assertHeld(); - await db + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) }) .where( @@ -941,7 +952,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - ); + )); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1009,7 +1020,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe && !(nativeExecution && "access" in nativeExecution.workspace && nativeExecution.workspace.access === "none")) { input.effects.onProviderResourceDisposition("stop_and_retain"); } - await db + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) }) .where( @@ -1017,7 +1028,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - ); + )); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 8b611e89799..b6fcedaab42 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -12,7 +12,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vites import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { environmentRunOrchestrator } from "../environment-run-orchestrator.js"; import { environmentService } from "../environments.js"; -import { executionWorkspaceService } from "../execution-workspaces.js"; +import { executionWorkspaceService, toExecutionWorkspace } from "../execution-workspaces.js"; import { instanceSettingsService } from "../instance-settings.js"; import { workspaceOperationService } from "../workspace-operations.js"; import { nativeWorkspaceSyncInternals, prepareNativeWorkspaceSync } from "../native-runtime/native-workspace-sync.js"; @@ -23,6 +23,7 @@ import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.t import * as managedRepositoryCheckout from "../managed-repository-checkout.js"; import { isolatedRemoteSourcePath } from "../isolated-remote-workspace-source.js"; import { cleanupExecutionWorkspaceArtifacts } from "../workspace-runtime.js"; +import { buildProjectMentionHref } from "@paperclipai/shared"; import { trace } from "@opentelemetry/api"; import { buildExecutionWorkspaceAdapterConfig } from "../execution-workspace-policy.js"; import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; @@ -364,7 +365,7 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", expect((await readIssue(issue.id)).executionWorkspaceId).toBeNull(); }); - it.each(["sandbox", "ssh", "local", null] as const)("uses the selected %s placement for controller provisioning", async driver => { + it.each(["sandbox", "ssh", "plugin", "local", null] as const)("uses the selected %s placement for controller provisioning", async driver => { const { input, source, issue, envOrchestrator } = await fixture(true); const marker = join(source, "host-provision-ran"); const provisionCommand = `printf host > '${marker}'`; @@ -377,11 +378,115 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", const bound = await readIssue(issue.id); const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, bound.executionWorkspaceId!)); expect(await readFile(join(workspace.cwd!, "README.md"), "utf8")).toBe("source content\n"); - if (driver === "sandbox" || driver === "ssh") await expect(readFile(marker, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + if (driver === "sandbox" || driver === "ssh" || driver === "plugin") await expect(readFile(marker, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); else expect(await readFile(marker, "utf8")).toBe("host"); expect(input.config.mergedConfig).toEqual(originalConfig); }); + it("keeps a plugin workspace ledger across runs without preparing Core repositories", async () => { + const { input, issue, envOrchestrator, source } = await fixture(); + input.agent.adapterType = "paperclip_runner"; + input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment, driver: "plugin" }; + const guestOnly = join(home, "guest-only", issue.projectId!); + await db.update(projectWorkspaces).set({ cwd: guestOnly }).where(eq(projectWorkspaces.id, issue.projectWorkspaceId!)); + const copy = vi.spyOn(input.services.executionWorkspacesSvc, "prepareTaskRepositoriesForAdmission").mockRejectedValue(new Error("Core must not prepare repositories")); + const restoredCopy = vi.spyOn(input.services.executionWorkspacesSvc, "prepareConfiguredRepositoryCopy").mockRejectedValue(new Error("Core must not restore repositories")); + const stopped = new Error("Plugin realization boundary"); + const realize = vi.spyOn(envOrchestrator, "realizeForRun").mockRejectedValue(stopped); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(stopped); + const firstId = (await readIssue(issue.id)).executionWorkspaceId!; + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, firstId)); + expect(workspace).toMatchObject({ cwd: `/workspaces/${issue.projectWorkspaceId}`, strategyType: "project_primary", providerType: "adapter_managed" }); + await reuse(input, toExecutionWorkspace(workspace)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(stopped); + expect((await readIssue(issue.id)).executionWorkspaceId).toBe(firstId); + expect(realize).toHaveBeenCalledTimes(2); + expect(copy).not.toHaveBeenCalled(); expect(restoredCopy).not.toHaveBeenCalled(); + await expect(access(guestOnly)).rejects.toMatchObject({ code: "ENOENT" }); + input.environment.selectedEnvironmentForConfig = input.environment.localEnvironment; + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ + resultJson: { reason: "plugin_workspace_owner_mismatch" }, + }); + expect(realize).toHaveBeenCalledTimes(2); + input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment, driver: "plugin" }; + const [localWorkspace] = await db.update(executionWorkspaces).set({ providerType: "local_fs", cwd: source, metadata: {} }) + .where(eq(executionWorkspaces.id, firstId)).returning(); + await reuse(input, toExecutionWorkspace(localWorkspace)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ + resultJson: { reason: "plugin_workspace_owner_mismatch" }, + }); + expect(realize).toHaveBeenCalledTimes(2); + expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n"); + }); + + it.each(["services", "fresh"] as const)("rejects unsupported %s before plugin workspace preparation or acquisition", async mode => { + const { input, envOrchestrator } = await fixture(); + input.agent.adapterType = "paperclip_runner"; + input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment, driver: "plugin" }; + if (mode === "services") input.config.mergedConfig.workspaceRuntime = { services: [{ name: "preview", command: "must-not-run-on-core" }] }; + else input.task.freshWorkspace = { mode: "fresh_isolated_workspace", retryIntentId: input.run.id, + sourceRunId: input.run.id, retainedSourceIds: [] }; + const acquire = vi.spyOn(envOrchestrator, "acquireForRun"); + const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun"); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ + resultJson: { reason: mode === "services" ? "plugin_workspace_services_unsupported" : "fresh_workspace_storage_unsupported" }, + }); + expect(resolve).not.toHaveBeenCalled(); + expect(acquire).not.toHaveBeenCalled(); + }); + + it("admits the metadata-only agent-default strategy for a plugin workspace", async () => { + const { input, envOrchestrator } = await fixture(); + input.agent.adapterType = "paperclip_runner"; + input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment, driver: "plugin" }; + input.policy.requestedExecutionWorkspaceMode = "agent_default"; + input.policy.effectiveExecutionWorkspaceMode = "agent_default"; + const stopped = new Error("Plugin agent-default admission reached"); + const realize = vi.spyOn(envOrchestrator, "realizeForRun").mockRejectedValue(stopped); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(stopped); + expect(realize).toHaveBeenCalledWith(expect.objectContaining({ executionWorkspace: expect.objectContaining({ strategy: "adapter_managed", cwd: `/workspaces/${input.task.issueId}` }), + persistedExecutionWorkspace: expect.objectContaining({ strategyType: "adapter_managed", providerType: "adapter_managed" }) })); + }); + + it("rejects Core Git worktree preparation for a plugin-owned workspace", async () => { + const { input, envOrchestrator } = await fixture(true); + input.agent.adapterType = "paperclip_runner"; + input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment, driver: "plugin" }; + const acquire = vi.spyOn(envOrchestrator, "acquireForRun"); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ + code: "workspace_validation_failed", resultJson: { reason: "plugin_workspace_strategy_unsupported" }, + }); + expect(acquire).not.toHaveBeenCalled(); + }); + + it.each([true, false])("forwards only authorized plugin project IDs without staging (remote enabled: %s)", async enabled => { + const { input, issue, envOrchestrator } = await fixture(); + vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC", "true"); + vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE", String(enabled)); + input.agent = { ...input.agent, adapterType: "paperclip_runner" }; + input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment, driver: "plugin" }; + const [readable, privateProject] = await db.insert(projects).values([ + { companyId: issue.companyId, name: "Readable reference" }, + { companyId: issue.companyId, name: "Private reference", visibility: "private" }, + ]).returning(); + const unavailableCwd = join(home, readable.id, "must-not-stage"); + await db.insert(projectWorkspaces).values({ companyId: issue.companyId, projectId: readable.id, name: "Provider-only storage", cwd: unavailableCwd }); + await db.update(issues).set({ description: [readable, privateProject].map(project => `[project](${buildProjectMentionHref(project.id)})`).join(" ") }).where(eq(issues.id, issue.id)); + const stopped = new Error("Fixture reached plugin realization"); + const realize = vi.spyOn(envOrchestrator, "realizeForRun").mockRejectedValue(stopped); + try { + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(stopped); + expect(realize).toHaveBeenCalledWith(expect.objectContaining({ + runnerTaskProjectIds: enabled ? [issue.projectId, readable.id] : undefined, + executionWorkspace: expect.objectContaining({ additionalWorkspaces: [] }), + })); + await expect(access(unavailableCwd)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC", "true"); + vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE", "true"); + } + }); + it("checks current repository admission before publishing configured sibling repositories", async () => { const { input, source, issue } = await fixture(); await db.insert(projectWorkspaces).values({ companyId: issue.companyId, projectId: issue.projectId!, @@ -561,6 +666,7 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", it.each([ { transport: "local", descriptor: "absent" }, { transport: "ssh", descriptor: "absent" }, + { transport: "task", descriptor: "absent" }, { transport: "daytona", descriptor: "valid" }, { transport: "daytona", descriptor: "absent" }, { transport: "daytona", descriptor: "corrupt" }, @@ -579,7 +685,8 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", : transport === "ssh" ? { kind: "remote", transport: "ssh", remoteCwd, spec: { host: "fixture.invalid", port: 22, username: "fixture", remoteWorkspacePath: remoteCwd, privateKey: null, knownHosts: null, strictHostKeyChecking: true, remoteCwd } } - : { kind: "remote", transport: "sandbox", providerKey: "daytona", remoteCwd }; + : { kind: "remote", transport: "sandbox", providerKey: "daytona", remoteCwd, + ...(transport === "task" ? { runnerTask: vi.fn() } : {}) }; const realize = input.services.envOrchestrator.realizeForRun; vi.spyOn(input.services.envOrchestrator, "realizeForRun").mockImplementation(async args => ({ ...await realize(args), executionTarget: target, @@ -604,7 +711,7 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", if (transport !== "daytona" || descriptor === "valid") { const result = await prepareHeartbeatWorkspace(db, input); expect(result.executionTarget).toEqual(target); - if (transport === "ssh") expect(reserve).not.toHaveBeenCalled(); + if (transport === "ssh" || transport === "task") expect(reserve).not.toHaveBeenCalled(); else expect(reserve).toHaveBeenCalledWith({ companyId: issue.companyId, issueId: issue.id, workspaceId: first.persistedExecutionWorkspace!.id, cwd: admittedRoot, agentId: input.agent.id, runId: input.run.id }); if (transport !== "daytona") { diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index a0b4fae0ff8..21e0d0c44da 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -215,6 +215,27 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace resolveReusedGitWorkspaceAnchor, appendRunEvent, } = input.services; + const pluginTaskWorkspace = selectedEnvironmentForConfig?.driver === "plugin" && agent.adapterType === "paperclip_runner"; + if (fresh && !isDotRun && pluginTaskWorkspace) { + throw new WorkspaceValidationFailure("A fresh attempt is unavailable for this workspace storage provider. The existing workspace is retained.", + { reason: "fresh_workspace_storage_unsupported" }); + } + const pluginConfig = parseObject(selectedEnvironmentForConfig?.config); + const taskWorkspaceOwner = pluginTaskWorkspace ? { agentId: agent.id, environmentId: selectedEnvironmentForConfig.id, + pluginKey: pluginConfig.pluginKey ?? null, driverKey: pluginConfig.driverKey ?? null } : null; + for (const workspace of [existingExecutionWorkspace, reusableExistingExecutionWorkspace]) { + const owner = parseObject(workspace?.metadata?.runnerTaskWorkspaceOwner); + if ((pluginTaskWorkspace && workspace && workspace.providerType !== "plugin" && Object.keys(owner).length === 0) + || (Object.keys(owner).length > 0 && (!taskWorkspaceOwner || Object.entries(taskWorkspaceOwner).some(([key, value]) => owner[key] !== value)))) { + throw new WorkspaceValidationFailure("The retained workspace belongs to a different agent, environment, or storage provider.", + { reason: "plugin_workspace_owner_mismatch" }); + } + } + const workspaceServices = parseObject(mergedConfig.workspaceRuntime).services; + if (pluginTaskWorkspace && Array.isArray(workspaceServices) && workspaceServices.length > 0) { + throw new WorkspaceValidationFailure("The selected plugin environment does not support Core workspace services.", + { reason: "plugin_workspace_services_unsupported" }); + } const { selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, workspace: resolvedWorkspace, @@ -249,7 +270,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null, workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] }; } - if (useIsolatedTaskDirectory && issueRef) { + if (useIsolatedTaskDirectory && issueRef && !pluginTaskWorkspace) { const cwd = await materializeIsolatedTaskDirectory({ companyId: agent.companyId, issueId: issueRef.id, @@ -282,7 +303,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace }, }); } - if (nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId) { + if (nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId && !pluginTaskWorkspace) { const cwd = await materializeNativeChatTaskRoot( nativeChatWorkspaceScope, ); @@ -309,7 +330,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace useProjectWorkspace: requestedExecutionWorkspaceMode !== "agent_default", configuredCwd: readNonEmptyString(mergedConfig.cwd), - anchorWorkspace: requestedShouldReuseExisting && reusableExistingExecutionWorkspace?.strategyType === "git_worktree" + anchorWorkspace: !pluginTaskWorkspace && requestedShouldReuseExisting && reusableExistingExecutionWorkspace?.strategyType === "git_worktree" ? await resolveReusedGitWorkspaceAnchor({ agent, workspace: reusableExistingExecutionWorkspace, @@ -346,6 +367,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace ? { ...workspace, additionalWorkspaces: [], + runnerTaskProjectIds: undefined, referencedProjectFailures: [], } : workspace; @@ -388,7 +410,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace sourceWorkspaceId: resolvedWorkspace.workspaceId, boundWorkspace: reusableExistingExecutionWorkspace ?? existingExecutionWorkspace, })); - const skipHostProvision = copyOnlyProvisioning || hostProvisionDriver === "sandbox" || hostProvisionDriver === "ssh"; + const skipHostProvision = copyOnlyProvisioning || hostProvisionDriver === "sandbox" || hostProvisionDriver === "ssh" || hostProvisionDriver === "plugin"; const hostExecutionWorkspaceConfig = isDotRun ? {} : stripHostWorkspaceProvision({ config: fresh ? { ...mergedConfig, workspaceStrategy: { type: "project_primary" } } : mergedConfig, @@ -403,6 +425,11 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace repoRef: resolvedWorkspace.repoRef, additionalWorkspaces: resolvedWorkspace.additionalWorkspaces, } satisfies ExecutionWorkspaceInput; + const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType(requestedExecutionWorkspaceMode, hostExecutionWorkspaceConfig); + if (pluginTaskWorkspace && !pluginPrepared && latestWorkspaceStrategyType === "git_worktree") { + throw new WorkspaceValidationFailure("The selected plugin environment does not support Core Git worktree preparation.", + { reason: "plugin_workspace_strategy_unsupported" }); + } if (!pluginPrepared) await timeWorkspace("workspace_base_validation", () => assertGitWorktreeBaseWorkspaceReady({ requestedExecutionWorkspaceMode, config: hostExecutionWorkspaceConfig, @@ -421,10 +448,6 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace Object.keys(workspaceStrategyForFingerprint).length > 0 ? workspaceStrategyForFingerprint : null; - const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType( - requestedExecutionWorkspaceMode, - hostExecutionWorkspaceConfig, - ); const selectedEnvironmentConfigForFingerprint = parseObject( selectedEnvironmentForConfig?.config, ); @@ -513,6 +536,11 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace requestedShouldReuseExisting, workspaceConfigFreshness, }); + if (pluginTaskWorkspace && !pluginPrepared && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace + && reusableExistingExecutionWorkspace?.strategyType === "git_worktree") { + throw new WorkspaceValidationFailure("The selected plugin environment cannot reuse a Core Git worktree.", + { reason: "plugin_workspace_strategy_unsupported" }); + } const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({ companyId: agent.companyId, heartbeatRunId: run.id, @@ -565,7 +593,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace warnings: [], created: false, branchCreatedByRuntime: false, baseRefSha: pluginPrepared.metadata?.workspaceProviderBaseSha as string | null, } as RealizedExecutionWorkspace, reusedExecutionWorkspace: Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } - : isDotRun ? { executionWorkspace: { ...executionWorkspaceBase, strategy: "project_primary" as const, cwd: resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: false, policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision( + : isDotRun || pluginTaskWorkspace ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace ? latestWorkspaceStrategyType : "project_primary" as const, cwd: resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: pluginTaskWorkspace && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision( { requestedShouldReuseExisting, existingExecutionWorkspaceId: @@ -814,6 +842,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace const nextExecutionWorkspaceMetadata = { ...baseExecutionWorkspaceMetadata, ...(pluginPrepared?.metadata ?? {}), + ...(taskWorkspaceOwner && !pluginPrepared ? { runnerTaskWorkspaceOwner: taskWorkspaceOwner } : {}), ...(copyOnlyProvisioning ? { [CONTROLLER_PROVISION_COPY_ONLY]: true } : {}), ...(persistedWorktreeInstanceRoot ? { @@ -839,7 +868,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace repoUrl: executionWorkspace.repoUrl, baseRef: executionWorkspace.repoRef, branchName: branchNameForInitialPersistence, - providerType: + providerType: pluginTaskWorkspace ? "adapter_managed" : executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs", @@ -868,7 +897,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace : requestedExecutionWorkspaceMode === "agent_default" ? "adapter_managed" : "shared_workspace", - strategyType: + strategyType: pluginTaskWorkspace ? latestWorkspaceStrategyType : executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "project_primary", @@ -881,7 +910,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace repoUrl: executionWorkspace.repoUrl, baseRef: executionWorkspace.repoRef, branchName: branchNameForInitialPersistence, - providerType: + providerType: pluginTaskWorkspace ? "adapter_managed" : executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs", @@ -977,14 +1006,14 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace } await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); const projectRepositoryPaths: string[] = []; - if (!pluginPrepared && !isDotRun && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) { + if (!pluginPrepared && !pluginTaskWorkspace && !isDotRun && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) { const taskRepositories = await timeWorkspace("workspace_repository_admission", () => executionWorkspacesSvc.prepareTaskRepositoriesForAdmission({ companyId: agent.companyId, issueId, workspaceId: persistedExecutionWorkspace!.id, cwd: executionWorkspace.cwd, agentId: agent.id, runId: run.id, responsibleUserId })); projectRepositoryPaths.push(...taskRepositories.map(repository => repository.relativePath)); } - if (!pluginPrepared && executionWorkspace.projectId + if (!pluginPrepared && !pluginTaskWorkspace && executionWorkspace.projectId && (resolvedWorkspace.source === "project_primary" || Boolean(reusableExistingExecutionWorkspace?.projectWorkspaceId)) && !resolvedWorkspace.baseCwdFallback && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) { const repositoryRows = await db.select().from(projectWorkspaces).where(and( @@ -1051,6 +1080,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace adapterType: agent.adapterType, adapterConfig: parseObject(agent.adapterConfig), admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode, + recoverPluginTask: runOptions.nativeRestartRecovery?.kind === "resume_dead_runner" || runOptions.nativeRestartRecovery?.kind === "bootstrap_incomplete", issueId: issueId ?? null, heartbeatRunId: run.id, agentId: agent.id, @@ -1147,6 +1177,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace executionWorkspace, effectiveExecutionWorkspaceMode, persistedExecutionWorkspace, + runnerTaskProjectIds: resolvedWorkspace.runnerTaskProjectIds, duplexObservabilityRecorder, })); nativeRunnerPreparationSpans.push({ @@ -1181,8 +1212,9 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); const workspaceRealization = realizationResult.workspaceRealization; const executionTarget = realizationResult.executionTarget; - if (!pluginPrepared && !isDotRun && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId) - && !(executionTarget?.kind === "remote" && executionTarget.transport === "ssh")) { + if (!pluginPrepared && !pluginTaskWorkspace && !isDotRun && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId) + && !(executionTarget?.kind === "remote" && (executionTarget.transport === "ssh" + || (executionTarget.transport === "sandbox" && executionTarget.runnerTask)))) { const workspaceId = persistedNativeExecutionInput?.binding.executionWorkspaceId ?? nativeRecoveryExecutionWorkspaceId!; const reference = readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync); if (!reference && executionTarget?.kind === "remote" && executionTarget.transport === "sandbox") { diff --git a/server/src/services/heartbeat/workspaces.test.ts b/server/src/services/heartbeat/workspaces.test.ts index bad33936d01..ee48bdf31d2 100644 --- a/server/src/services/heartbeat/workspaces.test.ts +++ b/server/src/services/heartbeat/workspaces.test.ts @@ -127,6 +127,37 @@ describe("heartbeat workspace module", () => { expect(await fs.stat(missingCwd).catch(() => null)).toBeNull(); }); + it("resolves plugin project identity without Core files", async () => { + const repoUrl = "/unavailable-controller-source.git"; + const missingCwd = path.join(testRoot, "exists-only-on-plugin"); + const configured = { ...workspace("selected", "project-one", missingCwd), repoUrl }; + const database = databaseWithResults([configured], [{ id: "project-one" }]); + const stat = vi.spyOn(fs, "stat"); const mkdir = vi.spyOn(fs, "mkdir"); + try { + const result = await createHeartbeatWorkspaceResolver(database.db).resolveWorkspaceForRun( + { ...agent(), adapterType: "paperclip_runner" }, { projectId: "project-one" }, null, + { executionEnvironmentDriver: "plugin" }); + expect(result).toMatchObject({ cwd: "/workspaces/selected", projectId: "project-one", workspaceId: "selected", repoUrl, + source: "project_primary", baseCwdFallback: false, additionalWorkspaces: [] }); + expect(stat).not.toHaveBeenCalled(); expect(mkdir).not.toHaveBeenCalled(); + } finally { stat.mockRestore(); mkdir.mockRestore(); } + }); + it("keeps a projectless plugin scope without creating an agent directory on Core", async () => { + const mkdir = vi.spyOn(fs, "mkdir"); + try { + const result = await createHeartbeatWorkspaceResolver(databaseWithResults().db).resolveWorkspaceForRun( + { ...agent(), adapterType: "paperclip_runner" }, {}, null, { executionEnvironmentDriver: "plugin" }); + expect(result).toMatchObject({ cwd: "/workspaces/agent-one", projectId: null, workspaceId: null }); + expect(mkdir).not.toHaveBeenCalled(); + } finally { mkdir.mockRestore(); } + }); + it("rejects plugin project scope outside the company", async () => { + const database = databaseWithResults([], []); + await expect(createHeartbeatWorkspaceResolver(database.db).resolveWorkspaceForRun( + { ...agent(), adapterType: "paperclip_runner" }, { projectId: "foreign" }, null, + { executionEnvironmentDriver: "plugin" })).rejects.toMatchObject({ code: "workspace_validation_failed" }); + }); + it("fails closed with workspace diagnostics when a configured directory is unavailable", async () => { const missingCwd = path.join(testRoot, "missing-configured-workspace"); const database = databaseWithResults([workspace("selected", "project-one", missingCwd)]); diff --git a/server/src/services/heartbeat/workspaces.ts b/server/src/services/heartbeat/workspaces.ts index 90ac28d9f04..fa12c39490a 100644 --- a/server/src/services/heartbeat/workspaces.ts +++ b/server/src/services/heartbeat/workspaces.ts @@ -1343,6 +1343,8 @@ export type ResolvedWorkspaceForRun = { * workspace only, exactly as before. */ additionalWorkspaces: ResolvedAdditionalWorkspace[]; + /** Authorized project IDs for plugin-owned storage; no Core filesystem staging. */ + runnerTaskProjectIds?: string[]; /** * Structured record of every referenced project that the run dropped or failed, paired with the * layer that dropped it. Run preparation reads this to emit the requested-vs-synced observability @@ -1354,7 +1356,7 @@ export type ResolvedWorkspaceForRun = { /** The anchor workspace shape, before the additional referenced workspaces are attached. */ type ResolvedAnchorWorkspaceForRun = Omit< ResolvedWorkspaceForRun, - "additionalWorkspaces" | "referencedProjectFailures" + "additionalWorkspaces" | "referencedProjectFailures" | "runnerTaskProjectIds" >; /** @@ -3404,7 +3406,7 @@ export function createHeartbeatWorkspaceResolver(db: Db) { agent: typeof agents.$inferSelect, context: Record, previousSessionParams: Record | null, - opts?: { useProjectWorkspace?: boolean | null; configuredCwd?: string | null }, + opts?: { useProjectWorkspace?: boolean | null; configuredCwd?: string | null; executionEnvironmentDriver?: string | null }, ): Promise { const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); @@ -3473,6 +3475,24 @@ export function createHeartbeatWorkspaceResolver(db: Db) { repoRef: readNonEmptyString(workspace.repoRef), })); + if (opts?.executionEnvironmentDriver === "plugin" && agent.adapterType === "paperclip_runner") { + if (workspaceProjectId) { + const project = await db.select({ id: projects.id }).from(projects).where(and( + eq(projects.companyId, agent.companyId), eq(projects.id, workspaceProjectId), + )).then(rows => rows[0] ?? null); + if (!project) throw new WorkspaceValidationFailure("The selected plugin project is unavailable.", { reason: "plugin_project_unavailable" }); + } + if (configuredSourceId && projectWorkspaceRows.length === 0) { + throw new WorkspaceValidationFailure("The selected project workspace is unavailable.", { reason: "configured_workspace_unavailable" }); + } + const selected = projectWorkspaceRows[0]; + // This is a logical scope. Preparation resolves the execution-host path. + return { cwd: `/workspaces/${selected?.id ?? issueId ?? agent.id}`, source: workspaceProjectId ? "project_primary" : issueId ? "task_session" : "agent_home", + projectId: workspaceProjectId ?? null, workspaceId: selected?.id ?? null, + repoUrl: selected?.repoUrl ?? null, repoRef: selected?.repoRef ?? null, + workspaceHints, warnings: [], baseCwdFallback: false, materializationFailures: [] }; + } + if (projectWorkspaceRows.length > 0 || configuredSourceId) { const preferredWorkspace = preferredProjectWorkspaceId ? (projectWorkspaceRows.find( @@ -3682,6 +3702,21 @@ export function createHeartbeatWorkspaceResolver(db: Db) { readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); const responsibleUserId = readNonEmptyString(context.responsibleUserId) ?? readNonEmptyString(context.responsible_user_id); + const referencedOptions: ResolveRunReferencedProjectsOptions = { + companyId: agent.companyId, + actor: { type: "agent", agentId: agent.id, companyId: agent.companyId, source: "agent_key", + runId: readNonEmptyString(context.executionIdentityRunId), + onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId }, + issues: issueService(db), projects: projectService(db), access: authorizationService(db), + }; + if (executionEnvironmentDriver === "plugin" && agent.adapterType === "paperclip_runner" && + issueId && isMultiProjectWorkspaceSyncRemoteEnabled()) { + const referenced = await resolveRunReferencedProjects(issueId, anchor.projectId, referencedOptions); + return { ...anchor, additionalWorkspaces: [], + runnerTaskProjectIds: [anchor.projectId, ...referenced.additional.map(project => project.projectId)] + .filter((id): id is string => id !== null), + referencedProjectFailures: referenced.failures, warnings: [...anchor.warnings, ...referenced.warnings] }; + } const resolveGitAuth = createGitRemoteAuthProvider(db, agent.companyId, { issueId, heartbeatRunId: readNonEmptyString(context.executionIdentityRunId), @@ -3698,18 +3733,7 @@ export function createHeartbeatWorkspaceResolver(db: Db) { executionEnvironmentDriver, ), remoteReferencedSyncEnabled: isMultiProjectWorkspaceSyncRemoteEnabled(), - companyId: agent.companyId, - actor: { - type: "agent", - agentId: agent.id, - companyId: agent.companyId, - source: "agent_key", - runId: readNonEmptyString(context.executionIdentityRunId), - onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId, - }, - issues: issueService(db), - projects: projectService(db), - access: authorizationService(db), + ...referencedOptions, resolveProjectWorkspace: (project) => resolveAdditionalProjectWorkspace( { companyId: agent.companyId, project }, diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index c3c5b01b505..23c66dd35ac 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -243,6 +243,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta enableChatConnectors: parsed.data.enableChatConnectors ?? false, enableGitHubReviewBots: parsed.data.enableGitHubReviewBots ?? false, enableMemoryConnectors: parsed.data.enableMemoryConnectors ?? false, + enablePaperclipNotebook: parsed.data.enablePaperclipNotebook ?? false, enableCases: parsed.data.enableCases ?? false, enableAgentChat: parsed.data.enableAgentChat ?? false, enableCombinedInboxTasks: parsed.data.enableCombinedInboxTasks ?? false, @@ -293,6 +294,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta enableChatConnectors: false, enableGitHubReviewBots: false, enableMemoryConnectors: false, + enablePaperclipNotebook: false, enableCases: false, enableAgentChat: false, enableCombinedInboxTasks: false, diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5351722b198..04e2c19a6d1 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -1,3 +1,5 @@ +import { writeExecutionControl } from "./execution-control-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { and, desc, eq, inArray, isNull, isNotNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { heartbeatRuns, issueRecoveryActions, workspaceOperations } from "@paperclipai/db"; @@ -318,6 +320,8 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ): Promise { try { const created = await db.transaction(async (tx) => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); const [superseded] = await tx .update(issueRecoveryActions) .set({ @@ -381,10 +385,10 @@ export function issueRecoveryActionService(db: DbOrTransaction) { if (existing.cause === "native_workspace_finalization_owner_unverified" && input.cause === existing.cause && input.fingerprint === existing.fingerprint && !existing.evidence?.workspaceOwnerStop && existing.nextAction !== input.nextAction) { - const [updated] = await db.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) + const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); if (updated) return toReadModel(updated); } return existing; @@ -404,7 +408,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { existing.attemptCount, Math.min(nextAttemptCount, effectiveMaxAttempts), ); - const [exhausted] = await db + const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx .update(issueRecoveryActions) .set({ status: "escalated", @@ -452,13 +456,13 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ]), ), ) - .returning(); + .returning()); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } return toReadModel(exhausted); } - const [updated] = await db + const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx .update(issueRecoveryActions) .set({ recoveryIssueId: input.preserveExistingOwner @@ -517,7 +521,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), ), ) - .returning(); + .returning()); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -525,10 +529,10 @@ export function issueRecoveryActionService(db: DbOrTransaction) { } try { - const [created] = await db + const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) - .returning(); + .returning()); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -565,7 +569,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { predicates.push(eq(issueRecoveryActions.fingerprint, input.fingerprint)); } - const [updated] = await dbOrTx + const [updated] = await writeExecutionControl(dbOrTx, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx .update(issueRecoveryActions) .set({ status: input.status, @@ -575,7 +579,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { updatedAt: now, }) .where(and(...predicates)) - .returning(); + .returning()); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 54fe6184b35..f1b16bd0e49 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -1,3 +1,4 @@ +import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { hasRequiredWorkspaceRecovery, preserveWorkspaceRestoreRecoveryMetadata, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources, type LegacyWorkspaceRestoreSource } from "./legacy-workspace-restore-recovery.js"; import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js"; @@ -108,6 +109,7 @@ export async function terminalizeLegacyExecution(input: { resultJson: preserveWorkspaceRestoreRecoveryMetadata(current.resultJson, input.recordRestoreFailureOnly ? { ...current.resultJson, ...patch.resultJson } : patch.resultJson), }; + await notifyExecutionRunWork(tx); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/local-ai-login.ts b/server/src/services/local-ai-login.ts index 71482abc4cb..2108a916baf 100644 --- a/server/src/services/local-ai-login.ts +++ b/server/src/services/local-ai-login.ts @@ -1,3 +1,4 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { realpathSync } from "node:fs"; import { mkdir, rm, writeFile } from "node:fs/promises"; @@ -120,6 +121,7 @@ export function localAiLoginService(db: Db) { const expiresAt = new Date(Date.now() + ATTEMPT_DURATION_MS); await prepareHome(id, intent.provider); try { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.localLoginCleanup); await tx.insert(adapterAuthSessions).values({ id, publicSessionId: id, companyId, environmentId: environment.id, adapterType, diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 76abcdefd31..c8f31024920 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -1,3 +1,4 @@ +import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { recordRequiredWorkspaceRestoreFailure } from "../legacy-execution-recovery.js"; @@ -55,6 +56,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: typeof hea let retained = !retainFiles || alreadyRetained ? run : await recordRequiredWorkspaceRestoreFailure(tx as unknown as Db, run, { workspaceRestoreFailure: "restore_failed" }, source); if (!retained || retainFiles && !hasRequiredWorkspaceRecovery(retained.resultJson)) return; if (options.allowPreterminal && coordinator.phase === "observed") { + await notifyExecutionRunWork(tx); const [settled] = await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, nextAttemptAt: null, recoveryState: "blocked", failureCode: coordinator.failureCode ?? "native_continuation_requires_reconciliation", updatedAt: new Date(), diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index da49986802b..ec391457b07 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -1,3 +1,4 @@ +import type { WorkspaceWorkers } from "../plugin-workspaces.js"; import { recoverLegacyUnsafeWorkspaceExports } from "./native-workspace-export-recovery.js"; import { hasPendingNativeChildCompletion } from "./native-child-completion-delivery.js"; import { dismissAutomaticCompletionReviews, decisionHasRetiredAutomaticReview } from "./automatic-completion-reviews.js"; @@ -626,6 +627,7 @@ export async function claimNativeSessionResumptions(input: { } type NativeReconciliationOptions = { + pluginWorkerManager?: WorkspaceWorkers; environmentRuntime?: EnvironmentRuntimeService; onWorkspaceSettled?: (input: { runId: string; @@ -1042,6 +1044,7 @@ async function reconcileNativeFinalizationsPass( db, runId: row.runId, environmentRuntime: options.environmentRuntime, + pluginWorkerManager: options.pluginWorkerManager, }); // Busy is ownership, not another failed export or retry-budget debit. if (!operation) continue; @@ -1051,11 +1054,13 @@ async function reconcileNativeFinalizationsPass( const unrecoverable = operation.stderrExcerpt?.includes( "workspace_sync_out_unrecoverable", ); + const pluginFailure = operation.stderrExcerpt?.trim(); + const pluginRepair = pluginFailure === "plugin_workspace_branch_mismatch" || pluginFailure === "plugin_workspace_owner_mismatch"; const failure = await recordNativeFinalizationFailure({ db, runId: row.runId, error: new Error( - unrecoverable + pluginRepair ? `native_${pluginFailure}` : unrecoverable ? "native_workspace_sync_out_unrecoverable" : "native_workspace_sync_out_failed", ), diff --git a/server/src/services/native-runtime/native-plugin-workspace-finalization.ts b/server/src/services/native-runtime/native-plugin-workspace-finalization.ts new file mode 100644 index 00000000000..3d94e756df0 --- /dev/null +++ b/server/src/services/native-runtime/native-plugin-workspace-finalization.ts @@ -0,0 +1,25 @@ +import type { Db, executionWorkspaces } from "@paperclipai/db"; +import { inspectPluginWorkspace, type WorkspaceWorkers } from "../plugin-workspaces.js"; + +/** Validate retained plugin files without interpreting a provider reference as a Core path. */ +export async function inspectPluginWorkspaceForFinalization( + db: Db, + workers: WorkspaceWorkers | undefined, + workspace: typeof executionWorkspaces.$inferSelect, + agentId: string, +): Promise | null> { + if (workspace.providerType === "plugin") { + const inspection = await inspectPluginWorkspace(db, workers, workspace); + if (workspace.strategyType === "git_worktree" + && (!workspace.branchName || inspection.branchName !== workspace.branchName)) { + throw new Error("plugin_workspace_branch_mismatch"); + } + return { providerType: workspace.providerType, inspection }; + } + const owner = workspace.metadata?.runnerTaskWorkspaceOwner; + if (workspace.providerType === "adapter_managed" && owner && typeof owner === "object" && !Array.isArray(owner)) { + if ((owner as Record).agentId !== agentId) throw new Error("plugin_workspace_owner_mismatch"); + return { providerType: workspace.providerType, owner }; + } + return null; +} diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index fb05f82252a..25e29408bef 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1,3 +1,4 @@ +import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; import { @@ -10,6 +11,7 @@ import { and, desc, eq, inArray, isNotNull, isNull, lte, or, sql } from "drizzle import type { Db } from "@paperclipai/db"; import { agents, + environmentLeases, heartbeatRunEvents, heartbeatRuns, issues, @@ -626,11 +628,16 @@ export async function claimNativeRestartRecoveries(input: { const remoteWorkspace = readNativeWorkspaceSyncReference( profile.nativeWorkspaceSync, ); + const [pluginTask] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, row.run.companyId), eq(environmentLeases.heartbeatRunId, row.run.id), + sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`, + )).limit(1); const remoteSandbox = profile.nativeWorkspaceSync != null; + const remoteProcess = remoteSandbox || !!pluginTask; const runnerPidAlive = - !remoteSandbox && processIsAlive(row.run.processPid); + !remoteProcess && processIsAlive(row.run.processPid); const runnerGroupAlive = - !remoteSandbox && processGroupIsAlive(row.run.processGroupId); + !remoteProcess && processGroupIsAlive(row.run.processGroupId); const observedRunnerStart = row.run.processPid && runnerPidAlive ? await observedProcessStart(row.run.processPid) @@ -753,7 +760,7 @@ export async function claimNativeRestartRecoveries(input: { } } const providerProcesses = await evaluateNativeProviderProcesses({ - identities: (remoteSandbox ? [] : providerProcessIdentities).filter( + identities: (remoteProcess ? [] : providerProcessIdentities).filter( (identity) => identity.pid !== row.run.processPid, ), }); @@ -855,6 +862,7 @@ export async function claimNativeRestartRecoveries(input: { recycledProviderPids: providerProcesses.recycledPids, stderrTail: row.run.stderrExcerpt, }); + await notifyExecutionRunWork(tx); await tx .update(nativeRunFinalizations) .set({ diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index cf17721d832..ae77de90638 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -1,3 +1,4 @@ +import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { reconcileMuseNativeTerminalRun } from "./muse-terminal-reconciliation.js"; import { assertNativeCaseHandoffResult } from "./native-case-handoff.js"; import { scheduledIssueMonitorWait } from "../issue-monitors.js"; @@ -420,6 +421,7 @@ async function recordRetryableFailure(input: { : "native_finalization_retry_exhausted" : input.failureCode; if (phase === "terminal_failure") await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.run.companyId); + await notifyExecutionRunWork(tx); await tx .update(nativeRunFinalizations) .set({ @@ -599,6 +601,9 @@ export async function recordNativeFinalizationFailure(input: { message, nextAction: input.failureScope === "workspace" + && (message === "native_plugin_workspace_branch_mismatch" || message === "native_plugin_workspace_owner_mismatch") + ? "Repair the plugin workspace branch or owner, then resume finalization without another provider turn." + : input.failureScope === "workspace" ? input.permanent ? "Restore the exact sandbox containing the unexported workspace changes, or resolve the task manually from durable evidence." : "Retry workspace export and merge from the retained sandbox; do not submit another provider turn." @@ -705,6 +710,7 @@ async function projectCommittedRun(input: { const [updatedRun] = await input.db.transaction(async transaction => { const tx = transaction as unknown as Db; await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.run.companyId); + await notifyExecutionRunWork(tx); return tx .update(heartbeatRuns) .set({ @@ -1617,13 +1623,12 @@ export async function finalizeNativeRun(input: { }) .where(eq(heartbeatRuns.id, run.id)) .returning(); - const [updatedRun] = input.projectRunStatus - ? await input.db.transaction(async transaction => { - const tx = transaction as unknown as Db; - await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); - return projectRun(tx); - }) - : await projectRun(input.db); + const [updatedRun] = await input.db.transaction(async transaction => { + const tx = transaction as unknown as Db; + if (input.projectRunStatus) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); + await notifyExecutionRunWork(tx); + return projectRun(tx); + }); if ( input.projectRunStatus && !alreadyEmittedByCommittedDecision && diff --git a/server/src/services/native-runtime/native-safe-replacement.ts b/server/src/services/native-runtime/native-safe-replacement.ts index 929cc58c510..03109f56def 100644 --- a/server/src/services/native-runtime/native-safe-replacement.ts +++ b/server/src/services/native-runtime/native-safe-replacement.ts @@ -1,3 +1,4 @@ +import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { logger } from "../../middleware/logger.js"; import { createHash, randomUUID } from "node:crypto"; import { stat } from "node:fs/promises"; @@ -34,6 +35,20 @@ const processAlive = (pid: number | null) => { } }; +export function nativeReplacementPending() { + return and( + eq(heartbeatRuns.status, "failed"), + eq(nativeRunFinalizations.phase, "terminal_failure"), + inArray( + nativeRunFinalizations.failureCode, + ["native_provider_terminal_failed", "native_session_cleanup_quarantined", "provider_transport_failed"], + ), + isNull(nativeRunFinalizations.resultId), + sql`coalesce(${nativeRunFinalizations.failureDetail}->>'successorRunId', '') = ''`, + sql`coalesce(${nativeRunFinalizations.failureDetail}->>'replacementDenied', '') = ''`, + ); +} + /** Retryable outbox: terminal_failure plus a new-format failure code is the durable candidate. */ export async function reconcileSafeNativeReplacements( db: Db, @@ -55,17 +70,7 @@ export async function reconcileSafeNativeReplacements( eq(nativeRunFinalizations.runId, heartbeatRuns.id), ) .where( - and( - eq(heartbeatRuns.status, "failed"), - eq(nativeRunFinalizations.phase, "terminal_failure"), - inArray( - nativeRunFinalizations.failureCode, - ["native_provider_terminal_failed", "native_session_cleanup_quarantined", "provider_transport_failed"], - ), - isNull(nativeRunFinalizations.resultId), - sql`coalesce(${nativeRunFinalizations.failureDetail}->>'successorRunId', '') = ''`, - sql`coalesce(${nativeRunFinalizations.failureDetail}->>'replacementDenied', '') = ''`, - ), + nativeReplacementPending(), ) .limit(25); let scheduled = 0; @@ -260,6 +265,7 @@ export async function reconcileSafeNativeReplacements( current.failureDetail?.replacementDenied ) return; + await notifyExecutionRunWork(tx); await tx .update(nativeRunFinalizations) .set({ @@ -401,6 +407,7 @@ export async function reconcileSafeNativeReplacements( idempotencyKey: `native-safe-replacement:${run.id}`, }) .returning(); + await notifyExecutionRunWork(tx); await tx.insert(heartbeatRuns).values({ id: successorRunId, companyId: run.companyId, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 67423e01554..98605fe1e89 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,3 +1,6 @@ +import * as environmentTaskRuntime from "../environment-task-runtime.js"; +import * as pluginTaskRuntime from "./plugin-runner-task.js"; +import { environmentTaskOperationSchema } from "@paperclipai/plugin-sdk"; import { trace } from "@opentelemetry/api"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { configuredEnvironment, configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js"; @@ -25,7 +28,7 @@ import { import { execFile, execFileSync, spawn, spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { inspect } from "node:util"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { heartbeatRuns, heartbeatRunEvents, @@ -9460,13 +9463,24 @@ describe("native session bounded recovery", () => { ); const transact = db.transaction.bind(db); const recoveryWriteAttempt = vi.fn(); - db.transaction = (async (operation) => { - if (state.execute.mock.calls.length > 0) { - recoveryWriteAttempt(); - if (ancillaryFailure === "recovery_write") throw secondaryFailure; - } - return transact(operation); - }) as typeof db.transaction; + db.transaction = (async (operation) => transact(async tx => operation(new Proxy(tx, { + get(target, key) { + if (key !== "update") return Reflect.get(target, key); + return (table: unknown) => { + const query = Reflect.apply(target.update, target, [table]); + return new Proxy(query, { get(update, property) { + if (property !== "set") return Reflect.get(update, property); + return (values: Record) => { + if (table === nativeRunFinalizations && values.phase === "terminal_failure") { + recoveryWriteAttempt(); + if (ancillaryFailure === "recovery_write") throw secondaryFailure; + } + return update.set(values); + }; + } }); + }; + }, + })))) as typeof db.transaction; const updateIssue = vi.fn(async () => null); const service = vi .spyOn(issueServiceModule, "issueService") @@ -11011,6 +11025,96 @@ describe("runnerd provider runtime wiring", () => { } }); + it.each(["verified", "wrong-state", "no-stop-proof"] as const)("admits only an authorized same-run plugin crash checkpoint: %s", async mode => { + const current = scopedNativeExecution(`plugin-crash-${mode}`); + const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!, + runnerInstanceId: "plugin-runner", environmentLeaseId: "old-task-lease", turnId: "old-turn", itemId: "old-item" }; + const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId); + await mkdir(join(root, "control-plane"), { recursive: true }); + const journal = JSON.stringify(durableControlPlaneState(identity)); + await writeFile(join(root, "control-plane/control-plane-state.json"), journal); + const resume = vi.spyOn(environmentTaskRuntime, "resolvePluginTaskResume").mockImplementation(async () => { + if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized"); + return "prior-task"; + }); + const task = vi.fn(async (operation: any) => { + environmentTaskOperationSchema.parse(operation); + if (operation.kind === "read_state") return { kind: "state", taskId: "next-task", state: { + ...durableRunnerState(identity, "ready"), ...(mode === "wrong-state" ? { itemId: "other" } : {}), + } }; + return { kind: "description", launchPreparationVersion: 1, projectAuthCheckVersion: 1, taskId: "next-task", + runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; + }); + state.createBackend.mockClear(); + try { + const backend = createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: identity.runnerInstanceId, + runnerIngressAuthorized: true, runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "new-task-lease", runnerTask: task }, + restartRecovery: { kind: "resume_dead_runner", runId: current.binding.runId, leaseOwner: "controller", controllerGeneration: 2, + providerAttempt: 2, restartKind: "hard", recoveryRequestId: null } }); + if (mode === "verified") { + await expect(backend).resolves.toBeDefined(); + expect(state.createBackend).toHaveBeenCalledOnce(); + expect(JSON.parse(await readFile(join(root, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "ready" }); + } else { + await expect(backend).rejects.toThrow(mode === "wrong-state" ? "checkpoint identity mismatch" : "not authorized"); + expect(state.createBackend).not.toHaveBeenCalled(); + } + expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); + if (mode === "no-stop-proof") expect(task).not.toHaveBeenCalled(); + else expect(task).toHaveBeenCalledWith({ kind: "read_state", identity, resumeFrom: "prior-task" }); + } finally { resume.mockRestore(); } + }); + + it.each(["codex", "opencode", "acpx"] as const)("uses a task-only %s driver without exposing Core paths or transferring the agent home", async (kind) => { + const taskExecution = { ...execution, provider: kind === "codex" ? execution.provider + : kind === "opencode" ? { kind, model: "openai/test", permissionMode: "deny" } + : { kind, agent: "claude", model: "claude-sonnet-4-6", permissionMode: "deny-all", claudeEffort: "high" } } as NativeExecutionInput; + const assignedMcpTools = { definitions: () => [], has: () => false, execute: vi.fn() }; + state.createAssignedMcpTools.mockResolvedValueOnce(assignedMcpTools); + const launcher = vi.fn(); + const registration = vi.fn(); + const driver = { prepareLaunch: vi.fn(), runnerProcessLauncher: launcher, controlPlaneRegistration: registration }; + const owner = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner").mockReturnValue({ launcher, registration, driver }); + const task = vi.fn(async (operation: unknown) => { + environmentTaskOperationSchema.parse(operation); + return { kind: "description", launchPreparationVersion: 1, projectAuthCheckVersion: 1, taskId: "task", runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; + }); + const target = { kind: "remote", transport: "sandbox", providerKey: "plugin", leaseId: "plugin-task-lease", + remoteCwd: "/unused", runnerTask: task, runnerTaskProjectIds: ["10000000-0000-4000-8000-000000000001"] } as const; + const updates: Array<{ table: unknown; values: Record }> = []; + state.createBackend.mockClear(); state.createTransport.mockClear(); + try { + await expect(createRunnerdBackend({ db: leaseDb(execution), execution: taskExecution, runnerInstanceId: "runner", runnerExecutionTarget: target })) + .rejects.toThrow("runner_ingress_unavailable"); + await createRunnerdBackend({ db: leaseDb(execution, {}, {}, updates), execution: taskExecution, runnerInstanceId: "runner", runnerExecutionTarget: target, + runnerIngressAuthorized: true, + runnerEnvironment: { HOME: "/controller/private", PATH: "/controller/bin", OPENAI_API_KEY: "fixture-secret", + PAPERCLIP_NATIVE_MCP_NAME: "assigned", PAPERCLIP_NATIVE_MCP_URL: "http://localhost/mcp/gateways/assigned-test", + PAPERCLIP_NATIVE_MCP_TOKEN: "fixture-mcp-secret" } }); + expect(state.toolAuthorityDefinitions).toHaveBeenCalledWith(expect.objectContaining({ assignedMcpTools })); + expect(state.createAssignedMcpTools).toHaveBeenCalledWith(expect.objectContaining({ gatewayPublicId: "assigned-test", bearerToken: "fixture-mcp-secret" })); + expect(owner).toHaveBeenCalledWith(expect.objectContaining({ task, projectIds: target.runnerTaskProjectIds, + credentialIdentity: undefined })); + expect(state.createBackend.mock.calls.at(-1)![1]).toMatchObject({ workingDirectoryAuthority: "remote_runner", environment: {} }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const options = state.createTransport.mock.calls.at(-1)![0]; + expect(options).toMatchObject({ ...driver, runnerArtifact: { version: "fixture" }, + prpIdentity: { environmentLeaseId: "plugin-task-lease" } }); + if (kind === "acpx") expect(options).toMatchObject({ acpxAgent: "claude", claudeEffort: "high" }); + for (const key of ["environment", "codexCommand", "codexArgs", "runnerRuntimePaths", "readRunnerState", "archiveExternalRunnerState"]) { + expect(options).not.toHaveProperty(key); + } + expect(task.mock.calls).toEqual([[{ kind: "describe" }]]); + expect(owner.mock.calls.at(-1)![0]).not.toHaveProperty("assets"); + const identity: Record = options.prpIdentity!; + await owner.mock.calls.at(-1)![0].onSessionRetained!({ schema: "paperclip.runner.durable.state.v1", lifecycle: "suspended", ...identity }); + const metadata = updates.find(update => update.values.metadata)?.values.metadata as SQL; + expect(JSON.parse(new PgDialect().sqlToQuery(metadata).params[0] as string)).toMatchObject({ + runnerTaskIdentity: { turnId: identity.turnId, itemId: identity.itemId }, + }); + } finally { owner.mockRestore(); } + }); + it("keeps assigned MCP credentials on the control plane for remote Codex", async () => { const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 73ae25afb56..962ab165994 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,4 +1,12 @@ +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import { writeExecutionControl } from "../execution-control-notifications.js"; +import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { createRunUsageRecorder } from "../usage-receipts.js"; +import { registerPluginTaskCredentialHandoff, type PluginTaskCredentialWritebackBinding } from "../plugin-task-credential-writeback.js"; +import { resolvePluginTaskResume } from "../environment-task-runtime.js"; +import { environmentTaskResultSchema } from "@paperclipai/plugin-sdk"; +import { preparePluginRunnerCredentials } from "./plugin-runner-credentials.js"; +import { createPluginRunnerTaskOwner } from "./plugin-runner-task.js"; import { applyObservabilityEnvironment, type ProviderObservabilityLaunch } from "@paperclipai/adapter-utils/agent-observability"; import { museRunnerBroker } from "../muse-runner-broker.js"; import { nativeCaseHandoffResult } from "./native-case-handoff.js"; @@ -1978,12 +1986,34 @@ function migrateLegacyRunnerdStateRoot(input: { return input.scoped; } +function savePluginRunnerCheckpoint(root: string, state: Record, identity: Record) { + const fields = ["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"]; + if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle)) + || fields.some(key => typeof identity[key] !== "string" || state[key] !== identity[key])) { + throw new Error("Plugin Runner checkpoint identity mismatch"); + } + const path = resolve(root, "plugin-runner-state.json"); + const temporary = `${path}.${randomUUID()}.tmp`; + // Keep only the verified lifecycle identity. Provider state stays on the execution host. + writeFileSync(temporary, JSON.stringify({ schema: state.schema, lifecycle: state.lifecycle, + ...Object.fromEntries(fields.map(key => [key, state[key]])) }), { flag: "wx", mode: 0o600 }); + renameSync(temporary, path); +} + function runnerdAuthorityLifecycle( root: string, identity: RunnerdDurableIdentity, ): "absent" | "suspended" | "not_suspended" | "indeterminate" { const runnerRoot = resolve(root, "runner"); - if (!existsSync(runnerRoot)) return "absent"; + if (!existsSync(runnerRoot)) { + try { + const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"), + NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8"))); + if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle)) + || !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate"; + return state.lifecycle === "suspended" ? "suspended" : "not_suspended"; + } catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; } + } if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate"; const statePath = resolve(runnerRoot, "runner-state.json"); if (!existsSync(statePath)) { @@ -4852,6 +4882,16 @@ async function migrateRunnerdStateRootForExecution(input: { } throw new Error("runner_state_identity_mismatch: session_scope_mismatch"); } + const taskTarget = input.runnerExecutionTarget; + if (input.restartRecovery?.kind === "resume_dead_runner" && durableIdentityMatchesExecution(identity, input.execution) + && taskTarget?.kind === "remote" && taskTarget.transport === "sandbox" && taskTarget.runnerTask && taskTarget.leaseId) { + const resumeFrom = await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId, + leaseId: taskTarget.leaseId, priorIdentity: identity }); + const result = environmentTaskResultSchema.parse(await taskTarget.runnerTask({ kind: "read_state", identity, resumeFrom })); + if (result.kind !== "state") throw new Error("Plugin Runner checkpoint unavailable"); + // The new lease and source termination receipt authorize this exact same-run recovery. + savePluginRunnerCheckpoint(scoped, result.state, identity); + } if (input.restartRecovery?.kind === "bootstrap_incomplete") { if (runnerdStateProvesIncompleteBootstrap(scoped)) { quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); @@ -7559,7 +7599,10 @@ export async function executePaperclipNativeSession(input: { /** Private grant materialization; never a user-configured host path. */ managedAiCredentialHome?: string; managedAiCredentialIdentity?: string; + managedAiAccountIdentity?: string; + managedAiCredentialWriteback?: PluginTaskCredentialWritebackBinding; billingIdentity?: Pick; + onTaskDispatched?: () => Promise | void; runnerExecutionTarget?: AdapterExecutionTarget | null; /** Resolved per-run authorization; not an independent instance setting. */ runnerIngressAuthorized?: boolean; @@ -8766,8 +8809,9 @@ async function executePaperclipNativeSessionWithinScope( controller, }); try { + const usesPluginTask = input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "sandbox" && !!input.runnerExecutionTarget.runnerTask; const prepareRunnerEnvironment = async () => { - if (input.managedGitHub && input.execution.schema !== "paperclip.native-execution-input.v6" && input.execution.schema !== "paperclip.native-execution-input.v8") { + if (!usesPluginTask && input.managedGitHub && input.execution.schema !== "paperclip.native-execution-input.v6" && input.execution.schema !== "paperclip.native-execution-input.v8") { if (!input.execution.workspace.cwd) throw new Error("native_workspace_required"); githubAccess ??= await timeController("github_access", () => createNativeGitHubAccess({ scope: input.execution.binding, @@ -8782,7 +8826,7 @@ async function executePaperclipNativeSessionWithinScope( } return input.runnerEnvironment; }; - const overlapGitHubPreparation = input.managedGitHub && input.useRunnerd && input.backend === undefined + const overlapGitHubPreparation = !usesPluginTask && input.managedGitHub && input.useRunnerd && input.backend === undefined && input.runnerExecutionTarget?.kind === "remote" && input.execution.provider.kind === "codex" && input.execution.schema !== "paperclip.native-execution-input.v6" && input.execution.schema !== "paperclip.native-execution-input.v8"; @@ -9226,7 +9270,7 @@ async function executePaperclipNativeSessionWithinScope( }; const stoppedLeaseRenewal = leaseRenewal.stop().catch(() => undefined); try { - await input.db + await writeExecutionControl(input.db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(nativeRunFinalizations) .set({ controlDeadlineAt: new Date( @@ -9238,7 +9282,7 @@ async function executePaperclipNativeSessionWithinScope( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq(nativeRunFinalizations.leaseOwner, leaseOwner), ), - ); + )); const failedAtMs = Date.now(); const executionFailureMessage = redactIdentityText( @@ -9280,7 +9324,7 @@ async function executePaperclipNativeSessionWithinScope( typeof stopIntent.intentAuditId === "string" && stopIntent.intentAuditId.length > 0 && ["pending", "acknowledged"].includes(String(stopIntent.dispatchState)); if (stoppedRun && (pendingStartupStop || hasAcknowledgedNativeStopIntent(stoppedRun) || hasAcknowledgedNativeReassignmentStopIntent(stoppedRun))) { - const [settled] = await input.db.update(nativeRunFinalizations).set({ + const [settled] = await writeExecutionControl(input.db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null, leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null, updatedAt: new Date(), @@ -9290,7 +9334,7 @@ async function executePaperclipNativeSessionWithinScope( eq(nativeRunFinalizations.leaseOwner, leaseOwner), eq(nativeRunFinalizations.attempt, attempt), isNull(nativeRunFinalizations.resultId), - )).returning({ runId: nativeRunFinalizations.runId }); + )).returning({ runId: nativeRunFinalizations.runId })); if (settled) { await stoppedLeaseRenewal; if (warmSessionId !== null && lifecyclePolicy.mode === "warm") { @@ -9377,6 +9421,7 @@ async function executePaperclipNativeSessionWithinScope( ), ) .for("update"); + await notifyExecutionRunWork(tx); const updated = await tx .update(nativeRunFinalizations) .set({ @@ -11413,6 +11458,10 @@ export async function createRunnerdBackend(input: { prepareRunnerEnvironment?: () => Promise; /** Private grant materialization; never a user-configured host path. */ managedAiCredentialHome?: string; + managedAiCredentialIdentity?: string; + managedAiAccountIdentity?: string; + managedAiCredentialWriteback?: PluginTaskCredentialWritebackBinding; + onTaskDispatched?: () => Promise | void; runnerExecutionTarget?: AdapterExecutionTarget | null; /** Resolved per-run authorization; not an independent instance setting. */ runnerIngressAuthorized?: boolean; @@ -11593,6 +11642,11 @@ async function createRunnerdBackendWithinSessionClaim( let recoveryPending = retainedTransition !== undefined; const target = input.runnerExecutionTarget ?? { kind: "local" as const }; const remoteTarget = target.kind === "remote" ? target : null; + const pluginTask = remoteTarget?.transport === "sandbox" ? remoteTarget.runnerTask : undefined; + if (pluginTask && input.runnerIngressAuthorized !== true) throw new Error("runner_ingress_unavailable"); + if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8" || input.execution.session.lifecyclePolicy.mode !== "per_turn")) { + throw new Error("Plugin Runner tasks currently require a per-turn workspace session"); + } const remoteCommandRunner = remoteTarget ? remoteTarget.transport === "ssh" ? createNativeSshCommandRunner({ @@ -11601,7 +11655,7 @@ async function createRunnerdBackendWithinSessionClaim( }) : remoteTarget.runner : null; - if (remoteTarget && !remoteCommandRunner) { + if (remoteTarget && !remoteCommandRunner && !pluginTask) { throw new Error( "runner_transport_ineligible: remote process runner is unavailable", ); @@ -11624,7 +11678,7 @@ async function createRunnerdBackendWithinSessionClaim( // Remote Codex already sends dynamic tool calls over authenticated PRP. Keep // the assigned gateway on the control plane instead of asking the sandbox to // reach the host's HTTP origin (which may be private or loopback-only). - const relayAssignedMcp = (remoteTarget !== null && input.execution.provider.kind === "codex") || input.execution.schema === "paperclip.native-execution-input.v6"; + const relayAssignedMcp = Boolean(pluginTask) || (remoteTarget !== null && input.execution.provider.kind === "codex") || input.execution.schema === "paperclip.native-execution-input.v6"; const assignedMcpUrl = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_URL; const assignedMcpToken = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_TOKEN; const assignedMcpName = input.runnerEnvironment?.PAPERCLIP_NATIVE_MCP_NAME; @@ -11683,7 +11737,7 @@ async function createRunnerdBackendWithinSessionClaim( assertBridgeAuthority: input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8" ? (database = input.db) => assertExternalNativeRunAuthority(input.db, input.execution, database) : undefined, workspaceRoot: input.dotWorkspaceRoot ?? remoteTarget?.remoteCwd ?? input.execution.workspace.cwd ?? undefined, executionTargetKind: target.kind, - readRemoteWorkspaceFile: remoteTarget && remoteCommandRunner + readRemoteWorkspaceFile: !pluginTask && remoteTarget && remoteCommandRunner ? (file) => readVerifiedRemoteWorkspaceFile({ runner: remoteCommandRunner, workspaceRoot: remoteTarget.remoteCwd, ...file }) : undefined, currentWakeComments: currentWakeComments ?? undefined, @@ -11716,10 +11770,83 @@ async function createRunnerdBackendWithinSessionClaim( const effectiveRunnerInstanceId = durableBinding?.runnerInstanceId ?? input.runnerInstanceId; const effectiveEnvironmentLeaseId = - durableBinding?.environmentLeaseId ?? + (pluginTask ? remoteTarget?.leaseId : undefined) ?? durableBinding?.environmentLeaseId ?? input.durableEnvironmentLeaseId ?? input.execution.binding.executionWorkspaceId; mkdirSync(root, { recursive: true, mode: 0o700 }); + if (pluginTask && remoteTarget?.transport === "sandbox" && input.execution.schema !== "paperclip.native-execution-input.v6" && input.execution.schema !== "paperclip.native-execution-input.v8") { + const provider = input.execution.provider; + if (provider.kind !== "codex" && provider.kind !== "opencode" && provider.kind !== "acpx") throw new Error("Plugin Runner tasks require a workspace harness"); + const description = environmentTaskResultSchema.parse(await pluginTask({ kind: "describe" })); + if (description.kind !== "description" || description.launchPreparationVersion !== 1 || description.projectAuthCheckVersion !== 1) throw new Error("Plugin Runner launch preparation v1 unavailable"); + const prior = durableBinding ? durableIdentity as RunnerdDurableIdentity : undefined; + const resumeFrom = prior && prior.environmentLeaseId !== remoteTarget.leaseId + ? await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId, + leaseId: remoteTarget.leaseId!, priorIdentity: prior }) : undefined; + const credentials = await preparePluginRunnerCredentials({ + provider: provider.kind === "acpx" ? provider.agent : provider.kind, + identity: input.managedAiCredentialIdentity, accountIdentity: input.managedAiAccountIdentity, + home: input.managedAiCredentialHome, environment: input.runnerEnvironment ?? {}, + }); + if (credentials.needsWriteback && !input.managedAiCredentialWriteback) throw new Error("Plugin Runner credential writeback authority unavailable"); + if (input.managedAiCredentialWriteback && credentials.accountIdentity) { + if (!remoteTarget.leaseId) throw new Error("Plugin Runner credential lease unavailable"); + await registerPluginTaskCredentialHandoff(input.db, { companyId: input.execution.binding.companyId, leaseId: remoteTarget.leaseId }, + credentials.accountIdentity, input.managedAiCredentialWriteback); + } + const owner = createPluginRunnerTaskOwner({ ...credentials, projectAuthCheckVersion: description.projectAuthCheckVersion, task: pluginTask, projectIds: remoteTarget.runnerTaskProjectIds ?? [], + harness: provider.kind === "acpx" ? provider.agent : provider.kind, resumeFrom, priorIdentity: prior ? { + runnerInstanceId: prior.runnerInstanceId, environmentLeaseId: prior.environmentLeaseId, + runId: prior.runId, normalizedSessionId: prior.normalizedSessionId, + turnId: typeof prior.turnId === "string" ? prior.turnId : "", + itemId: typeof prior.itemId === "string" ? prior.itemId : "", + } : undefined, + credentialIdentity: input.managedAiCredentialIdentity, + onSessionRetained: async state => { + savePluginRunnerCheckpoint(root, state, state); + await input.db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ + runnerTaskSessionRetained: true, runnerTaskIdentity: { + runnerId: state.runnerInstanceId, leaseId: state.environmentLeaseId, + runId: state.runId, sessionId: state.normalizedSessionId, turnId: state.turnId, itemId: state.itemId, + }, + })}::jsonb` }) + .where(and(eq(environmentLeases.id, remoteTarget.leaseId!), eq(environmentLeases.companyId, input.execution.binding.companyId))); + }, onDispatched: input.onTaskDispatched, + }); + const executeTool = (call: Parameters[0]) => { + const current = sessionToolAuthorityEpochs.get(sessionScopeId); + if (!current) throw new Error("native_session_tool_authority_unavailable"); + return current.execute(call); + }; + const backend = createNativeSessionBackend(input.execution, { + runnerInstanceId: prior?.runnerInstanceId ?? effectiveRunnerInstanceId, environment: {}, + workingDirectoryAuthority: "remote_runner", dynamicTools, dynamicToolHandler: executeTool, + completionFeedback: async result => { await authorityEpoch.definitions(); return nativeCompletionFeedback(input.db, input.execution.binding.runId, result); }, + codexTransportFactory: recovery => createRunnerdCodexTransport({ provider: provider.kind, stateDirectory: root, + ...owner.driver, runnerArtifact: description.runner, + ...(provider.kind === "acpx" ? { acpxAgent: provider.agent, + acpxCandidateProfile: provider.agent === "pi" ? undefined : resolveAcpxQualification(provider, process.env), + acpxPermissionMode: provider.permissionMode, acpxMode: provider.mode, piThinkingLevel: provider.piThinkingLevel, + claudeEffort: "claudeEffort" in provider ? provider.claudeEffort : undefined, + acpxPermissionModePinned: true } : {}), + ...(provider.kind === "opencode" ? { opencodePermissionMode: provider.permissionMode } : {}), + runnerReconnectGraceMs: 120_000, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + runtimeContext: "runtimeContext" in input.execution ? input.execution.runtimeContext : null, + externallySandboxed: true, baseInstructions: recovery?.baseInstructions, + resumeDynamicTools: dynamicTools, resumeCompletionContract: { revision: input.execution.completionContract.contract.revision, + criterionIds: input.execution.completionContract.contract.criteria.map(criterion => criterion.id) }, + resumeActiveTurnId: recovery?.persistedSession?.activeTurnId ?? null, resumeProviderSession: recovery?.persistedSession, + prpIdentity: { runnerInstanceId: prior?.runnerInstanceId ?? effectiveRunnerInstanceId, + environmentLeaseId: prior?.environmentLeaseId ?? effectiveEnvironmentLeaseId, + runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), + turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}` }, + }).transport, + }); + sessionToolAuthorityEpochs.get(sessionScopeId)?.revoke(); + sessionToolAuthorityEpochs.set(sessionScopeId, authorityEpoch); + return { ...backend, descriptor: () => backend.descriptor(), openSession: session => backend.openSession(session), + recoverSession: (snapshot, options) => backend.recoverSession!(snapshot, options), bindManagedSession: session => session }; + } if (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8") { if (target.kind === "remote" && input.dotWorkspaceRoot) throw new Error("dot_remote_workspace_access_unavailable"); const remoteDot = target.kind === "remote" diff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.ts index bdc1ec177ec..c674bcfb4e2 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts @@ -1,3 +1,4 @@ +import * as pluginWorkspaces from "../plugin-workspaces.js"; import * as platformProof from "./native-workspace-platform-proof.js"; import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; import { nativeSha256 } from "./canonical.js"; @@ -723,6 +724,46 @@ describe("native workspace finalization recovery", () => { } finally { release(); await first; spy.mockRestore(); } }); + it.each(["plugin", "adapter_managed"])("resumes accepted work without Core files for %s storage", async providerType => { + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, projectId, + name: "Plugin-only work", providerType, providerRef: "provider:opaque-worktree", cwd: "/workspaces/plugin-only", + mode: "isolated_workspace", strategyType: providerType === "plugin" ? "git_worktree" : "project_primary", + branchName: "task-branch", metadata: providerType === "plugin" ? {} : { runnerTaskWorkspaceOwner: { agentId } } }).returning(); + const seeded = await seedRun({ executionWorkspaceId: workspace.id, title: "Plugin result before controller restart" }); + const inspect = vi.spyOn(pluginWorkspaces, "inspectPluginWorkspace").mockResolvedValue({ kind: "inspection", + branchName: "task-branch", headSha: "a".repeat(40), hasDirtyTrackedFiles: true, hasUntrackedFiles: false, + hasLocalOnlyData: true, isMergedIntoBase: false, remotelyRecoverable: false }); + try { + expect(await resumeNativeWorkspaceFinalization({ db, runId: seeded.runId })).toMatchObject({ status: "succeeded" }); + if (providerType === "plugin") expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id })); + else expect(inspect).not.toHaveBeenCalled(); + } finally { inspect.mockRestore(); } + }); + + it.each(["synthetic plugin unavailable", "plugin_workspace_branch_mismatch", "plugin_workspace_owner_mismatch"])("defers plugin failure without aborting other result recovery: %s", async message => { + const [unavailable, healthy] = await db.insert(executionWorkspaces).values([ + { companyId, projectId, name: "Unavailable plugin", providerType: "plugin", providerRef: "provider:unavailable", + mode: "isolated_workspace", strategyType: "git_worktree", branchName: "task-branch" }, + { companyId, projectId, name: "Healthy task", providerType: "adapter_managed", cwd: "/workspaces/healthy-task", + mode: "shared_workspace", strategyType: "project_primary", metadata: { runnerTaskWorkspaceOwner: { agentId } } }, + ]).returning(); + const blocked = await seedRun({ executionWorkspaceId: unavailable.id, title: "Plugin is unavailable after restart" }); + const ready = await seedRun({ executionWorkspaceId: healthy.id, title: "Other result can recover" }); + const inspect = vi.spyOn(pluginWorkspaces, "inspectPluginWorkspace").mockRejectedValue(new Error(message)); + const code = message.startsWith("plugin_workspace_") ? message : "plugin_workspace_inspection_failed"; + try { + await expect(reconcileNativeFinalizations(db, [blocked.runId, ready.runId])).resolves.toBeDefined(); + const [retry] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, blocked.runId)); + expect(retry).toMatchObject({ phase: "retryable_failure", nextAttemptAt: expect.any(Date), resultId: expect.any(String), + failureCode: code === message ? `native_${code}` : "native_workspace_sync_out_failed" }); + if (code === message) expect(retry.failureDetail?.nextAction).toContain("Repair the plugin workspace"); + expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, blocked.runId))) + .toMatchObject([{ status: "failed", stderrExcerpt: `${code}\n` }]); + expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, ready.runId))) + .toMatchObject([{ status: "succeeded" }]); + } finally { inspect.mockRestore(); } + }); + it("records directory-only recovery without inventing an execution-workspace foreign key", async () => { const cwd = path.join(workspaceRoot, "directory-only"); await fs.mkdir(cwd); diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts index f0d7b657f14..9e0220004c0 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.ts @@ -1,3 +1,5 @@ +import { inspectPluginWorkspaceForFinalization } from "./native-plugin-workspace-finalization.js"; +import type { WorkspaceWorkers } from "../plugin-workspaces.js"; import { restoreNativeWorkspaceBestEffort } from "./native-workspace-best-effort.js"; import { withNativeWorkspaceFinalizationOwnership } from "./native-workspace-finalization-ownership.js"; import fs from "node:fs/promises"; @@ -51,9 +53,11 @@ export async function resumeNativeWorkspaceFinalization(input: { db: Db; runId: string; environmentRuntime?: EnvironmentRuntimeService; + pluginWorkerManager?: WorkspaceWorkers; }) { const bound = await input.db.select({ companyId: heartbeatRuns.companyId, + agentId: heartbeatRuns.agentId, runtimeMode: heartbeatRuns.runtimeMode, runnerProfileJson: heartbeatRuns.runnerProfileJson, issueId: nativeRunFinalizations.issueId, @@ -237,6 +241,22 @@ export async function resumeNativeWorkspaceFinalization(input: { return workspaceSyncFailure(code); } } + let pluginWorkspace: Record | null; + try { + pluginWorkspace = workspace + ? await inspectPluginWorkspaceForFinalization(input.db, input.pluginWorkerManager, workspace, bound.agentId) + : null; + } catch (error) { + await ownership.assertHeld(); + const message = error instanceof Error ? error.message : null; + const code = message === "plugin_workspace_branch_mismatch" || message === "plugin_workspace_owner_mismatch" + ? message : "plugin_workspace_inspection_failed"; + return { status: "failed", exitCode: 1, stderr: `${code}\n` }; + } + await ownership.assertHeld(); + if (pluginWorkspace) return { status: "succeeded", exitCode: 0, + system: "Native workspace finalization used the plugin workspace owner.\n", + metadata: { pluginWorkspace } }; if (!cwd) { return { status: "failed", diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.ts index d6b716e3412..f3e30fa033d 100644 --- a/server/src/services/native-runtime/native-workspace-sync.ts +++ b/server/src/services/native-runtime/native-workspace-sync.ts @@ -885,7 +885,7 @@ export async function prepareNativeWorkspaceSync(input: { onCheckpoint?: (metrics: WorkspaceCheckpointMetrics) => Promise; onEvent?: (event: AdapterRuntimeEvent) => Promise; }): Promise { - if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") { + if (input.target?.kind !== "remote" || input.target.transport !== "sandbox" || input.target.runnerTask) { return null; } const target = input.target; diff --git a/server/src/services/plugin-agent-runtime-state.ts b/server/src/services/plugin-agent-runtime-state.ts index 11615783b01..e8e64c54d3f 100644 --- a/server/src/services/plugin-agent-runtime-state.ts +++ b/server/src/services/plugin-agent-runtime-state.ts @@ -50,9 +50,12 @@ export async function pluginAgentRuntimeState(db: Db, pluginId: string, companyI versionSelections: skillVersionSelectionMap(preference.desiredSkillEntries, { versionPinsEnabled: settings.enableBetaSkills }), })); const connectionAssignments = await pluginAgentConnectionAssignments(db, companyId, agentId, { publicMcp: settings.enablePublicMcp, chatConnectors: settings.enableChatConnectors }); + const fallback = !state.instructionsRootPath ? await agentInstructionsService().exportFiles(state.agent, { rejectSymlinks: true }) : null; + const entryFile = fallback?.entryFile ?? state.instructionsEntryFile; + const entryContent = fallback?.files[entryFile]; const current = await readState(db, pluginId, companyId, agentId); if (!current) return null; - if (current.revision === state.revision) return { ...current, skills, connectionAssignments }; + if (current.revision === state.revision) return { ...current, instructionsEntryFile: entryFile, ...(entryContent === undefined ? {} : { instructionsEntryContent: entryContent }), skills, connectionAssignments }; } throw conflict("Agent runtime changed while resolving synchronization inputs; retry"); } diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 11de5463f08..c3c2c6b0b55 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -1,3 +1,4 @@ +import { getAppDefinitionForApplicationKey, isConnectorFeatureEnabled } from "@paperclipai/shared"; import { stableHash, descriptorHash } from "./tool-catalog-hash.js"; import { assertProfileEntryInputs } from "./tool-profile-entry-validation.js"; import { DAYTONA_TOOLS, daytonaClient, daytonaRisk, isDaytonaConnection, isDaytonaToolName } from "./daytona.js"; @@ -7182,6 +7183,7 @@ export function toolAccessService( unavailableServices: string[] = [], ): Promise { assertSupportedConnection(connection); + await assertConnectorConnectionSetupEnabled(connection); let headers = credentialHeaders ?? { ...projectedConnectionHeaders(connection), ...(await resolveCredentialHeaders(connection, actor)), @@ -7561,6 +7563,7 @@ export function toolAccessService( unavailableServices: string[] = [], ): Promise { assertSupportedConnection(connection); + await assertConnectorConnectionSetupEnabled(connection); if (connection.connectionPurpose === "ai") throw unprocessable("AI connections provide runtime authentication, not tool actions"); if (isDaytonaConnection(connection)) { const headers = credentialHeaders ?? await resolveCredentialHeaders(connection, actor); @@ -7681,6 +7684,7 @@ export function toolAccessService( ): Promise { const connection = await getConnectionRow(connectionId); if (connection.connectionPurpose === "ai") return { connection: toConnection(connection), runtimeSlot: null }; + await assertConnectorConnectionSetupEnabled(connection); const unavailableServices: string[] = []; try { assertSupportedConnection(connection); @@ -7828,6 +7832,7 @@ export function toolAccessService( ): Promise { const connection = await getConnectionRow(connectionId); if (connection.connectionPurpose === "ai") throw unprocessable("AI connections do not have a tool catalog"); + await assertConnectorConnectionSetupEnabled(connection); const refreshedAt = now(); const unavailableServices: string[] = []; let descriptors: McpToolDescriptor[]; @@ -12440,12 +12445,31 @@ export function toolAccessService( } async function assertExperimentalConnectorSetupEnabled(provider: unknown, existing = false) { - if (!existing && isMemoryConnectorId(provider) - && !(await instanceSettingsService(db).getExperimental()).enableMemoryConnectors) { + const app = typeof provider === "string" ? getConnectableAppDefinition(provider) : null; + if (!app?.experimentalFeatureKey && (existing || !isMemoryConnectorId(provider))) return; + const settings = await instanceSettingsService(db).getExperimental(); + if (app && !isConnectorFeatureEnabled(app, settings)) { + throw forbidden(`${app.name} is unavailable on this instance`, { code: "connector_disabled" }); + } + if (!existing && isMemoryConnectorId(provider) && !settings.enableMemoryConnectors) { throw forbidden("Enable memory connectors in Settings → Experimental to set up this connection", { code: "memory_connectors_disabled" }); } } + async function assertConnectorConnectionSetupEnabled(connection: { config?: Record; transportConfig?: Record; applicationId?: string; companyId?: string; status?: string }, applicationKey?: unknown) { + let resolvedApplicationKey = applicationKey; + if (resolvedApplicationKey == null && connection.applicationId && connection.companyId) { + const [application] = await db.select({ applicationKey: toolApplications.applicationKey }) + .from(toolApplications).where(and(eq(toolApplications.id, connection.applicationId), + eq(toolApplications.companyId, connection.companyId))).limit(1); + resolvedApplicationKey = application?.applicationKey; + } + const marker = connection.config?.sourceTemplateKey ?? connection.transportConfig?.sourceTemplateKey; + const app = resolvedApplicationKey != null ? getAppDefinitionForApplicationKey(resolvedApplicationKey) + : typeof marker === "string" ? getConnectableAppDefinition(marker) : null; + if (app) await assertExperimentalConnectorSetupEnabled(app.slug, connection.status !== "draft"); + } + async function connectGalleryApp( companyId: string, input: ConnectToolApp, @@ -12565,6 +12589,7 @@ export function toolAccessService( } } + await assertConnectorConnectionSetupEnabled(requestedResumeConnection ?? {}, existingApplication?.applicationKey); const requestedName = input.name ?? existingApplication?.name ?? @@ -12769,7 +12794,7 @@ export function toolAccessService( } } const requestedGrantKind = - retainedGrantKind ?? input.grantKind ?? "organization"; + retainedGrantKind ?? input.grantKind ?? (method?.grantKinds?.length === 1 ? method.grantKinds[0]! : "organization"); if (method?.grantKinds && !method.grantKinds.includes(requestedGrantKind)) { throw badRequest( `${galleryEntry?.name ?? "This app"} supports only ${method.grantKinds.join(" or ")} credentials`, @@ -14066,6 +14091,7 @@ export function toolAccessService( actor?: ActorInfo, ): Promise { const connection = await getConnectionRow(connectionId, companyId); + await assertConnectorConnectionSetupEnabled(connection); if (connection.status === "archived") throw conflict("Archived app connections cannot be finished"); if (connection.config.mcpPreserveAccess === true && isRemoteMcpConnectorMethod(connection.config.sourceTemplateKey, connection.config.connectionMethodKey)) { @@ -14561,6 +14587,7 @@ export function toolAccessService( ): Promise { const connection = await getConnectionRow(connectionId, companyId); assertSupportedConnection(connection); + await assertConnectorConnectionSetupEnabled(connection); await assertExperimentalConnectorSetupEnabled(connection.config.sourceTemplateKey, connection.status !== "draft"); if (connection.status === "archived") throw conflict("Archived app connections cannot be reconnected"); @@ -14745,6 +14772,7 @@ export function toolAccessService( ): Promise { let connection = await getConnectionRow(connectionId, companyId); assertSupportedConnection(connection); + await assertConnectorConnectionSetupEnabled(connection); await assertExperimentalConnectorSetupEnabled(connection.config.sourceTemplateKey, connection.status !== "draft"); if (connection.status === "archived") throw conflict("Archived app connections cannot start sign in"); @@ -16371,6 +16399,7 @@ export function toolAccessService( status: connection.status, enabled: connection.enabled, }; + await assertConnectorConnectionSetupEnabled(connection); const sourceTemplateKey = typeof connection.config.sourceTemplateKey === "string" ? connection.config.sourceTemplateKey @@ -18474,6 +18503,7 @@ export function toolAccessService( const config = normalizeGoogleSheetsConnectionConfig( input.config ?? input.transportConfig ?? {}, ); + await assertConnectorConnectionSetupEnabled({ config, applicationId, companyId }); // Validate company-scoped references before touching a caller-supplied // network endpoint. Besides failing fast, this keeps cross-company // authorization errors from being masked by DNS or SSRF validation. @@ -18489,6 +18519,7 @@ export function toolAccessService( if (applicationId) { const app = await assertApplication(companyId, applicationId); applicationNamespace = app.applicationKey ?? app.name; + await assertConnectorConnectionSetupEnabled({ config }, applicationNamespace); if ( (transport === "mcp_remote" && app.type !== "mcp_http") || (transport === "local_stdio" && app.type !== "mcp_stdio") diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index 73b9245366a..3e97a9fd7cc 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -29,6 +29,8 @@ import { emitConnectionInvoked } from "./connector-telemetry.js"; import { resolveManagedGitHubIdentitySelection } from "./git-credentials.js"; import { extractRemoteMcpPending } from "./remote-mcp-pending.js"; import { logger } from "../middleware/logger.js"; +import { getAppDefinitionForApplicationKey, isConnectorFeatureEnabled } from "@paperclipai/shared"; +import { instanceSettingsService } from "./instance-settings.js"; import { spawn } from "node:child_process"; import { createHash, randomBytes, randomUUID } from "node:crypto"; import { isDeepStrictEqual } from "node:util"; @@ -1437,8 +1439,11 @@ export function createToolGatewayService( return connectionRow ? [{ catalogEntry, ...connectionRow }] : []; }); + const featureSettings = connectionRows.some(({ application }) => getAppDefinitionForApplicationKey(application.applicationKey)?.experimentalFeatureKey) + ? await instanceSettingsService(db).getExperimental() : undefined; const eligibleRows = rows.filter( ({ catalogEntry, connection, application }) => + (!featureSettings || isConnectorFeatureEnabled(getAppDefinitionForApplicationKey(application.applicationKey), featureSettings)) && !isRetiredComposioConnection(connection) && !(isDaytonaConnection(connection) && !isCurrentDaytonaCatalogEntry(catalogEntry)) && !(isRailwayEndpoint(connection.config.url) && (isRailwayToolBlocked(catalogEntry.toolName) || (normalizeRailwayToolName(catalogEntry.toolName).startsWith(RAILWAY_TOOL_PREFIX) && connection.config.railwayApiStatus !== "available"))) && @@ -5340,15 +5345,23 @@ export function createToolGatewayService( } } + async function assertApplicationFeatureAvailable(applicationKey: unknown) { + const app = getAppDefinitionForApplicationKey(applicationKey); + if (app?.experimentalFeatureKey && !isConnectorFeatureEnabled(app, await instanceSettingsService(db).getExperimental())) { + throw new ToolGatewayHttpError(403, "This connector is unavailable on this instance.", "connector_unavailable"); + } + } + async function fullyAssignedMcpConnections(session: ToolGatewaySession) { if (!session.gatewayProfileId) return []; - return db - .select({ connection: toolConnections }) + const rows = await db + .select({ connection: toolConnections, applicationKey: toolApplications.applicationKey }) .from(toolProfileEntries) .innerJoin( toolConnections, eq(toolProfileEntries.connectionId, toolConnections.id), ) + .innerJoin(toolApplications, eq(toolConnections.applicationId, toolApplications.id)) .where( and( eq(toolProfileEntries.companyId, session.companyId), @@ -5356,12 +5369,16 @@ export function createToolGatewayService( eq(toolProfileEntries.selectorType, "connection"), eq(toolProfileEntries.effect, "include"), eq(toolConnections.companyId, session.companyId), + eq(toolApplications.companyId, session.companyId), eq(toolConnections.enabled, true), eq(toolConnections.status, "active"), inArray(toolConnections.transport, ["mcp_remote", "local_stdio"]), ), - ) - .then((rows) => rows.map((row) => row.connection)); + ); + const settings = rows.some(row => getAppDefinitionForApplicationKey(row.applicationKey)?.experimentalFeatureKey) + ? await instanceSettingsService(db).getExperimental() : undefined; + return rows.filter(row => !settings || isConnectorFeatureEnabled(getAppDefinitionForApplicationKey(row.applicationKey), settings)) + .map(row => row.connection); } async function callRemoteConnectionProtocol(input: { @@ -5472,6 +5489,10 @@ export function createToolGatewayService( params?: Record; callerHeaders?: Record; }): Promise { + const [application] = await db.select({ applicationKey: toolApplications.applicationKey }).from(toolApplications) + .where(and(eq(toolApplications.id, input.connection.applicationId), eq(toolApplications.companyId, input.session.companyId))).limit(1); + if (!application) throw new ToolGatewayHttpError(404, "MCP connection not found", "connection_not_found"); + await assertApplicationFeatureAvailable(application.applicationKey); if (input.connection.transport === "mcp_remote") { return callRemoteConnectionProtocol({ ...input, @@ -6039,6 +6060,7 @@ export function createToolGatewayService( session, tool, ); + await assertApplicationFeatureAvailable(tool.applicationKey); // Recheck immediately before dispatch, including previously approved calls // and connections whose stored grant/catalog predates scope reduction. assertGoogleChatToolArgumentsSupported(connection, entry.toolName, parameters); diff --git a/tests/e2e/chat-adapters-ui-providers.spec.ts b/tests/e2e/chat-adapters-ui-providers.spec.ts index 399a966d1bd..bdde190f2d7 100644 --- a/tests/e2e/chat-adapters-ui-providers.spec.ts +++ b/tests/e2e/chat-adapters-ui-providers.spec.ts @@ -180,6 +180,7 @@ test.describe.serial("native chat adapter UI", () => { await expect(page.getByLabel("App configuration access token", { exact: true })).toHaveCount(0); await expect(page.getByLabel("Signing Secret", { exact: true })).toHaveCount(0); await page.reload(); + await expect(page.getByRole("heading", { name: "Send a message to your agent", exact: true })).toBeVisible(); expect(mock.slackCreations).toBe(1); expect(mock.slackInstallations).toBe(1); mock.setWebhookVerified(); await expect(page.getByRole("heading", { name: "Success", exact: true })).toBeVisible(); diff --git a/tests/runner-e2e/automatic-repository-flow.ts b/tests/runner-e2e/automatic-repository-flow.ts index 2a1b4693568..89e57f4c823 100644 --- a/tests/runner-e2e/automatic-repository-flow.ts +++ b/tests/runner-e2e/automatic-repository-flow.ts @@ -24,7 +24,8 @@ export async function runAutomaticRepositoryFlow(input: { const files = automaticRepositoryFiles(nonce); let issue: Row | undefined; let runs: Row[] = []; - const observation: AutomaticRepositoryObservation = { reference: "", issueDone: false, runCount: 0, + const observation: AutomaticRepositoryObservation = { expectedModel: execution.profile.model, + expectedProvider: execution.profile.provider === "codex" ? "codex" : "acpx:claude", actualProviders: [], reference: "", issueDone: false, runCount: 0, successfulCommittedRuns: 0, sourceContinuation: null, successorSource: null, workspaceIds: [], readyRepository: false, savedContext: null, firstSavedContext: null, firstContextAttributed: false, firstContextObservedAt: null, successorStartedAt: null, predecessorFinishedAt: null, predecessorRetagged: true, continuationReceipt: null, @@ -56,6 +57,10 @@ export async function runAutomaticRepositoryFlow(input: { observation.firstContextObservedAt = Date.now(); } } + observation.actualProviders = runs.map(run => { + const provider = run.runnerProfileJson?.nativeExecutionInput?.provider; + return { model: provider?.model ?? null, kind: provider ? provider.kind === "acpx" ? `acpx:${provider.agent}` : provider.kind : null }; + }); observation.workspaceIds = runs.map(run => run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId ?? null); observation.monitorScheduled ||= Boolean(issue.monitorNextCheckAt || runs.some(run => String(run.resultJson?.finalizationReasonCode).startsWith("scheduled_monitor_"))); } diff --git a/tests/runner-e2e/automatic-repository.test.ts b/tests/runner-e2e/automatic-repository.test.ts index 825ed08a0f7..a55b97624c5 100644 --- a/tests/runner-e2e/automatic-repository.test.ts +++ b/tests/runner-e2e/automatic-repository.test.ts @@ -3,7 +3,8 @@ import { automaticRepositoryApiState, automaticRepositoryStopReason, gradeAutoma const sha = "a".repeat(40); const reference = "c".repeat(64); -const valid: AutomaticRepositoryObservation = { reference, issueDone: true, runCount: 2, +const valid: AutomaticRepositoryObservation = { expectedModel: "gpt-6.1-sol", expectedProvider: "codex", + actualProviders: [{ model: "gpt-6.1-sol", kind: "codex" }, { model: "gpt-6.1-sol", kind: "codex" }], reference, issueDone: true, runCount: 2, successfulCommittedRuns: 2, sourceContinuation: "same_agent", successorSource: "native_status_decision", workspaceIds: ["workspace", "workspace"], readyRepository: true, savedContext: `${reference}\n`, firstSavedContext: `${reference}\n`, firstContextAttributed: true, firstContextObservedAt: 1000, successorStartedAt: 2000, predecessorFinishedAt: 1900, predecessorRetagged: false, @@ -37,6 +38,9 @@ it.each>([ { workspaceIds: ["old", "new"] }, { savedContext: null }, { firstSavedContext: null }, { firstContextAttributed: false }, { firstSavedContext: `${"d".repeat(64)}\n` }, { firstSavedContext: "REFERENCE-fresh\n" }, { firstContextObservedAt: null }, { firstContextObservedAt: 2000 }, { firstContextObservedAt: 3000 }, { successorStartedAt: null }, + { actualProviders: [] }, { actualProviders: [{ model: "gpt-6.1-sol", kind: "codex" }] }, + { actualProviders: [{ model: "gpt-5.6-sol", kind: "codex" }, valid.actualProviders[1]!] }, + { actualProviders: [valid.actualProviders[0]!, { model: "gpt-6.1-sol", kind: "acpx:claude" }] }, { readyRepository: false }, { downloaded: null }, { predecessorFinishedAt: null }, { predecessorFinishedAt: 2001 }, { predecessorRetagged: true }, { continuationReceipt: null }, { continuationReceipt: { ...valid.continuationReceipt!, dispatchId: valid.continuationReceipt!.intentId } }, diff --git a/tests/runner-e2e/automatic-repository.ts b/tests/runner-e2e/automatic-repository.ts index 321ed86b907..b8cf9168b70 100644 --- a/tests/runner-e2e/automatic-repository.ts +++ b/tests/runner-e2e/automatic-repository.ts @@ -29,6 +29,9 @@ export const automaticRepositoryTask: RunnerTaskFixture = { }; export interface AutomaticRepositoryObservation { + expectedModel: string; + expectedProvider: string; + actualProviders: Array<{ model: string | null; kind: string | null }>; reference: string; issueDone: boolean; runCount: number; @@ -94,6 +97,9 @@ export function gradeAutomaticRepository(value: AutomaticRepositoryObservation) /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value.continuationReceipt.dispatchId) && value.continuationReceipt.intentId !== value.continuationReceipt.dispatchId, "The saved same-agent intent must bind a distinct successor dispatch after the unmodified predecessor finishes, without a timer or user message."), + check("configured-provider", value.actualProviders.length === 2 && value.actualProviders.every(provider => + provider.model === value.expectedModel && provider.kind === value.expectedProvider), + "Both admitted runs must retain the configured provider and model."), check("retained-workspace", value.workspaceIds.length === 2 && Boolean(value.workspaceIds[0]) && value.workspaceIds.every(id => id === value.workspaceIds[0]) && value.readyRepository, "Both admissions must retain the same task workspace and its prepared repository."), check("retained-context", value.firstContextAttributed && value.firstContextObservedAt !== null && value.successorStartedAt !== null && diff --git a/ui/public/brands/apps/manifest.json b/ui/public/brands/apps/manifest.json index b5ee462fd90..1710fb254a4 100644 --- a/ui/public/brands/apps/manifest.json +++ b/ui/public/brands/apps/manifest.json @@ -234,7 +234,10 @@ "provider": "Google Workspace", "catalogVisible": true, "localAsset": "/brands/apps/google-people.svg", - "aliases": ["google suite", "g suite"] + "aliases": [ + "google suite", + "g suite" + ] }, { "slug": "hugging-face", @@ -679,6 +682,13 @@ "catalogVisible": true, "localAsset": "/brands/apps/daytona.svg", "darkAsset": "/brands/apps/daytona-dark.svg" + }, + { + "slug": "paperclip-notebook", + "provider": "Paperclip Notebook", + "catalogVisible": true, + "localAsset": "/brands/apps/paperclip-notebook.svg", + "darkAsset": "/brands/apps/paperclip-notebook-dark.svg" } ] } diff --git a/ui/public/brands/apps/paperclip-notebook-dark.svg b/ui/public/brands/apps/paperclip-notebook-dark.svg new file mode 100644 index 00000000000..ce2f6baecd9 --- /dev/null +++ b/ui/public/brands/apps/paperclip-notebook-dark.svg @@ -0,0 +1 @@ + diff --git a/ui/public/brands/apps/paperclip-notebook.svg b/ui/public/brands/apps/paperclip-notebook.svg new file mode 100644 index 00000000000..565ee465849 --- /dev/null +++ b/ui/public/brands/apps/paperclip-notebook.svg @@ -0,0 +1 @@ + diff --git a/ui/src/features/connections/ConnectionSetupFlow.tsx b/ui/src/features/connections/ConnectionSetupFlow.tsx index 606fa741975..aafba8a76c8 100644 --- a/ui/src/features/connections/ConnectionSetupFlow.tsx +++ b/ui/src/features/connections/ConnectionSetupFlow.tsx @@ -7,6 +7,7 @@ import { connectionInstructionsConfig, defaultConnectionAgentInstructions, type import { RemoteMcpProductionSetup } from "./remote-mcp/RemoteMcpProductionSetup"; import { findAggregatorApp } from "@paperclipai/shared/aggregator-app-catalog"; import { useMemoryConnectorsEnabled } from "@/hooks/useMemoryConnectorsEnabled"; +import { useConnectorAvailability } from "@/hooks/useConnectorAvailability"; import { AiConnectionCredentialStep } from "@/components/ai-connections/AiConnectionCredentialStep"; import { ConnectionChoiceList } from "./ConnectionChoiceList"; import { useCallback, useEffect, useId, useMemo, useRef, useState, type ReactNode, type Ref } from "react"; @@ -51,6 +52,7 @@ import { connectionMethodSupportsAutomaticOAuth, credentialConfigPath, getAppDefinitionForUrl, + getAppDefinitionForApplicationKey, getConnectableAppDefinition, getAvailableConnectionMethods, getRecommendedConnectionMethod, @@ -584,8 +586,9 @@ export function ConnectionSetupFlow(props: ConnectionSetupFlowProps = {}) { const params = useParams<{ appKey?: string }>(); const { selectedCompanyId } = useCompany(); const memory = useMemoryConnectorsEnabled(); + const availability = useConnectorAvailability(); const interactionId = props.interactionId || searchParams.get("intent") || undefined; - const source = props.serviceSlug || searchParams.get("source") || params.appKey || searchParams.get("appKey"); + const source = (props.serviceSlug || searchParams.get("source") || params.appKey || searchParams.get("appKey"))?.trim() || undefined; const draftId = useMemo(() => { if (interactionId && isRemoteMcpConnectorId(source)) { try { return sessionStorage.getItem(`paperclip:mcp-intent-draft:${selectedCompanyId}:${interactionId}`); } catch { /* Storage may be disabled. */ } @@ -593,12 +596,29 @@ export function ConnectionSetupFlow(props: ConnectionSetupFlowProps = {}) { return null; }, [interactionId, selectedCompanyId, source]); const existingId = props.configuredConnection?.id || searchParams.get("resume") || searchParams.get("reconnect") || draftId; - const lookup = Boolean(existingId && (!source || isRemoteMcpConnectorId(source) || isMemoryConnectorId(source))); + const requestedDefinition = getConnectableAppDefinition(source ?? ""); + const lookup = Boolean(existingId && (!source || isRemoteMcpConnectorId(source) || isMemoryConnectorId(source) || requestedDefinition?.experimentalFeatureKey)); const existing = useQuery({ queryKey: ["tools", "connection", existingId], queryFn: () => toolsApi.getConnection(existingId!), enabled: lookup }); + const applications = useQuery({ queryKey: queryKeys.tools.applications(selectedCompanyId ?? "__none__"), + queryFn: () => toolsApi.listApplications(selectedCompanyId!), enabled: Boolean(existingId && selectedCompanyId) }); + const connections = useQuery({ queryKey: queryKeys.tools.connections(selectedCompanyId ?? "__none__"), + queryFn: () => toolsApi.listConnections(selectedCompanyId!), enabled: Boolean(existingId && selectedCompanyId) }); + const retained = existing.data ?? connections.data?.connections.find(item => item.id === existingId); + const application = applications.data?.applications.find(item => item.id === (retained?.applicationId ?? props.configuredConnection?.applicationId)); + const persistedDefinition = application?.applicationKey != null + ? getAppDefinitionForApplicationKey(application.applicationKey) + : getConnectableAppDefinition(String(retained?.config?.sourceTemplateKey ?? "")); + const gatedDefinitions = [persistedDefinition, requestedDefinition].filter(app => app?.experimentalFeatureKey); const provider = source || existing.data?.config?.sourceTemplateKey; const method = searchParams.get("method") || existing.data?.config?.connectionMethodKey; if (lookup && existing.isPending) return

Loading connection…

; if (lookup && existing.isError) return

Could not load this connection. Your saved access and credentials have not changed.

; + if (gatedDefinitions.length) { + if (!availability.loaded) return

Loading connection settings…

; + if (availability.failed) return

Could not load connector availability. Refresh to try again.

; + const disabled = gatedDefinitions.find(app => !availability.enabled(app)); + if (disabled) return

{disabled.name} is not available on this instance.

; + } if (isMemoryConnectorId(provider) && !(existing.data && existing.data.status !== "draft" && existing.data.config?.sourceTemplateKey === provider)) { if (!memory.loaded) return

Loading connection settings…

; if (!memory.enabled) return

Enable memory connectors in Settings → Experimental to set up this connection.

; @@ -648,6 +668,7 @@ function StandardConnectionSetupFlow({ const { selectedCompany, selectedCompanyId } = useCompany(); const { enabled: chatConnectorsEnabled, githubEnabled } = useChatConnectorsEnabled(); const { enabled: memoryConnectorsEnabled } = useMemoryConnectorsEnabled(); + const connectorAvailability = useConnectorAvailability(); const { setBreadcrumbs } = useBreadcrumbs(); const { pushToast } = useToast(); const [searchParams] = useSearchParams(); @@ -1038,12 +1059,13 @@ function StandardConnectionSetupFlow({ // Use the same visible catalog for cards and every branded URL shortcut. // Generic custom URLs remain usable without selecting a hidden provider. const visibleGalleryApps = useMemo( - () => (galleryQuery.data?.apps ?? []).filter((app) => memoryConnectorsEnabled || !isMemoryConnectorId(app.slug)).filter((app) => + () => (galleryQuery.data?.apps ?? []).filter((app) => (memoryConnectorsEnabled || !isMemoryConnectorId(app.slug)) + && connectorAvailability.enabled(app)).filter((app) => chatProviderVisible(app.slug, chatConnectorsEnabled, githubEnabled) || !app.methods.some((method) => method.purpose === "channel") || appSupportsToolCatalogSetup(app), ), - [galleryQuery.data, chatConnectorsEnabled, githubEnabled, memoryConnectorsEnabled], + [galleryQuery.data, chatConnectorsEnabled, githubEnabled, memoryConnectorsEnabled, connectorAvailability.enabled], ); const fullRequestedDefinition = requestedAppKey ? getConnectableAppDefinition(requestedAppKey) diff --git a/ui/src/hooks/useConnectorAvailability.ts b/ui/src/hooks/useConnectorAvailability.ts new file mode 100644 index 00000000000..5aff4f535ce --- /dev/null +++ b/ui/src/hooks/useConnectorAvailability.ts @@ -0,0 +1,15 @@ +import { useCallback } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { isConnectorFeatureEnabled, type AppDefinition } from "@paperclipai/shared"; +import { instanceSettingsApi } from "@/api/instanceSettings"; +import { queryKeys } from "@/lib/queryKeys"; + +export function useConnectorAvailability() { + const query = useQuery({ + queryKey: queryKeys.instance.experimentalSettings, + queryFn: () => instanceSettingsApi.getExperimental(), + }); + const enabled = useCallback((app: AppDefinition | null | undefined) => + isConnectorFeatureEnabled(app, query.isError ? undefined : query.data), [query.data, query.isError]); + return { enabled, loaded: query.isFetched, failed: query.isError }; +} diff --git a/ui/src/pages/InstanceExperimentalSettings.test.tsx b/ui/src/pages/InstanceExperimentalSettings.test.tsx index 0145120b416..073d1a85b47 100644 --- a/ui/src/pages/InstanceExperimentalSettings.test.tsx +++ b/ui/src/pages/InstanceExperimentalSettings.test.tsx @@ -92,6 +92,7 @@ function defaultExperimentalSettings(): InstanceExperimentalSettingsPayload { enableChatConnectors: false, enableGitHubReviewBots: false, enableMemoryConnectors: false, + enablePaperclipNotebook: false, enableCases: false, enableAgentChat: false, enableCombinedInboxTasks: false, @@ -1148,6 +1149,20 @@ describe("InstanceExperimentalSettings — operator-hidden cards", () => { expect(mockInstanceSettingsApi.updateExperimental).not.toHaveBeenCalled(); }); + it.each([ + { enabled: false, developerMode: false }, { enabled: true, developerMode: false }, + { enabled: false, developerMode: true }, { enabled: true, developerMode: true }, + ])("never offers a Notebook control with enabled=$enabled and developerMode=$developerMode", async ({ enabled, developerMode }) => { + await renderPage(undefined, { ...defaultExperimentalSettings(), enablePaperclipNotebook: enabled, + enablePaperclipDeveloperMode: developerMode, + managedKeys: { enablePaperclipNotebook: { managed: true, managedBy: "paperclip-cloud" } }, + }); + expect(container.textContent).toContain("Experimental features"); + expect(container.textContent).not.toContain("Paperclip Notebook"); + expect(container.querySelector('button[aria-label="Toggle Paperclip Notebook experimental setting"]')).toBeNull(); + expect(mockInstanceSettingsApi.updateExperimental).not.toHaveBeenCalled(); + }); + it("renders nothing for an operator-hidden toggle and keeps the rest", async () => { await renderPage(["instance.experimental.enableEnvironments"]); diff --git a/ui/src/pages/apps/AppsConnect.test.tsx b/ui/src/pages/apps/AppsConnect.test.tsx index e321f9881f1..551c025251d 100644 --- a/ui/src/pages/apps/AppsConnect.test.tsx +++ b/ui/src/pages/apps/AppsConnect.test.tsx @@ -665,6 +665,64 @@ describe("AppsConnect — Connect with a link (M4 frame)", () => { expect(finishAppMock).not.toHaveBeenCalled(); }); + it.each([ + "source=paperclip-notebook", "source=%20paperclip-notebook%20", "appKey=paperclip-notebook", "source=paperclip-notebook&resume=existing-notebook", + "source=paperclip-notebook&reconnect=existing-notebook", "reconnect=existing-notebook", + ])("blocks Notebook setup and existing authorization shortcuts while disabled: %s", async query => { + mockSearch.value = query; + getConnectionMock.mockResolvedValue({ id: "existing-notebook", status: "active", config: { sourceTemplateKey: "paperclip-notebook" } }); + listGalleryMock.mockResolvedValue({ apps: [getAppStoreDefinition("paperclip-notebook")] }); + await render(); + expect(container.textContent).toContain("Paperclip Notebook is not available on this instance"); + expect(connectAppMock).not.toHaveBeenCalled(); + expect(startOAuthMock).not.toHaveBeenCalled(); + expect(finishAppMock).not.toHaveBeenCalled(); + }); + + it.each(["path", "inline"])("blocks the Notebook %s setup route while disabled", async route => { + getConnectionMock.mockResolvedValue({ id: "existing-notebook", status: "active", config: { sourceTemplateKey: "paperclip-notebook" } }); + mockParams.appKey = route === "path" ? "paperclip-notebook" : undefined; + await render(undefined, false, route === "inline" ? : undefined); + expect(container.textContent).toContain("Paperclip Notebook is not available on this instance"); + expect(connectAppMock).not.toHaveBeenCalled(); + expect(startOAuthMock).not.toHaveBeenCalled(); + }); + + it("uses the saved application identity when its mutable provider marker has changed", async () => { + mockSearch.value = "source=notion&reconnect=existing-notebook"; + getConnectionMock.mockResolvedValue({ id: "existing-notebook", applicationId: "notebook-app", status: "active", config: { sourceTemplateKey: "notion" } }); + listConnectionsMock.mockResolvedValue({ connections: [{ id: "existing-notebook", applicationId: "notebook-app", status: "active", config: { sourceTemplateKey: "notion" } }] }); + listApplicationsMock.mockResolvedValue({ applications: [{ id: "notebook-app", applicationKey: "app-gallery:paperclip-notebook:11111111-1111-4111-8111-111111111111", metadata: { sourceTemplateKey: "notion" } }] }); + await render(); + expect(container.textContent).toContain("Paperclip Notebook is not available on this instance"); + expect(connectAppMock).not.toHaveBeenCalled(); + expect(startOAuthMock).not.toHaveBeenCalled(); + }); + + it("waits for Notebook settings and fails closed when they cannot be loaded", async () => { + mockSearch.value = "source=paperclip-notebook"; + let reject!: (error: Error) => void; + experimentalMock.mockImplementation(() => new Promise((_resolve, fail) => { reject = fail; })); + await render(); + expect(container.textContent).toContain("Loading connection settings"); + await act(async () => { reject(new Error("Settings unavailable")); }); + await flushReact(); + expect(container.querySelector('[role="alert"]')?.textContent).toContain("Could not load connector availability"); + expect(connectAppMock).not.toHaveBeenCalled(); + expect(startOAuthMock).not.toHaveBeenCalled(); + }); + + it("opens Notebook setup only with its explicit rollout opt-in", async () => { + mockSearch.value = "source=paperclip-notebook"; + experimentalMock.mockResolvedValue({ enablePaperclipNotebook: true }); + listGalleryMock.mockResolvedValue({ apps: [getAppStoreDefinition("paperclip-notebook")] }); + await render(); + expect(container.textContent).toContain("Paperclip Notebook"); + expect(container.textContent).not.toContain("not available on this instance"); + expect(listGalleryMock).toHaveBeenCalledWith("company-1"); + expect([...container.querySelectorAll("button")].some(button => /Continue to|Continue to sign in|Connect/.test(button.textContent ?? ""))).toBe(true); + }); + it.each(["mem0", "zep", "supermemory", "cognee", "honcho"])("blocks direct %s setup while memory connectors are off", async (provider) => { mockSearch.value = `source=${provider}`; await render(); diff --git a/ui/src/pages/apps/Browse.test.tsx b/ui/src/pages/apps/Browse.test.tsx index fb691451ca9..50f98f95dff 100644 --- a/ui/src/pages/apps/Browse.test.tsx +++ b/ui/src/pages/apps/Browse.test.tsx @@ -935,6 +935,68 @@ describe("Connectors landing page", () => { }, ); + it("uses availability metadata for a non-Notebook connector in cached gallery results", async () => { + const privateResearch = { ...getAppStoreDefinition("notion")!, slug: "private-research", name: "Private Research", experimentalFeatureKey: "enableCases" as const }; + listGalleryMock.mockResolvedValue({ apps: [privateResearch, getAppStoreDefinition("context7")] }); + const client = await renderBrowse(); + expect(container.querySelector('[data-app-slug="private-research"]')).toBeNull(); + expect(container.querySelector('[data-app-slug="context7"]')).not.toBeNull(); + await act(() => { client.setQueryData(queryKeys.instance.experimentalSettings, { enableCases: true }); }); + await flushReact(); + expect(container.querySelector('[data-app-slug="private-research"]')).not.toBeNull(); + }); + + it("hides cached Notebook catalog entries until explicitly enabled", async () => { + listGalleryMock.mockResolvedValue({ apps: ["paperclip-notebook", "notion"].map(getAppStoreDefinition) }); + const client = await renderBrowse(); + expect(container.querySelector('[data-app-slug="paperclip-notebook"]')).toBeNull(); + expect(container.querySelector('[data-app-slug="notion"]')).not.toBeNull(); + await act(() => { client.setQueryData(queryKeys.instance.experimentalSettings, { enablePaperclipNotebook: true }); }); + await flushReact(); + expect(container.querySelector('[data-app-slug="paperclip-notebook"]')).not.toBeNull(); + await act(() => { client.setQueryData(queryKeys.instance.experimentalSettings, { enablePaperclipNotebook: false }); }); + await flushReact(); + expect(container.querySelector('[data-app-slug="paperclip-notebook"]')).toBeNull(); + }); + + it("hides Notebook while settings load", async () => { + experimentalMock.mockImplementation(() => new Promise(() => {})); + listGalleryMock.mockResolvedValue({ apps: [getAppStoreDefinition("paperclip-notebook")] }); + await renderBrowse(); + expect(container.querySelector('[data-app-slug="paperclip-notebook"]')).toBeNull(); + }); + + it("hides Notebook when a settings refresh fails with a cached opt-in", async () => { + experimentalMock.mockResolvedValue({ enablePaperclipNotebook: true }); + listGalleryMock.mockResolvedValue({ apps: [getAppStoreDefinition("paperclip-notebook")] }); + const client = await renderBrowse(); + expect(container.querySelector('[data-app-slug="paperclip-notebook"]')).not.toBeNull(); + experimentalMock.mockRejectedValue(new Error("Settings unavailable")); + await act(async () => { await client.invalidateQueries({ queryKey: queryKeys.instance.experimentalSettings }); }); + await flushReact(); + expect(client.getQueryData(queryKeys.instance.experimentalSettings)).toEqual({ enablePaperclipNotebook: true }); + expect(container.querySelector('[data-app-slug="paperclip-notebook"]')).toBeNull(); + }); + + it("keeps Notebook removal available while hiding reconnect and new-account controls", async () => { + listGalleryMock.mockResolvedValue({ apps: [getAppStoreDefinition("paperclip-notebook")] }); + listApplicationsMock.mockResolvedValue({ applications: [application({ id: "notebook-app", name: "Paperclip Notebook", + applicationKey: "app-gallery:paperclip-notebook:one", metadata: { sourceTemplateKey: "paperclip-notebook" } })] }); + listConnectionsMock.mockResolvedValue({ connections: [connection({ id: "saved-notebook", applicationId: "notebook-app", + name: "Notebook account", config: { sourceTemplateKey: "paperclip-notebook" }, healthStatus: "error", requiresReauthorization: true })] }); + await renderBrowse(); + const row = container.querySelector('[data-app-slug="paperclip-notebook"]')!; + expect(row).not.toBeNull(); + expect([...row.querySelectorAll("button")].some(button => /^(Connect|Add connection|Reconnect|Finish setup)$/.test(button.textContent?.trim() ?? ""))).toBe(false); + await act(() => { row.querySelector('button[aria-label^="Manage "]')!.dispatchEvent(new PointerEvent("pointerdown", { bubbles: true })); }); + await flushReact(); + await act(() => { [...document.querySelectorAll('[role="menuitem"]')].find(item => item.textContent?.trim() === "Remove connection")!.click(); }); + await flushReact(); + await act(() => { [...document.querySelectorAll("button")].find(button => button.textContent?.trim() === "Remove connection")!.click(); }); + await flushReact(); + expect(archiveConnectionMock).toHaveBeenCalledWith("saved-notebook"); + }); + it("hides cached memory connectors until enabled and preserves saved MCP connections", async () => { const providers = ["mem0", "zep", "supermemory", "cognee", "honcho"]; listGalleryMock.mockResolvedValue({ apps: [...providers, "notion"].map(getAppStoreDefinition) }); diff --git a/ui/src/pages/apps/Browse.tsx b/ui/src/pages/apps/Browse.tsx index 84318d0d233..843a8ec52ac 100644 --- a/ui/src/pages/apps/Browse.tsx +++ b/ui/src/pages/apps/Browse.tsx @@ -32,6 +32,7 @@ import { import type { ComposioAppSnapshot, ToolApplication, ToolConnection } from "@paperclipai/shared"; import { getAppDefinitionForUrl, + getAppDefinitionForApplicationKey, isMemoryConnectorId, getAppStoreDefinition, isToolConnectionAttentionHealth, @@ -43,6 +44,7 @@ import { import { useNavigate } from "@/lib/router"; import { useChatConnectorsEnabled, chatProviderVisible } from "@/hooks/useChatConnectorsEnabled"; import { useMemoryConnectorsEnabled } from "@/hooks/useMemoryConnectorsEnabled"; +import { useConnectorAvailability } from "@/hooks/useConnectorAvailability"; import { appCopyFor } from "@/lib/app-gallery-copy"; import { useCompany } from "@/context/CompanyContext"; import { useBreadcrumbs } from "@/context/BreadcrumbContext"; @@ -366,6 +368,7 @@ export function Browse({ renderAccountDetails = (connection) => connection.conne const { userId: viewingUserId, settled: identitySettled } = useAccountIdentity(); const { enabled: chatConnectorsEnabled, githubEnabled } = useChatConnectorsEnabled(); const { enabled: memoryConnectorsEnabled } = useMemoryConnectorsEnabled(); + const connectorAvailability = useConnectorAvailability(); const { setBreadcrumbs } = useBreadcrumbs(); const [query, setQuery] = useState(""); const [page, setPage] = useState(1); @@ -522,6 +525,7 @@ export function Browse({ renderAccountDetails = (connection) => connection.conne (galleryQuery.data?.apps ?? []) as AppGalleryDisplayEntry[] ).filter((entry) => { if (!memoryConnectorsEnabled && isMemoryConnectorId(appDefinitionSlug(entry))) return false; + if (!connectorAvailability.enabled(entry)) return false; const definition = getAppStoreDefinition(appDefinitionSlug(entry)); return ( chatProviderVisible(appDefinitionSlug(entry), chatConnectorsEnabled, githubEnabled) || @@ -954,6 +958,8 @@ export function Browse({ renderAccountDetails = (connection) => connection.conne onRequestRemove={target => void requestConnectionRemoval(target)} preselectedAgentId={preselectedChatAgentId} chatConnectorsEnabled={chatConnectorsEnabled} + setupEnabled={connectorAvailability.enabled(row.entry ?? getConnectableAppDefinition(row.slug)) + && row.applications.every(application => connectorAvailability.enabled(getAppDefinitionForApplicationKey(application.applicationKey)))} githubReviewBotsEnabled={githubEnabled} onConnectAggregator={connectAggregator} onManageAggregator={(app, connectionId) => setAggregatorToManage({ app, connectionId })} @@ -1054,6 +1060,7 @@ export function ConnectorCard({ onRequestRemove, preselectedAgentId, chatConnectorsEnabled, + setupEnabled = true, githubReviewBotsEnabled = false, onConnectAggregator, onManageAggregator, @@ -1076,6 +1083,7 @@ export function ConnectorCard({ onRequestRemove: (target: ConnectionRemovalTarget) => void; preselectedAgentId?: string | null; chatConnectorsEnabled: boolean; + setupEnabled?: boolean; githubReviewBotsEnabled?: boolean; onConnectAggregator?: (app: AggregatorAppCatalogEntry) => void; onManageAggregator?: (app: AggregatorAppCatalogEntry, connectionId?: string) => void; @@ -1164,7 +1172,7 @@ export function ConnectorCard({ > Connect Copilot } - + } setExpandedOverride({ open, defaultExpanded })}> @@ -1198,6 +1206,7 @@ export function ConnectorCard({ details={renderAccountDetails?.(connection)} key={connection.id} row={row} + setupEnabled={setupEnabled} connection={connection} owner={connectionOwnerProfile(connection, userProfileById)} onNavigate={onNavigate} @@ -1347,6 +1356,7 @@ function ConnectionAccountRow({ discovery, onConfigureArcade, onConfigureExecutor, + setupEnabled = true, }: { details?: ReactNode; row: ConnectorRowModel; @@ -1359,6 +1369,7 @@ function ConnectionAccountRow({ discovery?: AggregatorAppsResponse; onConfigureArcade?: () => void; onConfigureExecutor?: () => void; + setupEnabled?: boolean; }) { const state = connectionState(connection); const actionHref = accountActionHref(row, connection); @@ -1371,7 +1382,7 @@ function ConnectionAccountRow({ return ( {details}{discovery?.discovery.message || discovery?.sync.status === "error" ?

{discovery.discovery.message ?? discovery.sync.error}

: null}} openLabel={`Open ${accountName} permissions`} onOpen={() => onNavigate(`/apps/${connection.id}/permissions`)}> - {state.kind === "attention" || state.kind === "draft" ? ( + {setupEnabled && (state.kind === "attention" || state.kind === "draft") ? (