diff --git a/.github/workflows/merge-queue-checks.yml b/.github/workflows/merge-queue-checks.yml new file mode 100644 index 00000000000..1e8bfb68498 --- /dev/null +++ b/.github/workflows/merge-queue-checks.yml @@ -0,0 +1,41 @@ +name: Merge queue checks + +# The master ruleset requires `ci / verify` and `ci / e2e`. pr.yml reports them +# for a pull request, and a pull request enters the merge queue only after both +# pass there. The queue then waits for the same two names on its merge-group +# commit, which no pull_request workflow can report. These jobs report them, so +# the queue does not stall. +# +# They run no tests on purpose: a queued merge gets the protection a direct +# merge has today, and the exact-source gate still tests every master commit +# before its image is published. To test the combined tree before it lands, +# replace these jobs with real ones that keep the same names. +# +# Nothing here reads the merge-group commit: no checkout, no secrets, no token +# permissions, GitHub-hosted runners. +# +# Inert until the merge queue is enabled for master: no merge_group event fires +# before then. +on: + merge_group: + +permissions: {} + +concurrency: + group: merge-queue-checks-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + name: ci / verify + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - run: echo "ci / verify passed on every pull request in this merge group before it was queued." + + e2e: + name: ci / e2e + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - run: echo "ci / e2e passed on every pull request in this merge group before it was queued." diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 513b9bdc52a..94672521794 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -251,6 +251,16 @@ jobs: - name: Test release verify workflow wiring run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/cloud-readiness-runner-loss.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs + - name: Test merge-queue workflow wiring + # This workflow loads from master and tests the caller's checkout. A + # rerun of an older pull request has no copy of these tests. + run: | + tests=() + for test in ./scripts/merge-group-image.test.mjs ./scripts/__tests__/merge-queue-checks-workflow.test.mjs; do + if [[ -f "$test" ]]; then tests+=("$test"); fi + done + if (( ${#tests[@]} > 0 )); then node "$CI_NODE_TEST" "${tests[@]}"; fi + - name: Test standalone package build concurrency run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/DESIGN.md b/DESIGN.md index 707a877b75e..aa2eba46b29 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -149,7 +149,10 @@ Runtime discovery must not replace the turn with a generic spinner or a A queued turn shows the same `Working` header and shimmering `Thinking` activity as a running turn, starting at message submission. Keep specific server holds, -such as `Waiting for maintenance` or agent setup, visible during admission. +such as `Waiting for maintenance`, agent setup, or another open task of the same +routine, visible during admission. Admission normally starts a run within seconds, +so this startup feedback lasts at most 30 seconds: a turn still queued after that +with no named hold says `Waiting to start`, without spinner or shimmer. A run that waits for one of its agent's concurrency slots is a hold too: show `Waiting for · N slots max · P of Q in line · Configure` (the shared `AgentQueueWait` component) with no spinner or elapsed timer, link Configure to the diff --git a/doc/RELEASING.md b/doc/RELEASING.md index 16fcb0f8b46..ac2b1a02405 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -388,6 +388,22 @@ Without a matching successful merge-group run, or on any mismatch or error, the push builds from source. The PID 1 probe, exact-source gate and signing apply on both paths. +The `master` ruleset requires `ci / verify` and `ci / e2e`, which `pr.yml` +reports only for pull requests. +[`merge-queue-checks.yml`](../.github/workflows/merge-queue-checks.yml) reports +both names on each merge-group commit so the queue does not stall. It runs no +tests: a pull request enters the queue only after both checks pass on it, and +the exact-source gate still tests every `master` commit before publication. +The queue can therefore land a commit while its image is still building. The +`master` push waits up to 15 minutes for that build before it builds from +source. Publication waits for the exact-source gate in any case. + +To turn the queue on, add **Require merge queue** to the `master` ruleset with +merge method Squash, build concurrency 5, group size 1 to 5 and a 60 minute +status check timeout. After the first queued merge, confirm that the `master` +commit SHA equals the SHA of its merge-group runs; promotion depends on it. +To turn it off, remove the rule. `master` then builds from source again. + Canary dispatches first look up the full-SHA master image. Reuse requires a valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching registry digest, and signed provenance from this repository's `docker.yml` diff --git a/doc/connections/CONNECTOR-PLAYBOOK.md b/doc/connections/CONNECTOR-PLAYBOOK.md index a5df04b1c6f..26eaaadc723 100644 --- a/doc/connections/CONNECTOR-PLAYBOOK.md +++ b/doc/connections/CONNECTOR-PLAYBOOK.md @@ -181,6 +181,16 @@ refs and the same ownership checks. A personal resolver must not fall back to a company credential. Ownership mismatch is `grant_credential_invalid` and tells the owner to reconnect. +Dedicated agent connections keep credentials on their agent grant. Background +catalog and health work may use a single active grant for that connection and +company. An agent caller must use its own grant. Missing, revoked, or ambiguous +agent authorizations fail before provider requests; they never fall back to a +personal, organization, or connection-owned credential. Renaming or pausing the +connection must preserve the grant and its secret bindings. A later successful +catalog refresh clears an earlier connection-health error without new consent. +An agent caller denied for lacking its own grant must not change shared health +or the catalog, including when that agent has permission to manage the connection. + Existing personal connections with company-scoped invocation credentials require their owner to reconnect and enter a fresh key or secret URL. Reconnect creates a user-owned credential and updates the grant and declarations while preserving diff --git a/doc/connections/GENERIC-REMOTE-MCP.md b/doc/connections/GENERIC-REMOTE-MCP.md index 026eab63090..91a4ee3fd90 100644 --- a/doc/connections/GENERIC-REMOTE-MCP.md +++ b/doc/connections/GENERIC-REMOTE-MCP.md @@ -91,7 +91,7 @@ this as a setup error and keeps the connection available for retry. See The connect screen states the sign-in choice in one line, **Paperclip asks the server how to sign in.**, with a **Change** link, the same shape as the access -line below it. **Change** lists five options, each with its fields directly +line below it. **Change** lists five options (six for organization connections), each with its fields directly beneath it when selected: - **Let Paperclip check** — the default; Paperclip probes and walks you through it. @@ -101,6 +101,10 @@ beneath it when selected: Paperclip first. - **Custom headers** — for servers that name their own headers. - **No sign-in needed** — the server is open to anyone with the address. +- **Machine sign-in** — organization connections only. Paperclip signs in as a + client you registered with the provider, using a Paperclip key or a client + secret, with no person involved. It runs its own setup and test in place of + **Check link**. See [Machine sign-in](./MCP-CLIENT-CREDENTIALS.md). **Done** collapses the list back to a one-line summary of the choice. It stays open while the chosen option still needs a value. diff --git a/doc/connections/MCP-CLIENT-CREDENTIALS.md b/doc/connections/MCP-CLIENT-CREDENTIALS.md new file mode 100644 index 00000000000..8fcfada8748 --- /dev/null +++ b/doc/connections/MCP-CLIENT-CREDENTIALS.md @@ -0,0 +1,129 @@ +# Machine sign-in (MCP OAuth client credentials) + +Machine sign-in connects an **organization-owned** remote MCP server where no +person signs in: service accounts, CI-style integrations, internal automation. +Paperclip signs in as a client the provider registered for it, using the +[MCP OAuth Client Credentials extension](https://modelcontextprotocol.io/extensions/auth/oauth-client-credentials) +(`io.modelcontextprotocol/oauth-client-credentials`, revision 2026-07-28), +which builds on [RFC 6749 §4.4](https://datatracker.ietf.org/doc/html/rfc6749#section-4.4) +and [RFC 7523](https://datatracker.ietf.org/doc/html/rfc7523). + +It works the same on Paperclip Cloud and self-hosted instances. + +## Setting it up + +**Apps → Connect an app → Connect your own MCP server**, paste the address, +press **Change** beside the sign-in line and choose **Machine sign-in**. The option +appears only when the connection is shared with the organization; personal +connections sign in as the person. + +1. **Sign-in service.** Leave it blank and Paperclip asks the server where it + signs in (its protected-resource metadata, then that service's metadata). + Otherwise enter the sign-in service's address, or its token URL when the + provider publishes no metadata. +2. **How Paperclip proves who it is.** + - **Paperclip key (recommended).** Press **Create key**. Paperclip creates a + signing key, keeps the private half in the company vault, and shows the + public key as PEM and as JSON. Register it with the provider for the + Paperclip client. The public key isn't secret. + - **Client secret.** For providers that only issue secrets. The secret goes + straight into the vault and is never shown again. +3. **Client ID and scopes.** Enter the client ID the provider shows for the + Paperclip client, and any scopes its docs list (space separated). +4. **Test connection.** Paperclip asks the sign-in service for a token, then + lists the server's tools with it. Success turns the connection on and goes to + the normal review step. A failed test keeps the draft and its key, so you + can fix the provider side and test again. + +Errors say what to change: an unaccepted key or secret, refused scopes, a +provider that doesn't accept keys (choose Client secret) or secrets (choose +Paperclip key), or a sign-in service Paperclip couldn't find. + +## How tokens are requested + +- `grant_type=client_credentials`, with `scope` and the RFC 8707 `resource` + (the MCP server address) on every request. +- **Paperclip key:** `private_key_jwt`. Each request carries a fresh assertion + signed with RS256 (or ES256 when the service accepts only that): + `iss` = `sub` = client ID, `aud` = the token endpoint, `exp` 60 seconds after + `iat`, a unique `jti`, and `kid` = the key's RFC 7638 thumbprint. Per the + extension, no `client_id` or secret is sent alongside it. +- **Client secret:** `client_secret_basic`, or `client_secret_post` when that + is all the service advertises. +- Requests that carry a 2026-07-28 client capability object declare + `io.modelcontextprotocol/oauth-client-credentials` for these connections only. + +Tokens live in the vault on the connection's organization grant, like any +other OAuth token, and are reused until a minute before they expire. There is +no refresh token: Paperclip asks for a new access token. The existing grant +refresh lease and in-process single flight mean concurrent agents, across +server processes, share one token request. A `401` from the server requests a +new token once; a second `401` reports the problem without touching the +pinned settings. Agent boot and time to first token do not change: a live +token is read the same way as for browser sign-in connections. + +## The sign-in service is pinned + +The connection records the sign-in service's issuer and token endpoint at +setup. Credentials and assertions go only to that token endpoint, never to a +deployment-wide OAuth client and never to a different service the server later +advertises: + +- Test and rotation check the server's protected-resource metadata; if it now + lists other sign-in services and not the pinned one, Paperclip refuses and + asks you to set up machine sign-in again. +- A rejected token at runtime never re-runs discovery or rewrites the + connection's endpoints. + +When you enter a token URL directly there is no issuer to compare; the token +URL itself is the pin. + +## Rotation without downtime + +- **New client secret:** enter it; Paperclip tests it against the sign-in + service first and only then replaces the stored secret (a new version of the + same vault entry). A wrong secret changes nothing. +- **New key:** start a key rotation; Paperclip creates a replacement key and + shows its public key while the current key keeps working. Register the new + public key with the provider, then complete the rotation: Paperclip tests the + new key and only then switches to it and deletes the old one. + +API: `POST /api/tool-connections/{id}/machine-sign-in/rotate` with +`{ "method": "client_secret", "clientSecret": "…" }` or +`{ "method": "paperclip_key" }`, then +`POST /api/tool-connections/{id}/machine-sign-in/rotate/complete` for a key. + +## Security + +- Private keys and client secrets exist only in the company vault + (`oauth.client_private_key`, `oauth.client_private_key_next`, + `oauth.client_secret`). Config, API responses, activity and logs carry only + the public key, key ID, issuer, method and scope count. +- Company scoped. Only connection managers can set it up, and only for + organization connections; the service refuses personal connections. +- Activity: `tool_connection.machine_sign_in_created`, `…_verified`, + `…_rotation_started`, `…_rotated`, and removal as `tool_connection.archived` + with `machineSignIn: true`. Removal destroys the vault entries the connection + owned. + +## API + +| Route | Use | +| --- | --- | +| `POST /api/companies/{companyId}/tools/machine-sign-in` | Create the draft (finds and pins the sign-in service, creates the key or stores the secret). | +| `GET /api/tool-connections/{id}/machine-sign-in` | Settings and public keys. | +| `POST /api/tool-connections/{id}/machine-sign-in/test` | Test with an optional client ID, scopes or secret; turns the connection on. | +| `POST /api/tool-connections/{id}/machine-sign-in/rotate` | Rotate the secret, or stage a new key. | +| `POST /api/tool-connections/{id}/machine-sign-in/rotate/complete` | Switch to the staged key. | + +Code: `server/src/services/tool-access/oauth/client-credentials.ts` (keys and +assertions), `client-credentials-service.ts` (setup, pinning, token requests), +the machine sign-in branch in `grant-refresh.ts`, and +`ui/src/features/connections/MachineSignInSetup.tsx`. Tests: +`server/src/__tests__/tool-access-client-credentials.test.ts`. + +## Not yet covered + +- A settings-page control for rotation (rotation is API-only today). +- Enterprise single sign-on and enterprise-managed authorization (ID-JAG) are + later phases. diff --git a/doc/connections/MCP-PROGRESS.md b/doc/connections/MCP-PROGRESS.md new file mode 100644 index 00000000000..83f51b13b4f --- /dev/null +++ b/doc/connections/MCP-PROGRESS.md @@ -0,0 +1,66 @@ +# Progress for long MCP tool calls + +When an agent run calls a connected app's tool and the app reports progress, +the task chat shows it on the tool's row while the call runs: a bar that fills +when the app knows the total (a quiet sweep when it does not), the app's latest +status line, and elapsed time. The row drops it when the call finishes. + +Spec: [MCP 2026-07-28, Progress](https://modelcontextprotocol.io/specification/2026-07-28/basic/utilities/progress). + +## How it works + +1. **Request.** For a `tools/call` made by an agent run, the gateway adds a + per-call `_meta.progressToken` (`paperclip-progress-`). This applies to + remote (Streamable HTTP) and local stdio connections, under every protocol + version. Calls not made by a run (board test calls, MCP Apps views) send no + token. +2. **Response stream.** The app sends `notifications/progress` (`progress`, + optional `total`, optional `message`) on the call's own response stream: SSE + events on the POST response for HTTP, stdout lines for stdio. Under + 2026-07-28, request-scoped notifications always stay on that stream. + `readMcpHttpResponse` hands each notification to an observer before it + reaches the result, and the observer can never fail or delay the call. +3. **Reporter** (`server/src/services/tool-call-progress.ts`). Each update is + checked (finite, non-negative, increasing; `total` kept only when positive and + at least `progress`) and its message reduced to plain single-line text of at + most 200 characters. The reporter then: + - publishes a `heartbeat.run.tool_progress` live event at most every 250 ms + (the latest update wins); + - saves the latest update on the `tool_invocations.progress` column at most + every second, and never onto a finished call; + - publishes a final event with `done: true` when the call ends. + Nothing the reporter does is awaited by the call. +4. **Delivery.** The event goes to the run's company only. The live socket + treats it like transcript output: a client receives it only for runs it + asked to follow, after the same run read check. +5. **Task chat.** `ToolCallProgressProvider` (one per chat) keeps the latest + update per call. A running tool row subscribes to it and finds its call by + run and tool name. Each update carries both the gateway name and the name + native runners give the tool (`assignedToolName`: shortened, with a hash), + since native transcripts show the latter. When a running row first mounts, the chat reads + `GET /api/issues/:issueId/tool-call-progress` (again on mount and on tab + focus), so a reload mid-call shows the latest saved state. The read also + marks recently finished calls `done`; a call is never dropped just because + the read does not list it, since its first save may still be in flight. Finished rows never fetch or re-render for + progress. + +## Speed + +The token adds a few bytes to the request. Parsing happens on the stream that +was already being read. Publishing and saving are throttled, fire-and-forget, +and stop when the call ends. Agent boot and time to first token are untouched: +the code runs only while a tool call is in flight. + +## Trying it + +`packages/kv-demo-mcp-server` has a `slow_with_progress` tool +(`steps`, `stepMs`, `withTotal`). Connect the demo server, then ask an agent to +call it with, say, `{ "steps": 20, "stepMs": 1000 }`; the tool row fills over +20 seconds. `withTotal: false` shows the indeterminate state. + +## Not covered + +- Matching uses the run and the tool name. If one run calls the same tool + twice at once, both rows show the newer call's progress. +- Progress from a call that became an MCP Task (see `MCP-TASKS.md`) is not shown. +- Progress is not forwarded to the agent's own MCP client. diff --git a/doc/connections/MCP-PROTOCOL-VERSIONS.md b/doc/connections/MCP-PROTOCOL-VERSIONS.md index 806707aa70e..044a4e3b358 100644 --- a/doc/connections/MCP-PROTOCOL-VERSIONS.md +++ b/doc/connections/MCP-PROTOCOL-VERSIONS.md @@ -169,5 +169,5 @@ Tracked as follow-ups: agent to retry. - `subscriptions/listen` change notifications (cache invalidation relies on `ttlMs`). -- The Tasks extension, Skills over MCP, progress and cancellation, icons, and - enterprise-managed authorization. +- Cancellation, icons, and enterprise-managed authorization. (Progress is + supported: see [MCP-PROGRESS.md](MCP-PROGRESS.md).) diff --git a/doc/connections/README.md b/doc/connections/README.md index 31fb2959d61..d66cb81371d 100644 --- a/doc/connections/README.md +++ b/doc/connections/README.md @@ -20,9 +20,14 @@ MCP protocol versions (2026-07-28 stateless MCP and older handshake servers): MCP extensions: [MCP Apps](./MCP-APPS.md) (interactive views in the task chat), [MCP Skills](./MCP-SKILLS.md) (skills that connected apps publish, reviewed by the board and delivered to agents), [MCP Tasks](./MCP-TASKS.md) (long -provider calls that survive timeouts, restarts, and deploys), and +provider calls that survive timeouts, restarts, and deploys), [MCP elicitation](./MCP-ELICITATION.md) (an app's question to the person, -answered in the task chat). +answered in the task chat), and [Machine sign-in](./MCP-CLIENT-CREDENTIALS.md) +(OAuth client credentials for organization connections where no person signs +in). + +[Machine sign-in](./MCP-CLIENT-CREDENTIALS.md) (OAuth client credentials for +organization connections where no person signs in). Long-term memory: [Experimental memory connectors](./MEMORY.md). diff --git a/docs/api/issues.md b/docs/api/issues.md index d7425507ec1..9be228b683f 100644 --- a/docs/api/issues.md +++ b/docs/api/issues.md @@ -374,6 +374,33 @@ GET /api/attachments/{attachmentId}/content DELETE /api/attachments/{attachmentId} ``` +## Tool Call Progress + +``` +GET /api/issues/{issueId}/tool-call-progress +``` + +Returns the latest progress of the task's connected-app tool calls that are still running, for a page that loads mid-call, and the calls that finished in the last hour, marked `done`. A client should drop a call's progress only on `done`: a running call may not have saved its first update yet. Live updates arrive as `heartbeat.run.tool_progress` events on the company live-events socket. + +- Board callers only. The caller needs company access and read access to the task; otherwise the response is `404`. +- Lists unfinished calls made by a run in the last 24 hours that reported progress (at most 50), then calls finished in the last hour (at most 50). + +Each item: + +| Field | Type | Meaning | +| --- | --- | --- | +| `invocationId` | string | The tool call. | +| `runId` | string | The run that made the call. | +| `toolName` | string | The tool name the agent called. | +| `startedAt` | string (ISO) | When the call started. | +| `updatedAt` | string (ISO) | When the app last reported progress. | +| `progress` | number | Progress so far. | +| `total` | number or null | Known end value; `null` when the app does not know it. | +| `message` | string or null | The app's latest status line: plain text, at most 200 characters. | +| `done` | boolean | `true` once the call has finished. | + +See `doc/connections/MCP-PROGRESS.md`. + ## Issue Lifecycle ``` diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index eaea1067bb6..7a362b0ae71 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -691,6 +691,7 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([ "0470_heartbeat_runs_queued_created_index.sql", "0472_runner_api_capture_pending_deletes.sql", "0476_wakeup_request_run_index.sql", + "0486_heartbeat_runs_company_finished_index.sql", "0469_zippy_photon.sql", diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 0c4c0fcd9e8..7e41782d377 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -54,5 +54,5 @@ export { issueRelations } from "./schema/issue_relations.js"; export { issueReferenceMentions } from "./schema/issue_reference_mentions.js"; export * from "./schema/index.js"; -export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; +export { databaseWorkScopeKey, primeDatabaseWorkTransactionId, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; export { withDatabaseOperation } from "./response-capture.js"; diff --git a/packages/db/src/migrations/0484_certain_zemo.sql b/packages/db/src/migrations/0484_certain_zemo.sql new file mode 100644 index 00000000000..8197a5714d9 --- /dev/null +++ b/packages/db/src/migrations/0484_certain_zemo.sql @@ -0,0 +1 @@ +ALTER TABLE "tool_invocations" ADD COLUMN "progress" jsonb; \ No newline at end of file diff --git a/packages/db/src/migrations/0485_moaning_ultimatum.sql b/packages/db/src/migrations/0485_moaning_ultimatum.sql new file mode 100644 index 00000000000..abdde7a9b72 --- /dev/null +++ b/packages/db/src/migrations/0485_moaning_ultimatum.sql @@ -0,0 +1,16 @@ +CREATE TABLE "automatic_run_brakes" ( + "company_id" uuid PRIMARY KEY NOT NULL, + "state" text DEFAULT 'closed' NOT NULL, + "reason" jsonb DEFAULT '{}'::jsonb NOT NULL, + "state_changed_at" timestamp with time zone DEFAULT now() NOT NULL, + "opened_at" timestamp with time zone, + "next_probe_at" timestamp with time zone, + "probe_run_id" uuid, + "probe_successes" integer DEFAULT 0 NOT NULL, + "release_window_started_at" timestamp with time zone, + "release_window_starts" integer DEFAULT 0 NOT NULL, + "next_creation_at" timestamp with time zone, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "automatic_run_brakes" ADD CONSTRAINT "automatic_run_brakes_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/db/src/migrations/0486_heartbeat_runs_company_finished_index.sql b/packages/db/src/migrations/0486_heartbeat_runs_company_finished_index.sql new file mode 100644 index 00000000000..9d1995ff234 --- /dev/null +++ b/packages/db/src/migrations/0486_heartbeat_runs_company_finished_index.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "heartbeat_runs_company_finished_idx" ON "heartbeat_runs" USING btree ("company_id","finished_at" DESC NULLS LAST) WHERE "heartbeat_runs"."finished_at" is not null; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0481_snapshot.json b/packages/db/src/migrations/meta/0484_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0481_snapshot.json rename to packages/db/src/migrations/meta/0484_snapshot.json index c4ad44771b7..1debf9ce54b 100644 --- a/packages/db/src/migrations/meta/0481_snapshot.json +++ b/packages/db/src/migrations/meta/0484_snapshot.json @@ -1,6 +1,6 @@ { - "id": "41eacc6a-6eab-4f01-965a-62aa0cd6778f", - "prevId": "55a8c37c-453c-4be4-b657-b703932836a0", + "id": "9389124c-379a-4b64-9c20-c7b37538e7c8", + "prevId": "ceaa38c4-681b-430e-9578-0854650d85b3", "version": "7", "dialect": "postgresql", "tables": { @@ -18092,6 +18092,12 @@ "notNull": true, "default": "'company'" }, + "system_purpose": { + "name": "system_purpose", + "type": "text", + "primaryKey": false, + "notNull": false + }, "owner_user_id": { "name": "owner_user_id", "type": "text", @@ -59601,6 +59607,12 @@ "primaryKey": false, "notNull": false }, + "progress": { + "name": "progress", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -60107,6 +60119,277 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tool_mcp_elicitations": { + "name": "tool_mcp_elicitations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "invocation_id": { + "name": "invocation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "catalog_tool_name": { + "name": "catalog_tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "call_identity": { + "name": "call_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "era": { + "name": "era", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sealed_request_state": { + "name": "sealed_request_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "questions": { + "name": "questions", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "arguments_identity": { + "name": "arguments_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "claimed_until": { + "name": "claimed_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_elicitations_call_idx": { + "name": "tool_mcp_elicitations_call_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "call_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_invocation_idx": { + "name": "tool_mcp_elicitations_invocation_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invocation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_waiting_uq": { + "name": "tool_mcp_elicitations_waiting_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_elicitations_company_id_companies_id_fk": { + "name": "tool_mcp_elicitations_company_id_companies_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": { + "name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_invocations", + "columnsFrom": [ + "invocation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_mcp_gateway_tokens": { "name": "tool_mcp_gateway_tokens", "schema": "", diff --git a/packages/db/src/migrations/meta/0480_snapshot.json b/packages/db/src/migrations/meta/0485_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0480_snapshot.json rename to packages/db/src/migrations/meta/0485_snapshot.json index b4ce9431fde..fcf5987ed8c 100644 --- a/packages/db/src/migrations/meta/0480_snapshot.json +++ b/packages/db/src/migrations/meta/0485_snapshot.json @@ -1,6 +1,6 @@ { - "id": "55a8c37c-453c-4be4-b657-b703932836a0", - "prevId": "f5f852bd-0568-4b2d-bc35-0dac5d9c8c5b", + "id": "78d10bc7-b440-4c00-a02e-90111c50071d", + "prevId": "9389124c-379a-4b64-9c20-c7b37538e7c8", "version": "7", "dialect": "postgresql", "tables": { @@ -6582,6 +6582,111 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.automatic_run_brakes": { + "name": "automatic_run_brakes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'closed'" + }, + "reason": { + "name": "reason", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state_changed_at": { + "name": "state_changed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_probe_at": { + "name": "next_probe_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "probe_run_id": { + "name": "probe_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "probe_successes": { + "name": "probe_successes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "release_window_started_at": { + "name": "release_window_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_window_starts": { + "name": "release_window_starts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_creation_at": { + "name": "next_creation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automatic_run_brakes_company_id_companies_id_fk": { + "name": "automatic_run_brakes_company_id_companies_id_fk", + "tableFrom": "automatic_run_brakes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.board_api_keys": { "name": "board_api_keys", "schema": "", @@ -18092,6 +18197,12 @@ "notNull": true, "default": "'company'" }, + "system_purpose": { + "name": "system_purpose", + "type": "text", + "primaryKey": false, + "notNull": false + }, "owner_user_id": { "name": "owner_user_id", "type": "text", @@ -55989,6 +56100,18 @@ "primaryKey": false, "notNull": false }, + "credential_version": { + "name": "credential_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "credential_hash": { + "name": "credential_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_by_user_id": { "name": "created_by_user_id", "type": "text", @@ -59589,6 +59712,12 @@ "primaryKey": false, "notNull": false }, + "progress": { + "name": "progress", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -60095,6 +60224,277 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tool_mcp_elicitations": { + "name": "tool_mcp_elicitations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "invocation_id": { + "name": "invocation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "catalog_tool_name": { + "name": "catalog_tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "call_identity": { + "name": "call_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "era": { + "name": "era", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sealed_request_state": { + "name": "sealed_request_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "questions": { + "name": "questions", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "arguments_identity": { + "name": "arguments_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "claimed_until": { + "name": "claimed_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_elicitations_call_idx": { + "name": "tool_mcp_elicitations_call_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "call_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_invocation_idx": { + "name": "tool_mcp_elicitations_invocation_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invocation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_waiting_uq": { + "name": "tool_mcp_elicitations_waiting_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_elicitations_company_id_companies_id_fk": { + "name": "tool_mcp_elicitations_company_id_companies_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": { + "name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_invocations", + "columnsFrom": [ + "invocation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_mcp_gateway_tokens": { "name": "tool_mcp_gateway_tokens", "schema": "", diff --git a/packages/db/src/migrations/meta/0479_snapshot.json b/packages/db/src/migrations/meta/0486_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0479_snapshot.json rename to packages/db/src/migrations/meta/0486_snapshot.json index 36323203dc1..e0af95ce1b3 100644 --- a/packages/db/src/migrations/meta/0479_snapshot.json +++ b/packages/db/src/migrations/meta/0486_snapshot.json @@ -1,6 +1,6 @@ { - "id": "f5f852bd-0568-4b2d-bc35-0dac5d9c8c5b", - "prevId": "349cdd89-db04-4340-a5d3-3b877bf7227d", + "id": "0c3e74be-7936-4c97-9798-505daf53c16b", + "prevId": "78d10bc7-b440-4c00-a02e-90111c50071d", "version": "7", "dialect": "postgresql", "tables": { @@ -6582,6 +6582,111 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.automatic_run_brakes": { + "name": "automatic_run_brakes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'closed'" + }, + "reason": { + "name": "reason", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state_changed_at": { + "name": "state_changed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_probe_at": { + "name": "next_probe_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "probe_run_id": { + "name": "probe_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "probe_successes": { + "name": "probe_successes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "release_window_started_at": { + "name": "release_window_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_window_starts": { + "name": "release_window_starts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_creation_at": { + "name": "next_creation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automatic_run_brakes_company_id_companies_id_fk": { + "name": "automatic_run_brakes_company_id_companies_id_fk", + "tableFrom": "automatic_run_brakes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.board_api_keys": { "name": "board_api_keys", "schema": "", @@ -18092,6 +18197,12 @@ "notNull": true, "default": "'company'" }, + "system_purpose": { + "name": "system_purpose", + "type": "text", + "primaryKey": false, + "notNull": false + }, "owner_user_id": { "name": "owner_user_id", "type": "text", @@ -36023,6 +36134,28 @@ "method": "btree", "with": {} }, + "heartbeat_runs_company_finished_idx": { + "name": "heartbeat_runs_company_finished_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finished_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"finished_at\" is not null", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_company_created_at_desc_idx": { "name": "heartbeat_runs_company_created_at_desc_idx", "columns": [ @@ -55953,6 +56086,153 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tailscale_connections": { + "name": "tailscale_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "secret_id": { + "name": "secret_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "credential_version": { + "name": "credential_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "credential_hash": { + "name": "credential_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oauth_state_hash": { + "name": "oauth_state_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_expires_at": { + "name": "oauth_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tailscale_connections_company_idx": { + "name": "tailscale_connections_company_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tailscale_connections_oauth_state_idx": { + "name": "tailscale_connections_oauth_state_idx", + "columns": [ + { + "expression": "oauth_state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tailscale_connections_company_id_companies_id_fk": { + "name": "tailscale_connections_company_id_companies_id_fk", + "tableFrom": "tailscale_connections", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tailscale_connections_secret_id_company_secrets_id_fk": { + "name": "tailscale_connections_secret_id_company_secrets_id_fk", + "tableFrom": "tailscale_connections", + "tableTo": "company_secrets", + "columnsFrom": [ + "secret_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_access_audit_events": { "name": "tool_access_audit_events", "schema": "", @@ -59454,6 +59734,12 @@ "primaryKey": false, "notNull": false }, + "progress": { + "name": "progress", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -59960,6 +60246,277 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tool_mcp_elicitations": { + "name": "tool_mcp_elicitations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "invocation_id": { + "name": "invocation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "catalog_tool_name": { + "name": "catalog_tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "call_identity": { + "name": "call_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "era": { + "name": "era", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sealed_request_state": { + "name": "sealed_request_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "questions": { + "name": "questions", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "arguments_identity": { + "name": "arguments_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "claimed_until": { + "name": "claimed_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_elicitations_call_idx": { + "name": "tool_mcp_elicitations_call_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "call_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_invocation_idx": { + "name": "tool_mcp_elicitations_invocation_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invocation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_waiting_uq": { + "name": "tool_mcp_elicitations_waiting_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_elicitations_company_id_companies_id_fk": { + "name": "tool_mcp_elicitations_company_id_companies_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": { + "name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_invocations", + "columnsFrom": [ + "invocation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_mcp_gateway_tokens": { "name": "tool_mcp_gateway_tokens", "schema": "", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 54d4380b27f..f3774187bbb 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3375,6 +3375,27 @@ "when": 1794611818460, "tag": "0483_busy_tombstone", "breakpoints": true + }, + { + "idx": 484, + "version": "7", + "when": 1794621009918, + "tag": "0484_certain_zemo", + "breakpoints": true + }, + { + "idx": 485, + "version": "7", + "when": 1794623155354, + "tag": "0485_moaning_ultimatum", + "breakpoints": true + }, + { + "idx": 486, + "version": "7", + "when": 1794623177597, + "tag": "0486_heartbeat_runs_company_finished_index", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/automatic_run_brakes.ts b/packages/db/src/schema/automatic_run_brakes.ts new file mode 100644 index 00000000000..d8e7ac4d49c --- /dev/null +++ b/packages/db/src/schema/automatic_run_brakes.ts @@ -0,0 +1,17 @@ +import { integer, jsonb, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core"; +import { companies } from "./companies.js"; + +export const automaticRunBrakes = pgTable("automatic_run_brakes", { + companyId: uuid("company_id").primaryKey().references(() => companies.id, { onDelete: "cascade" }), + state: text("state").$type<"closed" | "open" | "half_open">().notNull().default("closed"), + reason: jsonb("reason").$type>().notNull().default({}), + stateChangedAt: timestamp("state_changed_at", { withTimezone: true }).notNull().defaultNow(), + openedAt: timestamp("opened_at", { withTimezone: true }), + nextProbeAt: timestamp("next_probe_at", { withTimezone: true }), + probeRunId: uuid("probe_run_id"), + probeSuccesses: integer("probe_successes").notNull().default(0), + releaseWindowStartedAt: timestamp("release_window_started_at", { withTimezone: true }), + releaseWindowStarts: integer("release_window_starts").notNull().default(0), + nextCreationAt: timestamp("next_creation_at", { withTimezone: true }), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); diff --git a/packages/db/src/schema/heartbeat_runs.ts b/packages/db/src/schema/heartbeat_runs.ts index b2979d52b30..0b424df776d 100644 --- a/packages/db/src/schema/heartbeat_runs.ts +++ b/packages/db/src/schema/heartbeat_runs.ts @@ -248,6 +248,8 @@ export const heartbeatRuns = pgTable( table.status, table.processStartedAt, ), + companyFinishedIdx: index("heartbeat_runs_company_finished_idx").on(table.companyId, table.finishedAt.desc()) + .where(sql`${table.finishedAt} is not null`).concurrently(), companyCreatedAtDescIdx: index("heartbeat_runs_company_created_at_desc_idx").on( table.companyId, table.createdAt.desc(), diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 7eef891d0d3..fd8c162f7ee 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -26,6 +26,7 @@ export { budgetResumeClaims } from "./budget_resume_claims.js"; export { agentConfigRevisions } from "./agent_config_revisions.js"; export { agentApiKeys } from "./agent_api_keys.js"; export { agentRuntimeState } from "./agent_runtime_state.js"; +export { automaticRunBrakes } from "./automatic_run_brakes.js"; export { agentTaskSessions, agentSessionGoalActions } from "./agent_task_sessions.js"; export { agentWakeupRequests } from "./agent_wakeup_requests.js"; export { projects } from "./projects.js"; diff --git a/packages/db/src/schema/tool_access.ts b/packages/db/src/schema/tool_access.ts index 621597fd6d3..2c57cba3b1e 100644 --- a/packages/db/src/schema/tool_access.ts +++ b/packages/db/src/schema/tool_access.ts @@ -16,6 +16,7 @@ import { uuid, } from "drizzle-orm/pg-core"; import type { + ToolCallProgress, ConnectionAgentInstructions, ConnectionTokenIssuanceOutcome, ConnectionTokenIssuancePath, @@ -804,6 +805,8 @@ export const toolInvocations = pgTable( errorMessage: text("error_message"), startedAt: timestamp("started_at", { withTimezone: true }), completedAt: timestamp("completed_at", { withTimezone: true }), + /** The provider's latest progress while the call runs (MCP `notifications/progress`), bounded. */ + progress: jsonb("progress").$type(), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }, diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts index b98a868af96..704afac48a7 100644 --- a/packages/db/src/work-signals.test.ts +++ b/packages/db/src/work-signals.test.ts @@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/postgres-js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createDb, withDedicatedDbConnection } from "./client.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; -import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; +import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; const support = await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("database work signals", () => { @@ -145,6 +145,22 @@ const support = await getEmbeddedPostgresTestSupport(); } finally { release(); await serverDone?.catch(() => {}); unsubscribe(); } }); + it("resolves an uncertain transaction with a transaction id that a statement returned", async () => { + await expect(db.transaction(async tx => { + const [{ xid }] = await tx.execute(sql`select pg_current_xact_id()::text as xid`); + primeDatabaseWorkTransactionId(tx, xid as string); + await signalDatabaseWork(tx, "primed-disconnect"); + await tx.execute(sql`insert into work_signal_test values ('primed-disconnect')`); + const [{ pid }] = await tx.execute(sql`select pg_backend_pid() as pid`); + await db.execute(sql`select pg_terminate_backend(${pid as number})`); + await tx.execute(sql`select 1`); + })).rejects.toThrow(); + expect(databaseWorkPending(db, "primed-disconnect")).toBe(true); + await reconcileDatabaseWork(db, "primed-disconnect"); + expect(databaseWorkPending(db, "primed-disconnect")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'primed-disconnect'`)).toHaveLength(0); + }); + it("recovers after a real backend disconnect without replaying the queue write", async () => { await expect(db.transaction(async tx => { await signalDatabaseWork(tx, "disconnect"); @@ -196,6 +212,17 @@ describe("work signal query boundaries", () => { expect(write).not.toHaveBeenCalled(); expect(databaseWorkPending(connection, "failed")).toBe(false); }); + it("reads no transaction id when a statement of the transaction already returned it", async () => { + const execute = vi.fn(); + const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) }); + primeDatabaseWorkTransactionId(connection, "outside"); + await connection.transaction(async (tx: any) => { + primeDatabaseWorkTransactionId(tx, "42"); + await signalDatabaseWork(tx, "primed"); + await signalDatabaseWork(tx, "primed-again"); + }); + expect(execute).not.toHaveBeenCalled(); + }); it("rejects registration on an autocommit root", async () => { const connection = installDatabaseWorkSignals({ execute: vi.fn(), transaction: vi.fn() }); await expect(signalDatabaseWork(connection, "bad")).rejects.toThrow("inside a transaction"); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts index 05b83a417cf..aa312072cd7 100644 --- a/packages/db/src/work-signals.ts +++ b/packages/db/src/work-signals.ts @@ -88,6 +88,11 @@ export async function signalDatabaseWork(connection: object, topic: string, hint if (context.scope.notify) notifyAtCommit(context, notification(context.scope, topic)); } +export function primeDatabaseWorkTransactionId(connection: object, xid: string): void { + const state = contexts.get(connection)?.transaction; + if (state && !state.xid) state.xid = Promise.resolve(xid); +} + function notifyAtCommit(context: Context, payload: string): void { if (context.ended) throw new Error("Database work was signalled after its transaction or savepoint ended"); const key = `${DATABASE_WORK_CHANNEL}\u0000${payload}`; diff --git a/packages/kv-demo-mcp-server/README.md b/packages/kv-demo-mcp-server/README.md index ea1daaa90e1..88c18d64e11 100644 --- a/packages/kv-demo-mcp-server/README.md +++ b/packages/kv-demo-mcp-server/README.md @@ -1,7 +1,7 @@ # KV Demo MCP Server A standalone, self-contained MCP server for demos. One process exposes four -key/value MCP tools **and** a tiny web UI that renders the values those tools +key/value MCP tools (plus a progress demo tool) **and** a tiny web UI that renders the values those tools mutate — so you can call a tool from a Paperclip agent and watch the value appear in a browser tab. @@ -24,6 +24,10 @@ recorded walkthrough in [doc/MCP-DEMO-SCRIPT.md](../../doc/MCP-DEMO-SCRIPT.md). - `kv_set` (write) — set one key to a string value. - `kv_delete` (destructive) — delete a key. Carries `destructiveHint: true` so Paperclip's catalog quarantines it on first sight. +- `slow_with_progress` (read) — waits `steps` × `stepMs` and sends MCP + `notifications/progress` after each step (`withTotal: false` leaves the total + out). For the task chat's tool progress; see + [doc/connections/MCP-PROGRESS.md](../../doc/connections/MCP-PROGRESS.md). - A Values UI at `/` — an auto-refreshing HTML table over the same store. - A JSON state route at `/api/state` — what the UI polls. diff --git a/packages/kv-demo-mcp-server/src/http.test.ts b/packages/kv-demo-mcp-server/src/http.test.ts index 1134ee6acae..1fcb68bf5a3 100644 --- a/packages/kv-demo-mcp-server/src/http.test.ts +++ b/packages/kv-demo-mcp-server/src/http.test.ts @@ -83,4 +83,26 @@ describe("kv demo HTTP server", () => { expect(state.count).toBe(1); await client.close(); }); + + it("streams progress for slow_with_progress on the call's response, before the result", async () => { + const { base } = await startServer(); + const client = await mcpClient(base); + const updates: Array<{ progress: number; total?: number; message?: string }> = []; + const result = await client.callTool( + { name: "slow_with_progress", arguments: { steps: 3, stepMs: 5 } }, + undefined, + { onprogress: (update) => { updates.push(update); } }, + ); + expect(updates).toEqual([ + { progress: 1, total: 3, message: "Finished step 1 of 3" }, + { progress: 2, total: 3, message: "Finished step 2 of 3" }, + { progress: 3, total: 3, message: "Finished step 3 of 3" }, + ]); + expect(JSON.stringify(result.content)).toContain("completed"); + const indeterminate: Array<{ total?: number }> = []; + await client.callTool({ name: "slow_with_progress", arguments: { steps: 1, stepMs: 0, withTotal: false } }, undefined, + { onprogress: (update) => { indeterminate.push(update); } }); + expect(indeterminate[0]?.total).toBeUndefined(); + await client.close(); + }); }); diff --git a/packages/kv-demo-mcp-server/src/index.ts b/packages/kv-demo-mcp-server/src/index.ts index a67a5fb8ed9..51f451ff0c5 100644 --- a/packages/kv-demo-mcp-server/src/index.ts +++ b/packages/kv-demo-mcp-server/src/index.ts @@ -1,6 +1,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { KvStore } from "./store.js"; import { createToolDefinitions } from "./tools.js"; +import { registerSlowWithProgressTool } from "./slow-progress.js"; export interface CreateKvDemoMcpServerResult { server: McpServer; @@ -29,6 +30,7 @@ export function createKvDemoMcpServer(store: KvStore = new KvStore()): CreateKvD tool.execute, ); } + registerSlowWithProgressTool(server); return { server, store }; } @@ -36,6 +38,7 @@ export function createKvDemoMcpServer(store: KvStore = new KvStore()): CreateKvD export { KvStore } from "./store.js"; export type { KvEntry, KvStateSnapshot } from "./store.js"; export { createToolDefinitions } from "./tools.js"; +export { SLOW_WITH_PROGRESS_TOOL, registerSlowWithProgressTool } from "./slow-progress.js"; export { createKvDemoHttpServer } from "./http.js"; export type { KvDemoHttpServer, KvDemoHttpOptions } from "./http.js"; export { readConfigFromEnv } from "./config.js"; diff --git a/packages/kv-demo-mcp-server/src/slow-progress.ts b/packages/kv-demo-mcp-server/src/slow-progress.ts new file mode 100644 index 00000000000..aaf08c7c4a7 --- /dev/null +++ b/packages/kv-demo-mcp-server/src/slow-progress.ts @@ -0,0 +1,47 @@ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; + +/** + * A demo tool that takes a while and reports MCP progress as it goes, so a + * client can show it. Each step waits `stepMs`, then sends + * `notifications/progress` for the caller's `progressToken` (when it sent + * one). `withTotal: false` leaves `total` out, for an indeterminate display. + */ +export const SLOW_WITH_PROGRESS_TOOL = "slow_with_progress"; + +const schema = z.object({ + steps: z.number().int().min(1).max(100).default(5), + stepMs: z.number().int().min(0).max(10_000).default(1_000), + withTotal: z.boolean().default(true), +}); + +export function registerSlowWithProgressTool(server: McpServer): void { + server.registerTool( + SLOW_WITH_PROGRESS_TOOL, + { + description: "Wait for a few steps, reporting progress after each one. For progress demos.", + inputSchema: schema.shape, + annotations: { title: "Slow task with progress", readOnlyHint: true, openWorldHint: false }, + }, + async (input, extra) => { + const { steps, stepMs, withTotal } = schema.parse(input); + const progressToken = extra._meta?.progressToken; + for (let step = 1; step <= steps; step++) { + await new Promise((resolve) => setTimeout(resolve, stepMs)); + if (extra.signal.aborted) break; + if (progressToken !== undefined) { + await extra.sendNotification({ + method: "notifications/progress", + params: { + progressToken, + progress: step, + ...(withTotal ? { total: steps } : {}), + message: `Finished step ${step} of ${steps}`, + }, + }); + } + } + return { content: [{ type: "text", text: JSON.stringify({ steps, completed: true }) }] }; + }, + ); +} diff --git a/packages/kv-demo-mcp-server/src/tools.test.ts b/packages/kv-demo-mcp-server/src/tools.test.ts index df62a84e39f..8bca3567f1c 100644 --- a/packages/kv-demo-mcp-server/src/tools.test.ts +++ b/packages/kv-demo-mcp-server/src/tools.test.ts @@ -19,10 +19,10 @@ async function connectedClient() { } describe("kv demo MCP tools", () => { - it("exposes exactly kv_set, kv_get, kv_list, kv_delete", async () => { + it("exposes exactly kv_set, kv_get, kv_list, kv_delete and the progress demo tool", async () => { const { client } = await connectedClient(); const { tools } = await client.listTools(); - expect(tools.map((t) => t.name).sort()).toEqual(["kv_delete", "kv_get", "kv_list", "kv_set"]); + expect(tools.map((t) => t.name).sort()).toEqual(["kv_delete", "kv_get", "kv_list", "kv_set", "slow_with_progress"]); }); it("reflects kv_set writes in the shared store", async () => { diff --git a/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md b/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md index 1c1242934d0..3b61ad38db9 100644 --- a/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md +++ b/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md @@ -216,6 +216,55 @@ identities. The report's shape is `$defs.backgroundTasks` in generated from it and tested against `protocol/fixtures/provider-boundary/acpx-background-tasks.json`. +The Claude sidecar reports `{"live": N}` with N above zero only while N shell +tasks are running and nothing has made the count unknown; finished tasks do +not count. It reports `{"live": 0}` only while its provider process has never +had a background task and Claude has not acted between prompts. Every other +case is unknown and the field is omitted: once a background task has finished +or a provider-started cycle has run and no shell is running, and permanently +after a dropped, declined, failed or cancelled cycle, a task that is not a +shell, a malformed task update, a scheduling tool, a full ledger, or idle +activity that was not adopted. The ACP stream does not say which task a Claude +cycle reports on, so the sidecar does not guess: after a provider-started turn, +Runner keeps waiting only while a background command is still running, and a +command that finishes while Claude is already reporting on an earlier one gets +no turn of its own. An origin-tagged usage update that directly follows the +previous one with the same context size and cost and zero usage is an empty +result, not a cycle, and is ignored. + +A controller opts in per turn with `providerStartedTurns: true` on +`turn.start`. Only then, when that completed Claude turn reported no accepted +`paperclip_finish` or `paperclip_block` result for the run, its terminal reports +`backgroundTasks: {"live": N}` with N above zero, and the same restart +conditions hold, Runner sends the sidecar `session.provider_turn` with +`armed: true` for that terminal. Only if the sidecar replies `armed: true` does +Runner keep it and await a provider-started turn: the PRP `turn.completed` +carries `awaitingProviderTurn: true` and no `run.terminal` follows. Without the +opt-in, with a result, or when arming is refused, every event and the turn-end +restart are exactly as before. + +The sidecar adopts Claude's own cycle only while armed. A cycle that starts +after the terminal and before the arming command is held, with nothing lost, +for at most 5 s; arming adopts it and anything else drops it. When armed, the +sidecar emits `runtime.turn_started` with `{"origin": "provider"}` and a +sidecar-minted turn id. Runner admits it only while awaiting and only for +Claude, with the same settled-identity and receipt rules as a controller turn, +and publishes PRP `turn.started` with `origin: "provider"`. Any other +provider-started turn fails the session closed. A controller `turn.start` while +awaiting disarms and replaces the sidecar first. If the provider exits while armed, the +sidecar reports `runtime.diagnostic` with code `ACPX_PROVIDER_PROCESS_LOST`; that, +or a failed sidecar transport, ends the wait and Runner publishes the +`run.terminal` it held back for the completed turn. Park is refused while awaiting. +Runner waits at most 5 s for the sidecar to answer an arm or a disarm. No answer +is handled as a lost sidecar, and Runner still publishes the held `run.terminal`. +When a `run.attach` from a different run succeeds, the wait ends and Runner +discards the held `run.terminal`. If that attach fails, the wait and its held +`run.terminal` stay with the old run. A `run.attach` from the same run publishes +the held `run.terminal`. +Runner saves the wait with its held `run.terminal` in the provider state; a +Runner restart during the wait publishes that `run.terminal` once and resumes +in a fresh process. + The check also inspects the live provider after that drain, so a late active-goal notification cannot be hidden by the older executor journal. This hook does not detach a controller, stop a process, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs index c32d37d89de..d56ade1a9b5 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs @@ -26,10 +26,11 @@ impl AcpxCommandExecutor { pub(super) fn restart_session_after_turn(&mut self) -> Result<(), DurableRunnerError> { self.ensure_persistence_healthy()?; - if self - .session - .as_ref() - .is_some_and(AcpxProviderSession::continues_in_process) + if self.awaiting_provider_turn() + || self + .session + .as_ref() + .is_some_and(AcpxProviderSession::continues_in_process) || !self.generation_restartable(AcpxProviderSession::can_restart_after_turn) { return Ok(()); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index d29cd3ed265..43d40c98580 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -43,6 +43,8 @@ use crate::qualified_launch::{verify_executable_launch_artifact, verify_launch_a #[path = "acpx_park_preparation.rs"] mod park_preparation; +#[path = "acpx_provider_turn_wait.rs"] +mod provider_turn_wait; fn bounded_turn_timing(response: &Value, elapsed: Duration) -> Value { let mut receipt = serde_json::Map::new(); @@ -639,6 +641,8 @@ struct AcpxDurableState { /// clears it and resumes the saved session. Written only while set. #[serde(default, skip_serializing_if = "Option::is_none")] turn_end_restart_failed_run: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + provider_turn_wait: Option, } /// The saved launch names the same executable and sidecar files as this @@ -719,6 +723,7 @@ impl AcpxDurableState { open_semantic_calls: BTreeSet::new(), retired_semantic_calls: BTreeSet::new(), turn_end_restart_failed_run: None, + provider_turn_wait: None, } } @@ -806,6 +811,10 @@ impl AcpxDurableState { .turn_end_restart_failed_run .as_ref() .is_some_and(|run_id| *run_id != self.descriptor.run_id) + || self + .provider_turn_wait + .as_ref() + .is_some_and(|wait| !wait.is_consistent(self)) { return Err(DurableRunnerError::invalid( "ACPX durable provider state is malformed or inconsistent", @@ -1030,6 +1039,7 @@ pub struct AcpxCommandExecutor { failed_turn_end_restart: Option, /// The failed turn-end restart's cause, until the Runner records it. unreported_turn_end_restart_failure: Option, + turn_accepts_provider_turns: bool, } // The durable command's turn and the live provider callback have separate IDs. @@ -1071,6 +1081,7 @@ impl AcpxCommandExecutor { recovery_pending_effects: None, failed_turn_end_restart: None, unreported_turn_end_restart_failure: None, + turn_accepts_provider_turns: false, restore_error: None, persistence_error: None, #[cfg(test)] @@ -1284,6 +1295,14 @@ impl AcpxCommandExecutor { if self.session.is_some() { return Ok(()); } + if self.awaiting_provider_turn() { + let same_run = self + .state + .as_ref() + .is_some_and(|state| state.descriptor.run_id == self.context.run_id); + self.end_provider_turn_wait(same_run)?; + self.save_state()?; + } let Some(state) = self.state.as_ref() else { return Ok(()); }; @@ -1594,6 +1613,9 @@ impl AcpxCommandExecutor { fn save_state(&mut self) -> Result<(), DurableRunnerError> { self.ensure_persistence_healthy()?; + if let Some(state) = self.state.as_mut() { + state.leave_finished_provider_turn_wait()?; + } let result = self .state .as_ref() @@ -1865,6 +1887,7 @@ impl AcpxCommandExecutor { "run.attach requires the same settled ACPX provider profile and session", )); } + let different_run = state.descriptor.run_id != attachment_context.run_id; if !self.rebind_prepared_session(&descriptor, &tool_set, next_candidate.is_some()) { if let Some(session) = self.session.as_mut() { session @@ -1895,6 +1918,11 @@ impl AcpxCommandExecutor { state.semantic_result = None; state.pending_events.clear(); state.lifecycle = "suspended".to_owned(); + let ended_wait = if different_run { + state.provider_turn_wait.take() + } else { + None + }; // A new run starts a fresh provider generation; the previous run's // failed turn-end restart no longer applies to it. A same-run attach // keeps the restriction. @@ -1914,6 +1942,11 @@ impl AcpxCommandExecutor { if saved.is_ok() && clears_turn_end_restart_failure { self.failed_turn_end_restart = None; } + if let (true, Some(wait), Some(session)) = + (saved.is_ok(), ended_wait.as_ref(), self.session.as_mut()) + { + session.disarm_provider_turn(&wait.turn_id); + } saved } @@ -2239,6 +2272,8 @@ impl AcpxCommandExecutor { .map(|state| PathBuf::from(&state.descriptor.cwd)) .expect("ACPX state exists before turn start"); let measure = payload.get("performance").and_then(Value::as_bool) == Some(true); + self.turn_accepts_provider_turns = + payload.get("providerStartedTurns").and_then(Value::as_bool) == Some(true); let timing_started = std::time::Instant::now(); let start_result = self .session @@ -2475,10 +2510,7 @@ impl AcpxCommandExecutor { .as_ref() .and_then(|state| state.active_turn_id.clone()); let Some(turn_id) = turn_id else { - return Ok(CommandExecution::result(json!({ - "status": "already_settled", - "reason": reason, - }))); + return self.settled_without_turn(reason); }; self.session .as_mut() @@ -2509,10 +2541,7 @@ impl AcpxCommandExecutor { .as_ref() .is_some_and(|state| state.descriptor.agent == "pi"); if turn_id.is_none() && !stop_idle_pi { - return Ok(CommandExecution::result(json!({ - "status": "already_settled", - "reason": reason, - }))); + return self.settled_without_turn(reason); } let provider_lifetime_fence_candidates = { let session = self @@ -2879,6 +2908,9 @@ impl AcpxCommandExecutor { self.state .as_ref() .and_then(|state| state.goal_source_revision), + self.state + .as_ref() + .is_some_and(|state| state.provider_turn_wait.is_some()), ); let events = match events { Ok(events) => events, @@ -2913,6 +2945,25 @@ impl AcpxCommandExecutor { }; let mut provider_turn_settled = false; for event in events { + if let crate::acpx_provider_state::AcpxProviderStateEvent::ProviderTurnStarted { + turn_id, + } = &event + { + let state = self + .state + .as_mut() + .expect("ACPX state remains available while polling"); + if state.lifecycle != "session_open" || state.active_turn_id.is_some() { + return Err(DurableRunnerError::invalid( + "ACPX provider turn started outside an open idle session", + )); + } + state.active_turn_id = Some(turn_id.clone()); + state.attempted_turn_controls.clear(); + state.semantic_result = None; + state.lifecycle = "turn_active".to_owned(); + self.context.provider_turn_id = Some(turn_id.clone()); + } let provider_process_lost = matches!( &event, crate::acpx_provider_state::AcpxProviderStateEvent::TurnTerminal { @@ -2947,10 +2998,16 @@ impl AcpxCommandExecutor { })?; } } + let provider_lost = matches!( + &event, + crate::acpx_provider_state::AcpxProviderStateEvent::Diagnostic { code, .. } + if code == crate::generated_acpx_sidecar_contract::GENERATED_ACPX_PROVIDER_TURN_LOST_CODE + ); let state = self .state .as_mut() .expect("ACPX state remains available while polling"); + let mut accepted = Vec::with_capacity(normalized.len()); for mut event in normalized { if matches!( event.event_type.as_str(), @@ -2973,8 +3030,10 @@ impl AcpxCommandExecutor { if event.event_type == "run.result.proposed" { state.semantic_result = Some(event.payload.clone()); } - state.push(event)?; + accepted.push(event); } + let turn_ended = terminal.is_some(); + let mut run_terminal = None; if let Some(event_type) = terminal { state.active_turn_id = None; state.provider_exit_unconfirmed |= provider_process_lost; @@ -2992,15 +3051,12 @@ impl AcpxCommandExecutor { provider_turn_settled = true; // An ACP goal has session lifetime, not prompt lifetime. // Out-of-prompt goal updates remain observable after quiescence. - if !provider_process_lost + let goal_continues = !provider_process_lost && state .goal_projection .pointer("/goal/status") .and_then(Value::as_str) - == Some("active") - { - continue; - } + == Some("active"); let (turn_terminal_state, status) = match event_type.as_str() { "turn.completed" => ("completed", "succeeded"), "turn.cancelled" => ("cancelled", "cancelled"), @@ -3019,18 +3075,20 @@ impl AcpxCommandExecutor { .and_then(Value::as_str), status == "succeeded", ); - state.push(NormalizedProviderEvent { - event_type: "run.terminal".to_owned(), - priority: EventPriority::P0, - payload: json!({ + run_terminal = (!goal_continues).then(|| { + json!({ "schema": "paperclip.prp.terminal.v1", "status": status, "turnTerminalState": turn_terminal_state, "runTerminalState": status, "reportedWorkDisposition": disposition, "provider": "acpx", - }), - })?; + }) + }); + } + self.settle_turn_events(accepted, turn_ended, run_terminal)?; + if provider_lost { + self.end_provider_turn_wait(true)?; } } if provider_turn_settled { @@ -3088,6 +3146,7 @@ impl CommandExecutor for AcpxCommandExecutor { || !state.pending_runtime_requests.is_empty() || (!state.goal_projection.is_null() && !goal_is_dormant(&state.goal_projection)) || !state.descriptor.names_qualified_release() + || state.provider_turn_wait.is_some() || self.candidate_admission().is_some() || !acpx_release_profile(&state.descriptor.agent) .is_some_and(|profile| !profile.qualification_pending) @@ -3529,7 +3588,7 @@ mod tests { } } - fn descriptor(agent: &str) -> Value { + pub(super) fn descriptor(agent: &str) -> Value { let profile = acpx_release_profile(agent).unwrap(); let mut value = json!({ "kind": "acpx", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs index 3c79aaa114e..6da5c8d45f3 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs @@ -24,6 +24,8 @@ use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS, SHORT_STABLE #[path = "acpx_provider_generation.rs"] mod generation; +#[path = "acpx_provider_turn.rs"] +mod provider_turn; const MAX_ID_CHARS: usize = 240; const MAX_MODEL_CHARS: usize = 240; @@ -364,6 +366,8 @@ pub struct AcpxProviderSession { quiet_turn_observed: bool, continues_in_process: bool, turn_start_replaced_provider: bool, + provider_turn_offer: Option, + run_result_reported: Option, generation_has_run_turn: bool, restarted_after_turn_from: Option, prepared_generation: Option, @@ -429,6 +433,8 @@ impl AcpxProviderSession { quiet_turn_observed: false, continues_in_process: false, turn_start_replaced_provider: false, + provider_turn_offer: None, + run_result_reported: None, generation_has_run_turn: false, restarted_after_turn_from: None, prepared_generation: None, @@ -684,6 +690,7 @@ impl AcpxProviderSession { && self.background_work_is_quiet(); self.continues_in_process = false; self.quiet_turn_observed = false; + self.provider_turn_offer = None; let restart_now = self.generation_has_run_turn && !continue_in_process; if restart_now { if let Err(error) = self.restart_idle_provider(None) { @@ -857,7 +864,16 @@ impl AcpxProviderSession { timeout: Duration, ) -> Result>, LocalRunnerError> { Ok(self - .poll_output(timeout, false, None)? + .poll_output(timeout, false, None, false)? + .map(|(events, _)| events)) + } + + pub fn poll_awaited_event( + &mut self, + timeout: Duration, + ) -> Result>, LocalRunnerError> { + Ok(self + .poll_output(timeout, false, None, true)? .map(|(events, _)| events)) } @@ -866,6 +882,7 @@ impl AcpxProviderSession { timeout: Duration, disconnected: bool, last_goal_revision: Option, + awaited: bool, ) -> Result, Option)>, LocalRunnerError> { self.ensure_open()?; let event = match self.transport.poll_event(timeout) { @@ -873,6 +890,11 @@ impl AcpxProviderSession { Err(error) => return Err(self.fail_closed(error)), }; let Some(mut event) = event else { + if awaited { + if let Some(error) = self.transport.exited_error("provider turn wait") { + return Err(self.fail_closed(error)); + } + } return Ok(None); }; let mut terminal_goal = None; @@ -921,8 +943,18 @@ impl AcpxProviderSession { payload.insert("origin".to_owned(), origin); } } + if event.event_type == GeneratedAcpxSidecarEventType::RuntimeTurnStarted { + return match self.admit_provider_turn(&event, awaited) { + Ok(started) => Ok(Some((vec![started], terminal_goal))), + Err(error) => Err(self.fail_closed(error)), + }; + } let terminal_quiet = event.event_type == GeneratedAcpxSidecarEventType::RuntimeTurnTerminal && generation::reports_no_live_background_tasks(&event.payload); + let terminal_live = (event.event_type + == GeneratedAcpxSidecarEventType::RuntimeTurnTerminal) + .then(|| provider_turn::reported_live_background_tasks(&event.payload)) + .flatten(); let mut next_state = self.state.clone(); let events = match next_state.accept_event(&event) { Ok(events) => events, @@ -976,6 +1008,9 @@ impl AcpxProviderSession { if let Err(error) = validate_reserved_terminal_result(&next_state, result) { return Err(self.fail_closed(error)); } + if result.ok { + self.record_run_result(); + } if let Err(error) = next_reserved_bridge.apply_result(crate::provider_bridge::ToolResult { call_id: result.call_id.clone(), @@ -1082,9 +1117,17 @@ impl AcpxProviderSession { && self.cancelled_turn_id.as_deref() != Some(turn_id.as_str()); } } + let terminal_completed = self.completed_turn_observed; + let terminal_turn = reconciled_events.iter().find_map(|event| match event { + AcpxProviderStateEvent::TurnTerminal { turn_id, .. } => Some(turn_id.clone()), + _ => None, + }); self.state = next_state; self.tool_bridge = next_bridge; self.reserved_tool_bridge = next_reserved_bridge; + if let Some(turn_id) = terminal_turn { + self.offer_provider_turn(&turn_id, terminal_completed, terminal_live); + } Ok(Some((reconciled_events, terminal_goal))) } @@ -1103,12 +1146,17 @@ impl AcpxProviderSession { }; let duplicate = bridge.has_completed_call(&result.call_id); let detached = bridge.turn_closed(); + let reported_run_result = + is_reserved_terminal_operation(&result.operation_id) && !result.is_error; bridge.apply_result(result.clone()).map_err(|error| { LocalRunnerError::invalid(format!( "ACPX tool result for call {} operation {} is invalid: {error}", result.call_id, result.operation_id )) })?; + if reported_run_result { + self.record_run_result(); + } if duplicate { return Ok(()); } @@ -1406,6 +1454,7 @@ impl AcpxProviderSession { self.generation_has_run_turn = false; self.quiet_turn_observed = false; self.continues_in_process = false; + self.provider_turn_offer = None; Ok(()) } @@ -2314,6 +2363,8 @@ mod mode_tests { quiet_turn_observed: false, continues_in_process: false, turn_start_replaced_provider: false, + provider_turn_offer: None, + run_result_reported: None, generation_has_run_turn: false, restarted_after_turn_from: None, prepared_generation: None, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs index 58f537c07a8..91608c307d6 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs @@ -80,6 +80,9 @@ pub enum AcpxProviderStateEvent { status: AcpxTurnStatus, error: Option, }, + ProviderTurnStarted { + turn_id: String, + }, Process(Value), Goal(Value), Diagnostic { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn.rs new file mode 100644 index 00000000000..a1f67c12004 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn.rs @@ -0,0 +1,131 @@ +use std::time::Instant; + +use serde_json::{json, Value}; + +use super::AcpxProviderSession; +use crate::acpx_provider_state::AcpxProviderStateEvent; +use crate::acpx_sidecar_transport::AcpxSidecarEvent; +use crate::generated_acpx_sidecar_contract::{ + read_generated_acpx_background_tasks, GeneratedAcpxSidecarCommand, + GENERATED_ACPX_PROVIDER_TURN_ORIGIN, +}; +use crate::local_runner::LocalRunnerError; + +pub(super) fn reported_live_background_tasks(payload: &Value) -> Option { + payload + .get("backgroundTasks") + .and_then(read_generated_acpx_background_tasks) +} + +impl AcpxProviderSession { + pub fn run_result_reported(&self) -> bool { + self.run_result_reported.as_deref() == Some(self.state.run_id()) + } + + pub(super) fn record_run_result(&mut self) { + self.run_result_reported = Some(self.state.run_id().to_owned()); + } + + pub fn take_provider_turn_offer(&mut self) -> Option { + self.provider_turn_offer.take() + } + + pub fn arm_provider_turn(&mut self, turn_id: &str) -> bool { + self.request_provider_turn(turn_id, true) + .is_some_and(|response| response.get("armed").and_then(Value::as_bool) == Some(true)) + } + + pub fn disarm_provider_turn(&mut self, turn_id: &str) { + if !self.transport_terminated { + self.request_provider_turn(turn_id, false); + } + } + + fn request_provider_turn(&mut self, turn_id: &str, armed: bool) -> Option { + let deadline = Instant::now() + + super::ACPX_TURN_CANCEL_ACK_TIMEOUT.min(self.config.transport.request_timeout); + match self.transport.request_by( + GeneratedAcpxSidecarCommand::SessionProviderTurn, + json!({"turnId": turn_id, "armed": armed}), + deadline, + ) { + Ok(Ok(response)) => Some(response), + Ok(Err(_)) => None, + Err(error) => { + self.fail_closed(error); + None + } + } + } + + pub(super) fn offer_provider_turn( + &mut self, + turn_id: &str, + completed: bool, + live: Option, + ) { + let eligible = completed + && live.is_some_and(|live| live > 0) + && !self.run_result_reported() + && self.config.agent == "claude" + && self.config.candidate_admission.is_none() + && !self.runtime_retired + && self.failure.is_none() + && !self.state.has_pending_requests() + && !self.state.has_pending_tools() + && !self.state.settled_turn_identity_capacity_reached() + && !self.tool_bridge.replay_history_blocks_admission() + && !self.reserved_tool_bridge.replay_history_blocks_admission(); + self.provider_turn_offer = eligible.then(|| turn_id.to_owned()); + } + + pub(super) fn admit_provider_turn( + &mut self, + event: &AcpxSidecarEvent, + awaited: bool, + ) -> Result { + if !awaited + || self.config.agent != "claude" + || self.state.active_turn_id().is_some() + || self.state.has_pending_tools() + { + return Err(LocalRunnerError::invalid( + "ACPX sidecar started a provider turn outside an awaited continuation", + )); + } + if event.payload != json!({"origin": GENERATED_ACPX_PROVIDER_TURN_ORIGIN}) + || event.run_id.as_deref() != Some(self.state.run_id()) + { + return Err(LocalRunnerError::invalid( + "ACPX provider turn start violated its bounded contract", + )); + } + let turn_id = event.turn_id.clone().ok_or_else(|| { + LocalRunnerError::invalid("ACPX provider turn start omitted its turn binding") + })?; + self.state.validate_new_turn_identity(&turn_id)?; + let mut next_tool_bridge = self.tool_bridge.clone(); + next_tool_bridge.prepare_turn().map_err(|error| { + LocalRunnerError::invalid(format!( + "ACPX dynamic tool receipt rotation failed: {error}" + )) + })?; + let mut next_reserved_tool_bridge = self.reserved_tool_bridge.clone(); + next_reserved_tool_bridge.prepare_turn().map_err(|error| { + LocalRunnerError::invalid(format!( + "ACPX reserved tool receipt rotation failed: {error}" + )) + })?; + let mut next_state = self.state.clone(); + next_state.begin_turn(turn_id.clone())?; + self.state = next_state; + self.tool_bridge = next_tool_bridge; + self.reserved_tool_bridge = next_reserved_tool_bridge; + self.provider_turn_offer = None; + self.completed_turn_observed = false; + self.quiet_turn_observed = false; + self.continues_in_process = false; + self.cancelled_turn_id = None; + Ok(AcpxProviderStateEvent::ProviderTurnStarted { turn_id }) + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn_wait.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn_wait.rs new file mode 100644 index 00000000000..e9d35b1ebd4 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_turn_wait.rs @@ -0,0 +1,224 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use super::{AcpxCommandExecutor, AcpxDurableState}; +use crate::durable::{CommandExecution, DurableRunnerError, EventPriority}; +use crate::provider_events::NormalizedProviderEvent; +use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS}; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct AcpxProviderTurnWait { + pub(super) turn_id: String, + pub(super) held_run_terminal: Value, +} + +impl AcpxProviderTurnWait { + pub(super) fn is_consistent(&self, state: &AcpxDurableState) -> bool { + is_stable_id(&self.turn_id, DURABLE_STABLE_ID_CHARS) + && self.held_run_terminal.is_object() + && state.lifecycle == "session_open" + && state.active_turn_id.is_none() + && state.identity.is_some() + } + + pub(super) fn into_run_terminal(self) -> NormalizedProviderEvent { + run_terminal_event(self.held_run_terminal) + } +} + +fn run_terminal_event(payload: Value) -> NormalizedProviderEvent { + NormalizedProviderEvent { + event_type: "run.terminal".to_owned(), + priority: EventPriority::P0, + payload, + } +} + +impl AcpxDurableState { + pub(super) fn leave_finished_provider_turn_wait(&mut self) -> Result<(), DurableRunnerError> { + if self + .provider_turn_wait + .as_ref() + .is_none_or(|wait| wait.is_consistent(self)) + { + return Ok(()); + } + let wait = self + .provider_turn_wait + .take() + .expect("checked provider turn wait"); + if matches!(self.lifecycle.as_str(), "turn_starting" | "turn_active") { + return Ok(()); + } + self.push(wait.into_run_terminal()) + } +} + +impl AcpxCommandExecutor { + pub(super) fn awaiting_provider_turn(&self) -> bool { + self.state + .as_ref() + .is_some_and(|state| state.provider_turn_wait.is_some()) + } + + pub(super) fn settle_turn_events( + &mut self, + events: Vec, + turn_ended: bool, + run_terminal: Option, + ) -> Result<(), DurableRunnerError> { + let offer = turn_ended + .then(|| { + self.session + .as_mut() + .and_then(|session| session.take_provider_turn_offer()) + }) + .flatten(); + let wait = match (offer, run_terminal) { + (Some(turn_id), Some(held_run_terminal)) + if self.turn_accepts_provider_turns + && self + .session + .as_mut() + .is_some_and(|session| session.arm_provider_turn(&turn_id)) => + { + Ok(AcpxProviderTurnWait { + turn_id, + held_run_terminal, + }) + } + (_, run_terminal) => Err(run_terminal), + }; + let state = self + .state + .as_mut() + .expect("ACPX state remains available while polling"); + for mut event in events { + if wait.is_ok() && event.event_type == "turn.completed" { + event.payload["awaitingProviderTurn"] = json!(true); + } + state.push(event)?; + } + match wait { + Ok(wait) => state.provider_turn_wait = Some(wait), + Err(Some(payload)) => state.push(run_terminal_event(payload))?, + Err(None) => {} + } + Ok(()) + } + + pub(super) fn end_provider_turn_wait( + &mut self, + publish: bool, + ) -> Result<(), DurableRunnerError> { + let Some(wait) = self + .state + .as_mut() + .and_then(|state| state.provider_turn_wait.take()) + else { + return Ok(()); + }; + if let Some(session) = self.session.as_mut() { + session.disarm_provider_turn(&wait.turn_id); + } + if publish { + self.state + .as_mut() + .expect("ACPX state exists with a provider turn wait") + .push(wait.into_run_terminal())?; + } + Ok(()) + } + + pub(super) fn settled_without_turn( + &mut self, + reason: &str, + ) -> Result { + if self.awaiting_provider_turn() { + self.end_provider_turn_wait(true)?; + self.save_state()?; + } + Ok(CommandExecution { + result: json!({"status": "already_settled", "reason": reason}), + events: Vec::new(), + }) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::super::tests::descriptor; + use super::super::AcpxDurableState; + use super::AcpxProviderTurnWait; + use crate::provider_bridge::{authorized_tool_catalog_digest, AuthorizedToolSet}; + + fn state() -> AcpxDurableState { + let operations = Vec::new(); + AcpxDurableState::new( + serde_json::from_value(descriptor("claude")).unwrap(), + AuthorizedToolSet { + schema: "paperclip.runner.authorized-tools.v1".into(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }, + "test".into(), + ) + } + + fn wait() -> AcpxProviderTurnWait { + AcpxProviderTurnWait { + turn_id: "turn-1".into(), + held_run_terminal: json!({"schema":"paperclip.prp.terminal.v1","turnTerminalState":"completed"}), + } + } + + #[test] + fn state_outside_a_wait_keeps_the_previous_release_shape_in_both_directions() { + let saved = serde_json::to_value(state()).unwrap(); + assert!(saved.get("providerTurnWait").is_none()); + let resumed: AcpxDurableState = serde_json::from_value(saved).unwrap(); + assert_eq!(resumed.provider_turn_wait, None); + } + + #[test] + fn a_wait_round_trips_and_an_unknown_field_is_refused_like_a_previous_release_refuses_it() { + let mut waiting = state(); + waiting.provider_turn_wait = Some(wait()); + let saved = serde_json::to_value(&waiting).unwrap(); + assert_eq!(saved["providerTurnWait"]["turnId"], json!("turn-1")); + let resumed: AcpxDurableState = serde_json::from_value(saved.clone()).unwrap(); + assert_eq!(resumed.provider_turn_wait, Some(wait())); + let mut future = saved; + future["providerTurnWaitFromAFutureRelease"] = json!({}); + let refused = serde_json::from_value::(future).unwrap_err(); + assert!(refused.to_string().contains("unknown field"), "{refused}"); + } + + #[test] + fn leaving_the_open_idle_session_ends_the_wait_by_the_next_lifecycle() { + for (lifecycle, active, published) in [ + ("closed", None, true), + ("suspended", None, true), + ("prepared", None, true), + ("turn_starting", Some("turn-2"), false), + ("turn_active", Some("provider-1"), false), + ] { + let mut left = state(); + left.provider_turn_wait = Some(wait()); + left.lifecycle = lifecycle.into(); + left.active_turn_id = active.map(str::to_owned); + left.leave_finished_provider_turn_wait().unwrap(); + assert_eq!(left.provider_turn_wait, None, "{lifecycle}"); + let terminals = left + .pending_events + .iter() + .filter(|event| event.event_type == "run.terminal") + .count(); + assert_eq!(terminals, usize::from(published), "{lifecycle}"); + } + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index 350cece37b6..7efa6d3e2f6 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -484,6 +484,22 @@ impl AcpxSidecarTransport { } } + pub(crate) fn request_by( + &mut self, + command: GeneratedAcpxSidecarCommand, + params: Value, + deadline: Instant, + ) -> Result, LocalRunnerError> { + let bound = self.request_deadline; + self.request_deadline = Some(bound.map_or(deadline, |bound| bound.min(deadline))); + let outcome = self.request_outcome(command, params); + self.request_deadline = bound; + match outcome? { + CommandOutcome::Success(value) => Ok(Ok(value)), + CommandOutcome::Rejected { error, .. } => Ok(Err(error)), + } + } + pub(crate) fn request_turn_start( &mut self, params: Value, @@ -941,6 +957,17 @@ impl AcpxSidecarTransport { )) } + pub(crate) fn exited_error(&mut self, stage: &str) -> Option { + match self.process.try_wait() { + Ok(None) => None, + Ok(Some(_)) | Err(_) => { + let error = self.closed_error(stage); + self.poison(); + Some(error) + } + } + } + fn closed_error(&mut self, stage: &str) -> LocalRunnerError { self.drain_diagnostics(Duration::from_millis(20)); let suffix = self.diagnostic_suffix(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index f9b9e78563b..c66ae154349 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -123,6 +123,8 @@ fn run() -> Result<(), Box> { let mut provider_lost = false; let mut session_identity = Value::Null; let mut source_run = String::new(); + let mut pending_provider_run: Option = None; + let mut provider_rearmable = false; let mut source_pending_completion: Option = None; let mut source_goal_revision = 0_u64; let mut runtime_directory: Option = None; @@ -217,6 +219,94 @@ fn run() -> Result<(), Box> { continue; } } + if command == "turn.cancel" + && provider_turn_tokens() + .iter() + .any(|token| token == "provider-turn-open") + && request["params"]["turnId"] + .as_str() + .is_some_and(|turn| turn.starts_with("provider-")) + { + let turn = request["params"]["turnId"].as_str().unwrap().to_owned(); + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"id":id,"ok":true, + "result":{"cancelled":true,"sessionClosed":true}}), + )?; + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":next_sequence, + "eventType":"runtime.turn_terminal","runId":"run-1","turnId":turn, + "payload":{"status":"cancelled","stopReason":"cancelled"}}), + )?; + next_sequence += 1; + continue; + } + if command == "session.provider_turn" { + if let Some(journal) = source_journal.as_mut() { + write_json(journal, &request)?; + } + let tokens = provider_turn_tokens(); + let lost = if request["params"]["armed"] == json!(true) { + "provider-arm-reply-lost" + } else { + "provider-disarm-reply-lost" + }; + if tokens.iter().any(|token| token == lost) { + continue; + } + let armed = request["params"]["armed"] == json!(true) + && (pending_provider_run.is_some() || provider_rearmable) + && !tokens.iter().any(|token| token == "provider-arm-reject"); + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"id":id,"ok":true, + "result":{"armed":armed}}), + )?; + if request["params"]["armed"] == json!(false) + && tokens + .iter() + .any(|token| token == "provider-turn-after-disarm") + { + if let Some(run) = pending_provider_run.take() { + next_sequence = write_provider_turn(&mut stdout, next_sequence, &run)?; + } + } + if armed + && tokens + .iter() + .any(|token| token == "provider-sidecar-exit-after-arm") + { + std::thread::spawn(|| { + std::thread::sleep(Duration::from_millis(300)); + std::process::exit(0); + }); + } + if armed + && tokens + .iter() + .any(|token| token == "provider-lost-after-arm") + { + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":next_sequence, + "eventType":"runtime.diagnostic","runId":null,"turnId":null, + "payload":{"code":"ACPX_PROVIDER_PROCESS_LOST","message":"The provider exited."}}), + )?; + next_sequence += 1; + } + if armed && tokens.iter().any(|token| token == "provider-turn-hold") { + continue; + } + if let (true, Some(run)) = (armed, pending_provider_run.take()) { + next_sequence = write_provider_turn(&mut stdout, next_sequence, &run)?; + provider_rearmable = tokens.iter().any(|token| token == "provider-terminal-live"); + } else { + provider_rearmable = false; + } + pending_provider_run = None; + continue; + } if let Some(journal) = source_journal.as_mut() { write_json(journal, &request)?; if command == "run.attach" { @@ -452,6 +542,12 @@ fn run() -> Result<(), Box> { json!({"status":status}), )?; next_sequence += 1; + next_sequence = offer_provider_turn( + &mut stdout, + next_sequence, + &source_run, + &mut pending_provider_run, + )?; } if command == "tool.resolve" { let turn = source_pending_completion @@ -974,6 +1070,22 @@ fn run() -> Result<(), Box> { )?; next_sequence += 1; } + if command == "turn.start" + && matches!( + mode, + "turns-tool-terminal" + | "turns-reused-tool-id-terminal" + | "turns-tool-result-terminal" + | "turns-reserved-result-terminal" + ) + { + next_sequence = offer_provider_turn( + &mut stdout, + next_sequence, + "run-1", + &mut pending_provider_run, + )?; + } if command == "turn.start" && mode == "turns-multiple-tool-results-terminal" { for index in 1..=2 { write_turn_event( @@ -1448,6 +1560,134 @@ fn bootstrap_success( }) } +fn provider_turn_tokens() -> Vec { + BACKGROUND_TASKS + .get() + .and_then(Option::as_ref) + .and_then(|path| std::fs::read_to_string(path).ok()) + .map(|modes| modes.split_whitespace().map(str::to_owned).collect()) + .unwrap_or_default() +} + +fn offer_provider_turn( + output: &mut impl Write, + sequence: u64, + default_run: &str, + pending: &mut Option, +) -> io::Result { + let tokens = provider_turn_tokens(); + if !tokens.iter().any(|token| token == "provider-turn") { + return Ok(sequence); + } + if tokens.iter().any(|token| token == "provider-turn-unarmed") { + return write_provider_turn(output, sequence, default_run); + } + *pending = Some(default_run.to_owned()); + Ok(sequence) +} + +fn write_provider_turn( + output: &mut impl Write, + mut sequence: u64, + default_run: &str, +) -> io::Result { + let tokens = provider_turn_tokens(); + if !tokens.iter().any(|token| token == "provider-turn") { + return Ok(sequence); + } + let value = |prefix: &str, fallback: &str| { + tokens + .iter() + .find_map(|token| token.strip_prefix(prefix).map(str::to_owned)) + .unwrap_or_else(|| fallback.to_owned()) + }; + let turn_id = value("provider-turn-id=", "provider-1"); + let run_id = value("provider-run=", default_run); + let frame = |sequence: u64, event_type: &str, payload: Value| { + json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":sequence, + "eventType":event_type,"runId":run_id,"turnId":turn_id,"payload":payload}) + }; + write_json( + output, + &frame( + sequence, + "runtime.turn_started", + json!({"origin":"provider"}), + ), + )?; + sequence += 1; + if let Some(call_id) = tokens + .iter() + .find_map(|token| token.strip_prefix("provider-reuse-call=")) + { + write_json( + output, + &frame( + sequence, + "runtime.tool_called", + json!({"callId":call_id,"operationId":"issues.read","input":{"id":"issue-1"}}), + ), + )?; + sequence += 1; + } + if tokens.iter().any(|token| token == "provider-result") { + let result = valid_reserved_completion_result(); + write_json( + output, + &frame( + sequence, + "runtime.tool_called", + json!({"callId":"call-provider-finish","operationId":"paperclip_finish","input":result.clone()}), + ), + )?; + write_json( + output, + &frame( + sequence + 1, + "runtime.event", + json!({"type":"semantic_result","callId":"call-provider-finish","operationId":"paperclip_finish","ok":true,"result":result}), + ), + )?; + sequence += 2; + } + write_json( + output, + &frame( + sequence, + "runtime.event", + json!({"type":"text_delta","text":"Claude finished its own work."}), + ), + )?; + sequence += 1; + if tokens.iter().any(|token| token == "provider-turn-open") { + return Ok(sequence); + } + write_json( + output, + &frame( + sequence, + "runtime.event", + json!({"type":"status","tag":"usage_update","text":"terminal prompt usage", + "breakdown":{"inputTokens":6,"outputTokens":4,"cachedReadTokens":0,"cachedWriteTokens":0,"thoughtTokens":0}}), + ), + )?; + sequence += 1; + let live = if tokens.iter().any(|token| token == "provider-terminal-live") { + 1 + } else { + 0 + }; + write_json( + output, + &frame( + sequence, + "runtime.turn_terminal", + json!({"status":"completed","stopReason":"end_turn","backgroundTasks":{"live":live}}), + ), + )?; + Ok(sequence + 1) +} + fn write_turn_event( output: &mut impl Write, sequence: u64, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs index 1407ca9c356..d27871c36b1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs @@ -404,6 +404,20 @@ pub fn project_acpx_state_event( }), ) } + AcpxProviderStateEvent::ProviderTurnStarted { turn_id } => { + require_projected_turn(context, turn_id)?; + one( + "turn.started", + EventPriority::P0, + json!({ + "provider": "acpx", + "providerTurnId": turn_id, + "status": "inProgress", + "turn": {"id": turn_id, "status": "inProgress"}, + "origin": crate::generated_acpx_sidecar_contract::GENERATED_ACPX_PROVIDER_TURN_ORIGIN, + }), + ) + } AcpxProviderStateEvent::Process(details) => one( "harness.diagnostic", EventPriority::P1, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_quiet_turn_continuation.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_quiet_turn_continuation.rs index 0d69cc92bea..4663301d938 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_quiet_turn_continuation.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_quiet_turn_continuation.rs @@ -227,6 +227,196 @@ fn a_full_settled_turn_ledger_still_rotates_through_a_restart() { session.shutdown("test complete").unwrap(); } +fn provider_session(scratch: &Scratch, background: &str, agent: &str) -> AcpxProviderSession { + provider_session_in(scratch, background, agent, "turns-tool-result-terminal") +} + +fn provider_session_in( + scratch: &Scratch, + background: &str, + agent: &str, + mode: &str, +) -> AcpxProviderSession { + scratch.background(background); + AcpxProviderSession::start(&config(&scratch.0, mode, agent)).unwrap() +} + +fn arm(session: &mut AcpxProviderSession) -> bool { + session + .take_provider_turn_offer() + .is_some_and(|turn_id| session.arm_provider_turn(&turn_id)) +} + +fn poll_until_terminal( + session: &mut AcpxProviderSession, + turn_id: &str, +) -> Vec { + let mut seen = Vec::new(); + loop { + let events = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); + let done = events.iter().any(|event| { + matches!(event, AcpxProviderStateEvent::TurnTerminal { turn_id: settled, .. } if settled == turn_id) + }); + seen.extend(events); + if done { + return seen; + } + } +} + +#[test] +fn live_shell_tasks_at_a_completed_claude_terminal_await_and_admit_the_provider_started_turn() { + let scratch = Scratch::new(); + let mut session = provider_session(&scratch, "live provider-turn", "claude"); + let pid = session.process_id(); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + assert!(!session.continue_after_quiet_turn()); + + let started = session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + assert!(matches!( + started.as_slice(), + [AcpxProviderStateEvent::ProviderTurnStarted { turn_id }] if turn_id == "provider-1" + )); + assert_eq!(session.state().active_turn_id(), Some("provider-1")); + let provider = poll_until_terminal(&mut session, "provider-1"); + assert!(provider.iter().any(|event| matches!( + event, + AcpxProviderStateEvent::Activity(activity) if activity.event_type == "usage.reported" + ))); + assert_eq!(session.state().active_turn_id(), None); + assert!(session.take_provider_turn_offer().is_none()); + assert_eq!(session.process_id(), pid); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn a_provider_started_turn_that_leaves_shells_running_awaits_again() { + let scratch = Scratch::new(); + let mut session = provider_session( + &scratch, + "live provider-turn provider-terminal-live", + "claude", + ); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + poll_until_terminal(&mut session, "provider-1"); + assert_eq!( + session.take_provider_turn_offer().as_deref(), + Some("provider-1") + ); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn an_unexpected_provider_started_turn_fails_closed_like_any_idle_event() { + for (background, agent, expected) in [ + ( + "quiet provider-turn provider-turn-unarmed", + "claude", + "outside an awaited continuation", + ), + ( + "provider-turn provider-turn-unarmed", + "claude", + "outside an awaited continuation", + ), + ( + "live provider-turn provider-turn-unarmed", + "codex", + "outside an awaited continuation", + ), + ( + "live provider-turn provider-turn-id=turn-1", + "claude", + "reused a settled turn identity", + ), + ( + "live provider-turn provider-run=run-2", + "claude", + "bounded contract", + ), + ] { + let scratch = Scratch::new(); + let mut session = provider_session(&scratch, background, agent); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + let awaited = arm(&mut session); + let error = if awaited { + session.poll_awaited_event(Duration::from_secs(1)) + } else { + session.poll_event(Duration::from_secs(1)) + } + .unwrap_err(); + assert!( + error.to_string().contains(expected), + "{background} {agent}: {error}" + ); + let _ = session.shutdown("test complete"); + } +} + +#[test] +fn a_provider_started_turn_cannot_replay_a_settled_tool_call() { + let scratch = Scratch::new(); + let mut session = provider_session( + &scratch, + "live provider-turn provider-reuse-call=call-1", + "claude", + ); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + let error = session.poll_event(Duration::from_secs(1)).unwrap_err(); + assert!( + error + .to_string() + .contains("reused a completed tool call id"), + "{error}" + ); + let _ = session.shutdown("test complete"); +} + +#[test] +fn a_controller_turn_while_awaiting_restarts_the_provider_first() { + let scratch = Scratch::new(); + let mut session = provider_session(&scratch, "live provider-turn", "claude"); + let pid = session.process_id(); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .start_turn("turn-2", "Paperclip's next turn", &std::env::temp_dir()) + .unwrap(); + assert_ne!(session.process_id(), pid); + assert!(session.turn_start_replaced_provider()); + assert!(session.take_provider_turn_offer().is_none()); + session.shutdown("test complete").unwrap(); +} + #[test] fn the_generated_reader_matches_the_shared_background_task_vectors() { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -248,3 +438,105 @@ fn the_generated_reader_matches_the_shared_background_task_vectors() { ); } } + +#[test] +fn a_turn_that_reported_its_result_never_awaits_even_with_live_shells() { + let scratch = Scratch::new(); + let mut session = provider_session_in( + &scratch, + "live provider-turn", + "claude", + "turns-reserved-result-terminal", + ); + session + .start_turn( + "turn-1", + "Finish with a shell still running", + &std::env::temp_dir(), + ) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(session.run_result_reported()); + assert!(session.take_provider_turn_offer().is_none()); + assert!(session + .poll_event(Duration::from_millis(200)) + .unwrap() + .is_none()); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn a_provider_started_turn_that_reports_a_result_settles_and_does_not_await_again() { + let scratch = Scratch::new(); + let mut session = provider_session( + &scratch, + "live provider-turn provider-result provider-terminal-live", + "claude", + ); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + let events = poll_until_terminal(&mut session, "provider-1"); + assert!(events.iter().any(|event| matches!( + event, + AcpxProviderStateEvent::TurnTerminal { status, .. } + if *status == paperclip_runner_core::acpx_event_payload::AcpxTurnStatus::Completed + ))); + assert!(session.run_result_reported()); + assert!(session.take_provider_turn_offer().is_none()); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn a_sidecar_that_cannot_arm_leaves_runnerd_not_awaiting() { + let scratch = Scratch::new(); + let mut session = + provider_session(&scratch, "live provider-turn provider-arm-reject", "claude"); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + let offer = session + .take_provider_turn_offer() + .expect("eligible terminal"); + assert!(!session.arm_provider_turn(&offer)); + session.shutdown("test complete").unwrap(); +} + +#[test] +fn cancelling_a_provider_started_turn_settles_it_cancelled_and_retires_the_provider() { + let scratch = Scratch::new(); + let mut session = provider_session(&scratch, "live provider-turn provider-turn-open", "claude"); + session + .start_turn("turn-1", "Start background work", &std::env::temp_dir()) + .unwrap(); + poll_until_terminal(&mut session, "turn-1"); + assert!(arm(&mut session)); + session + .poll_awaited_event(Duration::from_secs(1)) + .unwrap() + .unwrap(); + assert_eq!(session.state().active_turn_id(), Some("provider-1")); + let response = session + .interrupt_turn("provider-1", "Operator stop") + .unwrap(); + assert_eq!(response["sessionClosed"], json!(true)); + assert!(session.runtime_retired()); + let events = poll_until_terminal(&mut session, "provider-1"); + assert!(events.iter().any(|event| matches!( + event, + AcpxProviderStateEvent::TurnTerminal { status, .. } + if *status == paperclip_runner_core::acpx_event_payload::AcpxTurnStatus::Cancelled + ))); + assert!(session.take_provider_turn_offer().is_none()); + assert!(session + .start_turn("turn-2", "Paperclip's next turn", &std::env::temp_dir()) + .is_err()); + let _ = session.shutdown("test complete"); +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_quiet_turn_restart.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_quiet_turn_restart.rs index 2f61aa3aa83..641f4150c01 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_quiet_turn_restart.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/support/acpx_quiet_turn_restart.rs @@ -75,8 +75,8 @@ fn a_quiet_completed_claude_turn_keeps_its_provider_for_the_next_turns() { } #[test] -fn live_or_unknown_background_work_restarts_at_the_turn_end_like_today() { - for mode in ["live", "", "malformed"] { +fn unknown_background_work_restarts_at_the_turn_end_like_today() { + for mode in ["", "malformed"] { let mut harness = Harness::new(); harness.fixture.background(mode); let pid = harness.opened["processId"].clone(); @@ -227,3 +227,643 @@ fn a_runner_restart_after_a_kept_quiet_provider_resumes_in_a_fresh_process() { ); harness.executor.shutdown().unwrap(); } + +fn complete_turn_accepting_provider_turns(harness: &mut Harness, turn: &str) -> Vec { + let started = harness.execute( + "turn.start", + json!({"turnId": turn, "text": "Complete the work.", "providerStartedTurns": true}), + ); + let seen = started + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + drain_until(harness, seen, "turn.completed") +} + +fn drain_until(harness: &mut Harness, mut seen: Vec, kind: &str) -> Vec { + let deadline = Instant::now() + Duration::from_secs(5); + while !seen.iter().any(|event| event["type"] == kind) { + assert!(Instant::now() < deadline, "expected {kind}: {seen:?}"); + seen.extend(harness.drain()); + } + seen +} + +#[test] +fn live_shells_at_a_completed_claude_turn_keep_the_provider_and_refuse_park() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold"); + let pid = harness.opened["processId"].clone(); + let events = complete_turn_accepting_provider_turns(&mut harness, "first"); + let completed = events + .iter() + .find(|event| event["type"] == "turn.completed") + .unwrap(); + assert_eq!(completed["payload"]["awaitingProviderTurn"], json!(true)); + assert!(!events.iter().any(|event| event["type"] == "run.terminal")); + let before = harness.fixture.journal().len(); + harness.executor.prepare_next_turn().unwrap(); + assert_eq!( + commands_after(&harness.fixture.journal(), before), + Vec::::new() + ); + assert_eq!(live_providers(&harness), [pid.clone()]); + assert!(!harness.executor.host_park_ready().unwrap()); + + let (next, _) = run_turn(&mut harness, "second"); + assert!( + replaced(&next), + "a controller turn restarts the awaited provider first" + ); + assert!(!alive(&pid)); + harness.executor.shutdown().unwrap(); + assert_eq!(live_providers(&harness), Vec::::new()); +} + +#[test] +fn the_provider_started_turn_reaches_the_controller_as_ordinary_turn_events() { + let mut harness = Harness::new(); + harness.fixture.background("live provider-turn"); + let pid = harness.opened["processId"].clone(); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + let events = drain_until(&mut harness, first, "run.terminal"); + let kinds = events + .iter() + .map(|event| event["type"].as_str().unwrap().to_owned()) + .collect::>(); + let first_completed = kinds + .iter() + .position(|kind| kind == "turn.completed") + .unwrap(); + let provider_started = events + .iter() + .position(|event| { + event["type"] == "turn.started" && event["payload"]["origin"] == "provider" + }) + .expect("provider-started turn"); + assert!(first_completed < provider_started); + assert_eq!( + events[provider_started]["payload"]["providerTurnId"], + json!("provider-1") + ); + let provider_completed = events + .iter() + .rposition(|event| event["type"] == "turn.completed") + .unwrap(); + assert!(provider_started < provider_completed); + assert_eq!( + events[provider_completed]["payload"]["providerTurnId"], + json!("provider-1") + ); + assert!(events[provider_completed]["payload"] + .get("awaitingProviderTurn") + .is_none()); + assert!(events[provider_started..provider_completed] + .iter() + .any(|event| event["type"] == "usage.reported")); + let run_terminal = kinds + .iter() + .position(|kind| kind == "run.terminal") + .unwrap(); + assert!(run_terminal > provider_completed); + assert_eq!( + kinds.iter().filter(|kind| *kind == "run.terminal").count(), + 1 + ); + harness.executor.prepare_next_turn().unwrap(); + assert!(harness.executor.host_park_ready().unwrap()); + assert_eq!(live_providers(&harness).len(), 1); + assert!(!alive(&pid) || live_providers(&harness) == [pid.clone()]); + harness.executor.shutdown().unwrap(); + assert_eq!(live_providers(&harness), Vec::::new()); +} + +#[test] +fn a_runner_restart_while_awaiting_resumes_like_today() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold"); + let pid = harness.opened["processId"].clone(); + complete_turn_accepting_provider_turns(&mut harness, "first"); + harness.executor.prepare_next_turn().unwrap(); + assert_eq!(live_providers(&harness), [pid.clone()]); + harness.executor.shutdown().unwrap(); + assert!(!alive(&pid)); + harness.executor = + NativeProviderCommandExecutor::with_runner_config(&harness.fixture.0, &harness.config); + harness.sequence = 90; + harness.fixture.background("quiet"); + let resumed = harness.drain(); + assert_eq!( + resumed + .iter() + .filter(|event| event["type"] == "run.terminal") + .count(), + 1, + "{resumed:?}" + ); + let (events, _) = run_turn(&mut harness, "second"); + assert!(events.iter().any(|event| event["type"] == "turn.completed")); + assert_eq!(harness.fixture.process_starts().len(), 2); + harness.executor.shutdown().unwrap(); + assert_eq!(live_providers(&harness), Vec::::new()); +} + +fn controller_view(events: &[Value]) -> Vec { + events + .iter() + .map(|event| { + let mut event = event.clone(); + for volatile in ["processId", "previousProcessId", "providerTurnId"] { + if let Some(payload) = event["payload"].as_object_mut() { + payload.remove(volatile); + } + } + event + }) + .collect() +} + +#[test] +fn without_the_controller_capability_live_shells_behave_exactly_as_before() { + let outcome = |background: &str| { + let mut harness = Harness::new(); + harness.fixture.background(background); + let pid = harness.opened["processId"].clone(); + let events = harness.complete_turn("first"); + let park_ready_before_restart = harness.executor.host_park_ready().unwrap(); + let before = harness.fixture.journal().len(); + harness.executor.prepare_next_turn().unwrap(); + let commands = commands_after(&harness.fixture.journal(), before); + let restarted = !alive(&pid); + let park_ready = harness.executor.host_park_ready().unwrap(); + harness.executor.shutdown().unwrap(); + ( + controller_view(&events), + commands, + restarted, + park_ready, + park_ready_before_restart, + ) + }; + let live = outcome("live provider-turn"); + let unknown = outcome(""); + assert_eq!(live.0, unknown.0); + assert_eq!(live.1, unknown.1); + assert!(live.2 && unknown.2); + assert_eq!(live.3, unknown.3); + assert_eq!(live.4, unknown.4); + assert!(live.4); + assert!(live.0.iter().any(|event| event["type"] == "run.terminal")); + assert!(!live + .0 + .iter() + .any(|event| event["payload"].get("awaitingProviderTurn").is_some())); + assert!(!live + .1 + .iter() + .any(|command| command == "session.provider_turn")); +} + +#[test] +fn a_provider_lost_while_awaiting_ends_the_wait_with_the_held_run_terminal() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold provider-lost-after-arm"); + let pid = harness.opened["processId"].clone(); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + let events = drain_until(&mut harness, first, "run.terminal"); + let completed = events + .iter() + .position(|event| event["type"] == "turn.completed") + .unwrap(); + assert_eq!( + events[completed]["payload"]["awaitingProviderTurn"], + json!(true) + ); + let terminals = events + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed") + ); + assert!(events + .iter() + .any(|event| event["payload"]["code"] == "ACPX_PROVIDER_PROCESS_LOST")); + harness.executor.prepare_next_turn().unwrap(); + assert!( + !alive(&pid), + "the lost provider is replaced at the turn end" + ); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn a_sidecar_that_dies_while_awaiting_ends_the_wait_with_the_held_run_terminal() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm"); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + let events = drain_until(&mut harness, first, "run.terminal"); + let terminals = events + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed") + ); + assert!(!events.iter().any(|event| event["type"] == "turn.failed")); + let _ = harness.executor.shutdown(); +} + +#[test] +fn cancelling_or_stopping_the_run_while_awaiting_disarms_and_publishes_the_held_run_terminal() { + for command in ["run.cancel", "turn.interrupt", "turn.stop"] { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold provider-turn-after-disarm"); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + assert!( + !first.iter().any(|event| event["type"] == "run.terminal"), + "{command}" + ); + let before = harness.fixture.journal().len(); + let cancelled = harness.execute( + command, + json!({"turnId": "first", "reason": "Operator stop"}), + ); + assert_eq!( + cancelled.result["status"], + json!("already_settled"), + "{command}" + ); + let mut later = cancelled + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + let journal = harness.fixture.journal(); + assert!( + journal[before..].iter().any(|line| { + line["command"] == "session.provider_turn" + && line["params"]["armed"] == json!(false) + }), + "{command}" + ); + let deadline = Instant::now() + Duration::from_millis(500); + while Instant::now() < deadline { + match harness.executor.poll_events() { + Ok(events) => { + later.extend( + events + .iter() + .map(|event| json!({"type":event.event_type,"payload":event.payload})), + ); + let _ = harness.executor.acknowledge_events(events.len()); + } + Err(_) => break, + } + } + assert!( + !later + .iter() + .any(|event| event["payload"]["origin"] == "provider"), + "{command}: {later:?}" + ); + let terminals = later + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1, "{command}: {later:?}"); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed"), + "{command}" + ); + let _ = harness.executor.shutdown(); + } +} + +#[test] +fn an_active_goal_at_a_live_shell_turn_end_neither_marks_nor_arms_a_wait() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold"); + harness.fixture.fault("goal"); + let before = harness.fixture.journal().len(); + let events = complete_turn_accepting_provider_turns(&mut harness, "first"); + let events = [events, harness.drain()].concat(); + let completed = events + .iter() + .find(|event| event["type"] == "turn.completed") + .unwrap(); + assert!(completed["payload"].get("awaitingProviderTurn").is_none()); + assert!(!events.iter().any(|event| event["type"] == "run.terminal")); + assert!(!commands_after(&harness.fixture.journal(), before) + .iter() + .any(|command| command == "session.provider_turn")); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn a_crash_after_cancelling_or_stopping_a_wait_still_delivers_the_held_run_terminal_once() { + for command in ["run.cancel", "turn.interrupt", "turn.stop"] { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold"); + complete_turn_accepting_provider_turns(&mut harness, "first"); + let lost_with_the_crash = harness.execute( + command, + json!({"turnId": "first", "reason": "Operator stop"}), + ); + assert_eq!( + lost_with_the_crash.result["status"], + json!("already_settled"), + "{command}" + ); + harness.executor = + NativeProviderCommandExecutor::with_runner_config(&harness.fixture.0, &harness.config); + harness.fixture.background("quiet"); + let recovered = harness.drain(); + assert_eq!( + recovered + .iter() + .filter(|event| event["type"] == "run.terminal") + .count(), + 1, + "{command}: {recovered:?}" + ); + let _ = harness.executor.shutdown(); + } +} + +fn awaiting_and_quiesced(harness: &mut Harness) { + harness + .fixture + .background("live provider-turn provider-turn-hold"); + let first = complete_turn_accepting_provider_turns(harness, "first"); + let events = [first, harness.drain()].concat(); + assert!(!events.iter().any(|event| event["type"] == "run.terminal")); + assert!(harness.fixture.state().get("providerTurnWait").is_some()); + harness.quiesce(); +} + +fn drain_briefly(harness: &mut Harness) -> Vec { + let mut seen = Vec::new(); + let deadline = Instant::now() + Duration::from_millis(500); + while Instant::now() < deadline { + seen.extend(harness.drain()); + } + seen +} + +#[test] +fn a_different_run_attaching_while_awaiting_ends_the_wait_without_the_old_run_terminal() { + let mut harness = Harness::new(); + awaiting_and_quiesced(&mut harness); + let pid = harness.opened["processId"].clone(); + let before = harness.fixture.journal().len(); + let (attached, _) = harness.attach("run-2", |_| {}); + let state = harness.fixture.state(); + assert_eq!(state["descriptor"]["runId"], "run-2"); + assert!(state.get("providerTurnWait").is_none(), "{state}"); + assert!( + !state["pendingEvents"] + .as_array() + .unwrap() + .iter() + .any(|event| event.to_string().contains("run.terminal")), + "{state}" + ); + let mut received = attached + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + received.extend(drain_briefly(&mut harness)); + assert!( + !received.iter().any(|event| event["type"] == "run.terminal"), + "{received:?}" + ); + let commands = commands_after(&harness.fixture.journal(), before); + assert!( + commands.iter().any(|command| command == "session.close"), + "{commands:?}" + ); + assert!( + !commands + .iter() + .any(|command| command == "session.provider_turn"), + "{commands:?}" + ); + assert!(!alive(&pid)); + let _ = harness.executor.shutdown(); +} + +#[test] +fn a_failed_different_run_attach_while_awaiting_keeps_the_wait_and_its_run_terminal() { + for ending in [ + None, + Some("run.cancel"), + Some("turn.interrupt"), + Some("turn.stop"), + ] { + let mut harness = Harness::new(); + awaiting_and_quiesced(&mut harness); + let pid = harness.opened["processId"].clone(); + std::process::Command::new("kill") + .args(["-9", &pid.to_string()]) + .status() + .unwrap(); + std::thread::sleep(Duration::from_millis(200)); + let payload = harness.attach_payload("run-2", |_| {}); + harness.sequence += 1; + let error = harness + .executor + .execute(&command(harness.sequence, "run.attach", payload)) + .unwrap_err(); + assert!( + format!("{error:?}").contains("failed to checkpoint ACPX before attaching a new run"), + "{ending:?}: {error:?}" + ); + let state = harness.fixture.state(); + assert_eq!(state["descriptor"]["runId"], "run-1", "{ending:?}"); + assert!( + state.get("providerTurnWait").is_some(), + "{ending:?}: {state}" + ); + let mut recovered = Vec::new(); + if let Some(ending) = ending { + let ended = harness.execute( + ending, + json!({"turnId": "first", "reason": "Operator stop"}), + ); + assert_eq!(ended.result["status"], json!("already_settled"), "{ending}"); + recovered.extend( + ended + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})), + ); + } + recovered.extend(drain_briefly(&mut harness)); + let terminals = recovered + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1, "{ending:?}: {recovered:?}"); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed"), + "{ending:?}" + ); + let state = harness.fixture.state(); + assert_eq!(state["descriptor"]["runId"], "run-1", "{ending:?}"); + assert!(state.get("providerTurnWait").is_none(), "{ending:?}"); + let _ = harness.executor.shutdown(); + } +} + +#[test] +fn a_same_run_attach_while_awaiting_publishes_the_held_run_terminal_once() { + let mut harness = Harness::new(); + awaiting_and_quiesced(&mut harness); + let attached = harness.execute( + "run.attach", + json!({"provider": harness.descriptor.clone()}), + ); + let mut received = attached + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + received.extend(drain_briefly(&mut harness)); + let terminals = received + .iter() + .filter(|event| event["type"] == "run.terminal") + .collect::>(); + assert_eq!(terminals.len(), 1, "{received:?}"); + assert_eq!( + terminals[0]["payload"]["turnTerminalState"], + json!("completed") + ); + let _ = harness.executor.shutdown(); +} + +fn execution_events(execution: &CommandExecution) -> Vec { + execution + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect() +} + +fn run_terminals(events: &[Value]) -> usize { + events + .iter() + .filter(|event| event["type"] == "run.terminal") + .count() +} + +#[test] +fn a_lost_disarm_reply_is_bounded_and_still_publishes_the_held_run_terminal_once() { + for command in ["run.cancel", "turn.interrupt"] { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-turn-hold provider-disarm-reply-lost"); + let pid = harness.opened["processId"].clone(); + let first = complete_turn_accepting_provider_turns(&mut harness, "first"); + let first = [first, harness.drain()].concat(); + assert_eq!(run_terminals(&first), 0, "{command}"); + let started = Instant::now(); + let ended = harness.execute( + command, + json!({"turnId": "first", "reason": "Operator stop"}), + ); + let elapsed = started.elapsed(); + assert_eq!( + ended.result["status"], + json!("already_settled"), + "{command}" + ); + assert!( + elapsed >= Duration::from_secs(4) && elapsed < Duration::from_secs(10), + "{command}: {elapsed:?}" + ); + let mut later = execution_events(&ended); + later.extend(drain_briefly(&mut harness)); + assert_eq!(run_terminals(&later), 1, "{command}: {later:?}"); + assert_eq!( + later + .iter() + .find(|event| event["type"] == "run.terminal") + .unwrap()["payload"]["turnTerminalState"], + json!("completed"), + "{command}" + ); + assert!( + harness.fixture.state().get("providerTurnWait").is_none(), + "{command}" + ); + let started = Instant::now(); + let closed = harness.execute("session.close", json!({})); + assert!( + started.elapsed() < Duration::from_secs(2), + "{command}: {:?}", + started.elapsed() + ); + assert_eq!(closed.result["status"], json!("closed"), "{command}"); + later.extend(execution_events(&closed)); + later.extend(drain_briefly(&mut harness)); + assert_eq!(run_terminals(&later), 1, "{command}: {later:?}"); + assert!(!alive(&pid), "{command}"); + let _ = harness.executor.shutdown(); + } +} + +#[test] +fn a_lost_arm_reply_is_bounded_and_the_turn_ends_with_its_run_terminal() { + let mut harness = Harness::new(); + harness + .fixture + .background("live provider-turn provider-arm-reply-lost"); + let started = Instant::now(); + let mut seen = execution_events(&harness.execute( + "turn.start", + json!({"turnId": "first", "text": "Complete the work.", "providerStartedTurns": true}), + )); + let deadline = started + Duration::from_secs(15); + while run_terminals(&seen) == 0 { + assert!(Instant::now() < deadline, "{seen:?}"); + seen.extend(harness.drain()); + } + assert!( + started.elapsed() < Duration::from_secs(10), + "{:?}", + started.elapsed() + ); + seen.extend(drain_briefly(&mut harness)); + let completed = seen + .iter() + .find(|event| event["type"] == "turn.completed") + .unwrap(); + assert!(completed["payload"].get("awaitingProviderTurn").is_none()); + assert_eq!(run_terminals(&seen), 1, "{seen:?}"); + assert!(harness.fixture.state().get("providerTurnWait").is_none()); + let _ = harness.executor.shutdown(); +} diff --git a/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn-wait.test.ts b/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn-wait.test.ts new file mode 100644 index 00000000000..967258fba11 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn-wait.test.ts @@ -0,0 +1,174 @@ +import { createHash } from "node:crypto"; +import { execFileSync, spawn } from "node:child_process"; +import { closeSync, openSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { QUALIFIED_ACPX_PROFILES } from "../drivers/acpx/qualified-profiles.js"; +import { createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type RunnerProviderLaunchRequirements } from "./runner-provider-launch.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding, type CapabilityRunnerdCodexTransportOptions } from "./runnerd-codex-transport.js"; + +const sha = (value: string | Buffer) => `sha256:${createHash("sha256").update(value).digest("hex")}`; +const quote = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`; +const kill = (pid: number) => { try { process.kill(-pid, "SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } }; + +type CoreEvent = { eventType: string; envelope: { payload?: Record } }; + +async function awaitedTurn(tokens: string, providerStartedTurns: boolean) { + const root = await realpath(await mkdtemp(join(tmpdir(), "runner-acpx-provider-wait-"))); + const journal = join(root, "calls"), fault = join(root, "fault"), wrapper = join(root, "sidecar"), script = join(root, "sidecar-entry"); + const background = join(root, "background"); + const binary = resolve(import.meta.dirname, "../../runner/target/debug/paperclip-runnerd"), sidecar = resolve(import.meta.dirname, "../../runner/target/debug/fake-acpx-sidecar"); + await mkdir(join(root, "acpx"), { mode: 0o700 }); await mkdir(join(root, "diagnostics"), { mode: 0o700 }); + await writeFile(script, "qualified fixture entry"); + await writeFile(background, tokens); + await writeFile(wrapper, `#!/bin/sh\nexec ${quote(sidecar)} --mode source-reconcile --profile-digest ${quote(QUALIFIED_ACPX_PROFILES.claude.commandDigest)} --journal ${quote(journal)} --source-fault ${quote(fault)} --background-tasks ${quote(background)} --spawn-owned-child\n`, { mode: 0o700 }); + let authority: DurablePrpControlPlane | null = null; + const server = createServer(); + server.on("upgrade", (request, socket, head) => authority ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); + await new Promise(done => server.listen(0, "127.0.0.1", done)); + const address = server.address(); if (!address || typeof address === "string") throw new Error("missing fixture listener"); + const identity = { runnerInstanceId: "runner-wait", environmentLeaseId: "lease-wait", runId: "run-wait", normalizedSessionId: "session-wait", turnId: "turn-wait", itemId: "item-wait" }; + const runnerPids = new Set(); + let requirements: RunnerProviderLaunchRequirements | undefined; + const options: CapabilityRunnerdCodexTransportOptions = { + provider: "acpx", acpxAgent: "claude", runnerBinary: binary, stateDirectory: root, prpIdentity: identity, + runnerArtifact: readRunnerdArtifactBinding(binary), codexSourceRecoveryVersion: 1, acpxSourceRecoveryVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, runnerProcessReconnectGraceMs: null, runnerReconnectGraceMs: 3_000, + controlPlaneRegistration: async next => { authority = next; return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: () => { authority = null; } }; }, + prepareLaunch: async launch => { + requirements ??= structuredClone(launch); + return { configurationId: `initial-${launch.identity.runId}`, requirementsDigest: runnerLaunchRequirementsDigest(launch), + provider: createRunnerProviderLaunch(launch, { options: {}, root, cwd: root, codexHome: root, codexCommand: "", providerNodeCommand: wrapper, opencodeProxyPath: "", acpxSidecarPath: script }) }; + }, + runnerProcessLauncher: spec => { + const stderr = openSync(join(root, "native.stderr"), "a", 0o600); + const child = spawn(binary, [...spec.args, "--acpx-launch-authority-digest", sha("fixture authority"), + "--acpx-sidecar-command", wrapper, "--acpx-sidecar-command-sha256", sha(execFileSync("cat", [wrapper])), + "--acpx-sidecar-script", script, "--acpx-sidecar-script-sha256", sha("qualified fixture entry")], + { env: { PATH: process.env.PATH, HOME: root, ...spec.environment }, stdio: ["ignore", "ignore", stderr], detached: true }); + closeSync(stderr); + if (child.pid) runnerPids.add(child.pid); + return { child, completion: new Promise((done, reject) => { child.once("error", reject); child.once("exit", (code, signal) => done({ code, signal, stdout: "", stderr: "" })); }) }; + }, + }; + const bundles = [createCapabilityRunnerdCodexTransport(options)]; + const bundle = bundles[0]!; + const calls = async () => (await readFile(journal, "utf8").catch(() => "")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line) as Record); + const coreEvents = async () => { + const core = JSON.parse(await readFile(join(root, "control-plane", "control-plane-state.json"), "utf8")) as { committedEvents?: CoreEvent[] }; + return (core.committedEvents ?? []).map(event => ({ eventType: event.eventType, + payload: ((event.envelope.payload ?? {}) as { payload?: Record }).payload ?? {} })); + }; + try { + const opened = await bundle.transport.request("thread/start", { cwd: root, model: "explicit-source-model", dynamicTools: [] }); + const thread = opened.thread as { id: string; sessionId: string }; + authority!.queueCommand("turn.start", { text: "Install and test.", turnId: "turn-own-1", ...(providerStartedTurns ? { providerStartedTurns: true } : {}) }, undefined, true); + await vi.waitFor(async () => expect((await coreEvents()).some(event => event.eventType === "turn.completed")).toBe(true), { timeout: 10_000, interval: 50 }); + const restartRunner = async () => { + const providerIdentity = JSON.parse(await readFile(join(root, "runner", "acpx-provider-state.json"), "utf8")).identity; + const pid = bundle.evidence().runnerPid!; + await bundle.detachControllerForRestart(); kill(pid); + for (const entry of await calls()) if (typeof entry.processStart === "number") kill(entry.processStart); + const recovered = createCapabilityRunnerdCodexTransport({ ...options, recoveryLaunch: { requirements: requirements! }, + resumeProviderSession: { driverSessionId: thread.id, providerSessionId: thread.sessionId, providerIdentity }, + onRecoveryAccountingEvent: async () => {}, authorityRunId: identity.runId }); + bundles.push(recovered); + await recovered.transport.request("thread/read", {}); + return recovered; + }; + return { coreEvents, calls, root, authority: authority!, restartRunner }; + } catch (error) { + throw new Error(`${String(error)}\n${await readFile(join(root, "native.stderr"), "utf8").catch(() => "")}`, { cause: error }); + } finally { + cleanups.push(async () => { + for (const opened of bundles.reverse()) await opened.transport.close().catch(() => undefined); + for (const pid of runnerPids) kill(pid); + for (const entry of await calls()) if (typeof entry.processStart === "number") kill(entry.processStart); + server.closeAllConnections(); await new Promise(done => server.close(() => done())); + await rm(root, { recursive: true, force: true }); + }); + } +} + +const cleanups: Array<() => Promise> = []; + +const alive = (pid: number) => { + try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; throw error; } + try { return !execFileSync("ps", ["-p", String(pid), "-o", "stat="], { encoding: "utf8" }).trim().startsWith("Z"); } + catch (error) { if ((error as { status?: number }).status === 1) return false; throw error; } +}; + +const runTerminals = (events: Array<{ eventType: string; payload: Record }>) => events.filter(event => event.eventType === "run.terminal"); +const commands = async (calls: () => Promise>>) => (await calls()).map(entry => entry.command).filter(Boolean); + +it.skipIf(process.platform === "win32")("ends a wait through the real runnerd when the sidecar exits while a descendant keeps its output open", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm", true); + const awaited = (await run.coreEvents()).find(event => event.eventType === "turn.completed"); + expect(awaited?.payload.awaitingProviderTurn).toBe(true); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 }); + const events = await run.coreEvents(); + expect(runTerminals(events)[0]!.payload).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" }); + expect(events.findIndex(event => event.eventType === "run.terminal")).toBeGreaterThan(events.findIndex(event => event.eventType === "turn.completed")); + expect(await commands(run.calls)).toContain("session.provider_turn"); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + +it.skipIf(process.platform === "win32")("ends a wait through the real runnerd when the sidecar reports the provider lost", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold provider-lost-after-arm", true); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 }); + const events = await run.coreEvents(); + expect(events.find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBe(true); + expect(events.some(event => event.payload.code === "ACPX_PROVIDER_PROCESS_LOST")).toBe(true); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + +it.skipIf(process.platform === "win32")("emits run.terminal with the completed turn and never arms without the controller opt-in", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm", false); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 5_000, interval: 50 }); + const events = await run.coreEvents(); + expect(events.find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBeUndefined(); + expect(await commands(run.calls)).not.toContain("session.provider_turn"); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + +it.skipIf(process.platform === "win32")("cancelling the run during a wait disarms the provider, publishes the held terminal and admits no later provider turn", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold provider-turn-after-disarm", true); + expect(runTerminals(await run.coreEvents())).toHaveLength(0); + run.authority.queueCommand("run.cancel", { reason: "Operator stop" }, undefined, true); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 }); + expect(runTerminals(await run.coreEvents())[0]!.payload).toMatchObject({ turnTerminalState: "completed" }); + expect((await run.calls()).some(entry => entry.command === "session.provider_turn" && (entry.params as { armed?: unknown }).armed === false)).toBe(true); + await new Promise(done => setTimeout(done, 1_000)); + const events = await run.coreEvents(); + expect(events.some(event => event.payload.origin === "provider")).toBe(false); + expect(runTerminals(events)).toHaveLength(1); + run.authority.queueCommand("session.close", { reason: "Run cancelled" }, undefined, true); + const processes = (await run.calls()).flatMap(entry => [entry.processStart, entry.childPid]).filter((pid): pid is number => typeof pid === "number"); + expect(processes.length).toBeGreaterThan(0); + await vi.waitFor(() => { for (const pid of processes) expect(alive(pid)).toBe(false); }, { timeout: 10_000, interval: 100 }); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + +it.skipIf(process.platform === "win32")("publishes the held run.terminal exactly once when runnerd restarts during the wait", async () => { + try { + const run = await awaitedTurn("live provider-turn provider-turn-hold", true); + expect((await run.coreEvents()).find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBe(true); + expect(runTerminals(await run.coreEvents())).toHaveLength(0); + await run.restartRunner(); + await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 }); + expect(runTerminals(await run.coreEvents())[0]!.payload).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" }); + await new Promise(done => setTimeout(done, 1_000)); + expect(runTerminals(await run.coreEvents())).toHaveLength(1); + } finally { for (const cleanup of cleanups.splice(0)) await cleanup(); } +}, 60_000); + diff --git a/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts b/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts new file mode 100644 index 00000000000..04fe166ad75 --- /dev/null +++ b/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { NativeExecutionInputV1 } from "./contracts/native-execution.js"; +import type { NativeSession } from "./contracts/native-session-backend.js"; +import { executeNativeSession } from "./native-session-runtime.js"; +import type { PrpEvent } from "./protocol/replay-contract.js"; + +const identity = { + runId: "run-provider-turn", + sessionId: "session-provider-turn", + companyId: "company-provider-turn", + issueId: "issue-provider-turn", + agentId: "agent-provider-turn", +}; + +const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { companyId: identity.companyId, runId: identity.runId, issueId: identity.issueId, agentId: identity.agentId, + executionWorkspaceId: "workspace-provider-turn" }, + task: { identifier: "PAP-PROVIDER", title: "Wait for a background install", description: null, + prompt: "# PAP-PROVIDER: Wait for a background install", workMode: "standard" }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + provider: { kind: "codex", model: null }, + completionContract: { id: "contract-provider-turn", sha256: "contract-provider-turn-sha", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Install and test", criteria: [{ id: "objective", requirement: "Install and test" }] } }, + interactionResponses: [], + credentialBindings: [], +}; + +const event = (sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record): PrpEvent => ({ + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-provider-turn:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-provider-turn", + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-1", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-11-12T00:00:00.000Z", + payload, +}); + +const usage = { provider: "acpx", model: null, providerSessionId: null, providerRequestId: null, + cumulative: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 1, providerCostUsd: 0 }, + runDeltaAvailable: true, + runDelta: { inputTokens: 6, outputTokens: 4, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 1, providerCostUsd: 0 } }; + +const turnOne = (awaiting: boolean): PrpEvent[] => [ + event(1, "turn.started", { provider: "acpx", providerTurnId: "turn-1", status: "inProgress", turn: { id: "turn-1", status: "inProgress" } }), + event(2, "usage.reported", usage), + event(3, "turn.completed", { provider: "acpx", providerTurnId: "turn-1", status: "completed", error: null, + ...(awaiting ? { awaitingProviderTurn: true } : {}) }), + ...(awaiting ? [] : [event(4, "run.terminal", { schema: "paperclip.prp.terminal.v1", status: "succeeded", turnTerminalState: "completed", + runTerminalState: "succeeded", reportedWorkDisposition: "done", provider: "acpx" })]), +]; + +const providerTurn: PrpEvent[] = [ + event(4, "turn.started", { provider: "acpx", providerTurnId: "provider-1", status: "inProgress", + turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }), + event(5, "turn.completed", { provider: "acpx", providerTurnId: "provider-1", status: "completed", error: null }), +]; + +async function finalize(stream: PrpEvent[]) { + const appended: PrpEvent[] = []; + const completeRun = vi.fn(async () => undefined); + const capabilities = { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + const session: NativeSession = { + identity: () => identity, + async capabilities() { return capabilities; }, + async *events() { for (const value of stream) yield structuredClone(value); }, + async startTurn() { return { turnId: "turn-1" }; }, + async result() { return null; }, + async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, + providerSessionId: "provider-session", cursor: String(stream.length), activeTurnId: "turn-1", pendingRuntimeRequests: [], lineage: [] }; }, + async close() {}, + }; + const outcome = await executeNativeSession({ + input, + backend: { async descriptor() { return { kind: "mock", name: "provider-turn", version: "1", capabilities }; }, async openSession() { return session; } }, + controlPlane: { + async openRun() {}, async checkpointSession() {}, + async appendEvent(value: PrpEvent) { + appended.push(structuredClone(value)); + return { cursor: value.sourceSeq, highestContiguousSourceSeq: value.sourceSeq, disposition: "committed" }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + completeRun, + }, + runnerInstanceId: "runner-provider-turn", controlPlaneInstanceId: "control-provider-turn", timeoutMs: 1000, + resolveMissingResult: async () => null, + }).then(() => "finalized", (error: unknown) => String(error instanceof Error ? error.message : error)); + return { outcome, appended, completeRun }; +} + +describe("today's controller consumer with provider-started turn sequences", () => { + it("finalizes a no-result turn with live shells exactly as before when the controller did not opt in", async () => { + const withoutOptIn = await finalize(turnOne(false)); + expect(withoutOptIn.outcome).toContain("native_finalization_missing"); + expect(withoutOptIn.completeRun).not.toHaveBeenCalled(); + expect(withoutOptIn.appended.map(value => value.eventType)).toEqual(["turn.started", "usage.reported", "turn.completed"]); + }); + + it("would end the run at the awaited terminal and never consume the provider-started turn, so the opt-in must stay off", async () => { + const optedIn = await finalize([...turnOne(true), ...providerTurn]); + expect(optedIn.outcome).toContain("native_finalization_missing"); + expect(optedIn.appended.some(value => value.payload.origin === "provider")).toBe(false); + }); +}); diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 3149ee1cc5f..47973de810c 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -973,6 +973,7 @@ export const LIVE_EVENT_TYPES = [ "heartbeat.run.event", "heartbeat.run.log", "heartbeat.run.activity", + "heartbeat.run.tool_progress", "agent.session.goal.changed", "agent.status", "agent.runtime_state.updated", diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 2d2b5ba044c..efab7ad18d8 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1458,6 +1458,9 @@ export type { RejectSecretProposalInput, ConnectToolAppAuthChallenge, ConnectToolAppResult, + MachineSignInPublicKey, + MachineSignInStatus, + MachineSignInSetupResult, ToolAppMetadataPreflightAttempt, ToolAppMetadataPreflightResult, ToolOAuthClientRegistrationSource, @@ -2302,6 +2305,15 @@ export { genericMcpOAuthClientSchema, type GenericMcpAuthMode, type GenericMcpOAuthClient, + MACHINE_SIGN_IN_METHODS, + machineSignInMethodSchema, + createMachineSignInSchema, + testMachineSignInSchema, + rotateMachineSignInSchema, + type MachineSignInMethod, + type CreateMachineSignIn, + type TestMachineSignIn, + type RotateMachineSignIn, reconnectToolAppSchema, createToolApplicationSchema, finishToolAppSchema, @@ -2966,5 +2978,6 @@ export { } from "./issue-activity-identity.js"; export { recognizeLinkPreview, type LinkPreviewTarget, type LinkPreviewState, type LinkPreview, type LinkPreviewResult } from "./link-preview.js"; export * from "./mcp-apps.js"; +export * from "./tool-call-progress.js"; export * from "./mcp-skills.js"; export type { TailscaleConnection, TailscaleDeviceLogin } from "./types/tailscale.js"; diff --git a/packages/shared/src/tool-call-progress.ts b/packages/shared/src/tool-call-progress.ts new file mode 100644 index 00000000000..89655767bfd --- /dev/null +++ b/packages/shared/src/tool-call-progress.ts @@ -0,0 +1,68 @@ +/** + * Live progress of one connected-app tool call, from the provider's MCP + * `notifications/progress`. Everything here is untrusted provider content: + * numbers are checked and the message is plain text, bounded in length. + */ +export const TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH = 200; + +export interface ToolCallProgress { + /** Increases with each update. Never shown as a raw number without `total`. */ + progress: number; + /** Known end value; absent when the provider does not know the total. */ + total: number | null; + /** The provider's latest status line, plain text. */ + message: string | null; + /** When the provider last reported progress (ISO). */ + updatedAt: string; +} + +/** A running tool call's progress, as the task chat reads it. */ +export interface ToolCallProgressSnapshot extends ToolCallProgress { + invocationId: string; + runId: string; + /** The tool name the agent called through Paperclip. */ + toolName: string; + /** The name native runners give that tool (shortened, with a hash); their transcripts show it. */ + nativeToolName?: string | null; + /** When the call started (ISO), for elapsed time. */ + startedAt: string; + /** True once the call has finished; the row then drops its progress. */ + done: boolean; +} + +function finite(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +/** Plain, single-line text: control characters removed, whitespace collapsed, bounded. */ +export function boundToolCallProgressMessage(value: unknown): string | null { + if (typeof value !== "string") return null; + // eslint-disable-next-line no-control-regex + const plain = value.replace(/[\u0000-\u001f\u007f-\u009f​-‏‪-‮⁦-⁩]+/g, " ").replace(/\s+/g, " ").trim(); + if (!plain) return null; + return plain.length > TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH + ? `${plain.slice(0, TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH - 1)}…` + : plain; +} + +/** + * The progress a provider reported, or null when it is not usable. `total` is + * kept only when it is positive and at least the progress so far. + */ +export function normalizeToolCallProgress(input: { progress?: unknown; total?: unknown; message?: unknown }, now = new Date()): ToolCallProgress | null { + const progress = finite(input.progress); + if (progress === null || progress < 0) return null; + const total = finite(input.total); + return { + progress, + total: total !== null && total > 0 && total >= progress ? total : null, + message: boundToolCallProgressMessage(input.message), + updatedAt: now.toISOString(), + }; +} + +/** A fraction in [0, 1] when the total is known, otherwise null (indeterminate). */ +export function toolCallProgressFraction(progress: Pick): number | null { + if (progress.total === null || progress.total <= 0) return null; + return Math.min(1, Math.max(0, progress.progress / progress.total)); +} diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index f63f67250d6..8abf1f4ed91 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -525,6 +525,9 @@ export type { ToolAccessSelector, ConnectToolAppAuthChallenge, ConnectToolAppResult, + MachineSignInPublicKey, + MachineSignInStatus, + MachineSignInSetupResult, ToolAppMetadataPreflightAttempt, ToolAppMetadataPreflightResult, FinishToolAppResult, diff --git a/packages/shared/src/types/tool-access.ts b/packages/shared/src/types/tool-access.ts index 4c05d176f9b..1d2c29fb7c0 100644 --- a/packages/shared/src/types/tool-access.ts +++ b/packages/shared/src/types/tool-access.ts @@ -1221,6 +1221,39 @@ export interface ConnectToolAppAuthChallenge { manualClientRequired?: boolean; } +/** Public half of a key Paperclip holds for machine sign-in. Safe to show and share. */ +export interface MachineSignInPublicKey { + keyId: string; + algorithm: "RS256" | "ES256"; + /** PEM (SPKI) for providers that take a certificate-style key. */ + pem: string; + /** JWK for providers that take a JSON key. */ + jwk: Record; + createdAt: string; +} + +/** Machine sign-in settings for one organization connection. Never includes secret or private key values. */ +export interface MachineSignInStatus { + connectionId: string; + method: "paperclip_key" | "client_secret"; + clientId: string | null; + /** The sign-in service this connection's credentials are bound to. */ + issuer: string | null; + tokenUrl: string; + scopes: string[]; + publicKey: MachineSignInPublicKey | null; + /** A replacement key waiting to be registered with the provider. */ + pendingPublicKey: MachineSignInPublicKey | null; + hasClientSecret: boolean; + lastTokenAt: string | null; +} + +export interface MachineSignInSetupResult { + connectionId: string; + connection: ToolConnection; + machineSignIn: MachineSignInStatus; +} + export interface ConnectToolAppResult { connectionId: string; application: ToolApplication; diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index a62024f564b..b0bc5449eb5 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -919,6 +919,15 @@ export { genericMcpOAuthClientSchema, type GenericMcpAuthMode, type GenericMcpOAuthClient, + MACHINE_SIGN_IN_METHODS, + machineSignInMethodSchema, + createMachineSignInSchema, + testMachineSignInSchema, + rotateMachineSignInSchema, + type MachineSignInMethod, + type CreateMachineSignIn, + type TestMachineSignIn, + type RotateMachineSignIn, reconnectToolAppSchema, finishToolAppSchema, finalizeOAuthAccessSchema, diff --git a/packages/shared/src/validators/tool-access.ts b/packages/shared/src/validators/tool-access.ts index cd28549a930..f6ada4ebdc2 100644 --- a/packages/shared/src/validators/tool-access.ts +++ b/packages/shared/src/validators/tool-access.ts @@ -372,6 +372,45 @@ export const genericMcpOAuthClientSchema = z.object({ export type GenericMcpOAuthClient = z.infer; +/** + * Machine sign-in (MCP OAuth client credentials) for organization-owned + * connections. "paperclip_key" means Paperclip holds a signing key and the admin + * registers its public half with the provider; "client_secret" stores a secret + * the provider issued. Secret values are write-only. + */ +export const MACHINE_SIGN_IN_METHODS = ["paperclip_key", "client_secret"] as const; +export const machineSignInMethodSchema = z.enum(MACHINE_SIGN_IN_METHODS); +export type MachineSignInMethod = z.infer; + +const machineSignInScopesSchema = z.string().trim().max(2048) + .refine((value) => !/[\u0000-\u001f"\\]/.test(value), "Scopes can only contain letters, numbers and punctuation, separated by spaces."); + +export const createMachineSignInSchema = z.object({ + link: z.string().trim().url("Enter the server's full address, starting with https://").max(4096), + name: z.string().trim().min(1).max(160).optional(), + /** The sign-in service's address, or its token URL. Omitted means "find it from the server". */ + authorizationServer: z.string().trim().url("Enter the sign-in service's full address, starting with https://").max(4096).optional(), + method: machineSignInMethodSchema, + clientId: z.string().trim().min(1).max(4096).optional(), + clientSecret: z.string().min(1).max(16384).optional(), + scopes: machineSignInScopesSchema.optional(), + applicationId: z.string().uuid().optional(), +}).strict(); +export type CreateMachineSignIn = z.infer; + +export const testMachineSignInSchema = z.object({ + clientId: z.string().trim().min(1).max(4096).optional(), + clientSecret: z.string().min(1).max(16384).optional(), + scopes: machineSignInScopesSchema.optional(), +}).strict(); +export type TestMachineSignIn = z.infer; + +export const rotateMachineSignInSchema = z.discriminatedUnion("method", [ + z.object({ method: z.literal("paperclip_key") }).strict(), + z.object({ method: z.literal("client_secret"), clientSecret: z.string().min(1).max(16384) }).strict(), +]); +export type RotateMachineSignIn = z.infer; + /** * Reject `headers.*` credential paths whose header name Paperclip refuses to * send, and any value that could split the outbound request. This runs at the diff --git a/scripts/__tests__/merge-queue-checks-workflow.test.mjs b/scripts/__tests__/merge-queue-checks-workflow.test.mjs new file mode 100644 index 00000000000..f2ec7f27e9c --- /dev/null +++ b/scripts/__tests__/merge-queue-checks-workflow.test.mjs @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const workflows = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".github", "workflows"); +const workflow = readFileSync(path.join(workflows, "merge-queue-checks.yml"), "utf8"); +const jobNames = [...workflow.matchAll(/^ name: (.+)$/gm)].map(match => match[1]); + +test("the merge queue reports the check names that pull requests report", () => { + const caller = readFileSync(path.join(workflows, "pr.yml"), "utf8"); + const trusted = readFileSync(path.join(workflows, "pr-trusted.yml"), "utf8"); + assert.match(caller, /^ ci:\n(?: {4}.*\n)*? {4}uses: .+\/pr-trusted\.yml@/m, "pr.yml reports its checks under the ci job"); + for (const job of ["verify", "e2e"]) { + assert.match(trusted, new RegExp(`^ ${job}:\\n name: ${job}$`, "m"), `pull requests report ci / ${job}`); + } + assert.deepEqual(jobNames, ["ci / verify", "ci / e2e"]); +}); + +test("the merge queue checks cannot read the queued commit or any credential", () => { + const triggers = workflow.split("\non:\n")[1].split("\n\n")[0]; + assert.equal(triggers.trim(), "merge_group:", "only the merge queue triggers these checks"); + assert.match(workflow, /^permissions: \{\}$/m); + assert.doesNotMatch(workflow, /^ +permissions:/m, "no job widens the empty token"); + assert.doesNotMatch(workflow, /secrets\.|uses:|environment:|id-token|runs-on\//); + assert.deepEqual([...workflow.matchAll(/^ runs-on: (.+)$/gm)].map(match => match[1]), ["ubuntu-latest", "ubuntu-latest"]); +}); diff --git a/scripts/merge-group-image.mjs b/scripts/merge-group-image.mjs index b22e311a468..19f8be48249 100644 --- a/scripts/merge-group-image.mjs +++ b/scripts/merge-group-image.mjs @@ -21,6 +21,10 @@ export const artifactName = arch => `merge-group-image-${arch}`; const architectures = ["amd64", "arm64"]; const indexTypes = ["application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json"]; const manifestTypes = ["application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json"]; +// How long a master push waits for a merge-queue build that is still running. +// A build takes about 12 minutes and the exact-source gate about 14. +const findWaitMs = 15 * 60_000; +const findPollMs = 20_000; const fullSha = /^[a-f0-9]{40}$/; const digestPattern = /^sha256:[a-f0-9]{64}$/; const sha256 = bytes => `sha256:${createHash("sha256").update(bytes).digest("hex")}`; @@ -43,6 +47,11 @@ export function selectVerifiedRun({ sha, runs }) { typeof run.head_branch === "string" && run.head_branch.startsWith("gh-readonly-queue/master/")); if (eligible.length === 0) return { reason: "no merge-queue image build exists for this commit" }; const passed = eligible.filter(run => run.status === "completed" && run.conclusion === "success").sort((a, b) => b.id - a.id); + // The queue can land a commit while its image is still building. `pending` + // tells the caller that waiting can still produce a usable run. + if (passed.length === 0 && eligible.some(run => run.status !== "completed")) { + return { pending: true, reason: "the merge-queue image build for this commit did not finish in time" }; + } if (passed.length === 0) return { reason: "the merge-queue image build for this commit did not succeed" }; return { runId: passed[0].id }; } @@ -163,8 +172,18 @@ async function main(command) { assert.ok(response.ok, `GitHub API ${path}: HTTP ${response.status}`); return response.json(); }; - const { workflow_runs: runs } = await api(`/actions/workflows/docker-merge-group.yml/runs?event=merge_group&head_sha=${sha}&per_page=20`); - const run = selectVerifiedRun({ sha, runs }); + // Publication waits for the exact-source gate in any case, so waiting here + // for an unfinished build costs nothing while the build is the shorter of + // the two. Past the deadline the push builds from source. + const deadline = Date.now() + findWaitMs; + let run; + for (;;) { + const { workflow_runs: runs } = await api(`/actions/workflows/docker-merge-group.yml/runs?event=merge_group&head_sha=${sha}&per_page=20`); + run = selectVerifiedRun({ sha, runs }); + if (!run.pending || Date.now() >= deadline) break; + console.log("The merge-queue image build for this commit is still running. Waiting."); + await new Promise(resolve => setTimeout(resolve, findPollMs)); + } if (!run.runId) { console.log(`No promotion: ${run.reason}. Building from source.`); return; } const { artifacts } = await api(`/actions/runs/${run.runId}/artifacts?per_page=100`); const artifact = selectArtifact({ artifacts, arch }); diff --git a/scripts/merge-group-image.test.mjs b/scripts/merge-group-image.test.mjs index 5654c65e415..c582abefae6 100644 --- a/scripts/merge-group-image.test.mjs +++ b/scripts/merge-group-image.test.mjs @@ -30,8 +30,17 @@ test("promotes only a successful merge-queue build of this repository for this e ["a queue for another branch", run({ head_branch: "gh-readonly-queue/release/pr-1-x" })], ["an ordinary branch", run({ head_branch: "master" })], ]) assert.match(selectVerifiedRun({ sha, runs: [candidate] }).reason, /no merge-queue image build/, label); - for (const candidate of [run({ conclusion: "failure" }), run({ conclusion: "cancelled" }), run({ status: "in_progress", conclusion: null })]) { - assert.match(selectVerifiedRun({ sha, runs: [candidate] }).reason, /did not succeed/); + for (const candidate of [run({ conclusion: "failure" }), run({ conclusion: "cancelled" })]) { + assert.deepEqual(selectVerifiedRun({ sha, runs: [candidate] }), { reason: "the merge-queue image build for this commit did not succeed" }); + } + // An unfinished build is worth waiting for, even beside a failed attempt. A + // finished one is used at once, and an unfinished run of another commit is not. + for (const status of ["queued", "in_progress"]) { + const unfinished = run({ id: 11, status, conclusion: null }); + assert.equal(selectVerifiedRun({ sha, runs: [unfinished] }).pending, true); + assert.equal(selectVerifiedRun({ sha, runs: [run({ conclusion: "failure" }), unfinished] }).pending, true); + assert.deepEqual(selectVerifiedRun({ sha, runs: [run(), unfinished] }), { runId: 10 }); + assert.equal(selectVerifiedRun({ sha, runs: [run({ id: 11, status, conclusion: null, head_sha: "b".repeat(40) })] }).pending, undefined); } assert.throws(() => selectVerifiedRun({ sha: "master", runs: [] })); }); diff --git a/server/src/__tests__/agent-connection-catalog.integration.test.ts b/server/src/__tests__/agent-connection-catalog.integration.test.ts new file mode 100644 index 00000000000..e92d11bafaa --- /dev/null +++ b/server/src/__tests__/agent-connection-catalog.integration.test.ts @@ -0,0 +1,214 @@ +import { randomUUID } from "node:crypto"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, authUsers, companies, companyMemberships, companySecretBindings, connectionGrants, createDb, principalPermissionGrants, toolApplications, toolCatalogEntries, toolConnections } from "@paperclipai/db"; +import { and, eq } from "drizzle-orm"; +import { syncConnectionCredentialBindings } from "../services/connection-credential-bindings.js"; +import { connectionManagementService } from "../services/connection-management.js"; +import { secretService } from "../services/secrets.js"; +import { toolAccessService } from "../services/tool-access.js"; +import type { PaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +describe("dedicated agent connection discovery", () => { + let database: Awaited>; + let db: ReturnType; + let provider: Server; + let origin: string; + let requests: Array; + let rejectCredential: boolean; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("agent-connection-catalog-"); + db = createDb(database.connectionString); + provider = createServer(async (req, res) => { + if (req.method !== "POST") { res.writeHead(404); res.end(); return; } + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const message = JSON.parse(Buffer.concat(chunks).toString()); + requests.push(req.headers.authorization); + if (rejectCredential || req.headers.authorization !== "Bearer fixture-agent-access") { + res.writeHead(401, { "www-authenticate": "Bearer", "content-type": "application/json" }); + res.end(JSON.stringify({ error: "unauthorized" })); + return; + } + if (message.method === "notifications/initialized") { + res.writeHead(202); res.end(); return; + } + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ jsonrpc: "2.0", id: message.id, result: message.method === "initialize" + ? { protocolVersion: "2025-03-26", capabilities: { tools: {} }, serverInfo: { name: "fixture", version: "1" } } + : { tools: [{ name: "get_fixture", description: "Read a fixture", inputSchema: { type: "object" }, annotations: { readOnlyHint: true } }] } })); + }); + await new Promise(resolve => provider.listen(0, "127.0.0.1", resolve)); + origin = `http://127.0.0.1:${(provider.address() as AddressInfo).port}`; + }); + + beforeEach(() => { + requests = []; + rejectCredential = false; + const localFetch = globalThis.fetch; + vi.spyOn(globalThis, "fetch").mockImplementation(async (url, init) => { + const href = String(url); + if (href.startsWith(`${origin}/`)) return localFetch(url, init); + if (href === "https://api.github.com/user") return Response.json({ id: 42, login: "fixture-bot" }); + if (href.startsWith("https://api.github.com/user/installations?")) return Response.json({ installations: [{ id: 101, repository_selection: "selected", account: { login: "example" } }] }); + if (href.startsWith("https://api.github.com/user/installations/101/repositories?")) return Response.json({ total_count: 1, repositories: [{ id: 1, full_name: "example/fixture" }] }); + throw new Error("Unexpected provider request"); + }); + }); + afterEach(() => vi.restoreAllMocks()); + afterAll(async () => { + if (provider) await new Promise(resolve => provider.close(() => resolve())); + await db?.$client.end({ timeout: 1 }); + await database?.cleanup(); + }); + + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Agent catalog fixture", issuePrefix: `AC${randomUUID().slice(0, 6)}` }).returning(); + const [agent, otherAgent] = await db.insert(agents).values(["Owner", "Other"].map(name => ({ companyId: company.id, name, role: "engineer", adapterType: "process" }))).returning(); + const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "GitHub", type: "mcp_http" }).returning(); + const [connection] = await db.insert(toolConnections).values({ + companyId: company.id, applicationId: application.id, name: "GitHub", uid: randomUUID(), + transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_agent", status: "active", enabled: true, + healthStatus: "failed", healthMessage: "OAuth credentials have expired and need to be reconnected.", + config: { sourceTemplateKey: "github", url: `${origin}/mcp`, oauth: { + strategy: "paperclip_cloud_connector", provider: "github", connectorProfile: "github.code", + tokenUrl: "https://github.com/login/oauth/access_token", expiresAt: null, + } }, + }).returning(); + const secret = await secretService(db).create(company.id, { name: "Agent access", key: randomUUID(), provider: "local_encrypted", value: "fixture-agent-access" }); + const [grant] = await db.insert(connectionGrants).values({ + companyId: company.id, connectionId: connection.id, kind: "agent", subjectAgentId: agent.id, status: "active", + credentialSecretRefs: [{ secretId: secret.id, configPath: "oauth.access_token", versionSelector: "latest", required: true }], + providerTenant: { oauth: { strategy: "paperclip_cloud_connector", accessTokenExpiresAt: null }, + github: { userId: "42", login: "fixture-bot", installationCount: 1, repositoryCount: 1, repositorySelection: "selected", installationIds: ["101"], installationOwnerLogins: ["example"] } }, + }).returning(); + await syncConnectionCredentialBindings(db, connection); + const connector = { setWebhookBinding: vi.fn(async () => undefined) } as unknown as PaperclipCloudConnector; + const service = toolAccessService(db, { deploymentMode: "local_trusted", deploymentExposure: "private", paperclipCloudConnector: connector }); + return { company, agent, otherAgent, connection, grant, secret, service }; + } + + it.each(["background", "system", "operator", "agent"] as const)("renames and refreshes an agent-owned GitHub connection without reconnect (%s)", async mode => { + const f = await fixture(); + const actor = mode === "background" ? undefined : mode === "system" + ? { actorType: "system" as const } : mode === "operator" + ? { actorType: "user" as const, actorId: "fixture-operator" } + : { actorType: "agent" as const, actorId: f.agent.id }; + await f.service.updateConnection(f.connection.id, { name: "Renamed bot" }); + const refreshed = await f.service.refreshCatalog(f.connection.id, actor); + expect(refreshed.connection).toMatchObject({ name: "Renamed bot", healthStatus: "ok", enabled: true }); + expect((await f.service.getConnection(f.connection.id, f.company.id)).requiresReauthorization).toBe(false); + expect(refreshed.catalog.map(tool => tool.toolName)).toEqual(["get_fixture"]); + expect(requests.length).toBeGreaterThan(0); + expect(requests.every(header => header === "Bearer fixture-agent-access")).toBe(true); + await expect(f.service.checkHealth(f.connection.id, actor)).resolves.toMatchObject({ connection: { healthStatus: "ok" } }); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(grant.status).toBe("active"); + expect(grant.credentialSecretRefs).toEqual(f.grant.credentialSecretRefs); + const bindings = await db.select().from(companySecretBindings).where(and(eq(companySecretBindings.targetId, f.connection.id), eq(companySecretBindings.secretId, f.secret.id))); + expect(bindings).toHaveLength(1); + }); + + describe.each(["refreshCatalog", "checkHealth"] as const)("%s caller isolation", operation => { + it.each(["another agent", "another company", "missing agent identity"])("denies %s without changing the owner's connection", async scope => { + const f = await fixture(); + const actorId = scope === "another company" ? (await fixture()).agent.id : scope === "missing agent identity" ? null : f.otherAgent.id; + await f.service.refreshCatalog(f.connection.id, { actorType: "agent", actorId: f.agent.id }); + const before = await db.select().from(toolConnections).where(eq(toolConnections.id, f.connection.id)); + const catalog = await db.select().from(toolCatalogEntries).where(eq(toolCatalogEntries.connectionId, f.connection.id)); + requests = []; + vi.mocked(fetch).mockClear(); + await expect(f.service[operation](f.connection.id, { actorType: "agent", actorId })) + .rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } }); + expect(requests).toEqual([]); + expect(fetch).not.toHaveBeenCalled(); + expect(await db.select().from(toolConnections).where(eq(toolConnections.id, f.connection.id))).toEqual(before); + expect(await db.select().from(toolCatalogEntries).where(eq(toolCatalogEntries.connectionId, f.connection.id))).toEqual(catalog); + await expect(f.service.checkHealth(f.connection.id, { actorType: "agent", actorId: f.agent.id })) + .resolves.toMatchObject({ connection: { healthStatus: "ok" } }); + }); + + it("keeps the owner's tools healthy when a permitted managing agent lacks the credential", async () => { + const f = await fixture(); + const userId = randomUUID(); + await db.insert(authUsers).values({ id: userId, name: "Manager", email: `${userId}@fixture.invalid`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values([ + { companyId: f.company.id, principalType: "user", principalId: userId, membershipRole: "owner", status: "active" }, + { companyId: f.company.id, principalType: "agent", principalId: f.otherAgent.id, status: "active" }, + ]); + await db.insert(principalPermissionGrants).values([ + { companyId: f.company.id, principalType: "agent", principalId: f.otherAgent.id, permissionKey: "tools:manage_connections" }, + { companyId: f.company.id, principalType: "user", principalId: userId, permissionKey: "tools:manage_connections" }, + ]); + const managementContext = await connectionManagementService(db).authorize({ + type: "agent", agentId: f.otherAgent.id, companyId: f.company.id, onBehalfOfUserId: userId, + }, "connections:configure", { type: "connection_management", companyId: f.company.id, connectionId: f.connection.id, operation }); + expect(managementContext.decision.allowed).toBe(true); + await f.service.refreshCatalog(f.connection.id, { actorType: "agent", actorId: f.agent.id }); + const before = await db.select().from(toolConnections).where(eq(toolConnections.id, f.connection.id)); + requests = []; + vi.mocked(fetch).mockClear(); + await expect(f.service[operation](f.connection.id, { actorType: "agent", actorId: f.otherAgent.id, managementContext })) + .rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } }); + expect(requests).toEqual([]); + expect(fetch).not.toHaveBeenCalled(); + expect(await db.select().from(toolConnections).where(eq(toolConnections.id, f.connection.id))).toEqual(before); + expect((await f.service.listCatalog(f.connection.id, f.company.id)).map(tool => tool.toolName)).toEqual(["get_fixture"]); + }); + }); + + it("does not fall back to personal or organization grants when the agent grant is revoked", async () => { + const f = await fixture(); + await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, f.grant.id)); + await db.insert(connectionGrants).values([ + { companyId: f.company.id, connectionId: f.connection.id, kind: "organization", status: "active", isDefault: true, credentialSecretRefs: f.grant.credentialSecretRefs }, + { companyId: f.company.id, connectionId: f.connection.id, kind: "user", subjectUserId: "fixture-operator", status: "active", credentialSecretRefs: f.grant.credentialSecretRefs }, + ]); + await expect(f.service.refreshCatalog(f.connection.id)) + .rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } }); + await expect(f.service.refreshCatalog(f.connection.id, { actorType: "user", actorId: "fixture-operator" })) + .rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } }); + expect((await f.service.getConnection(f.connection.id, f.company.id)).requiresReauthorization).toBe(true); + await expect(f.service.checkHealth(f.connection.id)) + .rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } }); + expect(requests).toEqual([]); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("uses the same agent grant for generic OAuth MCP health and discovery", async () => { + const f = await fixture(); + await f.service.updateConnection(f.connection.id, { config: { url: `${origin}/mcp`, oauth: { + provider: origin, tokenUrl: `${origin}/token`, expiresAt: null, + } } }); + await expect(f.service.refreshCatalog(f.connection.id)).resolves.toMatchObject({ connection: { healthStatus: "ok" } }); + await expect(f.service.checkHealth(f.connection.id)).resolves.toMatchObject({ connection: { healthStatus: "ok" } }); + expect(requests.length).toBeGreaterThan(0); + expect(requests.every(header => header === "Bearer fixture-agent-access")).toBe(true); + }); + + it("does not choose between multiple active agent authorizations in background work", async () => { + const f = await fixture(); + await db.insert(connectionGrants).values({ companyId: f.company.id, connectionId: f.connection.id, + kind: "agent", subjectAgentId: f.otherAgent.id, status: "active", credentialSecretRefs: f.grant.credentialSecretRefs }); + await expect(f.service.refreshCatalog(f.connection.id)) + .rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_ambiguous" } }); + await expect(f.service.checkHealth(f.connection.id)) + .rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_ambiguous" } }); + expect(requests).toEqual([]); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("still requires reconnect when the provider rejects the agent's credential", async () => { + const f = await fixture(); + rejectCredential = true; + await expect(f.service.refreshCatalog(f.connection.id)).rejects.toThrow(); + expect(requests.length).toBeGreaterThan(0); + expect(requests.every(header => header === "Bearer fixture-agent-access")).toBe(true); + const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(grant.status).toBe("needs_reauthorization"); + expect((await f.service.getConnection(f.connection.id, f.company.id)).requiresReauthorization).toBe(true); + }); +}); diff --git a/server/src/__tests__/assigned-work-stalled-queued-run.test.ts b/server/src/__tests__/assigned-work-stalled-queued-run.test.ts index 5b006a9c68b..f812d312305 100644 --- a/server/src/__tests__/assigned-work-stalled-queued-run.test.ts +++ b/server/src/__tests__/assigned-work-stalled-queued-run.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { activityLog, agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { activityLog, agents, agentWakeupRequests, automaticRunBrakes, companies, createDb, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat/queue.js"; @@ -13,6 +13,7 @@ import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY } from "../services/chat-control-re import { ASSIGNED_WORK_CONTINUED_ACTION, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; import { CONTINUATION_CAP_WINDOW_MS } from "../services/recovery/assigned-work-invariant.js"; import { logger } from "../middleware/logger.js"; +import { automaticRunBrake, automaticRunCreationHeldUntil } from "../services/recovery/automatic-run-brake.js"; const support = await getEmbeddedPostgresTestSupport(); const MINUTE = 60_000; @@ -25,7 +26,7 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o db = createDb(database.connectionString); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); - afterEach(() => { vi.restoreAllMocks(); }); + afterEach(() => { vi.restoreAllMocks(); automaticRunBrake.reset(); }); async function seed() { const prefix = `S${randomUUID().slice(0, 5)}`; @@ -104,7 +105,7 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o const readRun = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; const continuations = async () => db.select().from(activityLog).where(and(eq(activityLog.entityId, issue.id), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); - return { companyId, agentId, issue, queuedRun, otherStarts, olderRun, queue, sweep, runsOfTask, readRun, continuations }; + return { companyId, agentId, issue, other, queuedRun, otherStarts, olderRun, queue, sweep, runsOfTask, readRun, continuations }; } async function replacementOf(f: Awaited>, stalledId: string) { @@ -112,6 +113,32 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o return runs.filter(run => run.id !== stalledId); } + it("a replacement spends the paused company's creation slot", async () => { + const f = await seed(); + const stalled = await f.queuedRun(); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + await db.insert(automaticRunBrakes).values({ companyId: f.companyId, state: "open", openedAt: new Date(), stateChangedAt: new Date() }); + expect(await automaticRunCreationHeldUntil(db, f.companyId, { issueId: f.issue.id, sourceRunId: stalled.id })).toBeNull(); + + const check = await f.sweep().checkIssue({ companyId: f.companyId, issueId: f.issue.id }); + expect(check).toMatchObject({ outcome: "replaced_stalled_run" }); + const [brake] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, f.companyId)); + expect(brake!.nextCreationAt!.getTime()).toBeGreaterThan(Date.now()); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + it("an old finished request from a person does not exempt a replacement from the pause", async () => { + const f = await seed(); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "on_demand", triggerDetail: "manual", + reason: "task_retry", payload: { issueId: f.issue.id }, status: "completed", requestedByActorType: "user", requestedByActorId: "person-1", + requestedAt: new Date(Date.now() - 4 * 60 * MINUTE), finishedAt: new Date(Date.now() - 4 * 60 * MINUTE) }); + const stalled = await f.queuedRun(); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + await db.insert(automaticRunBrakes).values({ companyId: f.companyId, state: "open", openedAt: new Date(), stateChangedAt: new Date() }); + expect(await f.sweep().checkIssue({ companyId: f.companyId, issueId: f.issue.id })).toMatchObject({ outcome: "replaced_stalled_run" }); + const [brake] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, f.companyId)); + expect(brake!.nextCreationAt!.getTime()).toBeGreaterThan(Date.now()); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + it("cancels an automatic run the claim keeps refusing and queues exactly one claimable replacement", async () => { const f = await seed(); const stalled = await f.queuedRun(); @@ -191,6 +218,40 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o expect(await f.continuations()).toHaveLength(0); }, 60_000); + it("leaves an automatic run that waits behind work on tasks people created", async () => { + // Work people wait on goes first by design; a system run behind it is waiting its turn, not stalled. + const f = await seed(); + await db.update(issues).set({ createdByUserId: "person-1" }).where(eq(issues.id, f.other.id)); + const waiting = await f.queuedRun({ refused: false }); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 3, 10 * MINUTE); + const sweep = f.sweep(); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id })) + .toMatchObject({ verdict: { kind: "ok", clause: "live_run", runId: waiting.id } }); + expect((await f.readRun(waiting.id)).status).toBe("queued"); + expect(await f.continuations()).toHaveLength(0); + }, 60_000); + + it("leaves an automatic run that waits behind continuations of a person's Retry", async () => { + // The dispatcher ranks a recovery run as person work when its source chain leads to a person's request. + const f = await seed(); + const waiting = await f.queuedRun({ refused: false }); + const personSourceAt = new Date(Date.now() - 30 * MINUTE); + const [personSource] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, invocationSource: "on_demand", + triggerDetail: "manual", status: "failed", createdAt: personSourceAt, startedAt: personSourceAt, finishedAt: personSourceAt, + contextSnapshot: { issueId: f.other.id, taskId: f.other.id, wakeReason: "task_retry" } }).returning(); + for (let index = 0; index < OLDEST_DISPATCH_SHARE + 3; index++) { + const startedAt = new Date(Date.now() - 10 * MINUTE + index * 1_000); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, invocationSource: "automation", triggerDetail: "system", + status: "succeeded", createdAt: new Date(startedAt.getTime() - 1_000), startedAt, finishedAt: new Date(startedAt.getTime() + 500), + contextSnapshot: { issueId: f.other.id, taskId: f.other.id, wakeReason: "issue_recovery_action_restored", previousRunId: personSource!.id } }); + } + const sweep = f.sweep(); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id })) + .toMatchObject({ verdict: { kind: "ok", clause: "live_run", runId: waiting.id } }); + expect((await f.readRun(waiting.id)).status).toBe("queued"); + expect(await f.continuations()).toHaveLength(0); + }, 60_000); + it("leaves a run that waits behind a full agent", async () => { const f = await seed(); const waiting = await f.queuedRun(); diff --git a/server/src/__tests__/automatic-run-brake-import.test.ts b/server/src/__tests__/automatic-run-brake-import.test.ts new file mode 100644 index 00000000000..a46adb46a18 --- /dev/null +++ b/server/src/__tests__/automatic-run-brake-import.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; + +describe("automatic-run brake module graph", () => { + it("loads with the server's heartbeat and recovery modules in the order the server imports them", async () => { + await expect(import("../services/heartbeat.js")).resolves.toBeDefined(); + const brake = await import("../services/recovery/automatic-run-brake.js"); + const rules = await brake.automaticRunOutcomeRules(); + expect(rules.isTransientSetupCode("environment_provider_unavailable")).toBe(true); + expect(rules.isPersonOwnedFailureCode("budget_hard_stop")).toBe(true); + }, 600_000); +}); diff --git a/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts b/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts new file mode 100644 index 00000000000..a8830f57ea7 --- /dev/null +++ b/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts @@ -0,0 +1,495 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + activityLog, agentWakeupRequests, agents, automaticRunBrakes, companies, createDb, heartbeatRuns, issueRecoveryActions, issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { AUTOMATIC_RUN_BRAKE, automaticRunBrake } from "../services/recovery/automatic-run-brake.js"; +import { recoveryService } from "../services/recovery/service.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION } from "../services/recovery/assigned-work-sweep.js"; +import { taskAutomaticAttempts } from "../services/recovery/task-automatic-attempts.js"; +import { CONTINUATION_CAP } from "../services/recovery/assigned-work-invariant.js"; +import { deliverReconciledExecutions } from "../services/execution-recovery-resolution.js"; +import { continueHeldExecutions, resetBacklogContinuationPacingForTests } from "../services/fresh-continuation-fallback.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; + +suite("automatic-run brake in the recovery passes", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let companyId = "", agentId = ""; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-automatic-run-brake-passes-"); + db = createDb(temporary.connectionString); + }, 60_000); + + afterEach(async () => { + automaticRunBrake.reset(); + resetBacklogContinuationPacingForTests(db); + await db.execute(sql`truncate table automatic_run_brakes, issue_recovery_actions, native_run_finalizations, activity_log, + agent_wakeup_requests, heartbeat_runs, issues, agents, companies cascade`); + }, 120_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }, 120_000); + + async function seedCompany() { + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Passes", issuePrefix: `P${companyId.replaceAll("-", "").slice(0, 6)}`, + requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "Builder", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + } + + const ago = (ms: number) => new Date(Date.now() - ms); + + async function openBrake(options: { startingAutomaticRun?: boolean } = { startingAutomaticRun: true }) { + const failures = await db.insert(heartbeatRuns).values(Array.from({ length: 20 }, (_, index) => ({ companyId, agentId, + invocationSource: "automation", triggerDetail: "system", runtimeMode: "legacy", status: "failed", + errorCode: "environment_provider_unavailable", executionStage: "preparing", createdAt: ago(4 * MINUTE + index), + startedAt: ago(3 * MINUTE + index), finishedAt: ago(MINUTE - index * 100), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() } }))).returning(); + if (options.startingAutomaticRun) { + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", triggerDetail: "system", + status: "running", runtimeMode: "legacy", executionStage: "preparing", startedAt: ago(20_000), + contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: failures[0]!.id, issueId: randomUUID() } }); + } + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("open"); + return failures; + } + + async function recover(failures: Array<{ id: string }>) { + await db.update(heartbeatRuns).set({ createdAt: sql`created_at - interval '1 hour'`, startedAt: sql`started_at - interval '1 hour'`, + finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id))); + await db.insert(heartbeatRuns).values(Array.from({ length: 2 }, () => ({ companyId, agentId, invocationSource: "automation", + triggerDetail: "system", runtimeMode: "legacy", status: "succeeded", executionStage: "dispatching", createdAt: ago(MINUTE), + startedAt: ago(50_000), finishedAt: ago(1_000), contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() } }))); + await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at - ${`${AUTOMATIC_RUN_BRAKE.minOpenMs} milliseconds`}::interval` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("half_open"); + } + + async function strandedTask() { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue the work", status: "in_progress", assigneeAgentId: agentId, + updatedAt: ago(3 * HOUR), createdAt: ago(5 * HOUR) }); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "on_demand", status: "interrupted", + errorCode: "server_shutdown_interrupted", startedAt: ago(4 * HOUR), finishedAt: ago(3 * HOUR), livenessState: "needs_followup", + createdAt: ago(4 * HOUR), contextSnapshot: { issueId, taskId: issueId }, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + return issueId; + } + + function wakeQueue() { + const enqueueWakeup = vi.fn(async (wakeAgentId: string, opts?: { payload?: Record | null; reason?: string | null }) => { + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: wakeAgentId, source: "automation", + triggerDetail: "system", reason: opts?.reason ?? null, payload: opts?.payload ?? null, status: "queued" }).returning(); + return { id: wake!.id } as never; + }); + return enqueueWakeup; + } + + const ledger = async (issueId: string) => (await taskAutomaticAttempts(db, companyId, issueId, new Date())).all.length; + const continuedActivity = async (issueId: string) => db.select().from(activityLog) + .where(and(eq(activityLog.entityId, issueId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); + + it("the assigned-work sweep defers continuations while open without spending the task's attempts, and resumes by itself", async () => { + await seedCompany(); + const issueId = await strandedTask(); + const failures = await openBrake(); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + for (let pass = 0; pass < 3; pass += 1) { + const check = await recovery.assignedWork.checkIssue({ companyId, issueId }); + expect(check).toMatchObject({ outcome: "deferred_rate_limit" }); + expect(check!.retryAt!.getTime()).toBeGreaterThan(Date.now() + 30_000); + } + expect(enqueueWakeup).not.toHaveBeenCalled(); + expect(await continuedActivity(issueId)).toHaveLength(0); + expect(await ledger(issueId)).toBe(0); + expect(await recovery.assignedWork.report(companyId)).toMatchObject({ automaticWork: { state: "open", + summary: expect.stringMatching(/^Automatic work is paused for this company since .* because the last 5 automatic runs could not get a sandbox\.$/) } }); + + await recover(failures); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + expect(await ledger(issueId)).toBe(1); + + await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at + - ${`${AUTOMATIC_RUN_BRAKE.releaseSteps * AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("closed"); + expect((await recovery.assignedWork.report(companyId)).automaticWork).toMatchObject({ state: "closed", + summary: "Automatic work is running normally." }); + }, 120_000); + + it("the sweep still continues one task a minute while open when no automatic run is starting", async () => { + await seedCompany(); + const [first, second] = [await strandedTask(), await strandedTask()]; + await openBrake({ startingAutomaticRun: false }); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: first })).toMatchObject({ outcome: "repaired" }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: second })).toMatchObject({ outcome: "deferred_rate_limit" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + }, 120_000); + + it("a task that loses the creation slot to another sweep check gains no extra wait", async () => { + await seedCompany(); + const [first, second] = [await strandedTask(), await strandedTask()]; + await openBrake({ startingAutomaticRun: false }); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + const checks = await Promise.all([first, second].map(issueId => recovery.assignedWork.checkIssue({ companyId, issueId }))); + expect(checks.map(check => check!.outcome).sort()).toEqual(["deferred_rate_limit", "repaired"]); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + const loser = checks[0]!.outcome === "deferred_rate_limit" ? first : second; + expect(await ledger(loser)).toBe(0); + }, 120_000); + + it("leaves every pass exactly as before when the company has no pause record", async () => { + await seedCompany(); + const issueId = await strandedTask(); + const action = await pendingContinuation(); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + await deliverReconciledExecutions(db, wake as never); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + expect(wake).toHaveBeenCalledTimes(1); + expect(wake.mock.calls[0]![1]).toMatchObject({ reason: "issue_recovery_action_restored", contextSnapshot: { recoveryActionId: action.id } }); + expect(await db.select().from(automaticRunBrakes)).toHaveLength(0); + }, 120_000); + + it("tries again on another agent when the waiting automatic run cannot start, and closes once it works", async () => { + await seedCompany(); + const otherAgent = randomUUID(); + await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + const failures = await openBrake({ startingAutomaticRun: false }); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", triggerDetail: "system", status: "queued", + createdAt: ago(10 * MINUTE), contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: failures[0]!.id, issueId: randomUUID() } }); + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Review the work", status: "in_progress", assigneeAgentId: otherAgent, + updatedAt: ago(3 * HOUR), createdAt: ago(5 * HOUR) }); + await db.insert(heartbeatRuns).values({ companyId, agentId: otherAgent, invocationSource: "on_demand", status: "interrupted", + errorCode: "server_shutdown_interrupted", startedAt: ago(4 * HOUR), finishedAt: ago(3 * HOUR), livenessState: "needs_followup", + createdAt: ago(4 * HOUR), contextSnapshot: { issueId, taskId: issueId }, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup.mock.calls[0]![0]).toBe(otherAgent); + + await db.update(heartbeatRuns).set({ finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id))); + await db.insert(heartbeatRuns).values({ companyId, agentId: otherAgent, invocationSource: "automation", triggerDetail: "system", + status: "succeeded", runtimeMode: "legacy", executionStage: "dispatching", startedAt: ago(40_000), finishedAt: ago(1_000), + contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID(), issueId } }); + await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at - ${`${AUTOMATIC_RUN_BRAKE.minOpenMs} milliseconds`}::interval` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("half_open"); + await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at + - ${`${AUTOMATIC_RUN_BRAKE.releaseSteps * AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("closed"); + }, 120_000); + + it("a delivery its agent refuses gives the creation slot back to the next continuation in the same pass", async () => { + await seedCompany(); + const otherAgent = randomUUID(); + await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + const refused = await pendingContinuation(); + const accepted = await pendingContinuation(otherAgent, 3 * MINUTE); + await openBrake({ startingAutomaticRun: false }); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + if (wakeAgentId === agentId) return null; + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + await deliverReconciledExecutions(db, wake as never); + expect(wake.mock.calls.map(call => call[0])).toEqual([agentId, otherAgent]); + const delivery = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]! + .evidence.continuationDelivery; + expect(await delivery(refused.id)).toBe("pending"); + expect(await delivery(accepted.id)).toBe("delivered"); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull(); + }, 120_000); + + it("a capped task gives the creation slot back to the next task the sweep checks", async () => { + await seedCompany(); + const [capped, next] = [await strandedTask(), await strandedTask()]; + await openBrake({ startingAutomaticRun: false }); + await db.insert(activityLog).values(Array.from({ length: CONTINUATION_CAP }, (_, index) => ({ companyId, actorType: "system", + actorId: "assigned-work-invariant", action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: capped, + createdAt: ago(10 * MINUTE + index), details: {} }))); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: capped })).toMatchObject({ outcome: "deferred_cap" }); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull(); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: next })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + }, 120_000); + + for (const createdBeforeError of [true, false]) { + it(`a delivery whose wake fails ${createdBeforeError ? "after creating its run keeps" : "before creating a run gives back"} the creation slot`, async () => { + await seedCompany(); + const otherAgent = randomUUID(); + await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + await pendingContinuation(); + await pendingContinuation(otherAgent, 3 * MINUTE); + await openBrake({ startingAutomaticRun: false }); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + if (wakeAgentId === agentId && !createdBeforeError) throw new Error("wake refused before creating a run"); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + if (wakeAgentId === agentId) throw new Error("dispatch failed after the run was committed"); + return run!; + }); + await deliverReconciledExecutions(db, wake as never); + expect(wake.mock.calls.map(call => call[0])).toEqual(createdBeforeError ? [agentId] : [agentId, otherAgent]); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull(); + }, 120_000); + + it(`a sweep whose wake fails ${createdBeforeError ? "after creating its run keeps" : "before creating a run gives back"} the creation slot`, async () => { + await seedCompany(); + const [first, second] = [await strandedTask(), await strandedTask()]; + await openBrake({ startingAutomaticRun: false }); + const enqueueWakeup = vi.fn(async (wakeAgentId: string, opts?: { payload?: Record | null; contextSnapshot?: Record }) => { + const issueId = String(opts?.payload?.issueId); + if (issueId === first && !createdBeforeError) throw new Error("wake refused before creating a run"); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts?.contextSnapshot ?? { issueId } }).returning(); + if (issueId === first) throw new Error("dispatch failed after the run was committed"); + return { id: run!.id } as never; + }); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + await recovery.assignedWork.checkIssue({ companyId, issueId: first }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId: second })) + .toMatchObject({ outcome: createdBeforeError ? "deferred_rate_limit" : "repaired" }); + }, 120_000); + } + + async function pendingContinuation(ownerAgentId = agentId, endedAgoMs = 2 * MINUTE) { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue the work", status: "todo", assigneeAgentId: ownerAgentId }); + const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: ownerAgentId, issueId, invocationSource: "assignment", status: "failed", + errorCode: "server_shutdown_interrupted", startedAt: ago(endedAgoMs + MINUTE), finishedAt: ago(endedAgoMs), createdAt: ago(endedAgoMs + 2 * MINUTE), + contextSnapshot: { issueId, wakeReason: "issue_assigned" } }).returning(); + const recordedAt = new Date().toISOString(); + const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", outcome: "handed_back", resolvedAt: new Date(), ownerType: "board", returnOwnerAgentId: ownerAgentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${source!.id}`, nextAction: "Continue the task.", + attemptCount: 1, evidence: { runId: source!.id, originalFailureCode: "server_shutdown_interrupted", continuationDelivery: "pending", + freshContinuation: { schema: "paperclip.fresh-continuation.v1", reason: "exact_resumption_unavailable", recordedAt }, + freshContinuationSettlement: { recordedAt }, + executionReconciliation: { runId: source!.id, observedOutcome: "unknown", disposition: "continue_fresh", + actorId: "execution-recovery", recordedAt } } }).returning(); + return action!; + } + + it("the continuation delivery leaves pending continuations pending while open and delivers them after", async () => { + await seedCompany(); + const action = await pendingContinuation(); + const failures = await openBrake(); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + const delivery = async () => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0]! + .evidence.continuationDelivery; + await deliverReconciledExecutions(db, wake as never); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect(await delivery()).toBe("pending"); + await recover(failures); + await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running"))); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(await delivery()).toBe("delivered"); + }, 120_000); + + async function heldBacklog() { + const issueId = randomUUID(), runId = randomUUID(); + const finishedAt = ago(HOUR); + await db.insert(issues).values({ id: issueId, companyId, title: "Ship the notes", status: "blocked", assigneeAgentId: agentId, + statusVersion: 4, blockedTransitionAt: finishedAt }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "assignment", issueId, nativeIssueId: issueId, + runtimeMode: "native", status: "failed", errorCode: "native_restart_recovery_blocked", error: "native_restart_recovery_blocked", + startedAt: new Date(finishedAt.getTime() - 10 * MINUTE), finishedAt, createdAt: new Date(finishedAt.getTime() - 11 * MINUTE), + contextSnapshot: { issueId, wakeReason: "issue_assigned" }, nativePhase: "terminal_failure" }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "terminal_failure", recoveryState: "blocked", + failureCode: "native_continuation_requires_reconciliation", leaseOwner: null }); + const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", outcome: "blocked", resolvedAt: finishedAt, ownerType: "board", returnOwnerAgentId: agentId, + cause: "native_continuation_requires_reconciliation", fingerprint: `native-continuation:${runId}`, nextAction: "Continue.", + maxAttempts: 3, createdAt: finishedAt, evidence: { runId, originalFailureCode: "native_restart_recovery_blocked", + nativeFailureBlock: { runId, statusVersion: 4 }, automaticRecovery: { policy: "preserve_without_replay_v1", runId, replay: "blocked", + actionOutcome: "unknown", recordedAt: finishedAt.toISOString() } } }).returning(); + return { runId, actionId: action!.id }; + } + + function runWake() { + return vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + } + + const continuationRecords = async (runId: string) => db.select().from(issueRecoveryActions) + .where(sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${runId}`); + + it("the held backlog records its continuation for free and the delivery creates the run only after the pause", async () => { + await seedCompany(); + const held = await heldBacklog(); + const failures = await openBrake(); + expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 }); + expect(await continuationRecords(held.runId)).toHaveLength(1); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull(); + await recover(failures); + await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running"))); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + }, 120_000); + + it("one held task spends one creation slot while open", async () => { + await seedCompany(); + await heldBacklog(); + await openBrake({ startingAutomaticRun: false }); + expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 }); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull(); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull(); + }, 120_000); + + const userWake = (issueId: string, values: Partial) => db.insert(agentWakeupRequests).values({ + companyId, agentId, source: "on_demand", triggerDetail: "manual", reason: "task_retry", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: "board", payload: { issueId }, ...values }); + + const deliveryShapes: Array<[string, (issueId: string, sourceRunId: string) => Promise]> = [ + ["a Retry waiting behind the task", issueId => userWake(issueId, {})], + ["a move-forward Retry queued for the task", issueId => userWake(issueId, { status: "queued", + payload: { issueId, taskRetry: { schema: "paperclip.task-retry.v1", sourceRunId: randomUUID() } } })], + ["a Retry folded into the continuation", issueId => userWake(issueId, { status: "coalesced", finishedAt: new Date(), + payload: { issueId, coalescedIntoFreshContinuationRun: randomUUID() } })], + ["a person's message", issueId => userWake(issueId, { reason: "issue_commented", status: "skipped", finishedAt: new Date() })], + ["a Retry refused at dispatch while the task was held", issueId => db.insert(heartbeatRuns).values({ companyId, agentId, issueId, + invocationSource: "on_demand", triggerDetail: "manual", status: "cancelled", errorCode: "execution_reconciliation_required", + createdAt: ago(60_000), startedAt: null, finishedAt: ago(59_000), + contextSnapshot: { issueId, wakeReason: "task_retry", taskRetryRequestId: randomUUID() } })], + ["a continuation of a person's Retry", (_issueId, sourceRunId) => db.update(heartbeatRuns) + .set({ contextSnapshot: sql`${heartbeatRuns.contextSnapshot} || '{"wakeReason":"task_retry"}'::jsonb` }).where(eq(heartbeatRuns.id, sourceRunId))], + ]; + + for (const [shape, apply] of deliveryShapes) { + it(`delivers at once while paused: ${shape}`, async () => { + await seedCompany(); + const action = await pendingContinuation(); + await openBrake(); + const sourceRunId = (action.evidence.executionReconciliation as { runId: string }).runId; + await apply(action.sourceIssueId, sourceRunId); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect((await db.select().from(automaticRunBrakes))[0]).toMatchObject({ state: "open", nextCreationAt: null }); + }, 120_000); + } + + it("a held automatic continuation does not hide a person's continuation for the same agent", async () => { + await seedCompany(); + const personAction = await pendingContinuation(); + await userWake(personAction.sourceIssueId, {}); + const automaticAction = await pendingContinuation(); + await openBrake(); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(wake.mock.calls[0]![1]).toMatchObject({ payload: { issueId: personAction.sourceIssueId } }); + const delivery = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]! + .evidence.continuationDelivery; + expect(await delivery(personAction.id)).toBe("delivered"); + expect(await delivery(automaticAction.id)).toBe("pending"); + }, 120_000); + + it("held continuations take turns across passes so they cannot fill every page", async () => { + await seedCompany(); + const recent = await pendingContinuation(agentId, 2 * HOUR); + const older = await pendingContinuation(agentId, 3 * HOUR); + await openBrake(); + const wake = runWake(); + const attempted = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]! + .evidence.continuationDeliveryAttemptedAt; + await deliverReconciledExecutions(db, wake as never); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect(await attempted(recent.id)).toBeTruthy(); + expect(await attempted(older.id)).toBeTruthy(); + }, 120_000); + + it("still pauses the delivery of a continuation no person asked for", async () => { + await seedCompany(); + const action = await pendingContinuation(); + await openBrake(); + await userWake(action.sourceIssueId, { status: "completed", requestedAt: ago(HOUR), finishedAt: ago(HOUR) }); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + }, 120_000); + + const sweepShapes: Array<[string, (issueId: string) => Promise]> = [ + ["the task's last run was a person's Retry", async issueId => { + const [run] = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId, taskId: issueId, wakeReason: "task_retry" } }).where(eq(heartbeatRuns.id, run!.id)); + }], + ["a person assigned the task after its last run", issueId => userWake(issueId, { reason: "issue_assigned", status: "failed", finishedAt: new Date() })], + ["a person commented after its last run", issueId => userWake(issueId, { reason: "issue_commented", status: "skipped", finishedAt: new Date() })], + ]; + + for (const [shape, apply] of sweepShapes) { + it(`the sweep continues at once while paused: ${shape}`, async () => { + await seedCompany(); + const issueId = await strandedTask(); + await openBrake(); + await apply(issueId); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(1); + }, 120_000); + } + + it("the held backlog continues a task a person asked for while paused, and its delivery is not paused", async () => { + await seedCompany(); + const held = await heldBacklog(); + await openBrake(); + const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, held.runId)); + await userWake(source!.issueId!, {}); + expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 }); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + const records = await continuationRecords(held.runId); + expect(records.map(record => [record.evidence.freshContinuation ? 'record' : 'hold', record.evidence.continuationDelivery ?? null])) + .not.toContainEqual(['record', 'pending']); + expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull(); + }, 120_000); +}); diff --git a/server/src/__tests__/automatic-run-brake.test.ts b/server/src/__tests__/automatic-run-brake.test.ts new file mode 100644 index 00000000000..48afc8a7c2a --- /dev/null +++ b/server/src/__tests__/automatic-run-brake.test.ts @@ -0,0 +1,441 @@ +import { randomUUID } from "node:crypto"; +import { eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agentWakeupRequests, agents, automaticRunBrakes, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { + AUTOMATIC_RUN_BRAKE, automaticRunOutcomeRules, automaticWorkStatusText, classifyAutomaticRunOutcome, createAutomaticRunBrake, + isAutomaticRun, +} from "../services/recovery/automatic-run-brake.js"; +import { taskAutomaticAttempts } from "../services/recovery/task-automatic-attempts.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const MINUTE = 60_000; + +type Run = typeof heartbeatRuns.$inferSelect; + +suite("automatic-run brake", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-automatic-run-brake-"); + db = createDb(temporary.connectionString); + }, 60_000); + + afterEach(async () => { + await db.execute(sql`truncate table automatic_run_brakes, agent_wakeup_requests, heartbeat_runs, agents, companies cascade`); + }, 120_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }, 120_000); + + async function seedCompany() { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Brake", issuePrefix: `B${companyId.replaceAll("-", "").slice(0, 6)}`, + requireBoardApprovalForNewAgents: false }); + const agentIds: string[] = []; + for (let index = 0; index < 2; index += 1) { + const agentId = randomUUID(); + await db.insert(agents).values({ id: agentId, companyId, name: `Worker ${index}`, role: "engineer", status: "idle", + adapterType: "codex_local", adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + agentIds.push(agentId); + } + return { companyId, agentIds }; + } + + async function insertRuns(companyId: string, agentId: string, count: number, values: (index: number) => Partial) { + if (count === 0) return [] as Run[]; + return db.insert(heartbeatRuns).values(Array.from({ length: count }, (_, index) => ({ companyId, agentId, + invocationSource: "automation", triggerDetail: "system", runtimeMode: "legacy", + contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() }, ...values(index) }))).returning(); + } + + const ago = (ms: number) => new Date(Date.now() - ms); + + function failedBeforeStarting(companyId: string, agentId: string, count: number, errorCode = "environment_provider_unavailable") { + return insertRuns(companyId, agentId, count, index => ({ status: "failed", errorCode, executionStage: "preparing", + createdAt: ago(4 * MINUTE + index), startedAt: ago(3 * MINUTE + index), finishedAt: ago(MINUTE - index * 100), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } })); + } + + function interruptedWhileWorking(companyId: string, agentId: string, count: number) { + const codes = ["controller_lease_lost", "server_shutdown_interrupted", "process_lost", "native_session_interrupted"]; + return insertRuns(companyId, agentId, count, index => ({ status: index % 2 ? "failed" : "interrupted", errorCode: codes[index % codes.length], + executionStage: "dispatching", lastOutputAt: ago(40_000), createdAt: ago(20 * MINUTE), startedAt: ago(19 * MINUTE), + finishedAt: ago(30_000 - index * 500), contextSnapshot: { wakeReason: "issue_assigned", issueId: randomUUID() } })); + } + + function healthy(companyId: string, agentId: string, count: number, status: "succeeded" | "failed" = "succeeded") { + return insertRuns(companyId, agentId, count, index => ({ status, errorCode: status === "failed" ? "adapter_failed" : null, + executionStage: "dispatching", lastOutputAt: ago(10_000), createdAt: ago(5 * MINUTE), startedAt: ago(4 * MINUTE), + finishedAt: ago(5_000 - index * 10) })); + } + + function queuedAutomatic(companyId: string, agentId: string, count: number) { + return insertRuns(companyId, agentId, count, index => ({ status: "queued", createdAt: ago(30_000 - index), + contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID(), issueId: randomUUID() } })); + } + + async function forgetRuns(ids: string[]) { + await db.update(heartbeatRuns).set({ createdAt: sql`created_at - interval '1 hour'`, startedAt: sql`started_at - interval '1 hour'`, + finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, ids)); + } + + async function ageBrake(companyId: string, ms: number) { + const shift = `${ms} milliseconds`; + await db.update(automaticRunBrakes).set({ + stateChangedAt: sql`state_changed_at - ${shift}::interval`, openedAt: sql`opened_at - ${shift}::interval`, + nextProbeAt: sql`next_probe_at - ${shift}::interval`, nextCreationAt: sql`next_creation_at - ${shift}::interval`, + releaseWindowStartedAt: sql`release_window_started_at - ${shift}::interval`, + }).where(eq(automaticRunBrakes.companyId, companyId)); + } + + const brakeRow = async (companyId: string) => (await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId)))[0]; + + async function openBrake(companyId: string, agentId: string, brake = createAutomaticRunBrake()) { + const failures = await failedBeforeStarting(companyId, agentId, 20); + expect((await brake.refresh(db, companyId)).state).toBe("open"); + return failures; + } + + async function heldCount(brake: ReturnType, runs: Run[], options: { personOrigin?: boolean } = {}) { + let started = 0; + for (const run of runs) if (!await brake.holdAutomaticRunStart(db, run, options)) started += 1; + return started; + } + + function countingDb() { + let queries = 0; + const counted = new Set(["select", "insert", "update", "delete", "execute"]); + const proxy = new Proxy(db, { + get(target, property, receiver) { + const value = Reflect.get(target, property, receiver); + if (typeof value !== "function") return value; + return counted.has(String(property)) ? (...args: unknown[]) => { queries += 1; return value.apply(target, args); } : value.bind(target); + }, + }); + return { db: proxy as typeof db, queries: () => queries }; + } + + it("classifies automatic runs and outcomes with the shared person and restart rules", async () => { + const rules = await automaticRunOutcomeRules(); + expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "issue_continuation_needed" } })).toBe(true); + expect(isAutomaticRun({ id: "a", companyId: "c", scheduledRetryReason: "transient_failure" })).toBe(true); + expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "task_retry", previousRunId: "x" } })).toBe(false); + expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "issue_assigned" } })).toBe(false); + const base = { status: "failed", runtimeMode: "legacy", startedAt: new Date(), hasOutput: false, providerWorkStarted: null }; + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "environment_provider_unavailable", executionStage: "preparing" }, rules)).toBe("failed_before_starting"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "controller_lease_lost", executionStage: "dispatching" }, rules)).toBe("neutral"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "process_lost", executionStage: "preparing" }, rules)).toBe("neutral"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "budget_hard_stop", executionStage: "preparing" }, rules)).toBe("healthy"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "adapter_failed", executionStage: "dispatching" }, rules)).toBe("healthy"); + expect(classifyAutomaticRunOutcome({ ...base, errorCode: "adapter_failed", executionStage: null, providerWorkStarted: "false" }, rules)).toBe("failed_before_starting"); + expect(classifyAutomaticRunOutcome({ ...base, status: "running", errorCode: null, executionStage: "preparing" }, rules)).toBe("neutral"); + expect(classifyAutomaticRunOutcome({ ...base, status: "cancelled", errorCode: "environment_provider_unavailable", executionStage: "preparing" }, rules)).toBe("neutral"); + const preDispatch = { version: 1, providerWorkStarted: false }; + for (const runtimeMode of ["legacy", "native"]) { + const shape = { ...base, runtimeMode, executionStage: runtimeMode === "legacy" ? "preparing" : null, + providerWorkStarted: runtimeMode === "legacy" ? "false" : null, nativePreDispatchFailure: runtimeMode === "native" ? preDispatch : undefined }; + for (const errorCode of ["environment_provider_unavailable", "setup_failed", "adapter_failed"]) { + expect(classifyAutomaticRunOutcome({ ...shape, errorCode }, rules)).toBe("failed_before_starting"); + } + for (const errorCode of ["controller_lease_lost", "process_lost", "native_session_interrupted", "server_shutdown_interrupted"]) { + expect(classifyAutomaticRunOutcome({ ...shape, errorCode }, rules)).toBe("neutral"); + } + const working = { ...base, runtimeMode, executionStage: runtimeMode === "legacy" ? "dispatching" : null, hasOutput: true }; + expect(classifyAutomaticRunOutcome({ ...working, errorCode: "adapter_failed" }, rules)).toBe("healthy"); + expect(classifyAutomaticRunOutcome({ ...working, errorCode: "environment_provider_unavailable" }, rules)).toBe("failed_before_starting"); + } + }); + + it("opens on native runs that failed before dispatch", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + await insertRuns(companyId, agentId!, 20, index => ({ runtimeMode: "native", status: "failed", + errorCode: index % 2 ? "adapter_failed" : "setup_failed", createdAt: ago(4 * MINUTE), startedAt: ago(3 * MINUTE), + finishedAt: ago(MINUTE - index * 100), nativePhase: "terminal_failure", + resultJson: { nativePreDispatchFailure: { version: 1, providerWorkStarted: false }, failureCause: index % 2 ? "40P01" : "adapter_reported_error" } })); + expect((await createAutomaticRunBrake().refresh(db, companyId))).toMatchObject({ state: "open", + reason: { recentAutomaticRuns: 20, failedBeforeStarting: 20, trigger: "failed_before_starting" } }); + }, 60_000); + + it("sees failures that finished in the window behind more than 500 newer runs", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + const failures = await failedBeforeStarting(companyId, agentId!, 20, "setup_failed"); + await db.update(heartbeatRuns).set({ createdAt: sql`now() - interval '90 minutes'`, startedAt: sql`now() - interval '3 minutes'` }) + .where(inArray(heartbeatRuns.id, failures.map(run => run.id))); + await queuedAutomatic(companyId, otherAgent!, 600); + await insertRuns(companyId, otherAgent!, 50, () => ({ status: "succeeded", createdAt: ago(MINUTE), contextSnapshot: { wakeReason: "issue_assigned" } })); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open"); + }, 120_000); + + it("opens when automatic runs fail before starting, holds all but one probe, and never holds a person's work", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + const brake = createAutomaticRunBrake(); + await failedBeforeStarting(companyId, agentId!, 20); + const snapshot = await brake.refresh(db, companyId); + expect(snapshot).toMatchObject({ state: "open", reason: { recentAutomaticRuns: 20, failedBeforeStarting: 20 } }); + expect(automaticWorkStatusText(snapshot)).toMatch(/^Automatic work is paused for this company since .* because the last 5 automatic runs could not get a sandbox/); + expect(automaticWorkStatusText(snapshot)).not.toMatch(/brake|probe|_/); + + const queued = [...await queuedAutomatic(companyId, agentId!, 25), ...await queuedAutomatic(companyId, otherAgent!, 25)]; + const attemptsBefore = await Promise.all(queued.map(run => taskAutomaticAttempts(db, companyId, String(run.contextSnapshot!.issueId), new Date()))); + expect(await heldCount(brake, queued, { personOrigin: false })).toBe(1); + expect(await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false })).toBe(0); + const after = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.id, queued.map(run => run.id))); + expect(after.every(run => run.status === "queued" && run.errorCode === null)).toBe(true); + const attemptsAfter = await Promise.all(queued.map(run => taskAutomaticAttempts(db, companyId, String(run.contextSnapshot!.issueId), new Date()))); + expect(attemptsAfter.map(attempts => attempts.all.length)).toEqual(attemptsBefore.map(attempts => attempts.all.length)); + + const [retry] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", contextSnapshot: { wakeReason: "task_retry", previousRunId: queued[0]!.id } })); + const [assignment] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", contextSnapshot: { wakeReason: "issue_assigned" } })); + const [joinedByPerson] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", + contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID() } })); + await db.insert(agentWakeupRequests).values({ companyId, agentId: agentId!, source: "on_demand", triggerDetail: "manual", + reason: "issue_commented", status: "queued", requestedByActorType: "user", requestedByActorId: "board", runId: joinedByPerson!.id }); + const [continuationOfPersonRetry] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", + contextSnapshot: { wakeReason: "transient_failure_retry", previousRunId: retry!.id } })); + for (const run of [retry!, assignment!, joinedByPerson!, continuationOfPersonRetry!]) { + expect(await brake.holdAutomaticRunStart(db, run)).toBe(false); + } + expect(await brake.holdAutomaticRunStart(db, queued[3]!, { personOrigin: true })).toBe(false); + }, 60_000); + + it("stays closed for mixed outcomes, person-owned failures and task failures after the agent started", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const brake = createAutomaticRunBrake(); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + expect(await brakeRow(companyId)).toBeUndefined(); + await failedBeforeStarting(companyId, agentId!, 8, "setup_failed"); + await healthy(companyId, agentId!, 4, "failed"); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + await insertRuns(companyId, agentId!, 30, index => ({ status: "failed", errorCode: index % 2 ? "budget_hard_stop" : "claude_auth_required", + executionStage: "preparing", createdAt: ago(3 * MINUTE), startedAt: ago(2 * MINUTE), finishedAt: ago(MINUTE), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } })); + await healthy(companyId, agentId!, 30, "failed"); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + expect(await brakeRow(companyId)).toBeUndefined(); + }, 60_000); + + it("ignores failures older than the window", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const failures = await failedBeforeStarting(companyId, agentId!, 40); + await db.update(heartbeatRuns).set({ finishedAt: sql`now() - interval '11 minutes'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id))); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("closed"); + }, 60_000); + + it("stays closed through a release swap that interrupts working runs, and opens when their continuations cannot get a sandbox", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + const brake = createAutomaticRunBrake(); + await interruptedWhileWorking(companyId, agentId!, 40); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + const continuations = [...await queuedAutomatic(companyId, agentId!, 20), ...await queuedAutomatic(companyId, otherAgent!, 20)]; + expect(await heldCount(createAutomaticRunBrake(), continuations)).toBe(40); + expect((await brake.reserveCreation(db, companyId, { issueId: randomUUID(), sourceRunId: randomUUID() })).heldUntil).toBeNull(); + + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "environment_provider_unavailable", executionStage: "preparing", + startedAt: sql`now() - interval '20 seconds'`, finishedAt: sql`now() - interval '5 seconds'`, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }) + .where(inArray(heartbeatRuns.id, continuations.map(run => run.id))); + expect((await brake.refresh(db, companyId)).state).toBe("open"); + }, 60_000); + + it("opens on five sandbox refusals in a row, below the run count", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const brake = createAutomaticRunBrake(); + await failedBeforeStarting(companyId, agentId!, 4); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + await failedBeforeStarting(companyId, agentId!, 1); + expect((await brake.refresh(db, companyId)).state).toBe("open"); + }, 60_000); + + it("waits out a failed probe, then releases at a doubling rate after two probes reach the agent, and closes", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + const failures = await openBrake(companyId, agentId!); + await forgetRuns(failures.map(run => run.id)); + const queued = [...await queuedAutomatic(companyId, agentId!, 30), ...await queuedAutomatic(companyId, otherAgent!, 30)]; + const probe = async () => { + const brake = createAutomaticRunBrake(); + for (const run of queued) if (!await brake.holdAutomaticRunStart(db, run, { personOrigin: false })) return run; + return null; + }; + const first = await probe(); + expect(first).not.toBeNull(); + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "environment_provider_unavailable", executionStage: "preparing", + startedAt: sql`now()`, finishedAt: sql`now()`, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }) + .where(eq(heartbeatRuns.id, first!.id)); + queued.splice(queued.indexOf(first!), 1); + expect(await probe()).toBeNull(); + expect((await brakeRow(companyId))).toMatchObject({ state: "open", probeSuccesses: 0, probeRunId: null }); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs); + + for (let success = 0; success < AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease; success += 1) { + const run = await probe(); + expect(run).not.toBeNull(); + await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()`, executionStage: "dispatching" }) + .where(eq(heartbeatRuns.id, run!.id)); + queued.splice(queued.indexOf(run!), 1); + await createAutomaticRunBrake().refresh(db, companyId); + } + expect(await brakeRow(companyId)).toMatchObject({ state: "open", probeSuccesses: 2 }); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("half_open"); + + const startedPerWindow: number[] = []; + for (let step = 0; step < AUTOMATIC_RUN_BRAKE.releaseSteps; step += 1) { + const startedNow = await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false }) + + await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false }); + startedPerWindow.push(startedNow); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.releaseIntervalMs); + } + expect(startedPerWindow).toEqual([2, 4, 8, 16, 32, 64]); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("closed"); + expect(await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false })).toBe(queued.length); + }, 120_000); + + it("stays open when runs that started fine are followed by newer failures before starting", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const failures = await openBrake(companyId, agentId!); + await forgetRuns(failures.map(run => run.id)); + await insertRuns(companyId, agentId!, 2, () => ({ status: "running", executionStage: "dispatching", lastOutputAt: ago(80_000), + startedAt: ago(90_000), createdAt: ago(100_000) })); + await failedBeforeStarting(companyId, agentId!, 10); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open"); + }, 60_000); + + it("reopens from a gradual release when runs fail before starting again", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const failures = await openBrake(companyId, agentId!); + await forgetRuns(failures.map(run => run.id)); + const recovered = await healthy(companyId, agentId!, 2); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("half_open"); + await forgetRuns(recovered.map(run => run.id)); + const again = await failedBeforeStarting(companyId, agentId!, 5); + await db.update(heartbeatRuns).set({ finishedAt: sql`now()` }).where(inArray(heartbeatRuns.id, again.map(run => run.id))); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open"); + }, 60_000); + + it("frees the probe slot when the probe is cancelled, deleted or never starts", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + await openBrake(companyId, agentId!); + const queued = [...await queuedAutomatic(companyId, agentId!, 3), ...await queuedAutomatic(companyId, otherAgent!, 3)]; + const brake = createAutomaticRunBrake(); + expect(await brake.holdAutomaticRunStart(db, queued[0]!, { personOrigin: false })).toBe(false); + expect(await heldCount(createAutomaticRunBrake(), queued.slice(1), { personOrigin: false })).toBe(0); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, queued[3]!, { personOrigin: false })).toBe(false); + expect((await brakeRow(companyId))?.probeRunId).toBe(queued[3]!.id); + + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: sql`now()` }).where(eq(heartbeatRuns.id, queued[3]!.id)); + await createAutomaticRunBrake().refresh(db, companyId); + expect((await brakeRow(companyId))?.probeRunId).toBeNull(); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, queued[4]!, { personOrigin: false })).toBe(false); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, queued[4]!.id)); + await createAutomaticRunBrake().refresh(db, companyId); + expect((await brakeRow(companyId))?.probeRunId).toBeNull(); + }, 60_000); + + it("lets another run take the probe when the reserved probe never starts within the claim bound", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + await openBrake(companyId, agentId!); + const [first] = await queuedAutomatic(companyId, agentId!, 1); + const [second] = await queuedAutomatic(companyId, otherAgent!, 1); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, first!, { personOrigin: false })).toBe(false); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, second!, { personOrigin: false })).toBe(true); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeClaimBoundMs + 1_000); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, second!, { personOrigin: false })).toBe(false); + expect((await brakeRow(companyId))?.probeRunId).toBe(second!.id); + await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()` }).where(eq(heartbeatRuns.id, second!.id)); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeClaimBoundMs + 1_000); + expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, first!, { personOrigin: false })).toBe(true); + }, 60_000); + + it("closes by itself when no automatic run started or finished for the quiet bound, even with runs waiting", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + const failures = await openBrake(companyId, agentId!); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.quietCloseMs - MINUTE); + expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open"); + await forgetRuns(failures.map(run => run.id)); + await queuedAutomatic(companyId, agentId!, 5); + await ageBrake(companyId, 2 * MINUTE); + expect((await createAutomaticRunBrake().refresh(db, companyId))).toMatchObject({ state: "closed", reason: { trigger: "no_automatic_work" } }); + }, 60_000); + + it("lets two servers agree on the state and start one probe between them", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + await failedBeforeStarting(companyId, agentId!, 20); + const [left, right] = [createAutomaticRunBrake(), createAutomaticRunBrake()]; + const [leftState, rightState] = await Promise.all([left.refresh(db, companyId), right.refresh(db, companyId)]); + expect([leftState.state, rightState.state]).toEqual(["open", "open"]); + expect(leftState.stateChangedAt?.getTime()).toBe(rightState.stateChangedAt?.getTime()); + const queued = [...await queuedAutomatic(companyId, agentId!, 25), ...await queuedAutomatic(companyId, otherAgent!, 25)]; + const results = await Promise.all(queued.map((run, index) => (index % 2 ? left : right).holdAutomaticRunStart(db, run, { personOrigin: false }))); + expect(results.filter(held => !held)).toHaveLength(1); + }, 60_000); + + it("allows one new automatic run a minute while open and no automatic run is starting, across servers", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + await openBrake(companyId, agentId!); + const work = { issueId: randomUUID(), sourceRunId: randomUUID() }; + const [left, right] = [createAutomaticRunBrake(), createAutomaticRunBrake()]; + const decisions = await Promise.all([left, right, left, right].map(async brake => (await brake.reserveCreation(db, companyId, work)).heldUntil)); + expect(decisions.filter(held => held === null)).toHaveLength(1); + const later = decisions.find(Boolean)!; + expect(later.getTime()).toBeGreaterThan(Date.now() + 50_000); + await queuedAutomatic(companyId, agentId!, 3); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.creationIntervalMs); + expect((await createAutomaticRunBrake().reserveCreation(db, companyId, work)).heldUntil).toBeNull(); + await insertRuns(companyId, agentId!, 1, () => ({ status: "running", executionStage: "preparing", startedAt: ago(20_000) })); + await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.creationIntervalMs); + expect((await createAutomaticRunBrake().reserveCreation(db, companyId, work)).heldUntil).not.toBeNull(); + }, 60_000); + + it("keeps a pass over hundreds of held runs at a bounded number of queries", async () => { + const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany(); + await openBrake(companyId, agentId!); + const queued = [...await queuedAutomatic(companyId, agentId!, 125), ...await queuedAutomatic(companyId, otherAgent!, 125)]; + const counting = countingDb(); + const brake = createAutomaticRunBrake(); + let started = 0; + for (let pass = 0; pass < 100; pass += 1) { + for (const run of queued) { + if (await brake.holdAutomaticRunStart(counting.db, run, { personOrigin: false })) continue; + started += 1; + await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()` }).where(eq(heartbeatRuns.id, run.id)); + } + } + expect(started).toBe(1); + expect(counting.queries()).toBeLessThanOrEqual(6); + + const fallback = countingDb(); + const second = createAutomaticRunBrake(); + for (let pass = 0; pass < 100; pass += 1) for (const run of queued) await second.holdAutomaticRunStart(fallback.db, run); + expect(fallback.queries()).toBeLessThanOrEqual(queued.length + 5); + }, 120_000); + + it("reads as closed and logs once when its run rules cannot load", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(); + await failedBeforeStarting(companyId, agentId!, 20); + const loadRules = vi.fn(async () => { throw new Error("module failed to load"); }); + const brake = createAutomaticRunBrake({ loadRules }); + const queued = await queuedAutomatic(companyId, agentId!, 3); + expect((await brake.refresh(db, companyId)).state).toBe("closed"); + expect((await brake.reserveCreation(db, companyId, { issueId: randomUUID(), sourceRunId: randomUUID() })).heldUntil).toBeNull(); + expect(await heldCount(brake, queued)).toBe(3); + expect(await brakeRow(companyId)).toBeUndefined(); + }, 60_000); + + it("keeps its in-process memory bounded across many companies and runs", async () => { + const brake = createAutomaticRunBrake(); + const { companyId, agentIds: [agentId] } = await seedCompany(); + for (let index = 0; index < AUTOMATIC_RUN_BRAKE.maxCompanies + 100; index += 1) await brake.read(db, randomUUID()); + await openBrake(companyId, agentId!, brake); + for (let index = 0; index < AUTOMATIC_RUN_BRAKE.maxRuns + 200; index += 1) { + await brake.holdAutomaticRunStart(db, { id: randomUUID(), companyId, contextSnapshot: { wakeReason: "issue_continuation_needed" } }); + } + expect(brake.cacheSizes()).toEqual({ companies: AUTOMATIC_RUN_BRAKE.maxCompanies, runs: AUTOMATIC_RUN_BRAKE.maxRuns }); + }, 180_000); +}); diff --git a/server/src/__tests__/environment-retained-orphan-listing-cost.test.ts b/server/src/__tests__/environment-retained-orphan-listing-cost.test.ts new file mode 100644 index 00000000000..767c09a0ab8 --- /dev/null +++ b/server/src/__tests__/environment-retained-orphan-listing-cost.test.ts @@ -0,0 +1,299 @@ +import { randomUUID } from "node:crypto"; +import { and, asc, sql, type SQL } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + agents, + companies, + createDb, + environmentLeases, + environments, + executionWorkspaces, + heartbeatRuns, + issueRecoveryActions, + issues, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { + nextRetainedOrphanDueAt, + releasableRetainedOrphanCondition, + retainedOrphanSandboxCondition, + softlyHeldSql, +} from "../services/environment-retained-orphan-settlement.js"; +import { abandonedWorkspaceFilesHeldOnLeaseRowSql } from "../services/remote-workspace-abandoned-files-hold.js"; +import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const KIND = OWNABLE_KINDS.find(kind => kind.id === "environment_retained_orphan_sandbox")!; +const TERMINAL = ["succeeded", "failed", "cancelled", "timed_out", "interrupted"]; + +function previousAlwaysHeldSql(): SQL { + return sql`(${environmentLeases.metadata} ?| array['nativeWorkspaceExportResume', 'sandboxStopAndRetain', + 'workspaceRestoreRecovery', 'workspaceRecoveryCleanup', 'runnerTaskSourceConsumer', 'reusableSandboxKeptAfterReplacement'] + or ${abandonedWorkspaceFilesHeldOnLeaseRowSql()} + or coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') = 'true' + or coalesce(${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,state}' = 'pending', false) + or exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId} + and run.status not in ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')) + or exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId} + and run.runtime_mode = 'native' + and run.runner_profile_json#>>'{nativeWorkspaceSync,schema}' in + ('paperclip.native-workspace-sync/v1', 'paperclip.native-workspace-sync/v2', 'paperclip.native-workspace-sync/v3') + and run.runner_profile_json#>>'{nativeWorkspaceSync,state}' = 'prepared' + and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,descriptorSha256}' ~ '^[0-9a-f]{64}$', false) + and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,baselineSha256}' ~ '^[0-9a-f]{64}$', false) + and coalesce(jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,finalHostSha256}'), 'null') = 'null' + and jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,workspaceId}') = 'string' + and jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,remoteCwd}') = 'string' + and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,resourceDisposition}', 'destroy') + in ('keep_running', 'stop_and_retain', 'destroy') + and run.runner_profile_json#>>'{nativeWorkspaceSync,leaseId}' = ${environmentLeases.id}::text + and run.runner_profile_json#>>'{nativeWorkspaceSync,providerLeaseId}' = ${environmentLeases.providerLeaseId} + and coalesce(run.runner_profile_json#>>'{nativeExecutionInput,workspace,access}', '') <> 'none') + or exists (select 1 from environment_leases other where other.id <> ${environmentLeases.id} + and other.provider = ${environmentLeases.provider} and other.provider_lease_id = ${environmentLeases.providerLeaseId} + and other.status in ('active', 'retained', 'pending_cleanup')))`; +} + +function previousReleasableCondition(now?: Date) { + return and(retainedOrphanSandboxCondition(), sql`not ${previousAlwaysHeldSql()}`, sql`not (${softlyHeldSql(now)})`); +} + +function random(seed: number) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +type PlanNode = { "Relation Name"?: string; "Actual Loops"?: number; "Actual Rows"?: number; + "Rows Removed by Filter"?: number; Plans?: PlanNode[] }; + +function heartbeatRunRowsRead(node: PlanNode): number { + const own = node["Relation Name"] === "heartbeat_runs" + ? (node["Actual Loops"] ?? 0) * ((node["Actual Rows"] ?? 0) + (node["Rows Removed by Filter"] ?? 0)) : 0; + return own + (node.Plans ?? []).reduce((total, child) => total + heartbeatRunRowsRead(child), 0); +} + +suite("retained orphan sandbox listing cost", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + const tenants: Array<{ companyId: string; agentId: string; issueIds: string[]; workspaceIds: string[] }> = []; + let environmentId = ""; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-retained-orphan-listing-"); + db = createDb(temporary.connectionString); + environmentId = randomUUID(); + await db.insert(environments).values({ id: environmentId, name: "Fake Sandbox", driver: "sandbox", + status: "active", config: { provider: "fake", image: "ubuntu:24.04" } }); + for (const prefix of ["ROA", "ROB"]) { + const companyId = randomUUID(); + const agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: prefix, issuePrefix: prefix, + requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "idle", + adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} }); + const workspaceIds: string[] = []; + for (let index = 0; index < 5; index += 1) { + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, mode: "isolated_workspace", + strategyType: "git_worktree", name: `${prefix} workspace ${index}` }).returning({ id: executionWorkspaces.id }); + workspaceIds.push(workspace!.id); + } + const issueIds: string[] = []; + for (let index = 0; index < 6; index += 1) { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: `${prefix} issue ${index}`, + executionWorkspaceId: index < 3 ? workspaceIds[index] : null }); + issueIds.push(issueId); + } + tenants.push({ companyId, agentId, issueIds, workspaceIds }); + } + }, 60_000); + + afterEach(async () => { + await db.delete(issueRecoveryActions); + await db.delete(environmentLeases); + await db.delete(heartbeatRuns); + }); + + afterAll(async () => { + await temporary?.cleanup(); + }); + + async function seedVariedFixture(seed: number, count: number) { + const next = random(seed); + const pick = (values: readonly T[]): T => values[Math.floor(next() * values.length)]!; + const chance = (probability: number) => next() < probability; + const runs: Array = []; + const leases: Array = []; + const base = Date.now() - 24 * 60 * 60_000; + for (let index = 0; index < count; index += 1) { + const tenant = pick(tenants); + const leaseId = randomUUID(); + const provider = pick(["fake", "fake", "fake", "local", null]); + const providerLeaseId = chance(0.9) ? `sbx-${Math.floor(next() * 300)}` : null; + let runId: string | null = null; + if (chance(0.9)) { + runId = randomUUID(); + const sync = pick(["none", "prepared", "prepared", "access_none", "other_sandbox", "bad_descriptor", "final"]); + const runnerProfileJson = sync === "none" ? {} : { + ...(sync === "access_none" ? { nativeExecutionInput: { workspace: { access: "none" } } } : {}), + nativeWorkspaceSync: { + schema: pick(["paperclip.native-workspace-sync/v3", "paperclip.native-workspace-sync/v1", "unknown"]), + state: "prepared", descriptorSha256: sync === "bad_descriptor" ? "zz" : "a".repeat(64), + baselineSha256: "b".repeat(64), finalHostSha256: sync === "final" ? "c".repeat(64) : null, + workspaceId: randomUUID(), remoteCwd: "/workspace", resourceDisposition: pick(["destroy", "keep_running", "odd"]), + leaseId, providerLeaseId: sync === "other_sandbox" ? "sbx-elsewhere" : providerLeaseId, + }, + }; + runs.push({ id: runId, companyId: tenant.companyId, agentId: tenant.agentId, issueId: pick(tenant.issueIds), + nativeIssueId: chance(0.2) ? pick(tenant.issueIds) : null, invocationSource: "on_demand", + status: chance(0.06) ? pick(["running", "queued", "scheduled_retry"]) : pick(TERMINAL), nextEventSeq: 1, + runtimeMode: pick(["native", "native", "legacy"]), runnerProfileJson }); + } + const terminationRunId = chance(0.85) ? runId : randomUUID(); + const metadata: Record = {}; + if (chance(0.75)) metadata.orphanedRunLeaseRecovery = { recoveredAt: "x" }; + if (chance(0.85)) metadata.sandboxStopAndRetainReceipt = { stoppedAt: "x" }; + if (chance(0.75)) metadata.remoteExecutionTermination = { state: pick(["stopped", "stopped", "running"]), + leaseId: chance(0.85) ? leaseId : randomUUID(), runId: terminationRunId }; + for (const key of ["nativeWorkspaceExportResume", "sandboxStopAndRetain", "workspaceRestoreRecovery", + "workspaceRecoveryCleanup", "runnerTaskSourceConsumer", "reusableSandboxKeptAfterReplacement"]) { + if (chance(0.04)) metadata[key] = { at: "x" }; + } + if (chance(0.1)) metadata.pendingCleanupInFlight = pick([true, false, "true", "maybe"]); + if (chance(0.06)) metadata.legacyShutdownWorkspaceSettlement = { state: pick(["pending", "done"]) }; + const expiresRoll = next(); + leases.push({ id: leaseId, companyId: tenant.companyId, environmentId, + executionWorkspaceId: chance(0.7) ? pick(tenant.workspaceIds) : null, + issueId: chance(0.9) ? pick(tenant.issueIds) : null, heartbeatRunId: runId, + status: pick(["released", "released", "released", "failed", "failed", "retained", "retained", "expired", "active", "pending_cleanup"]), + leasePolicy: pick(["ephemeral", "ephemeral", "reuse_by_environment", "reuse_by_environment", "retain_on_failure"]), + provider, providerLeaseId, cleanupStatus: pick(["success", "success", "success", "success", "failed", null]), + metadata: chance(0.03) ? null : metadata, + expiresAt: expiresRoll < 0.3 ? null : new Date(Date.now() + (expiresRoll < 0.6 ? -1 : 1) * 60 * 60_000), + createdAt: new Date(base + Math.floor(next() * 5) * 1000) }); + } + for (let index = 0; index < runs.length; index += 200) await db.insert(heartbeatRuns).values(runs.slice(index, index + 200)); + for (let index = 0; index < leases.length; index += 200) { + await db.insert(environmentLeases).values(leases.slice(index, index + 200)); + } + for (const tenant of tenants) { + for (const issueId of tenant.issueIds) { + if (!chance(0.25)) continue; + await db.insert(issueRecoveryActions).values({ companyId: tenant.companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", status: pick(["active", "resolved", "cancelled", "escalated"]), + cause: "fixture", fingerprint: randomUUID(), nextAction: "Recover the workspace." }); + } + } + } + + async function previousPage(limit: number, offset: number, now?: Date) { + const rows = await db.select({ id: environmentLeases.id }).from(environmentLeases) + .where(previousReleasableCondition(now)).orderBy(asc(environmentLeases.id)).limit(limit + 1).offset(offset); + return { truncated: rows.length > limit, ids: rows.slice(0, limit).map(row => row.id) }; + } + + it("lists exactly the rows the previous listing listed, page by page, over a varied fixture", async () => { + for (const seed of [11, 29, 47, 83]) { + await seedVariedFixture(seed, 600); + const expected = await previousPage(10_000, 0); + expect(expected.ids.length).toBeGreaterThan(0); + expect(expected.ids.length).toBeLessThan(600); + for (const limit of [7, 200]) { + for (let offset = 0; ; offset += limit) { + const [listed, previous] = await Promise.all([KIND.listInProgress!(db, limit, offset), previousPage(limit, offset)]); + expect(listed.records.map(record => record.recordId)).toEqual(previous.ids); + expect(listed.truncated).toBe(previous.truncated); + if (!previous.truncated) break; + } + } + const now = new Date(); + const [current] = await db.select({ count: sql`count(*)::int` }).from(environmentLeases) + .where(releasableRetainedOrphanCondition(now)); + expect(current!.count).toBe(expected.ids.length); + const [previousDue] = await db.select({ due: sql`min(${environmentLeases.expiresAt})` }) + .from(environmentLeases).where(and(retainedOrphanSandboxCondition(), sql`not ${previousAlwaysHeldSql()}`, + sql`${softlyHeldSql(now)}`, sql`${environmentLeases.expiresAt} is not null`)); + expect((await nextRetainedOrphanDueAt(db, now))?.getTime() ?? null) + .toBe(previousDue?.due ? new Date(previousDue.due).getTime() : null); + await db.delete(issueRecoveryActions); + await db.delete(environmentLeases); + await db.delete(heartbeatRuns); + } + }, 120_000); + + async function seedHistory(count: number) { + const tenant = tenants[0]!; + await db.execute(sql` + with history as ( + select gen_random_uuid() as run_id, gen_random_uuid() as lease_id, g from generate_series(1, ${count}) g + ), inserted as ( + insert into heartbeat_runs (id, company_id, agent_id, issue_id, invocation_source, status, next_event_seq, + runtime_mode, runner_profile_json) + select run_id, ${tenant.companyId}::uuid, ${tenant.agentId}::uuid, ${tenant.issueIds[0]}::uuid, 'on_demand', + 'succeeded', 1, 'native', jsonb_build_object('nativeWorkspaceSync', jsonb_build_object( + 'schema', 'paperclip.native-workspace-sync/v3', 'state', 'finalized', 'leaseId', lease_id::text)) + from history returning id + ) + insert into environment_leases (id, company_id, environment_id, execution_workspace_id, issue_id, heartbeat_run_id, + status, lease_policy, provider, provider_lease_id, cleanup_status, metadata, expires_at) + select lease_id, ${tenant.companyId}::uuid, ${environmentId}::uuid, ${tenant.workspaceIds[0]}::uuid, + ${tenant.issueIds[0]}::uuid, run_id, + case when g % 100 < 71 then 'expired' when g % 100 < 96 then 'released' else 'failed' end, + case when g % 3 = 0 then 'reuse_by_environment' else 'ephemeral' end, + 'fake', 'sbx-history-' || (g % 50), 'success', jsonb_build_object('sandboxStopAndRetainReceipt', '{}'::jsonb), + now() + interval '1 day' + from history`); + await db.execute(sql`analyze heartbeat_runs; analyze environment_leases`); + } + + async function listingPlan() { + const query = db.select({ id: environmentLeases.id }).from(environmentLeases) + .where(releasableRetainedOrphanCondition()).orderBy(asc(environmentLeases.id)).limit(201).offset(0).toSQL(); + const rows = await db.$client.unsafe(`explain (analyze, format json) ${query.sql}`, query.params as never[]); + return (rows[0] as unknown as { "QUERY PLAN": Array<{ Plan: PlanNode }> })["QUERY PLAN"][0]!.Plan; + } + + it("reads heartbeat runs only for retained orphan candidates, not for the whole lease history", async () => { + await seedHistory(2_000); + await seedVariedFixture(5, 20); + const [{ candidates }] = await db.select({ candidates: sql`count(*)::int` }).from(environmentLeases) + .where(retainedOrphanSandboxCondition()) as Array<{ candidates: number }>; + const plan = await listingPlan(); + expect(heartbeatRunRowsRead(plan)).toBeLessThanOrEqual(10 * candidates + 50); + }, 120_000); + + it("lets a schema change take the heartbeat_runs lock within a 5 second lock timeout while listings run", async () => { + await seedHistory(5_000); + let stop = false; + let listings = 0; + const load = (async () => { + while (!stop) { + await KIND.listInProgress!(db, 200, 0); + listings += 1; + } + })(); + const ddl = createDb(temporary!.connectionString); + try { + await new Promise(resolve => setTimeout(resolve, 200)); + const started = Date.now(); + await ddl.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '5s'`); + await tx.execute(sql`alter table heartbeat_runs add column retained_orphan_listing_probe jsonb`); + await tx.execute(sql`alter table heartbeat_runs drop column retained_orphan_listing_probe`); + }); + expect(Date.now() - started).toBeLessThan(5_000); + } finally { + stop = true; + await load; + await ddl.$client.end({ timeout: 1 }); + } + expect(listings).toBeGreaterThan(0); + }, 120_000); +}); diff --git a/server/src/__tests__/github-health-recovery.test.ts b/server/src/__tests__/github-health-recovery.test.ts index 410d6ce7c6c..5cf14b3b99d 100644 --- a/server/src/__tests__/github-health-recovery.test.ts +++ b/server/src/__tests__/github-health-recovery.test.ts @@ -65,6 +65,7 @@ function githubConnectionHealth() { discovery: {} as never, setupPolicy: { assertConnectorConnectionSetupEnabled: async () => undefined } as never, resolveCredentialHeaders: vi.fn() as never, + vaultGrantForConnection: vi.fn() as never, refreshManagedGitHubGrantAccess: (async () => { await loadGitHubGrantMetadata("ghu_secret", suspendedGitHub); }) as never, diff --git a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts index b805d99e9a6..9ff9edb5694 100644 --- a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts +++ b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts @@ -1,9 +1,11 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomBytes } from "node:crypto"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, inArray, sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; -import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints } from "@paperclipai/db"; +import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints, databaseWorkPending, + reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { appendHeartbeatRunEvent, appendHeartbeatRunEventBatch } from "../services/heartbeat-run-events.js"; import { nativeSha256 } from "../services/native-runtime/canonical.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -293,6 +295,7 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { locked(); await released; }); + void holder.catch(() => undefined); await lockedPromise; return { release: () => { release(); return holder; } }; }; @@ -485,17 +488,143 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { const runId = await makeRun(); const [issue] = await db.insert(issues).values({ companyId, title: "Progress task" }).returning(); await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); - const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); - // BEGIN, lock, duplicate probe, the wake's transaction id, the append, COMMIT. - expect(first.statements).toHaveLength(6); - const wake = first.statements.findIndex(text => text.includes("pg_current_xact_id")); - expect(wake).toBeGreaterThan(0); - expect(wake).toBeLessThan(first.statements.findIndex(text => text.includes("insert into heartbeat_run_events"))); - expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 }); - const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); - expect(replay.statements.some(text => text.includes("pg_current_xact_id"))).toBe(false); - expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 }); - expect(await readEvents(runId)).toEqual([first.result.row]); + const wakes: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, event => { wakes.push(event); }); + try { + const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); + expect(first.statements).toHaveLength(4); + expect(first.statements[0]).toMatch(/^begin/i); + expect(first.statements[1]).toMatch(/^with "snapshot_run" as .*pg_current_xact_id\(\).* for no key update.*"replay_found"/); + expect(first.statements[2]).toMatch(/insert into heartbeat_run_events/); + expect(first.statements[3]).toMatch(/^commit/i); + expect(wakes).toEqual(["intent", "settled"]); + expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 }); + wakes.length = 0; + const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); + expect(wakes).toEqual([]); + expect(replay.statements).toHaveLength(4); + expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 }); + expect(await readEvents(runId)).toEqual([first.result.row]); + } finally { unsubscribe(); } + }); + + it("gives the wake the transaction id of the transaction that took the run lock", async () => { + const runId = await makeRun(); + const [issue] = await db.insert(issues).values({ companyId, title: "Primed wake" }).returning(); + await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); + const topic = DELIVERY_QUEUES.chatNativeProgress; + const options = db.$client.options as { debug?: unknown }; + const sent: Array<{ text: string; parameters: unknown[] }> = []; + let ownXid: string | undefined; + options.debug = (_connection: number, text: string, parameters: unknown[]) => { + sent.push({ text: text.trim().replace(/\s+/g, " "), parameters }); + }; + try { + await expect(db.transaction(async tx => { + await appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 1, "item.completed")); + expect(sent.filter(statement => /^select pg_current_xact_id\(\)/.test(statement.text))).toEqual([]); + expect(sent.filter(statement => statement.text.includes("pg_current_xact_id()"))).toHaveLength(1); + const [own] = await tx.execute<{ xid: string; pid: number }>(sql`select pg_current_xact_id()::text as xid, pg_backend_pid() as pid`); + ownXid = own!.xid; + await otherDb.execute(sql`select pg_terminate_backend(${own!.pid})`); + await tx.execute(sql`select 1`); + })).rejects.toThrow(); + expect(databaseWorkPending(db, topic)).toBe(true); + sent.length = 0; + await reconcileDatabaseWork(db, topic); + } finally { options.debug = undefined; } + const statusRead = sent.find(statement => statement.text.includes("pg_xact_status")); + expect(ownXid).toMatch(/^\d+$/); + expect(statusRead?.parameters).toEqual([ownXid]); + expect(databaseWorkPending(db, topic)).toBe(false); + expect(await readEvents(runId)).toEqual([]); + }); + + it("primes nothing when the lock read fails, so the transaction is not left uncertain", async () => { + const runId = await makeRun(); + const [issue] = await db.insert(issues).values({ companyId, title: "Failed lock read" }).returning(); + await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); + let release!: () => void, locked!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const lockedRun = new Promise(resolve => { locked = resolve; }); + const holder = otherDb.transaction(async tx => { + await tx.execute(sql`select id from heartbeat_runs where id = ${runId} for update`); + locked(); + await released; + }); + void holder.catch(() => undefined); + await lockedRun; + const wakes: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, wake => { wakes.push(wake); }); + const options = db.$client.options as { debug?: unknown }; + const sent: string[] = []; + try { + await expect(db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '100ms'`); + await appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 1, "item.completed")); + })).rejects.toThrow(); + expect(databaseWorkPending(db, DELIVERY_QUEUES.chatNativeProgress)).toBe(false); + await expect(db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '100ms'`); + await expect(tx.transaction(savepoint => + appendHeartbeatRunEvent(savepoint as unknown as typeof db, nativeEvent(runId, 1, "item.completed")))).rejects.toThrow(); + options.debug = (_connection: number, text: string) => { sent.push(text.trim().replace(/\s+/g, " ")); }; + await signalDatabaseWork(tx, "lock-read-failed"); + options.debug = undefined; + throw new Error("roll back"); + })).rejects.toThrow("roll back"); + } finally { + options.debug = undefined; + unsubscribe(); + release(); + await holder; + } + expect(sent.filter(text => /^select pg_current_xact_id\(\)/.test(text))).toHaveLength(1); + await reconcileDatabaseWork(db, "lock-read-failed"); + expect(databaseWorkPending(db, "lock-read-failed")).toBe(false); + expect(wakes).toEqual([]); + expect(databaseWorkPending(db, DELIVERY_QUEUES.chatNativeProgress)).toBe(false); + expect(await readEvents(runId)).toEqual([]); + }); + + it("finds a replay that another writer committed while the lock read waited, and registers no wake for it", async () => { + const runId = await makeRun(); + const [issue] = await db.insert(issues).values({ companyId, title: "Progress race" }).returning(); + await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); + const event = nativeEvent(runId, 1, "item.completed"); + let release!: () => void, locked!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const lockedRun = new Promise(resolve => { locked = resolve; }); + const holder = otherDb.transaction(async tx => { + await tx.execute(sql`select id from heartbeat_runs where id = ${runId} for update`); + await tx.insert(heartbeatRunEvents).values({ companyId, agentId, runId, seq: 1, eventType: event.eventType, stream: "system", + level: "info", payload: event.payload, sourceInstanceId: "native-writer", sourceEventId: event.nativeSource.sourceEventId, + sourceSeq: 1, sourcePayloadSha256: nativeSha256(event.nativeSource.canonicalPayload), protocolSchemaVersion: 1 }); + await tx.update(heartbeatRuns).set({ nextEventSeq: 2 }).where(eq(heartbeatRuns.id, runId)); + locked(); + await released; + }); + void holder.catch(() => undefined); + await lockedRun; + const wakes: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, wake => { wakes.push(wake); }); + try { + const appended = sentStatements(db.$client, () => appendHeartbeatRunEvent(db, event)); + void appended.catch(() => undefined); + await vi.waitFor(async () => { + const [waiting] = await otherDb.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and query like '%"locked_run"%'`); + expect(waiting?.count).toBe(1); + }, { timeout: 10_000, interval: 20 }); + release(); + await holder; + const { result, statements } = await appended; + expect(result).toMatchObject({ disposition: "duplicate", row: { seq: 1, sourceSeq: 1 } }); + expect(statements.some(text => /^select \* from heartbeat_run_events/.test(text))).toBe(true); + expect(wakes).toEqual([]); + } finally { unsubscribe(); } + expect((await readEvents(runId)).map(row => row.seq)).toEqual([1]); + expect((await readRun(runId)).nextEventSeq).toBe(2); }); it("keeps one ordered row per source event when two servers append the same stream", async () => { @@ -601,11 +730,11 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { expect(replay.highestContiguousSourceSeq).toBe(kind === "native" ? 1 : 0); expect((await readRun(runId)).nextEventSeq).toBe(2); expect(await readEvents(runId)).toEqual([committed.row]); - const lockReads = statements.filter(query => query.startsWith("select ") && - query.includes('from "heartbeat_runs"') && query.endsWith("for no key update")); + const lockReads = statements.filter(query => /^(select|with) /.test(query) && + query.includes('from "heartbeat_runs"') && query.includes("for no key update")); expect(lockReads).toHaveLength(2); for (const query of lockReads) { - expect(query).toMatch(/^select "company_id", "agent_id"(, \("native_source_cursors" ->> \$1\)::bigint as "source_cursor")? from "heartbeat_runs" /); + expect(query).toMatch(/select "company_id", "agent_id"(, \("native_source_cursors" ->> \$\d+\)::bigint as "source_cursor")?(, pg_current_xact_id\(\)::text as "xid")?(, "next_event_seq")? from "heartbeat_runs" /); expect(query).not.toMatch(/context_snapshot|runner_profile_json|result_json|stdout_excerpt/); } }); @@ -631,9 +760,11 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { holderLocked(); await held; }); + void holder.catch(() => undefined); await locked; let secondSettled = false; const second = appendHeartbeatRunEvent(db, native(2)).finally(() => { secondSettled = true; }); + void second.catch(() => undefined); try { // A controller checkpoint row references the run row by foreign key. // Its insert must not wait for the open append. diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index e77682e455d..9cad1909412 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -1755,6 +1755,50 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { }, { timeout: 10_000 }); }); + describe("a queued system recovery run re-checks that its task still needs it when its turn comes", () => { + async function queueRecovery(task: Partial) { + const { companyId, agentId } = await seedCompanyAndAgent(); + const [issue] = await db.insert(issues).values({ companyId, title: "Waited behind people's work", + status: "in_progress", assigneeAgentId: agentId, ...task }).returning(); + const queued = await seedQueuedRun({ companyId, agentId, issueId: issue.id, + wakeReason: "issue_recovery_action_restored", invocationSource: "automation" }); + return { companyId, agentId, issueId: issue.id, ...queued }; + } + async function expectDroppedQuietly(runId: string, wakeupRequestId: string) { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(run).toMatchObject({ status: "cancelled", errorCode: "recovery_no_longer_needed", + resultJson: expect.objectContaining({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }) }); + expect(countExecuteCallsForRun(runId)).toBe(0); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)))[0]?.status) + .toBe("skipped"); + } + + it("drops it when the task was reassigned while it waited", async () => { + const { issueId, runId, wakeupRequestId } = await queueRecovery({}); + await db.update(issues).set({ assigneeAgentId: null, assigneeUserId: "responsible-user" }).where(eq(issues.id, issueId)); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + await expectDroppedQuietly(runId, wakeupRequestId); + }); + + it("drops it when a newer run already continued the task", async () => { + const { companyId, agentId, issueId, runId, wakeupRequestId } = await queueRecovery({}); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "assignment", triggerDetail: "system", + status: "succeeded", createdAt: new Date(Date.now() + 1_000), startedAt: new Date(Date.now() + 1_000), + finishedAt: new Date(Date.now() + 2_000), contextSnapshot: { issueId, wakeReason: "issue_commented" } }); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + await expectDroppedQuietly(runId, wakeupRequestId); + }); + + it("starts it when the task still needs it", async () => { + const { runId } = await queueRecovery({ status: "todo" }); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + expect(countExecuteCallsForRun(runId)).toBe(1); + }, 60_000); + }); + it("cancels queued max-turn continuations when another continuation owns the issue lock", async () => { const { companyId, agentId } = await seedCompanyAndAgent(); const issueId = randomUUID(); diff --git a/server/src/__tests__/legacy-workspace-rehome.test.ts b/server/src/__tests__/legacy-workspace-rehome.test.ts index d8025a9dbdc..10efdea679f 100644 --- a/server/src/__tests__/legacy-workspace-rehome.test.ts +++ b/server/src/__tests__/legacy-workspace-rehome.test.ts @@ -12,6 +12,9 @@ import { REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE } from "../services/remote-work import { LEGACY_WORKSPACE_REHOME_OUTCOME, rehomeStrandedLegacyWorkspaceTasks } from "../services/recovery/legacy-workspace-rehome.js"; import { recoveryActionSweepCondition } from "../services/recovery/blocker-stall-wakeups.js"; import { recoveryService } from "../services/recovery/service.js"; +import { assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; +import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js"; +import { QUEUED_RECOVERY_RUNS_PER_AGENT } from "../services/recovery-queue-admission.js"; import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip; @@ -225,6 +228,47 @@ describePostgres("stranded subtasks bound to a parent's held workspace", () => { expect(last.nextWakeRetryAt).toBeNull(); }, 60_000); + it("holds a move's wake while its agent's recovery queue is full, and the assigned-work sweep sees it as pending", async () => { + const f = await seedSharedFamily(); + const hero = await f.strand(f.hero.id); + expect((await rehomeStrandedLegacyWorkspaceTasks(db, null, { companyId: f.companyId })).rehomed).toEqual([f.hero.id]); + const queuedIds: string[] = []; + for (let index = 0; index < QUEUED_RECOVERY_RUNS_PER_AGENT; index += 1) { + const [busy] = await db.insert(issues).values({ companyId: f.companyId, title: `Busy ${index}`, status: "in_progress", + assigneeAgentId: f.agentId }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, issueId: busy!.id, status: "queued", + invocationSource: "automation", triggerDetail: "system", + contextSnapshot: { issueId: busy!.id, wakeReason: "issue_recovery_action_restored" } }).returning(); + queuedIds.push(run!.id); + } + const woken: string[] = []; + const wake = async (_agentId: string, options: { payload: Record }) => { + woken.push(String(options.payload.issueId)); return { id: randomUUID() }; + }; + const held = await rehomeStrandedLegacyWorkspaceTasks(db, wake as never, { companyId: f.companyId }); + expect(woken).toEqual([]); + const pending = (await action(hero.actionId)).evidence.legacyWorkspaceRehome as Record; + // Waiting its turn is not a refusal: no failure is counted. + expect(pending).toMatchObject({ wake: "pending" }); + expect(pending.wakeFailures).toBeUndefined(); + expect(held.nextWakeRetryAt).toBe(Number(pending.wakeRetryAt)); + + // The sweep reads the held wake as the task's pending path, so it queues no uncapped continuation. + const sweep = assignedWorkSweep(db, { enqueueWakeup: async () => null }); + const facts = (await sweep.loadFacts(f.companyId, [await issue(f.hero.id)])).get(f.hero.id)!; + expect(facts.pendingWakes).toBeGreaterThan(0); + expect(evaluateAssignedWork(facts)).toMatchObject({ kind: "ok", clause: "pending_intent" }); + + // A queued recovery run leaves the queue; the held wake is delivered on its next turn. + await db.update(heartbeatRuns).set({ status: "succeeded", startedAt: new Date(), finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, queuedIds[0]!)); + await db.update(issueRecoveryActions).set({ evidence: { ...(await action(hero.actionId)).evidence, + legacyWorkspaceRehome: { ...pending, wakeRetryAt: Date.now() - 1 } } }).where(eq(issueRecoveryActions.id, hero.actionId)); + await rehomeStrandedLegacyWorkspaceTasks(db, wake as never, { companyId: f.companyId }); + expect(woken).toEqual([f.hero.id]); + expect((await action(hero.actionId)).evidence).toMatchObject({ legacyWorkspaceRehome: { wake: "delivered" } }); + }, 60_000); + it("pages past escalations the guards reject", async () => { const f = await seedSharedFamily(); // Thirty escalations that match the query but fail a guard (another setup error). diff --git a/server/src/__tests__/mcp-http.test.ts b/server/src/__tests__/mcp-http.test.ts index 5271f2df485..4ea44109087 100644 --- a/server/src/__tests__/mcp-http.test.ts +++ b/server/src/__tests__/mcp-http.test.ts @@ -119,3 +119,93 @@ describe("parseMcpHttpResponseBody", () => { expect(() => parseMcpHttpResponseBody("event: ping\n\n", "text/event-stream")).toThrow(); }); }); + +describe("MCP progress notifications", () => { + it("sends the progress token in the request's _meta and keeps other metadata", async () => { + const { mcpHttpRequestInit } = await import("../services/mcp-http.js"); + const init = mcpHttpRequestInit({ "MCP-Protocol-Version": "2026-07-28" }, { + id: "call-1", method: "tools/call", params: { name: "export", arguments: {}, _meta: { other: true } }, + }, { progressToken: "token-1" }); + const meta = JSON.parse(init.body).params._meta; + expect(meta.progressToken).toBe("token-1"); + expect(meta.other).toBe(true); + expect(meta["io.modelcontextprotocol/protocolVersion"]).toBe("2026-07-28"); + const without = mcpHttpRequestInit(undefined, { id: "call-2", method: "tools/call", params: { name: "export" } }); + expect(JSON.parse(without.body).params._meta?.progressToken).toBeUndefined(); + }); + + it("reads progress interleaved with the result on one SSE stream, before the result", async () => { + const { mcpProgressNotificationOf } = await import("../services/mcp-http.js"); + const frames = [ + { jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "token-1", progress: 1, total: 3, message: "Exporting rows" } }, + { jsonrpc: "2.0", method: "notifications/message", params: { level: "info", data: "unrelated" } }, + { jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "someone-else", progress: 9 } }, + { jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "token-1", progress: 2, total: 3 } }, + { jsonrpc: "2.0", id: "call-1", result: { content: [{ type: "text", text: "done" }] } }, + { jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "token-1", progress: 3, total: 3 } }, + ]; + // Split mid-event so frames cross chunk boundaries. + const text = frames.map((frame) => `event: message\r\ndata: ${JSON.stringify(frame)}\r\n\r\n`).join(""); + const encoded = new TextEncoder().encode(text); + const body = new ReadableStream({ + start(controller) { + for (let i = 0; i < encoded.length; i += 37) controller.enqueue(encoded.slice(i, i + 37)); + controller.close(); + }, + }); + const seen: unknown[] = []; + let resultSeenBeforeProgress = false; + const result = await readMcpHttpResponse(new Response(body, { headers: { "content-type": "text/event-stream" } }), "call-1", { + onNotification: (message) => { + const update = mcpProgressNotificationOf(message, "token-1"); + if (update) seen.push(update); + }, + onProgress: (progress) => { if (progress.matchedResponse && seen.length === 0) resultSeenBeforeProgress = true; }, + }); + expect(result).toMatchObject({ id: "call-1", result: { content: [{ text: "done" }] } }); + expect(resultSeenBeforeProgress).toBe(false); + expect(seen).toEqual([ + { progress: 1, total: 3, message: "Exporting rows" }, + { progress: 2, total: 3, message: undefined }, + ]); + }); + + it("never lets a failing progress observer fail the call", async () => { + const text = [ + { jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "t", progress: 1 } }, + { jsonrpc: "2.0", id: 7, result: { ok: true } }, + ].map((frame) => `data: ${JSON.stringify(frame)}\n\n`).join(""); + const result = await readMcpHttpResponse(new Response(text, { headers: { "content-type": "text/event-stream" } }), 7, { + onNotification: () => { throw new Error("observer broke"); }, + }); + expect(result).toMatchObject({ result: { ok: true } }); + }); + + it("ignores requests, responses and other tokens", async () => { + const { mcpProgressNotificationOf } = await import("../services/mcp-http.js"); + expect(mcpProgressNotificationOf({ method: "notifications/progress", params: { progressToken: "t", progress: 1 } }, undefined)).toBeNull(); + expect(mcpProgressNotificationOf({ id: 1, method: "notifications/progress", params: { progressToken: "t", progress: 1 } }, "t")).toBeNull(); + expect(mcpProgressNotificationOf({ method: "notifications/progress", params: { progressToken: "x", progress: 1 } }, "t")).toBeNull(); + expect(mcpProgressNotificationOf({ method: "notifications/progress", params: { progressToken: "t", progress: 1 } }, "t")).toEqual({ progress: 1, total: undefined, message: undefined }); + }); + + it("does not count handled progress against the result's size limit", async () => { + const note = `data: ${JSON.stringify({ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "t", progress: 1, message: "x".repeat(200) } })}\n\n`; + const text = note.repeat(50) + `data: ${JSON.stringify({ jsonrpc: "2.0", id: 1, result: { ok: true } })}\n\n`; + expect(Buffer.byteLength(text)).toBeGreaterThan(5_000); + const result = await readMcpHttpResponse(new Response(text, { headers: { "content-type": "text/event-stream" } }), 1, { maxBytes: 1_000, limitPerMessage: true }); + expect(result).toMatchObject({ result: { ok: true } }); + // Callers without an overall deadline (catalog discovery) keep the total limit, + // so a stream of small notifications cannot keep them waiting forever. + await expect(readMcpHttpResponse(new Response(text, { headers: { "content-type": "text/event-stream" } }), 1, { maxBytes: 1_000 })) + .rejects.toMatchObject({ reason: "too_large" }); + }); + + it("still rejects one SSE message over the limit, complete or not", async () => { + const big = `data: ${JSON.stringify({ jsonrpc: "2.0", id: 1, result: { text: "x".repeat(2_000) } })}`; + for (const text of [`${big}\n\n`, big]) { + await expect(readMcpHttpResponse(new Response(text, { headers: { "content-type": "text/event-stream" } }), 1, { maxBytes: 1_000, limitPerMessage: true })) + .rejects.toMatchObject({ reason: "too_large" }); + } + }); +}); diff --git a/server/src/__tests__/mcp-tasks-demo-server.test.ts b/server/src/__tests__/mcp-tasks-demo-server.test.ts index 0d111e47d5f..ffe1cb95911 100644 --- a/server/src/__tests__/mcp-tasks-demo-server.test.ts +++ b/server/src/__tests__/mcp-tasks-demo-server.test.ts @@ -15,10 +15,12 @@ import { toolMcpTasks, } from "@paperclipai/db"; import { and, eq, lte, ne } from "drizzle-orm"; +import type { AskUserQuestionsPayload } from "@paperclipai/shared"; import { forgetMcpHttpSessions } from "../services/mcp-http.js"; import { createToolGatewayService } from "../services/tool-gateway.js"; import { toolAccessService } from "../services/tool-access.js"; import { instanceSettingsService } from "../services/instance-settings.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { adapterExecutionControls, createAdapterExecutionControl } from "../services/adapter-execution-control.js"; import { createTasksDemoHttpServer, @@ -144,11 +146,14 @@ const support = await getEmbeddedPostgresTestSupport(); expect(first).toMatchObject({ reasonCode: "elicitation_required" }); const [card] = await db.select().from(issueThreadInteractions).where(and(eq(issueThreadInteractions.companyId, org.id), eq(issueThreadInteractions.issueId, issue.id))); expect(card).toMatchObject({ kind: "ask_user_questions", status: "pending" }); - await db.update(issueThreadInteractions).set({ status: "answered", result: { version: 1, answers: [ - { questionId: "decision", optionIds: ["choice_1"] }, + const decision = (card!.payload as unknown as AskUserQuestionsPayload).questions.find((question) => question.id === "decision")!; + const approve = decision.options.find((option) => option.label === "approve"); + expect(approve).toBeDefined(); + await issueThreadInteractionService(db).answerQuestions({ id: issue.id, companyId: org.id }, card!.id, { answers: [ + { questionId: decision.id, optionIds: [approve!.id] }, { questionId: "note", optionIds: [], otherText: "go ahead" }, - ] } }).where(eq(issueThreadInteractions.id, card!.id)); - const pending = call("needs_approval", { action: "rotate the demo key" }); + ] }, { userId: actor.actorId }); + const pending = call("needs_approval", { action: "rotate the demo key" }).catch((error) => error); await vi.waitFor(() => expect(demoTask("needs_approval")?.status).toBe("working"), { timeout: 5_000 }); clock += 3_000; expect(await pending).toMatchObject({ status: "completed", result: { content: "Approved: rotate the demo key was carried out. Note: go ahead" } }); diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts index 2555d75ac59..a0f3f21f052 100644 --- a/server/src/__tests__/recovery-wake-herd.test.ts +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -10,6 +10,7 @@ import { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_DELIVERY_BOUND_MS, RECOVERY_CONTINUATION_MAX_HOLD_MS, deliverReconciledExecutions, } from "../services/execution-recovery-resolution.js"; +import { QUEUED_RECOVERY_RUNS_PER_AGENT } from "../services/recovery-queue-admission.js"; import { PENDING_CONTINUATION_BOUND_MS, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js"; import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; @@ -239,9 +240,12 @@ suite("recovery wake herd", () => { it("gives the longest-waiting queued run a free slot after three fresher starts passed it over", async () => { const agentId = await seedCompany(1); - const old = await queuedRecoveryRun(agentId, 150, 152); - const people: Array = []; - for (let index = 0; index < 3; index += 1) people.push(await queuedPersonRetry(agentId)); + // A person's run on a todo task, waiting 150 minutes behind fresher person runs on in-progress tasks. + const old = await queuedPersonRetry(agentId); + await db.update(issues).set({ status: "todo" }).where(eq(issues.id, old.issueId!)); + await db.update(heartbeatRuns).set({ createdAt: new Date(Date.now() - 150 * 60_000) }).where(eq(heartbeatRuns.id, old.id)); + const fresher: Array = []; + for (let index = 0; index < 3; index += 1) fresher.push(await queuedPersonRetry(agentId)); const issue = await task(agentId); for (let index = 0; index < 3; index += 1) { const startedAt = new Date(Date.now() - (3 - index) * 60_000); @@ -253,7 +257,29 @@ suite("recovery wake herd", () => { try { await heartbeat.resumeQueuedRuns(); expect((await readRun(old.id)).startedAt).not.toBeNull(); - for (const run of people) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + for (const run of fresher) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + + it("keeps a long-waiting recovery run behind people's queued work, even when the oldest-run share is due", async () => { + const agentId = await seedCompany(1); + const old = await queuedRecoveryRun(agentId, 150, 152); + const people: Array = []; + for (let index = 0; index < 3; index += 1) people.push(await queuedPersonRetry(agentId)); + const issue = await task(agentId); + for (let index = 0; index < 3; index += 1) { + const startedAt = new Date(Date.now() - (3 - index) * 60_000); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "on_demand", + triggerDetail: "manual", status: "succeeded", createdAt: new Date(startedAt.getTime() - 1_000), startedAt, + finishedAt: new Date(startedAt.getTime() + 1_000), contextSnapshot: { issueId: issue.id, wakeReason: "issue_commented" } }); + } + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + expect(await readRun(old.id)).toMatchObject({ status: "queued", startedAt: null }); + expect((await Promise.all(people.map(run => readRun(run.id)))).filter(run => run.startedAt)).toHaveLength(1); } finally { await heartbeat.drainActiveRunExecutions(); } @@ -285,6 +311,11 @@ suite("recovery wake herd", () => { } const deliveryOf = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!.evidence.continuationDelivery; + // The agent starts its queued recovery runs, which frees their turns in the queue. + const startQueuedRecovery = async (agentId: string) => db.update(heartbeatRuns) + .set({ status: "succeeded", startedAt: new Date(), finishedAt: new Date() }) + .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' in ('issue_recovery_action_restored', 'issue_continuation_needed')`)); const finishQueued = async (agentId: string) => db.update(heartbeatRuns).set({ status: "succeeded", startedAt: new Date(), finishedAt: new Date() }) .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"))); @@ -379,6 +410,7 @@ suite("recovery wake herd", () => { await pendingContinuation(agentId, 1); await deliverReconciledExecutions(db, wake as never); if (await deliveryOf(old.action.id) === "delivered") admittedAt = pass; + await startQueuedRecovery(agentId); } expect(admittedAt).toBeGreaterThan(0); expect(admittedAt).toBeLessThanOrEqual(bound); @@ -576,6 +608,10 @@ suite("recovery wake herd", () => { const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString(); await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, held.action.id)); + // Past the bound it is still waiting its turn behind the agent's queued recovery run. + const waiting = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!; + expect(waiting.pendingWakes).toBe(facts.pendingWakes); + await startQueuedRecovery(agentId); const stale = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!; expect(stale.pendingWakes).toBe(facts.pendingWakes - 1); }, 60_000); @@ -627,7 +663,10 @@ suite("recovery wake herd", () => { const wake = fakeWake(); await deliverReconciledExecutions(db, wake as never); expect(deliveredIds(wake)).toEqual([fresh.action.id, backlog[0]!.action.id]); - for (let pass = 0; pass < backlog.length; pass += 1) await deliverReconciledExecutions(db, wake as never); + for (let pass = 0; pass < backlog.length; pass += 1) { + await startQueuedRecovery(agentId); + await deliverReconciledExecutions(db, wake as never); + } expect(await Promise.all(backlog.map(record => deliveryOf(record.action.id)))).toEqual(backlog.map(() => "delivered")); expect(wake).toHaveBeenCalledTimes(backlog.length + 1); }, 60_000); @@ -641,8 +680,38 @@ suite("recovery wake herd", () => { for (let pass = 0; pass < records.length; pass += 1) { await deliverReconciledExecutions(db, wake as never); expect(wake).toHaveBeenCalledTimes(pass + 1); + await startQueuedRecovery(agentId); } expect(deliveredIds(wake)).toEqual(records.map(record => record.action.id)); + }, 60_000); + + it("queues at most a few recovery runs per agent, holds the rest, and admits the next when one leaves the queue", async () => { + // paperclip.staging, 2026-11-13: 139 continuations for one agent entered its queue within an hour, + // because a continuation held past the hold bound was admitted however deep the queue was. + const agentId = await seedCompany(20); + await queueOrdinaryWork(agentId, 25); + const records: Array>> = []; + for (let index = 0; index < 12; index += 1) { + const record = await pendingContinuation(agentId, index % 2 ? 2 : 200); + await holdPastBound(record, 60 * 60_000); + records.push(record); + } + const wake = fakeWake(); + for (let pass = 0; pass < 12; pass += 1) await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(QUEUED_RECOVERY_RUNS_PER_AGENT); + expect((await Promise.all(records.map(record => deliveryOf(record.action.id)))).filter(state => state === "pending")) + .toHaveLength(records.length - QUEUED_RECOVERY_RUNS_PER_AGENT); + expect(await executionControlSchedule(db).continuation()).not.toBeNull(); + + for (let pass = 0; pass < 2 * records.length && wake.mock.calls.length < records.length; pass += 1) { + await startQueuedRecovery(agentId); + await deliverReconciledExecutions(db, wake as never); + const [queued] = await db.select({ count: sql`count(*)::int` }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_recovery_action_restored'`)); + expect(Number(queued!.count)).toBeLessThanOrEqual(QUEUED_RECOVERY_RUNS_PER_AGENT); + } + expect(await Promise.all(records.map(record => deliveryOf(record.action.id)))).toEqual(records.map(() => "delivered")); await deliverReconciledExecutions(db, wake as never); expect(wake).toHaveBeenCalledTimes(records.length); }, 60_000); @@ -882,12 +951,20 @@ suite("recovery wake herd", () => { contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: "issue_recovery_action_restored" } }).returning(); await db.update(agentWakeupRequests).set({ runId: run!.id }).where(eq(agentWakeupRequests.id, request!.id)); } - const people = [await queuedPersonRetry(agentId), await queuedPersonRetry(agentId)]; + // Recovery's share comes out of agent work; with a person's run waiting it would wait too. + const work: Array = []; + for (let index = 0; index < 2; index += 1) { + const busy = await task(agentId); + const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "assignment", + triggerDetail: "system", status: "queued", contextSnapshot: { issueId: busy.id, taskId: busy.id, wakeReason: "issue_assigned" } }) + .returning(); + work.push(row!); + } const heartbeat = heartbeatService(db); try { await heartbeat.resumeQueuedRuns(); expect((await readRun(system.id)).startedAt).not.toBeNull(); - for (const run of people) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + for (const run of work) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); } finally { await heartbeat.drainActiveRunExecutions(); } diff --git a/server/src/__tests__/tool-access-client-credentials.test.ts b/server/src/__tests__/tool-access-client-credentials.test.ts new file mode 100644 index 00000000000..c39fb3d0b42 --- /dev/null +++ b/server/src/__tests__/tool-access-client-credentials.test.ts @@ -0,0 +1,682 @@ +import { createPublicKey, randomUUID, verify, type KeyObject } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + activityLog, + applyPendingMigrations, + authUsers, + companies, + companySecretBindings, + companySecrets, + connectionGrants, + createDb, + secretAccessEvents, + toolAccessAuditEvents, + toolApplications, + toolCatalogEntries, + toolConnections, +} from "@paperclipai/db"; +import { and, eq } from "drizzle-orm"; +import { toolAccessService } from "../services/tool-access.js"; +import { secretService } from "../services/secrets.js"; +import { toolAccessRoutes } from "../routes/tool-access.js"; +import { errorHandler } from "../middleware/index.js"; +import { createOAuthClientProtocol } from "../services/tool-access/oauth/client.js"; +import { + CLIENT_ASSERTION_LIFETIME_SECONDS, + generateClientKeyPair, + signClientAssertion, +} from "../services/tool-access/oauth/client-credentials.js"; +import { MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID, mcpClientCapabilities } from "../services/mcp-http.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const ISSUER = "https://auth.example.test"; +const TOKEN_URL = `${ISSUER}/oauth/token`; +const OTHER_ISSUER = "https://other-auth.example.test"; +const MCP_URL = "https://mcp.example.test/mcp"; +const CLIENT_ID = "paperclip-automation"; +const CLIENT_SECRET = "s3cret-value-never-shown"; + +function decodeJwt(token: string) { + const [header, claims, signature] = token.split("."); + return { + header: JSON.parse(Buffer.from(header!, "base64url").toString()) as Record, + claims: JSON.parse(Buffer.from(claims!, "base64url").toString()) as Record, + signingInput: `${header}.${claims}`, + signature: Buffer.from(signature!, "base64url"), + }; +} + +function verifyJwt(token: string, key: KeyObject): boolean { + const { header, signingInput, signature } = decodeJwt(token); + return verify( + "sha256", + Buffer.from(signingInput), + header.alg === "ES256" ? { key, dsaEncoding: "ieee-p1363" } : key, + signature, + ); +} + +function json(payload: unknown, status = 200, headers: Record = {}) { + return new Response(JSON.stringify(payload), { status, headers: { "content-type": "application/json", ...headers } }); +} + +/** + * A fake sign-in service and MCP server behind the outbound HTTP transport. + * The sign-in service verifies real signatures against keys an "admin" + * registered, exactly as a provider would. + */ +function fakeProvider() { + const state = { + advertisedIssuers: [ISSUER], + registeredKeys: new Map(), + clientSecret: CLIENT_SECRET, + tokenRequests: [] as Array<{ url: string; body: URLSearchParams; authorization: string | null }>, + otherIssuerRequests: 0, + mcpRequests: [] as Array<{ authorization: string | null; body: Record }>, + acceptedTokens: new Set(), + issued: 0, + tokenDelayMs: 0, + rejectAllTokens: false, + /** Other clients the sign-in service knows (id → secret). */ + extraClients: new Map(), + /** Clients whose tokens the MCP server refuses. */ + mcpDeniedClients: new Set(), + tokenClients: new Map(), + /** Serve protected-resource metadata only at the address the challenge names. */ + metadataOnlyAtHint: false, + /** Runs once when the server first receives a signed-in request (to stage races). */ + onAuthorizedMcpRequest: null as null | (() => Promise), + }; + const metadata = (issuer: string) => ({ + issuer, + token_endpoint: `${issuer}/oauth/token`, + grant_types_supported: ["client_credentials"], + token_endpoint_auth_methods_supported: ["private_key_jwt", "client_secret_basic", "client_secret_post"], + token_endpoint_auth_signing_alg_values_supported: ["RS256", "ES256"], + }); + const handle = async (input: string | URL | Request, init?: RequestInit): Promise => { + const url = String(input instanceof Request ? input.url : input); + const headers = new Headers(init?.headers); + if (url.startsWith(OTHER_ISSUER)) { + state.otherIssuerRequests += 1; + if (url.includes("/.well-known/")) return json(metadata(OTHER_ISSUER)); + return json({ access_token: "stolen", expires_in: 3600 }); + } + if (url === `${new URL(MCP_URL).origin}/meta/custom-resource` && state.metadataOnlyAtHint) { + return json({ resource: MCP_URL, authorization_servers: state.advertisedIssuers }); + } + if (url === `${new URL(MCP_URL).origin}/.well-known/oauth-protected-resource/mcp` + || url === `${new URL(MCP_URL).origin}/.well-known/oauth-protected-resource`) { + if (state.metadataOnlyAtHint) return new Response("not found", { status: 404 }); + return json({ resource: MCP_URL, authorization_servers: state.advertisedIssuers }); + } + if (url === `${ISSUER}/.well-known/oauth-authorization-server`) return json(metadata(ISSUER)); + if (url.startsWith(ISSUER) && url.includes("/.well-known/")) return new Response("not found", { status: 404 }); + if (url === TOKEN_URL) { + const body = new URLSearchParams(String(init?.body)); + state.tokenRequests.push({ url, body, authorization: headers.get("authorization") }); + if (state.tokenDelayMs) await new Promise((resolve) => setTimeout(resolve, state.tokenDelayMs)); + if (body.get("grant_type") !== "client_credentials") return json({ error: "unsupported_grant_type" }, 400); + const assertion = body.get("client_assertion"); + let authenticated = false; + let clientId = CLIENT_ID; + if (assertion) { + const { header, claims } = decodeJwt(assertion); + const key = state.registeredKeys.get(String(header.kid)); + authenticated = Boolean(key && verifyJwt(assertion, key) + && claims.iss === CLIENT_ID && claims.sub === CLIENT_ID && claims.aud === TOKEN_URL + && typeof claims.exp === "number" && claims.exp * 1000 > Date.now()); + } else { + const basic = headers.get("authorization"); + const expected = `Basic ${Buffer.from(`${CLIENT_ID}:${state.clientSecret}`).toString("base64")}`; + authenticated = basic === expected; + for (const [id, secret] of state.extraClients) { + if (basic === `Basic ${Buffer.from(`${id}:${secret}`).toString("base64")}`) { + authenticated = true; + clientId = id; + } + } + } + if (!authenticated) return json({ error: "invalid_client" }, 401); + state.issued += 1; + const token = `machine-token-${state.issued}`; + state.acceptedTokens.add(token); + state.tokenClients.set(token, clientId); + return json({ access_token: token, token_type: "Bearer", expires_in: 3600, scope: body.get("scope") ?? undefined }); + } + if (url === MCP_URL) { + const body = JSON.parse(String(init?.body ?? "{}")) as Record; + const authorization = headers.get("authorization"); + state.mcpRequests.push({ authorization, body }); + const bearer = authorization?.replace(/^Bearer /, "") ?? ""; + if (state.rejectAllTokens || !state.acceptedTokens.has(bearer) || state.mcpDeniedClients.has(state.tokenClients.get(bearer) ?? "")) { + return new Response("", { + status: 401, + headers: { "www-authenticate": `Bearer resource_metadata="${new URL(MCP_URL).origin}${state.metadataOnlyAtHint ? "/meta/custom-resource" : "/.well-known/oauth-protected-resource/mcp"}"` }, + }); + } + if (state.onAuthorizedMcpRequest) { + const hook = state.onAuthorizedMcpRequest; + state.onAuthorizedMcpRequest = null; + await hook(); + } + if (body.method === "server/discover") { + return json({ jsonrpc: "2.0", id: body.id, result: { supportedVersions: ["2026-07-28"], capabilities: {} } }); + } + if (body.method === "tools/list") { + return json({ jsonrpc: "2.0", id: body.id, result: { tools: [ + { name: "list_reports", description: "List reports", inputSchema: { type: "object", properties: {} }, annotations: { readOnlyHint: true } }, + ] } }); + } + return json({ jsonrpc: "2.0", id: body.id, result: {} }); + } + throw new Error(`unexpected request ${url}`); + }; + return { state, handle }; +} + +describe("client credentials protocol", () => { + it("creates keys without blocking other work on the server", async () => { + let otherWorkRan = false; + const generating = generateClientKeyPair("RS256"); + setImmediate(() => { otherWorkRan = true; }); + await generating; + expect(otherWorkRan).toBe(true); + }); + + it("signs an RS256 assertion with iss = sub = client ID, the token endpoint as audience, a short expiry and a unique jti", async () => { + const { privateKeyPem, publicKey } = await generateClientKeyPair("RS256"); + const now = new Date("2026-11-13T12:00:00Z"); + const first = signClientAssertion({ privateKeyPem, algorithm: "RS256", keyId: publicKey.keyId, clientId: CLIENT_ID, audience: TOKEN_URL, now }); + const second = signClientAssertion({ privateKeyPem, algorithm: "RS256", keyId: publicKey.keyId, clientId: CLIENT_ID, audience: TOKEN_URL, now }); + const decoded = decodeJwt(first); + expect(decoded.header).toEqual({ alg: "RS256", typ: "JWT", kid: publicKey.keyId }); + expect(decoded.claims).toMatchObject({ iss: CLIENT_ID, sub: CLIENT_ID, aud: TOKEN_URL, iat: now.getTime() / 1000 }); + expect(Number(decoded.claims.exp) - Number(decoded.claims.iat)).toBe(CLIENT_ASSERTION_LIFETIME_SECONDS); + expect(CLIENT_ASSERTION_LIFETIME_SECONDS).toBeLessThanOrEqual(60); + expect(decoded.claims.jti).not.toEqual(decodeJwt(second).claims.jti); + expect(verifyJwt(first, createPublicKey(publicKey.publicKeyPem))).toBe(true); + // The JWK the admin registers verifies the same signature. + expect(verifyJwt(first, createPublicKey({ key: publicKey.publicJwk, format: "jwk" }))).toBe(true); + expect(publicKey.publicJwk).toMatchObject({ kid: publicKey.keyId, alg: "RS256", use: "sig", kty: "RSA" }); + expect(JSON.stringify(publicKey)).not.toContain("PRIVATE KEY"); + expect(publicKey.publicJwk).not.toHaveProperty("d"); + }); + + it("signs ES256 assertions in JOSE (raw r||s) form", async () => { + const { privateKeyPem, publicKey } = await generateClientKeyPair("ES256"); + const token = signClientAssertion({ privateKeyPem, algorithm: "ES256", keyId: publicKey.keyId, clientId: CLIENT_ID, audience: TOKEN_URL }); + expect(decodeJwt(token).signature).toHaveLength(64); + expect(verifyJwt(token, createPublicKey(publicKey.publicKeyPem))).toBe(true); + }); + + it("sends only the signed assertion as client authentication", async () => { + const sent: Array<{ body: URLSearchParams; headers: Headers }> = []; + const { exchangeOAuthToken } = createOAuthClientProtocol({ + assertOAuthEndpointUrl: (_kind, value) => String(value), + fetchRemoteHttpUrl: async (_url, init) => { + sent.push({ body: new URLSearchParams(String(init?.body)), headers: new Headers(init?.headers) }); + return json({ access_token: "t", expires_in: 60 }); + }, + }); + await exchangeOAuthToken({ + tokenUrl: TOKEN_URL, clientId: CLIENT_ID, grantType: "client_credentials", scopes: ["reports.read"], + clientAssertion: "a.b.c", resource: MCP_URL, + }); + const { body, headers } = sent[0]!; + expect(body.get("grant_type")).toBe("client_credentials"); + expect(body.get("client_assertion_type")).toBe("urn:ietf:params:oauth:client-assertion-type:jwt-bearer"); + expect(body.get("client_assertion")).toBe("a.b.c"); + expect(body.get("scope")).toBe("reports.read"); + expect(body.get("resource")).toBe(MCP_URL); + expect(body.has("client_id")).toBe(false); + expect(body.has("client_secret")).toBe(false); + expect(headers.get("authorization")).toBeNull(); + }); + + it("declares the extension only when a connection uses machine sign-in", () => { + expect(JSON.stringify(mcpClientCapabilities("2026-07-28"))).not.toContain(MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID); + expect(mcpClientCapabilities("2026-07-28", { declareClientCredentials: true })).toMatchObject({ + extensions: { [MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID]: {} }, + }); + }); +}); + +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TOOL_ACCESS_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +describeEmbeddedPostgres("machine sign-in connections", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + const transport = { + remoteHttpEndpointLookup: async () => [{ address: "8.8.8.8", family: 4 as const }], + remoteHttpRequest: async (url: string, init: RequestInit) => fetch(url, init), + }; + const service = () => toolAccessService(db, transport); + const actor = { actorType: "user" as const, actorId: "board-user" }; + + beforeAll(async () => { + if (externalDatabaseUrl) { + await applyPendingMigrations(externalDatabaseUrl); + db = createDb(externalDatabaseUrl, { maxConnections: 6 }); + return; + } + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-client-credentials-"); + db = createDb(tempDb.connectionString, { maxConnections: 6 }); + }, 120_000); + + afterEach(async () => { + vi.restoreAllMocks(); + await db.delete(secretAccessEvents); + await db.delete(companySecretBindings); + await db.delete(activityLog); + await db.delete(toolAccessAuditEvents); + await db.delete(toolCatalogEntries); + await db.delete(connectionGrants); + await db.delete(toolConnections); + await db.delete(companySecrets); + await db.delete(toolApplications); + await db.delete(companies); + await db.delete(authUsers); + }); + + afterAll(async () => { + await db?.$client.end({ timeout: 1 }); + await tempDb?.cleanup(); + }); + + async function createCompany() { + const [company] = await db.insert(companies).values({ + name: `Machine sign-in ${randomUUID()}`, + issuePrefix: `MS${randomUUID().slice(0, 6).toUpperCase()}`, + }).returning(); + return company!; + } + + function useProvider() { + const provider = fakeProvider(); + vi.spyOn(globalThis, "fetch").mockImplementation(provider.handle); + return provider; + } + + function registerKey(provider: ReturnType, key: { keyId: string; pem: string }) { + provider.state.registeredKeys.set(key.keyId, createPublicKey(key.pem)); + } + + async function organizationGrant(connectionId: string) { + const [grant] = await db.select().from(connectionGrants) + .where(and(eq(connectionGrants.connectionId, connectionId), eq(connectionGrants.kind, "organization"))); + return grant!; + } + + async function setupKeyConnection(companyId: string, provider: ReturnType) { + const svc = service(); + const created = await svc.createMachineSignIn(companyId, { link: MCP_URL, method: "paperclip_key", scopes: "reports.read" }, actor); + registerKey(provider, created.machineSignIn.publicKey!); + const result = await svc.testMachineSignIn(companyId, created.connectionId, { clientId: CLIENT_ID }, actor); + return { svc, created, result }; + } + + it("sets up a Paperclip key connection: finds the sign-in service, shows the public key, signs in and lists tools", async () => { + const company = await createCompany(); + const provider = useProvider(); + const svc = service(); + const created = await svc.createMachineSignIn(company.id, { link: MCP_URL, method: "paperclip_key", scopes: "reports.read" }, actor); + expect(created.machineSignIn).toMatchObject({ method: "paperclip_key", issuer: ISSUER, tokenUrl: TOKEN_URL, clientId: null, scopes: ["reports.read"] }); + expect(created.machineSignIn.publicKey?.pem).toContain("BEGIN PUBLIC KEY"); + expect(created.connection.status).toBe("draft"); + // The private key is in the vault, nowhere else. + const serialized = JSON.stringify(created); + expect(serialized).not.toContain("PRIVATE KEY"); + const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId)); + expect(JSON.stringify(row!.config)).not.toContain("PRIVATE KEY"); + expect(row!.credentialSecretRefs.map((ref) => ref.configPath)).toEqual(["oauth.client_private_key"]); + + // Testing before the key is registered fails clearly and keeps the draft and its key. + await expect(svc.testMachineSignIn(company.id, created.connectionId, { clientId: CLIENT_ID }, actor)) + .rejects.toMatchObject({ status: 422, details: expect.objectContaining({ code: "client_credentials_rejected" }) }); + expect((await svc.getMachineSignIn(company.id, created.connectionId)).publicKey?.keyId).toBe(created.machineSignIn.publicKey!.keyId); + + registerKey(provider, created.machineSignIn.publicKey!); + const result = await svc.testMachineSignIn(company.id, created.connectionId, { clientId: CLIENT_ID }, actor); + expect(result.connection.status).toBe("active"); + expect(result.catalog.map((entry) => entry.name)).toContain("list_reports"); + + const accepted = provider.state.tokenRequests.at(-1)!; + expect(accepted.body.get("scope")).toBe("reports.read"); + expect(accepted.body.get("resource")).toBe(MCP_URL); + const { header, claims } = decodeJwt(accepted.body.get("client_assertion")!); + expect(header.kid).toBe(created.machineSignIn.publicKey!.keyId); + expect(claims).toMatchObject({ iss: CLIENT_ID, sub: CLIENT_ID, aud: TOKEN_URL }); + const authorized = provider.state.mcpRequests.filter((call) => call.authorization === "Bearer machine-token-1"); + expect(authorized.length).toBeGreaterThan(0); + const listCall = provider.state.mcpRequests.find((call) => call.body.method === "tools/list" && call.authorization); + expect(JSON.stringify(listCall?.body)).toContain(MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID); + const grant = await organizationGrant(created.connectionId); + expect(grant.status).toBe("active"); + expect(grant.credentialSecretRefs.map((ref) => ref.configPath)).toContain("oauth.access_token"); + }); + + it("uses a client secret with client_secret_basic and never returns the secret", async () => { + const company = await createCompany(); + const provider = useProvider(); + const svc = service(); + const created = await svc.createMachineSignIn(company.id, { + link: MCP_URL, method: "client_secret", authorizationServer: ISSUER, clientId: CLIENT_ID, clientSecret: CLIENT_SECRET, + }, actor); + expect(created.machineSignIn).toMatchObject({ method: "client_secret", hasClientSecret: true, publicKey: null }); + const result = await svc.testMachineSignIn(company.id, created.connectionId, {}, actor); + expect(result.connection.status).toBe("active"); + const tokenRequest = provider.state.tokenRequests.at(-1)!; + expect(tokenRequest.authorization).toBe(`Basic ${Buffer.from(`${CLIENT_ID}:${CLIENT_SECRET}`).toString("base64")}`); + expect(tokenRequest.body.has("client_secret")).toBe(false); + for (const value of [created, result, await svc.getMachineSignIn(company.id, created.connectionId)]) { + expect(JSON.stringify(value)).not.toContain(CLIENT_SECRET); + } + const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId)); + expect(JSON.stringify(row)).not.toContain(CLIENT_SECRET); + }); + + it("follows the server's challenge to metadata published outside the well-known addresses", async () => { + const company = await createCompany(); + const provider = useProvider(); + provider.state.metadataOnlyAtHint = true; + const created = await service().createMachineSignIn(company.id, { link: MCP_URL, method: "paperclip_key" }, actor); + expect(created.machineSignIn).toMatchObject({ issuer: ISSUER, tokenUrl: TOKEN_URL }); + }); + + it("accepts a token URL directly when the provider publishes no sign-in metadata", async () => { + const company = await createCompany(); + useProvider(); + const created = await service().createMachineSignIn(company.id, { + link: MCP_URL, method: "client_secret", authorizationServer: TOKEN_URL, clientId: CLIENT_ID, clientSecret: CLIENT_SECRET, + }, actor); + expect(created.machineSignIn).toMatchObject({ issuer: null, tokenUrl: TOKEN_URL }); + }); + + it("refuses to follow the server to a different sign-in service and never sends credentials there", async () => { + const company = await createCompany(); + const provider = useProvider(); + const svc = service(); + const created = await svc.createMachineSignIn(company.id, { link: MCP_URL, method: "paperclip_key" }, actor); + registerKey(provider, created.machineSignIn.publicKey!); + provider.state.advertisedIssuers = [OTHER_ISSUER]; + const tokenRequestsBefore = provider.state.tokenRequests.length; + await expect(svc.testMachineSignIn(company.id, created.connectionId, { clientId: CLIENT_ID }, actor)) + .rejects.toMatchObject({ status: 422, details: expect.objectContaining({ code: "client_credentials_issuer_changed" }) }); + expect(provider.state.tokenRequests.length).toBe(tokenRequestsBefore); + expect(provider.state.otherIssuerRequests).toBe(0); + expect((await svc.getMachineSignIn(company.id, created.connectionId)).tokenUrl).toBe(TOKEN_URL); + }); + + it("keeps the pinned token endpoint when an active server's challenge points elsewhere", async () => { + const company = await createCompany(); + const provider = useProvider(); + const { svc, created } = await setupKeyConnection(company.id, provider); + provider.state.advertisedIssuers = [OTHER_ISSUER]; + provider.state.rejectAllTokens = true; + await expect(svc.checkHealth(created.connectionId, actor)).rejects.toMatchObject({ + message: expect.stringContaining("didn't accept this connection's machine sign-in"), + }); + const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId)); + expect((row!.config.oauth as Record).tokenUrl).toBe(TOKEN_URL); + expect((row!.config.oauth as Record).issuer).toBe(ISSUER); + expect(provider.state.otherIssuerRequests).toBe(0); + }); + + it("reuses a cached token and shares one token request across concurrent callers", async () => { + const company = await createCompany(); + const provider = useProvider(); + const { created } = await setupKeyConnection(company.id, provider); + const grant = await organizationGrant(created.connectionId); + const issuedAfterSetup = provider.state.issued; + + // A live token is reused: no token request at all. + const first = service(); + await first.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, actor }); + expect(provider.state.issued).toBe(issuedAfterSetup); + + // Shortly before expiry, many agents on two server processes ask at once. + await db.update(connectionGrants).set({ + providerTenant: { ...(grant.providerTenant ?? {}), oauth: { ...((grant.providerTenant ?? {}).oauth as object ?? {}), accessTokenExpiresAt: new Date(Date.now() + 10_000).toISOString() } }, + }).where(eq(connectionGrants.id, grant.id)); + provider.state.tokenDelayMs = 150; + const second = service(); + const input = { companyId: company.id, connectionId: created.connectionId, grantId: grant.id, actor }; + await Promise.all([ + first.refreshOAuthGrantCredentials(input), first.refreshOAuthGrantCredentials(input), first.refreshOAuthGrantCredentials(input), + second.refreshOAuthGrantCredentials(input), second.refreshOAuthGrantCredentials(input), + ]); + expect(provider.state.issued).toBe(issuedAfterSetup + 1); + const refreshed = await organizationGrant(created.connectionId); + const expiresAt = Date.parse(String((refreshed.providerTenant?.oauth as Record).accessTokenExpiresAt)); + expect(expiresAt).toBeGreaterThan(Date.now() + 3_000_000); + }); + + it("requests a new token when the server rejects the current one", async () => { + const company = await createCompany(); + const provider = useProvider(); + const { svc, created } = await setupKeyConnection(company.id, provider); + const issuedAfterSetup = provider.state.issued; + provider.state.acceptedTokens.clear(); + const health = await svc.checkHealth(created.connectionId, actor); + expect(health.connection.healthStatus).toBe("ok"); + expect(provider.state.issued).toBe(issuedAfterSetup + 1); + expect(provider.state.mcpRequests.at(-1)?.authorization).toBe(`Bearer machine-token-${issuedAfterSetup + 1}`); + }); + + it("rotates the key without downtime: the old key works until the new one is registered and confirmed", async () => { + const company = await createCompany(); + const provider = useProvider(); + const { svc, created } = await setupKeyConnection(company.id, provider); + const oldKeyId = created.machineSignIn.publicKey!.keyId; + const started = await svc.rotateMachineSignIn(company.id, created.connectionId, { method: "paperclip_key" }, actor); + expect(started.completed).toBe(false); + const pending = started.machineSignIn.pendingPublicKey!; + expect(pending.keyId).not.toBe(oldKeyId); + expect(started.machineSignIn.publicKey?.keyId).toBe(oldKeyId); + + // Tokens keep coming from the current key while the new one is pending. + const grant = await organizationGrant(created.connectionId); + await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor }); + expect(decodeJwt(provider.state.tokenRequests.at(-1)!.body.get("client_assertion")!).header.kid).toBe(oldKeyId); + + // Completing before the provider knows the new key fails and changes nothing. + await expect(svc.completeMachineSignInRotation(company.id, created.connectionId, actor)) + .rejects.toMatchObject({ details: expect.objectContaining({ code: "client_credentials_rejected" }) }); + expect((await svc.getMachineSignIn(company.id, created.connectionId)).publicKey?.keyId).toBe(oldKeyId); + + registerKey(provider, pending); + const completed = await svc.completeMachineSignInRotation(company.id, created.connectionId, actor); + expect(completed.machineSignIn.publicKey?.keyId).toBe(pending.keyId); + expect(completed.machineSignIn.pendingPublicKey).toBeNull(); + provider.state.registeredKeys.delete(oldKeyId); + await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor }); + expect(decodeJwt(provider.state.tokenRequests.at(-1)!.body.get("client_assertion")!).header.kid).toBe(pending.keyId); + const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId)); + expect(row!.credentialSecretRefs.map((ref) => ref.configPath)).not.toContain("oauth.client_private_key_next"); + }); + + it("signs with a key ID that matches the key a refresh actually reads", async () => { + const company = await createCompany(); + const provider = useProvider(); + const { svc, created } = await setupKeyConnection(company.id, provider); + // A rotation lands between a refresh loading the settings and reading the + // vault: the vault holds the new key while config still names the old one. + const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId)); + const keyRef = row!.credentialSecretRefs.find((ref) => ref.configPath === "oauth.client_private_key")!; + const next = await generateClientKeyPair("RS256"); + registerKey(provider, { keyId: next.publicKey.keyId, pem: next.publicKey.publicKeyPem }); + await secretService(db).rotate(keyRef.secretId, { value: next.privateKeyPem }); + const grant = await organizationGrant(created.connectionId); + const before = provider.state.issued; + await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor }); + expect(provider.state.issued).toBe(before + 1); + expect(decodeJwt(provider.state.tokenRequests.at(-1)!.body.get("client_assertion")!).header.kid).toBe(next.publicKey.keyId); + }); + + it("never promotes a staged key that changed after it was registered", async () => { + const company = await createCompany(); + const provider = useProvider(); + const { svc, created } = await setupKeyConnection(company.id, provider); + const started = await svc.rotateMachineSignIn(company.id, created.connectionId, { method: "paperclip_key" }, actor); + registerKey(provider, started.machineSignIn.pendingPublicKey!); + // Another rotation overwrites the staged private key in the vault. + const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId)); + const staged = row!.credentialSecretRefs.find((ref) => ref.configPath === "oauth.client_private_key_next")!; + const other = await generateClientKeyPair("RS256"); + registerKey(provider, { keyId: other.publicKey.keyId, pem: other.publicKey.publicKeyPem }); + await secretService(db).rotate(staged.secretId, { value: other.privateKeyPem }); + await expect(svc.completeMachineSignInRotation(company.id, created.connectionId, actor)) + .rejects.toMatchObject({ status: 409, details: expect.objectContaining({ code: "client_credentials_rotation_changed" }) }); + const status = await svc.getMachineSignIn(company.id, created.connectionId); + expect(status.publicKey?.keyId).toBe(created.machineSignIn.publicKey!.keyId); + // The current key still signs in. + const grant = await organizationGrant(created.connectionId); + await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor }); + expect(decodeJwt(provider.state.tokenRequests.at(-1)!.body.get("client_assertion")!).header.kid).toBe(created.machineSignIn.publicKey!.keyId); + }); + + it("keeps working credentials when a retest's client can't use the server", async () => { + const company = await createCompany(); + const provider = useProvider(); + const svc = service(); + const created = await svc.createMachineSignIn(company.id, { + link: MCP_URL, method: "client_secret", clientId: CLIENT_ID, clientSecret: CLIENT_SECRET, + }, actor); + await svc.testMachineSignIn(company.id, created.connectionId, {}, actor); + provider.state.extraClients.set("other-client", "other-secret"); + provider.state.mcpDeniedClients.add("other-client"); + await expect(svc.testMachineSignIn(company.id, created.connectionId, { clientId: "other-client", clientSecret: "other-secret" }, actor)) + .rejects.toMatchObject({ details: expect.objectContaining({ code: "client_credentials_rejected" }) }); + const status = await svc.getMachineSignIn(company.id, created.connectionId); + expect(status.clientId).toBe(CLIENT_ID); + // The working connection still serves tools with the original client. + const health = await svc.checkHealth(created.connectionId, actor); + expect(health.connection.healthStatus).toBe("ok"); + expect(provider.state.tokenClients.get(provider.state.mcpRequests.at(-1)!.authorization!.replace("Bearer ", ""))).toBe(CLIENT_ID); + + // A retest that passes gives the organization grant the new token's expiry. + const grant = await organizationGrant(created.connectionId); + await db.update(connectionGrants).set({ + providerTenant: { ...(grant.providerTenant ?? {}), oauth: { ...((grant.providerTenant ?? {}).oauth as object ?? {}), accessTokenExpiresAt: "2000-01-01T00:00:00.000Z" } }, + }).where(eq(connectionGrants.id, grant.id)); + await svc.testMachineSignIn(company.id, created.connectionId, {}, actor); + const retested = await organizationGrant(created.connectionId); + expect(Date.parse(String((retested.providerTenant?.oauth as Record).accessTokenExpiresAt))).toBeGreaterThan(Date.now() + 3_000_000); + }); + + it("never reactivates a removed connection", async () => { + const company = await createCompany(); + const provider = useProvider(); + const { svc, created } = await setupKeyConnection(company.id, provider); + await svc.archiveConnection(created.connectionId, company.id, actor); + await expect(svc.testMachineSignIn(company.id, created.connectionId, { clientSecret: CLIENT_SECRET }, actor)) + .rejects.toMatchObject({ details: expect.objectContaining({ code: "client_credentials_connection_removed" }) }); + const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId)); + expect(row!.status).toBe("archived"); + }); + + it("does not undo a rotation that lands while a test is running", async () => { + const company = await createCompany(); + const provider = useProvider(); + const { svc, created } = await setupKeyConnection(company.id, provider); + provider.state.clientSecret = "switched-secret"; + // While the test talks to the server, an admin switches to a client secret. + provider.state.onAuthorizedMcpRequest = async () => { + await svc.rotateMachineSignIn(company.id, created.connectionId, { method: "client_secret", clientSecret: "switched-secret" }, actor); + }; + await expect(svc.testMachineSignIn(company.id, created.connectionId, {}, actor)) + .rejects.toMatchObject({ status: 409, details: expect.objectContaining({ code: "client_credentials_changed" }) }); + const status = await svc.getMachineSignIn(company.id, created.connectionId); + expect(status.method).toBe("client_secret"); + const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId)); + expect(row!.credentialSecretRefs.map((ref) => ref.configPath)).not.toContain("oauth.client_private_key"); + // The next token request uses the secret, not the deleted key. + const grant = await organizationGrant(created.connectionId); + await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor }); + expect(provider.state.tokenRequests.at(-1)!.authorization).toBe(`Basic ${Buffer.from(`${CLIENT_ID}:switched-secret`).toString("base64")}`); + }); + + it("tries a new client secret before replacing the old one", async () => { + const company = await createCompany(); + const provider = useProvider(); + const svc = service(); + const created = await svc.createMachineSignIn(company.id, { + link: MCP_URL, method: "client_secret", clientId: CLIENT_ID, clientSecret: CLIENT_SECRET, + }, actor); + await svc.testMachineSignIn(company.id, created.connectionId, {}, actor); + await expect(svc.rotateMachineSignIn(company.id, created.connectionId, { method: "client_secret", clientSecret: "wrong-new-secret" }, actor)) + .rejects.toMatchObject({ details: expect.objectContaining({ code: "client_credentials_rejected" }) }); + const grant = await organizationGrant(created.connectionId); + // The old secret still works. + await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor }); + provider.state.clientSecret = "rotated-secret"; + const rotated = await svc.rotateMachineSignIn(company.id, created.connectionId, { method: "client_secret", clientSecret: "rotated-secret" }, actor); + expect(rotated.completed).toBe(true); + await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor }); + expect(provider.state.tokenRequests.at(-1)!.authorization).toBe(`Basic ${Buffer.from(`${CLIENT_ID}:rotated-secret`).toString("base64")}`); + }); + + it("refuses machine sign-in on a personal connection", async () => { + const company = await createCompany(); + useProvider(); + const svc = service(); + const created = await svc.createMachineSignIn(company.id, { link: MCP_URL, method: "client_secret", clientId: CLIENT_ID, clientSecret: CLIENT_SECRET }, actor); + await db.update(toolConnections).set({ credentialPolicy: "per_user" }).where(eq(toolConnections.id, created.connectionId)); + await expect(svc.testMachineSignIn(company.id, created.connectionId, {}, actor)) + .rejects.toMatchObject({ status: 422, details: expect.objectContaining({ code: "client_credentials_personal_refused" }) }); + }); + + it("logs setup, verification, rotation and removal without any secret material", async () => { + const company = await createCompany(); + const provider = useProvider(); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.actor = { type: "board", userId: "board-user", userName: "Board User", userEmail: null, isInstanceAdmin: true, source: "local_implicit" }; + next(); + }); + app.use("/api", toolAccessRoutes(db, transport)); + app.use(errorHandler); + + const created = await request(app).post(`/api/companies/${company.id}/tools/machine-sign-in`) + .send({ link: MCP_URL, method: "paperclip_key", clientId: CLIENT_ID }); + expect(created.status).toBe(201); + registerKey(provider, created.body.machineSignIn.publicKey); + const connectionId = created.body.connectionId as string; + expect((await request(app).post(`/api/tool-connections/${connectionId}/machine-sign-in/test`).send({})).status).toBe(200); + expect((await request(app).post(`/api/tool-connections/${connectionId}/machine-sign-in/rotate`).send({ method: "client_secret", clientSecret: CLIENT_SECRET })).status).toBe(200); + expect((await request(app).get(`/api/tool-connections/${connectionId}/machine-sign-in`)).body).toMatchObject({ method: "client_secret", hasClientSecret: true }); + expect((await request(app).delete(`/api/tool-connections/${connectionId}`)).status).toBe(200); + + const rows = await db.select().from(activityLog).where(eq(activityLog.entityId, connectionId)); + expect(rows.map((row) => row.action)).toEqual(expect.arrayContaining([ + "tool_connection.machine_sign_in_created", + "tool_connection.machine_sign_in_verified", + "tool_connection.machine_sign_in_rotated", + "tool_connection.archived", + ])); + expect(rows.find((row) => row.action === "tool_connection.archived")?.details).toMatchObject({ machineSignIn: true }); + const logged = JSON.stringify(rows); + expect(logged).not.toContain(CLIENT_SECRET); + expect(logged).not.toContain("PRIVATE KEY"); + expect(logged).not.toContain("machine-token-"); + // Removal destroys the vault entries the connection owned. + const remaining = await db.select({ id: companySecrets.id, status: companySecrets.status }).from(companySecrets) + .where(eq(companySecrets.companyId, company.id)); + expect(remaining.filter((secret) => secret.status === "active")).toEqual([]); + }); + + it("refuses a validation error in plain words", async () => { + const company = await createCompany(); + useProvider(); + await expect(service().createMachineSignIn(company.id, { + link: "https://mcp.example.test/mcp?api_key=abc", method: "client_secret", + }, actor)).rejects.toMatchObject({ status: 400, message: expect.stringContaining("Remove the key from the server address") }); + }); +}); diff --git a/server/src/__tests__/tool-call-progress-store.test.ts b/server/src/__tests__/tool-call-progress-store.test.ts new file mode 100644 index 00000000000..8df91536dbf --- /dev/null +++ b/server/src/__tests__/tool-call-progress-store.test.ts @@ -0,0 +1,74 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issues, toolInvocations, type Db } from "@paperclipai/db"; +import { listToolCallProgress, saveToolCallProgress } from "../services/tool-call-progress.js"; +import { assignedToolName } from "../services/native-runtime/assigned-mcp-tools.js"; +import { startRecoveryPostgres } from "./helpers/recovery-postgres.js"; + +describe("saved tool call progress", () => { + let temporary: Awaited>; + let db: Db; + + beforeAll(async () => { + temporary = await startRecoveryPostgres("tool-call-progress-"); + db = createDb(temporary.connectionString); + }, 120_000); + afterAll(async () => { await temporary?.cleanup(); }); + + async function task(prefix: string) { + const companyId = randomUUID(); + const issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: prefix, issuePrefix: prefix }); + const [agent] = await db.insert(agents).values({ companyId, name: "Agent", adapterType: "process" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent!.id, status: "running" }).returning(); + await db.insert(issues).values({ id: issueId, companyId, title: "Export" }); + const call = async (values: Partial = {}) => (await db.insert(toolInvocations).values({ + companyId, issueId, runId: run!.id, agentId: agent!.id, toolName: "kv_demo__slow_with_progress", + status: "executing", startedAt: new Date("2026-11-13T10:00:00Z"), ...values, + }).returning())[0]!; + return { companyId, issueId, runId: run!.id, call }; + } + const progress = (value: number) => ({ progress: value, total: 10, message: "Exporting", updatedAt: "2026-11-13T10:00:05.000Z" }); + + it("returns a task's running calls with their latest progress, and nothing from another company or task", async () => { + const mine = await task("PRGA"); + const theirs = await task("PRGB"); + const running = await mine.call(); + await saveToolCallProgress(db, mine.companyId, running.id)(progress(3)); + await saveToolCallProgress(db, mine.companyId, running.id)(progress(4)); + const other = await theirs.call(); + await saveToolCallProgress(db, theirs.companyId, other.id)(progress(9)); + // A write naming the wrong company never lands. + await saveToolCallProgress(db, mine.companyId, other.id)(progress(1)); + const now = new Date("2026-11-13T10:01:00Z"); + + expect(await listToolCallProgress(db, mine.companyId, mine.issueId, now)).toEqual([{ + invocationId: running.id, runId: mine.runId, toolName: "kv_demo__slow_with_progress", + nativeToolName: assignedToolName("kv_demo__slow_with_progress"), + startedAt: "2026-11-13T10:00:00.000Z", progress: 4, total: 10, message: "Exporting", + updatedAt: "2026-11-13T10:00:05.000Z", done: false, + }]); + expect(await listToolCallProgress(db, mine.companyId, theirs.issueId, now)).toEqual([]); + expect(await listToolCallProgress(db, theirs.companyId, mine.issueId, now)).toEqual([]); + expect((await db.select().from(toolInvocations).where(eq(toolInvocations.id, other.id)))[0]!.progress).toMatchObject({ progress: 9 }); + }); + + it("never writes progress onto a finished call, and reports it as done", async () => { + const mine = await task("PRGC"); + const running = await mine.call({ createdAt: new Date() }); + await saveToolCallProgress(db, mine.companyId, running.id)(progress(2)); + expect(await listToolCallProgress(db, mine.companyId, mine.issueId)).toHaveLength(1); + await db.update(toolInvocations).set({ status: "succeeded", completedAt: new Date() }).where(eq(toolInvocations.id, running.id)); + // A late, throttled save after the result must not overwrite it. + await saveToolCallProgress(db, mine.companyId, running.id)(progress(8)); + expect((await db.select().from(toolInvocations).where(eq(toolInvocations.id, running.id)))[0]!.progress).toMatchObject({ progress: 2 }); + // A chat that missed the end learns it; a call that never saved progress is reported too. + const notSaved = await mine.call({ createdAt: new Date() }); + await db.update(toolInvocations).set({ status: "succeeded", completedAt: new Date() }).where(eq(toolInvocations.id, notSaved.id)); + const listed = await listToolCallProgress(db, mine.companyId, mine.issueId); + expect(listed.map((entry) => [entry.invocationId, entry.done, entry.progress]).sort()).toEqual( + [[running.id, true, 2], [notSaved.id, true, 0]].sort(), + ); + }); +}); diff --git a/server/src/__tests__/tool-call-progress.test.ts b/server/src/__tests__/tool-call-progress.test.ts new file mode 100644 index 00000000000..edf8a108de1 --- /dev/null +++ b/server/src/__tests__/tool-call-progress.test.ts @@ -0,0 +1,174 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH } from "@paperclipai/shared"; +import { + createToolCallProgressReporter, + TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS, + TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS, +} from "../services/tool-call-progress.js"; +import { assignedToolName } from "../services/native-runtime/assigned-mcp-tools.js"; + +function reporter() { + const published: Array<{ companyId: string; type: string; payload: Record }> = []; + const saved: unknown[] = []; + const instance = createToolCallProgressReporter({ + companyId: "company-a", + runId: "run-a", + issueId: "issue-a", + agentId: "agent-a", + invocationId: "invocation-a", + toolName: "kv_demo__slow_with_progress", + startedAt: new Date("2026-11-13T10:00:00Z"), + publish: (event) => { published.push(event); }, + save: async (progress) => { saved.push(progress); }, + }); + return { instance, published, saved }; +} + +describe("tool call progress reporter", () => { + beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-11-13T10:00:01Z")); }); + afterEach(() => { vi.useRealTimers(); }); + + it("publishes at most one update per interval and always the latest", () => { + const { instance, published } = reporter(); + for (let step = 1; step <= 50; step++) instance.report({ progress: step, total: 50, message: `Step ${step}` }); + // The first update goes out at once; the rest wait for the interval. + expect(published.map((event) => event.payload.progress)).toEqual([1]); + vi.advanceTimersByTime(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS); + expect(published.map((event) => event.payload.progress)).toEqual([1, 50]); + vi.advanceTimersByTime(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS * 4); + expect(published).toHaveLength(2); + // A steady stream for one second publishes about four times, never per update. + for (let tick = 0; tick < 100; tick++) { + instance.report({ progress: 51 + tick, total: 200 }); + vi.advanceTimersByTime(10); + } + expect(published.length - 2).toBeLessThanOrEqual(1_000 / TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS + 1); + }); + + it("saves on the invocation less often than it publishes", async () => { + const { instance, saved } = reporter(); + for (let tick = 0; tick < 200; tick++) { + instance.report({ progress: tick, total: 200 }); + await vi.advanceTimersByTimeAsync(10); + } + expect(saved.length).toBeLessThanOrEqual(2_000 / TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS + 1); + expect(saved.length).toBeGreaterThanOrEqual(2); + }); + + it("ties every event to its company, run and tool call", () => { + const { instance, published } = reporter(); + instance.report({ progress: 1, total: 4, message: "Reading" }); + expect(published[0]).toEqual({ + companyId: "company-a", + type: "heartbeat.run.tool_progress", + payload: { + invocationId: "invocation-a", + runId: "run-a", + issueId: "issue-a", + agentId: "agent-a", + toolName: "kv_demo__slow_with_progress", + nativeToolName: assignedToolName("kv_demo__slow_with_progress"), + startedAt: "2026-11-13T10:00:00.000Z", + updatedAt: "2026-11-13T10:00:01.000Z", + progress: 1, + total: 4, + message: "Reading", + done: false, + }, + }); + }); + + it("treats provider text as untrusted: plain, single-line and bounded", () => { + const { instance, published } = reporter(); + instance.report({ progress: 1, message: `bold\n\u0007line‮${"x".repeat(1_000)}` }); + const message = published[0]!.payload.message as string; + expect(message.startsWith("bold line ")).toBe(true); + expect(message).not.toMatch(/[\n\u0007‮]/); + expect(message.length).toBe(TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH); + }); + + it("drops unusable and out-of-order updates", () => { + const { instance, published } = reporter(); + instance.report({ progress: "1", total: 2 }); + instance.report({ progress: Number.NaN }); + instance.report({ progress: -1 }); + expect(published).toEqual([]); + instance.report({ progress: 5, total: 2 }); + expect(published[0]!.payload).toMatchObject({ progress: 5, total: null }); + vi.advanceTimersByTime(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS); + instance.report({ progress: 3, total: 10 }); + vi.advanceTimersByTime(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS); + expect(published).toHaveLength(1); + }); + + it("on finish stops pending work and tells viewers to drop the progress", () => { + const { instance, published, saved } = reporter(); + instance.report({ progress: 1, total: 3 }); + instance.report({ progress: 2, total: 3 }); + instance.finish(); + vi.advanceTimersByTime(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS * 3); + expect(published.map((event) => [event.payload.progress, event.payload.done])).toEqual([[1, false], [2, true]]); + expect(saved).toHaveLength(1); + instance.report({ progress: 3, total: 3 }); + expect(published).toHaveLength(2); + }); + + it("a call that never reported progress finishes silently", () => { + const { instance, published } = reporter(); + instance.finish(); + expect(published).toEqual([]); + }); + + it("never throws into the call when publishing or saving fails", async () => { + const instance = createToolCallProgressReporter({ + companyId: "c", runId: "r", issueId: null, agentId: null, invocationId: "i", toolName: "t", + publish: () => { throw new Error("socket gone"); }, + save: async () => { throw new Error("db gone"); }, + }); + expect(() => instance.report({ progress: 1 })).not.toThrow(); + expect(() => instance.finish()).not.toThrow(); + await vi.runAllTimersAsync(); + }); + + it("keeps one save in flight, only the newest update waits, and finish drops it", async () => { + let release!: () => void; + const saves: number[] = []; + const instance = createToolCallProgressReporter({ + companyId: "c", runId: "r", issueId: null, agentId: null, invocationId: "i", toolName: "t", + publish: () => undefined, + save: (progress) => { saves.push(progress.progress); return new Promise((resolve) => { release = resolve; }); }, + }); + instance.report({ progress: 1 }); + // The database is slow: three more save intervals pass while the first write runs. + for (let step = 2; step <= 4; step++) { + vi.advanceTimersByTime(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS); + instance.report({ progress: step }); + } + vi.advanceTimersByTime(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS); + expect(saves).toEqual([1]); + release(); + await vi.advanceTimersByTimeAsync(0); + expect(saves).toEqual([1, 4]); + instance.report({ progress: 5 }); + vi.advanceTimersByTime(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS); + instance.finish(); + release(); + await vi.advanceTimersByTimeAsync(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS * 3); + expect(saves).toEqual([1, 4]); + }); + + it("names the tool the way native runners name it, so their rows can find the call", () => { + const instance = createToolCallProgressReporter({ + companyId: "c", runId: "r", issueId: null, agentId: null, invocationId: "i", + toolName: "mcp.kv-demo-1a2b3c4d:slow-with-progress", + publish: (event) => { names.push(event.payload.nativeToolName); }, + }); + const names: unknown[] = []; + instance.report({ progress: 1 }); + expect(names).toEqual(["app_mcp_kv_demo_1a2b3c4d_slow_with_progress_889fb9a8d98e"]); + }); + + it("gives each call its own progress token", () => { + expect(reporter().instance.token).not.toBe(reporter().instance.token); + }); +}); diff --git a/server/src/__tests__/tool-gateway-service.test.ts b/server/src/__tests__/tool-gateway-service.test.ts index bf5f679e858..9f7967cb5e9 100644 --- a/server/src/__tests__/tool-gateway-service.test.ts +++ b/server/src/__tests__/tool-gateway-service.test.ts @@ -3297,7 +3297,7 @@ describeEmbeddedPostgres("tool gateway service", () => { const calls = provider.mock.calls.map(([, init]) => JSON.parse(String(init.body))) .filter((payload) => payload.method === "tools/call"); expect(calls).toHaveLength(1); - expect(calls[0].params).toEqual({ name: "search_messages", arguments: parameters }); + expect(calls[0].params).toEqual({ name: "search_messages", arguments: parameters, _meta: { progressToken: expect.stringMatching(/^paperclip-progress-/) } }); }); it("explains Google Workspace preview enrollment when a tool call is denied", async () => { diff --git a/server/src/__tests__/tool-gateway.test.ts b/server/src/__tests__/tool-gateway.test.ts index 9d64161f75e..1fc5b2701c1 100644 --- a/server/src/__tests__/tool-gateway.test.ts +++ b/server/src/__tests__/tool-gateway.test.ts @@ -6,6 +6,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import express from "express"; import { and, eq, sql } from "drizzle-orm"; +import type { LiveEvent } from "@paperclipai/shared"; +import { subscribeCompanyLiveEvents } from "../services/live-events.js"; +import { assignedToolName as nativeAssignedToolName } from "../services/native-runtime/assigned-mcp-tools.js"; import request from "supertest"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { @@ -3584,6 +3587,7 @@ rl.on("line", (line) => { "kv_get", "kv_list", "kv_set", + "slow_with_progress", ]); await allowAllToolsForAgent(db, company.id, agent.id); @@ -3618,6 +3622,61 @@ rl.on("line", (line) => { } }); + it("shows a run's viewers the provider's progress for a long call, saves it on the call, and returns the same result", async () => { + const company = await createCompany(db); + const other = await createCompany(db); + const agent = await createAgent(db, company.id); + const { issue, run } = await createIssueAndRun(db, company.id, agent.id); + const kvDemo: KvDemoHttpServer = createKvDemoHttpServer(); + const port = await kvDemo.listen(0, "127.0.0.1"); + const events: LiveEvent[] = []; + const otherEvents: LiveEvent[] = []; + const stop = subscribeCompanyLiveEvents(company.id, (event) => { events.push(event); }); + const stopOther = subscribeCompanyLiveEvents(other.id, (event) => { otherEvents.push(event); }); + try { + const access = toolAccessService(db); + const connection = await access.createConnection(company.id, { + name: "KV demo progress fixture", connectionPurpose: "tool", transport: "mcp_remote", authKind: "none", + ownership: "customer", connectionKind: "managed", + config: { url: `http://127.0.0.1:${port}/mcp` }, transportConfig: { url: `http://127.0.0.1:${port}/mcp` }, + credentialSecretRefs: [], enabled: true, status: "active", + }); + await access.refreshCatalog(connection.id, { actorType: "user", actorId: "board" }); + await allowAllToolsForAgent(db, company.id, agent.id); + const gateway = createTestToolGatewayService(db); + const session = await gateway.createSession({ companyId: company.id, agentId: agent.id, runId: run.id, issueId: issue.id }); + const slowTool = (await gateway.listToolsForSession(session.token)) + .find((tool) => tool.upstreamToolName === "slow_with_progress"); + expect(slowTool).toBeTruthy(); + + const result = await gateway.executeTool({ + sessionToken: session.token, + tool: slowTool!.name, + parameters: { steps: 6, stepMs: 200 }, + }); + expect(result).toMatchObject({ status: "completed", result: { data: { isError: false, transport: "mcp_http" } } }); + + const progress = events.filter((event) => event.type === "heartbeat.run.tool_progress"); + // Throttled: six provider updates over ~1.2s, a handful of events, the last one marks the end. + expect(progress.length).toBeGreaterThanOrEqual(3); + expect(progress.length).toBeLessThanOrEqual(7); + expect(progress.every((event) => event.companyId === company.id && event.payload.runId === run.id + && event.payload.issueId === issue.id && event.payload.toolName === slowTool!.name + && event.payload.nativeToolName === nativeAssignedToolName(slowTool!.name))).toBe(true); + expect(progress[0]!.payload).toMatchObject({ progress: 1, total: 6, message: "Finished step 1 of 6", done: false }); + expect(progress.at(-1)!.payload).toMatchObject({ done: true }); + expect(otherEvents.filter((event) => event.type === "heartbeat.run.tool_progress")).toEqual([]); + const [invocation] = await db.select().from(toolInvocations) + .where(eq(toolInvocations.id, String(progress[0]!.payload.invocationId))); + expect(invocation).toMatchObject({ companyId: company.id, runId: run.id, status: "succeeded" }); + expect(invocation!.progress).toMatchObject({ total: 6 }); + } finally { + stop(); + stopOther(); + await kvDemo.close(); + } + }); + it("expires legacy managed-connector approvals before provider dispatch", async () => { const company = await createCompany(db); const agent = await createAgent(db, company.id); diff --git a/server/src/realtime/live-event-delivery.test.ts b/server/src/realtime/live-event-delivery.test.ts index 7d3dd8c40b3..44f27a90395 100644 --- a/server/src/realtime/live-event-delivery.test.ts +++ b/server/src/realtime/live-event-delivery.test.ts @@ -303,3 +303,15 @@ it("keeps a read set at 249 ms and replaces it at 250 ms", async () => { f.delivery.dispose(); } finally { clock.mockRestore(); } }); +it("sends tool call progress only to clients watching that run", async () => { + const f = fixture(); f.subscribe(["run-1"]); + const progress = (id: number, runId: string): LiveEvent => ({ id, companyId: "company", type: "heartbeat.run.tool_progress", + createdAt: "2026-11-13T00:00:00Z", payload: { runId, agentId: "agent", invocationId: `call-${id}`, progress: id, total: 10, message: "Exporting" } }); + f.delivery.enqueue(progress(1, "run-1")); f.delivery.enqueue(progress(2, "run-2")); await settle(); + const delivered = f.frames.filter(frame => frame.type === "heartbeat.run.tool_progress"); + expect(delivered.map(frame => frame.payload.runId)).toEqual(["run-1"]); + // Another run's progress carries no content to this client. + expect(JSON.stringify(f.frames)).not.toContain("call-2"); + expect(f.decorate).not.toHaveBeenCalled(); + f.delivery.dispose(); +}); diff --git a/server/src/realtime/live-event-delivery.ts b/server/src/realtime/live-event-delivery.ts index 3c49e992bfb..2641be7d70d 100644 --- a/server/src/realtime/live-event-delivery.ts +++ b/server/src/realtime/live-event-delivery.ts @@ -16,7 +16,10 @@ export function sharedDeliveryRead(reads: LiveDeliveryReads | undefined, key: pending.catch(() => { if (reads.get(key) === pending) reads.delete(key); }); return pending; } -const transcript = (event: LiveEvent) => event.type === "heartbeat.run.event" || event.type === "heartbeat.run.log"; +// Per-run streams: sent only to clients that asked for the run. Tool progress +// follows its run's transcript, since only an open transcript shows it. +const transcript = (event: LiveEvent) => event.type === "heartbeat.run.event" || event.type === "heartbeat.run.log" + || event.type === "heartbeat.run.tool_progress"; const runId = (event: LiveEvent) => typeof event.payload.runId === "string" ? event.payload.runId : null; interface Socket { readyState: number; diff --git a/server/src/realtime/live-events-ws.ts b/server/src/realtime/live-events-ws.ts index b09a85d65d1..e5d3d23031b 100644 --- a/server/src/realtime/live-events-ws.ts +++ b/server/src/realtime/live-events-ws.ts @@ -315,7 +315,7 @@ export function setupLiveEventsWebSocketServer( : payload.entityType === "issue" && typeof payload.entityId === "string" ? payload.entityId : null; const runId = typeof payload.runId === "string" ? payload.runId : payload.entityType === "heartbeat_run" && typeof payload.entityId === "string" ? payload.entityId : null; - if (event.type === "heartbeat.run.event" || event.type === "heartbeat.run.log") { + if (event.type === "heartbeat.run.event" || event.type === "heartbeat.run.log" || event.type === "heartbeat.run.tool_progress") { const allowed = await sharedDeliveryRead(reads, "transcript-scope", async () => (await access.decide({ actor: context!.actor, action: "company_scope:read", resource: { type: "company", companyId: event.companyId } })).allowed); diff --git a/server/src/routes/mcp-apps.ts b/server/src/routes/mcp-apps.ts index 1a253223f24..e8a1537eaf9 100644 --- a/server/src/routes/mcp-apps.ts +++ b/server/src/routes/mcp-apps.ts @@ -15,6 +15,7 @@ import { assertBoard, getAccessibleResource } from "./authz.js"; import { connectionRequestUserId } from "./connection-management-authz.js"; import { authorizationService } from "../services/authorization.js"; import { mcpAppViewsService, toMcpAppViewSummary } from "../services/mcp-app-views.js"; +import { listToolCallProgress } from "../services/tool-call-progress.js"; import { ToolGatewayHttpError, type ToolGatewayService } from "../services/tool-gateway.js"; const uuidSchema = z.string().uuid(); @@ -34,7 +35,7 @@ export function mcpAppsRoutes(db: Db, toolGateway: McpAppsGateway) { const router = Router(); const views = mcpAppViewsService(db); - router.use("/issues/:issueId/mcp-app-views", async (req, res, next) => { + const readableTask = async (req: Request, res: Response, next: () => void) => { assertBoard(req); if (!uuidSchema.safeParse(req.params.issueId).success) { res.status(404).json({ error: "Task not found" }); @@ -57,7 +58,9 @@ export function mcpAppsRoutes(db: Db, toolGateway: McpAppsGateway) { res.locals.mcpAppIssue = issue; res.setHeader("Cache-Control", "no-store"); next(); - }); + }; + router.use("/issues/:issueId/mcp-app-views", readableTask); + router.use("/issues/:issueId/tool-call-progress", readableTask); async function viewFor(req: Request, res: Response) { const issue = res.locals.mcpAppIssue as { id: string; companyId: string }; @@ -162,6 +165,12 @@ export function mcpAppsRoutes(db: Db, toolGateway: McpAppsGateway) { res.status(201).json({ commentId: comment.id }); }); + // The latest progress of the task's running app tool calls (and recently finished ones, as done), for a reload mid-call. + router.get("/issues/:issueId/tool-call-progress", async (_req, res) => { + const issue = res.locals.mcpAppIssue as { id: string; companyId: string }; + res.json(await listToolCallProgress(db, issue.companyId, issue.id)); + }); + router.get("/issues/:issueId/mcp-app-views", async (req, res) => { const issue = res.locals.mcpAppIssue as { id: string; companyId: string }; res.json(await views.listForIssue(issue.companyId, issue.id)); diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 69c73db0b19..754c7ab8649 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -291,6 +291,9 @@ import { workspaceFileResourceQuerySchema, // Tool access connectToolAppSchema, + createMachineSignInSchema, + testMachineSignInSchema, + rotateMachineSignInSchema, configureRailwaySshSchema, createToolApplicationSchema, updateToolApplicationSchema, @@ -1447,6 +1450,7 @@ const browserUseOperations = [ // MCP Apps views in the task chat. Board only; reads follow task read access. const mcpAppsOperations = [ ["get", "/api/issues/{issueId}/mcp-app-views", "List a task's MCP Apps views", undefined], + ["get", "/api/issues/{issueId}/tool-call-progress", "Read the latest progress of a task's running app tool calls", undefined], ["get", "/api/issues/{issueId}/mcp-app-views/{viewId}", "Read an MCP Apps view with its document", undefined], ["post", "/api/issues/{issueId}/mcp-app-views/{viewId}/tool-calls", "Call a tool from an MCP Apps view", mcpAppToolCallSchema], ["put", "/api/issues/{issueId}/mcp-app-views/{viewId}/model-context", "Store an MCP Apps view's context for the task's next run", mcpAppModelContextUpdateSchema], @@ -12544,6 +12548,68 @@ registerCurrentRoute({ summary: "Archive a tool connection", }); +const MACHINE_SIGN_IN_ERRORS = { + 400: r.badRequest, + 401: r.unauthorized, + 403: r.forbidden, + 404: r.notFound, + 422: r.unprocessable, +}; + +registerCurrentRoute({ + "x-paperclip-transport": "rest", + "x-paperclip-connection-management": "connections:configure", + method: "post", + path: "/api/companies/{companyId}/tools/machine-sign-in", + tags: ["tool-access"], + summary: "Start machine sign-in (OAuth client credentials) for an organization connection", + description: "Creates a draft organization connection to a remote MCP server that signs in as a registered client, with no person involved. With method paperclip_key, Paperclip creates a signing key, keeps the private half in the company vault, and returns the public key to register with the provider. Secrets are write-only.", + body: createMachineSignInSchema, + responses: { 201: r.ok(), ...MACHINE_SIGN_IN_ERRORS }, +}); + +registerCurrentRoute({ + "x-paperclip-transport": "rest", + "x-paperclip-connection-management": "connections:configure", + method: "get", + path: "/api/tool-connections/{connectionId}/machine-sign-in", + tags: ["tool-access"], + summary: "Read a connection's machine sign-in settings (no secret values)", + responses: { 200: r.ok(), ...MACHINE_SIGN_IN_ERRORS }, +}); + +registerCurrentRoute({ + "x-paperclip-transport": "rest", + "x-paperclip-connection-management": "connections:configure", + method: "post", + path: "/api/tool-connections/{connectionId}/machine-sign-in/test", + tags: ["tool-access"], + summary: "Test machine sign-in against the provider and server, then turn the connection on", + body: testMachineSignInSchema, + responses: { 200: r.ok(), ...MACHINE_SIGN_IN_ERRORS }, +}); + +registerCurrentRoute({ + "x-paperclip-transport": "rest", + "x-paperclip-connection-management": "connections:configure", + method: "post", + path: "/api/tool-connections/{connectionId}/machine-sign-in/rotate", + tags: ["tool-access"], + summary: "Replace the machine sign-in key or client secret without downtime", + body: rotateMachineSignInSchema, + responses: { 200: r.ok(), ...MACHINE_SIGN_IN_ERRORS }, +}); + +registerCurrentRoute({ + "x-paperclip-transport": "rest", + "x-paperclip-connection-management": "connections:configure", + method: "post", + path: "/api/tool-connections/{connectionId}/machine-sign-in/rotate/complete", + tags: ["tool-access"], + summary: "Switch to the new machine sign-in key once the provider accepts it", + responses: { 200: r.ok(), ...MACHINE_SIGN_IN_ERRORS }, +}); + registerCurrentRoute({ "x-paperclip-transport": "rest", "x-paperclip-connection-management": "connections:configure", diff --git a/server/src/routes/tool-access.ts b/server/src/routes/tool-access.ts index 8faa88e2409..146d1f61b4a 100644 --- a/server/src/routes/tool-access.ts +++ b/server/src/routes/tool-access.ts @@ -23,6 +23,10 @@ import { type ToolConnection, type ToolConnectionCreateCapabilities, connectToolAppSchema, + createMachineSignInSchema, + testMachineSignInSchema, + rotateMachineSignInSchema, + type MachineSignInStatus, configureRailwaySshSchema, createConnectionGrantDelegationSchema, createToolStdioCommandTemplateSchema, @@ -105,6 +109,21 @@ const ORGANIZATION_GRANT_DENIAL_REASON = type Heartbeat = ReturnType; /** Allowlist (e.g. Google Sheets allowed spreadsheet ids) lives in connection config. */ +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +/** Activity for machine sign-in carries identifiers only: never a secret, key or token. */ +function machineSignInActivityDetails(status: MachineSignInStatus) { + return { + method: status.method, + issuer: status.issuer, + keyId: status.publicKey?.keyId ?? null, + pendingKeyId: status.pendingPublicKey?.keyId ?? null, + scopeCount: status.scopes.length, + }; +} + function allowlistIds(config: Record | null | undefined): string[] { const raw = config?.allowedSpreadsheetIds; if (!Array.isArray(raw)) return []; @@ -1005,6 +1024,87 @@ function connectorEnrollmentPrincipal(req: Request): string { } }); + // Machine sign-in (MCP OAuth client credentials): organization connections + // that authenticate as a registered client, with no person signing in. + router.post("/companies/:companyId/tools/machine-sign-in", managementGuard("connections:configure", "POST /companies/:companyId/tools/machine-sign-in"), validate(createMachineSignInSchema), async (req, res) => { + const companyId = req.params.companyId as string; + assertToolAppMutationAccess(req, companyId); + if (!await isToolConnectionManagerQuiet(req, companyId)) throw forbidden(ORGANIZATION_GRANT_DENIAL_REASON); + try { + const result = await svc.createMachineSignIn(companyId, req.body, connectionRequestActor(req)); + await logActivity(db, { + companyId, + ...connectionRequestActor(req), + action: "tool_connection.machine_sign_in_created", + entityType: "tool_connection", + entityId: result.connectionId, + details: machineSignInActivityDetails(result.machineSignIn), + }); + res.status(201).json(result); + } catch (error) { + svc.ensureNoDuplicateNameError(error); + } + }); + + router.get("/tool-connections/:connectionId/machine-sign-in", managementGuard("connections:configure", "GET /tool-connections/:connectionId/machine-sign-in"), async (req, res) => { + const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found"); + if (!existing) return; + await assertToolConnectionConfigureAccess(req, existing); + res.json(await svc.getMachineSignIn(existing.companyId, existing.id)); + }); + + router.post("/tool-connections/:connectionId/machine-sign-in/test", managementGuard("connections:configure", "POST /tool-connections/:connectionId/machine-sign-in/test"), validate(testMachineSignInSchema), async (req, res) => { + const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found"); + if (!existing) return; + await assertToolConnectionConfigureAccess(req, existing); + const result = await svc.testMachineSignIn(existing.companyId, existing.id, req.body, connectionRequestActor(req)); + await logActivity(db, { + companyId: existing.companyId, + ...connectionRequestActor(req), + action: "tool_connection.machine_sign_in_verified", + entityType: "tool_connection", + entityId: existing.id, + details: { + ...machineSignInActivityDetails(await svc.getMachineSignIn(existing.companyId, existing.id)), + clientSecretReplaced: Boolean(req.body.clientSecret), + catalogEntryCount: result.catalog.length, + }, + }); + res.json(result); + }); + + router.post("/tool-connections/:connectionId/machine-sign-in/rotate", managementGuard("connections:configure", "POST /tool-connections/:connectionId/machine-sign-in/rotate"), validate(rotateMachineSignInSchema), async (req, res) => { + const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found"); + if (!existing) return; + await assertToolConnectionConfigureAccess(req, existing); + const result = await svc.rotateMachineSignIn(existing.companyId, existing.id, req.body, connectionRequestActor(req)); + await logActivity(db, { + companyId: existing.companyId, + ...connectionRequestActor(req), + action: result.completed ? "tool_connection.machine_sign_in_rotated" : "tool_connection.machine_sign_in_rotation_started", + entityType: "tool_connection", + entityId: existing.id, + details: machineSignInActivityDetails(result.machineSignIn), + }); + res.json(result); + }); + + router.post("/tool-connections/:connectionId/machine-sign-in/rotate/complete", managementGuard("connections:configure", "POST /tool-connections/:connectionId/machine-sign-in/rotate/complete"), async (req, res) => { + const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found"); + if (!existing) return; + await assertToolConnectionConfigureAccess(req, existing); + const result = await svc.completeMachineSignInRotation(existing.companyId, existing.id, connectionRequestActor(req)); + await logActivity(db, { + companyId: existing.companyId, + ...connectionRequestActor(req), + action: "tool_connection.machine_sign_in_rotated", + entityType: "tool_connection", + entityId: existing.id, + details: machineSignInActivityDetails(result.machineSignIn), + }); + res.json(result); + }); + router.post( "/companies/:companyId/tools/connections/:connectionId/start-authorization", managementGuard("connections:configure", "POST /companies/:companyId/tools/connections/:connectionId/start-authorization", { personal: true }), validate(startConnectionAuthorizationSchema), @@ -2286,7 +2386,11 @@ function connectorEnrollmentPrincipal(req: Request): string { action: "tool_connection.archived", entityType: "tool_connection", entityId: connection.id, - details: { transport: connection.transport, ...removal }, + details: { + transport: connection.transport, + ...(asRecord(asRecord(existing.config).oauth).grantType === "client_credentials" ? { machineSignIn: true } : {}), + ...removal, + }, }); if (applicationBefore.status !== "archived" && applicationAfter.status === "archived") { await logActivity(db, { diff --git a/server/src/services/environment-retained-orphan-settlement.ts b/server/src/services/environment-retained-orphan-settlement.ts index 53fa0be16bc..911a295299c 100644 --- a/server/src/services/environment-retained-orphan-settlement.ts +++ b/server/src/services/environment-retained-orphan-settlement.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, ne, notInArray, or, sql } from "drizzle-orm"; +import { and, eq, inArray, ne, notInArray, or, sql, type SQL } from "drizzle-orm"; import { environmentLeases, heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; import { nativeWorkspaceExportPendingForLease } from "./native-runtime/native-workspace-finalization-state.js"; @@ -97,7 +97,7 @@ function orphanedRunSandboxCondition() { * `heldBy`, so a held orphan is not listed (and not polled) until its last * reference clears or its retention window ends. The settle re-checks all of * them, plus the pending native export, under the row lock. */ -function softlyHeldSql(now?: Date) { +export function softlyHeldSql(now?: Date) { const at = now ? sql`${now.toISOString()}::timestamptz` : sql`now()`; const issue = sql`coalesce(run.native_issue_id, run.issue_id, ${environmentLeases.issueId})`; return sql`(${environmentLeases.expiresAt} is null or ${environmentLeases.expiresAt} > ${at}) and ( @@ -116,8 +116,7 @@ function alwaysHeldSql() { or ${abandonedWorkspaceFilesHeldOnLeaseRowSql()} or coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') = 'true' or coalesce(${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,state}' = 'pending', false) - or exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId} - and run.status not in ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')) + or coalesce((select run.status not in ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted') -- A prepared native workspace bound to this exact sandbox, for a run with -- workspace access. This is deliberately no wider than the settle's own -- export check: an orphan the settle would release must never be hidden @@ -125,21 +124,23 @@ function alwaysHeldSql() { -- are listed and held by the settle until their export settles. -- The descriptor must also be one the settle's reader accepts -- (readNativeWorkspaceSyncReference); an invalid one holds nothing. - or exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId} - and run.runtime_mode = 'native' - and run.runner_profile_json#>>'{nativeWorkspaceSync,schema}' in - ('paperclip.native-workspace-sync/v1', 'paperclip.native-workspace-sync/v2', 'paperclip.native-workspace-sync/v3') - and run.runner_profile_json#>>'{nativeWorkspaceSync,state}' = 'prepared' - and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,descriptorSha256}' ~ '^[0-9a-f]{64}$', false) - and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,baselineSha256}' ~ '^[0-9a-f]{64}$', false) - and coalesce(jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,finalHostSha256}'), 'null') = 'null' - and jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,workspaceId}') = 'string' - and jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,remoteCwd}') = 'string' - and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,resourceDisposition}', 'destroy') - in ('keep_running', 'stop_and_retain', 'destroy') - and run.runner_profile_json#>>'{nativeWorkspaceSync,leaseId}' = ${environmentLeases.id}::text - and run.runner_profile_json#>>'{nativeWorkspaceSync,providerLeaseId}' = ${environmentLeases.providerLeaseId} - and coalesce(run.runner_profile_json#>>'{nativeExecutionInput,workspace,access}', '') <> 'none') + or (run.runtime_mode = 'native' + and profile.sync->>'schema' in + ('paperclip.native-workspace-sync/v1', 'paperclip.native-workspace-sync/v2', 'paperclip.native-workspace-sync/v3') + and profile.sync->>'state' = 'prepared' + and coalesce(profile.sync->>'descriptorSha256' ~ '^[0-9a-f]{64}$', false) + and coalesce(profile.sync->>'baselineSha256' ~ '^[0-9a-f]{64}$', false) + and coalesce(jsonb_typeof(profile.sync->'finalHostSha256'), 'null') = 'null' + and jsonb_typeof(profile.sync->'workspaceId') = 'string' + and jsonb_typeof(profile.sync->'remoteCwd') = 'string' + and coalesce(profile.sync->>'resourceDisposition', 'destroy') in ('keep_running', 'stop_and_retain', 'destroy') + and profile.sync->>'leaseId' = ${environmentLeases.id}::text + and profile.sync->>'providerLeaseId' = ${environmentLeases.providerLeaseId} + and coalesce(profile.access, '') <> 'none') + from heartbeat_runs run + cross join lateral (select run.runner_profile_json->'nativeWorkspaceSync' as sync, + run.runner_profile_json#>>'{nativeExecutionInput,workspace,access}' as access offset 0) profile + where run.id = ${environmentLeases.heartbeatRunId}), false) or exists (select 1 from environment_leases other where other.id <> ${environmentLeases.id} and other.provider = ${environmentLeases.provider} and other.provider_lease_id = ${environmentLeases.providerLeaseId} and other.status in ('active', 'retained', 'pending_cleanup')))`; @@ -148,15 +149,31 @@ function alwaysHeldSql() { /** Retained orphans that nothing appears to reference now. */ // The listing is built once, so it reads the database clock. export function releasableRetainedOrphanCondition(now?: Date) { - return and(retainedOrphanSandboxCondition(), sql`not ${alwaysHeldSql()}`, sql`not (${softlyHeldSql(now)})`); + return onlyForRetainedOrphans(sql`not ${alwaysHeldSql()} and not (${softlyHeldSql(now)})`); +} + +function retainedOrphanCandidateCondition() { + return and(eq(environmentLeases.cleanupStatus, "success"), sql`${environmentLeases.heartbeatRunId} is not null`, + sql`${environmentLeases.provider} is not null`, sql`${environmentLeases.providerLeaseId} is not null`, + or(and(eq(environmentLeases.status, "released"), eq(environmentLeases.leasePolicy, "ephemeral"), + sql`${environmentLeases.metadata} ? ${ORPHANED_SANDBOX_RECOVERY_KEY}`, + sql`${environmentLeases.metadata} ? 'sandboxStopAndRetainReceipt'`), + and(eq(environmentLeases.leasePolicy, "reuse_by_environment"), inArray(environmentLeases.status, ["released", "failed"]), + sql`coalesce(${environmentLeases.metadata}#>>'{remoteExecutionTermination,state}' = 'stopped', false)`), + and(eq(environmentLeases.status, "retained"), eq(environmentLeases.leasePolicy, "reuse_by_environment"), + sql`${environmentLeases.executionWorkspaceId} is not null`))); +} + +function onlyForRetainedOrphans(condition: SQL, candidate: SQL | undefined = retainedOrphanSandboxCondition()) { + return and(retainedOrphanCandidateCondition(), sql`(case when ${candidate} then ${condition} else false end)`); } /** When the next retained orphan's retention window ends while only soft * references hold it; the reconciler wakes then to release it. */ export async function nextRetainedOrphanDueAt(db: Db, now: Date): Promise { const [row] = await db.select({ due: sql`min(${environmentLeases.expiresAt})` }).from(environmentLeases) - .where(and(retainedOrphanSandboxCondition(), sql`not ${alwaysHeldSql()}`, sql`${softlyHeldSql(now)}`, - sql`${environmentLeases.expiresAt} is not null`)); + .where(onlyForRetainedOrphans(sql`not ${alwaysHeldSql()} and ${softlyHeldSql(now)}`, + and(retainedOrphanSandboxCondition(), sql`${environmentLeases.expiresAt} is not null`))); return row?.due ? new Date(row.due) : null; } diff --git a/server/src/services/execution-projection-agent-queue.test.ts b/server/src/services/execution-projection-agent-queue.test.ts index b1e31cce202..ea86124347b 100644 --- a/server/src/services/execution-projection-agent-queue.test.ts +++ b/server/src/services/execution-projection-agent-queue.test.ts @@ -80,6 +80,44 @@ const minutesAgo = (minutes: number) => new Date(now.getTime() - minutes * 60_00 }); }); + it("names the routine's open execution that holds a queued routine task, before the agent's limit", async () => { + const f = await fixture({ maxConcurrentRuns: 20, running: 0 }); + const routineId = randomUUID(); + const holderIssueId = randomUUID(), holderRunId = randomUUID(); + const waitingIssueId = randomUUID(), waitingRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: holderRunId, companyId: f.companyId, agentId: f.agentId, status: "running", + startedAt: minutesAgo(18), createdAt: minutesAgo(30), issueId: holderIssueId, contextSnapshot: { issueId: holderIssueId } }); + await db.insert(issues).values([ + { id: holderIssueId, companyId: f.companyId, identifier: "Q-3093", title: "Nightly test gardening", status: "in_progress", + assigneeAgentId: f.agentId, originKind: "routine_execution", originId: routineId, executionRunId: holderRunId }, + { id: waitingIssueId, companyId: f.companyId, identifier: "Q-3636", title: "Nightly test gardening", status: "in_progress", + assigneeAgentId: f.agentId, originKind: "routine_execution", originId: routineId }, + ]); + await db.insert(heartbeatRuns).values({ id: waitingRunId, companyId: f.companyId, agentId: f.agentId, status: "queued", + createdAt: minutesAgo(8), issueId: waitingIssueId, contextSnapshot: { issueId: waitingIssueId } }); + + expect((await executionProjectionsForRuns(db, f.companyId, [waitingRunId], now)).get(waitingRunId)).toMatchObject({ + phase: "queued", + label: "Waiting for Q-3093 to finish", + cause: "routine_execution_in_progress", + }); + // The holder's own run is not waiting on itself. + expect((await executionProjectionsForRuns(db, f.companyId, [holderRunId], now)).get(holderRunId)?.cause).not.toBe("routine_execution_in_progress"); + + // A private holder is not named to readers of the waiting task. + await db.update(issues).set({ visibility: "private" }).where(eq(issues.id, holderIssueId)); + const privateHold = (await executionProjectionsForRuns(db, f.companyId, [waitingRunId], now)).get(waitingRunId); + expect(privateHold).toMatchObject({ label: "Waiting for another task in this routine to finish", cause: "routine_execution_in_progress" }); + expect(JSON.stringify(privateHold)).not.toContain("Q-3093"); + await db.update(issues).set({ visibility: "open" }).where(eq(issues.id, holderIssueId)); + + // Once the holder closes, the routine slot is free and the run is plainly queued. + await db.update(issues).set({ status: "done" }).where(eq(issues.id, holderIssueId)); + expect((await executionProjectionsForRuns(db, f.companyId, [waitingRunId], now)).get(waitingRunId)).toMatchObject({ + label: "Queued", cause: null, + }); + }); + it("leaves out queued runs that a preview worktree's cutoff keeps from starting", async () => { const f = await fixture({ maxConcurrentRuns: 1, running: 1 }); const copiedOld = await f.queue({ priority: "critical", createdMinutesAgo: 120 }); diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 714cc9546d8..0954d4dcf44 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -24,6 +24,7 @@ import { isAgentAwaitingSetup } from "../modules/agent-lifecycle/index.js"; import { NATIVE_RESPONSE_WAKE_WAITING_REASON } from "./native-runtime/status-arbiter.js"; import { agentQueueLinesForRuns } from "./heartbeat/queued-run-line.js"; const MAX_PREDECESSOR_CLEANUP_CHECKS = 16; +const ANOTHER_ROUTINE_TASK = "another task in this routine"; const text = (v: unknown) => (typeof v === "string" ? v : null); const executionRunColumns = { id: heartbeatRuns.id, @@ -211,6 +212,32 @@ export async function executionProjectionsForRuns( : new Map>(); const issueIdOf = (run: Run) => run.nativeIssueId ?? text(run.contextSnapshot?.issueId); const queuedIssueIds = [...new Set(queuedRuns.map(issueIdOf).filter((id): id is string => !!id))]; + // A routine runs one execution at a time (issues_open_routine_execution_uq): + // a never-started run of a routine task waits while another open task of + // the same routine holds an execution. One read for the batch. + const routineHolderByIssue = new Map(); + const unstartedIssueIds = [...new Set(queuedRuns.filter(run => !run.startedAt).map(issueIdOf).filter((id): id is string => !!id))]; + if (unstartedIssueIds.length) { + const holders = await read(async () => db.execute<{ issueId: string; holderIdentifier: string | null }>(sql` + -- A private holder is never named: this projection has no reader to check access for. + select waiting.id as "issueId", case when holder.visibility = 'open' then holder.identifier end as "holderIdentifier" + from ${issues} waiting + join ${issues} holder on holder.company_id = waiting.company_id + and holder.origin_kind = 'routine_execution' + and holder.origin_id = waiting.origin_id + and holder.origin_fingerprint is not distinct from waiting.origin_fingerprint + and holder.id <> waiting.id + and holder.hidden_at is null + and holder.execution_run_id is not null + and holder.status in ('backlog', 'todo', 'in_progress', 'in_review', 'blocked') + where waiting.company_id = ${companyId} + and waiting.origin_kind = 'routine_execution' + and waiting.origin_id is not null + and waiting.id in (${sql.join(unstartedIssueIds.map(id => sql`${id}::uuid`), sql`, `)})`)); + for (const row of [...holders]) { + if (!routineHolderByIssue.has(row.issueId)) routineHolderByIssue.set(row.issueId, row.holderIdentifier ?? ANOTHER_ROUTINE_TASK); + } + } if (queuedIssueIds.length) { const issueKey = sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.contextSnapshot}->>'issueId')`; // Two set-based reads for the whole batch, then one bounded, parallel @@ -285,6 +312,7 @@ export async function executionProjectionsForRuns( setupByAgent.get(run.agentId), currentResponseWait(run), agentQueueByRun.get(run.id), + run.status === "queued" && !run.startedAt ? routineHolderByIssue.get(issueIdOf(run) ?? "") : undefined, ), ); } @@ -313,6 +341,8 @@ export function projectExecution( setupAgent?: { lifecycleState: string; lifecycleHolds: readonly string[]; retryRequired?: boolean }, responseWaitCurrent = false, agentQueue?: ExecutionProjection["agentQueue"], + /** The open task of the same routine that holds the routine's one execution. */ + routineHolder?: string, ): ExecutionProjection { const detail = coordinator?.failureDetail ?? {}; const successorRunId = text(detail.successorRunId); @@ -561,6 +591,12 @@ export function projectExecution( projection.nextAction = "The previous run on this task is still releasing its runner. This turn starts automatically when it settles."; return set("queued", "Waiting for previous run to stop"); } + // Checked before the agent's limit: a free slot would not start this run. + if (run.status === "queued" && !run.startedAt && routineHolder) { + projection.cause = "routine_execution_in_progress"; + projection.nextAction = `This routine runs one task at a time and ${routineHolder === ANOTHER_ROUTINE_TASK ? "another of its tasks" : routineHolder} is still open with its run. This task starts automatically when that one finishes.`; + return set("queued", `Waiting for ${routineHolder} to finish`); + } if (run.status === "queued" && !run.startedAt && agentQueue) { projection.cause = "agent_at_capacity"; projection.agentQueue = agentQueue; diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index acc9a9132b7..250c985095d 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -39,10 +39,13 @@ import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { isReattachPhaseActive } from "./heartbeat/run-admission.js"; import { provenGoneLegacyControllerBootId } from "./legacy-controller-lease.js"; +import { reserveAutomaticRunCreation } from "./recovery/automatic-run-brake.js"; +import { recordContinuationDeliveryAttempt } from "./recovery/continuation-delivery-attempt.js"; import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; import { alias } from "drizzle-orm/pg-core"; import { RECOVERY_BACKLOG_AGE_MS, RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; +import { agentRecoveryQueueIsFull } from "./recovery-queue-admission.js"; import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js"; import { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_BACKLOG_SHARE_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, @@ -394,6 +397,7 @@ export async function deliverReconciledExecutions( ] : [ sql`${pastBound} desc`, sql`${overdueTurn} asc nulls last`, + sql`${attemptedAt} asc nulls first`, sql`${sourceRun.finishedAt} desc nulls last`, sql`${issueRecoveryActions.updatedAt} asc`, ]; @@ -444,23 +448,19 @@ export async function deliverReconciledExecutions( ] : [ sql`${candidates.pastBound} desc`, sql`${candidates.heldAt} asc nulls last`, + sql`${candidates.attemptedAt} asc nulls first`, sql`${candidates.sourceFinishedAt} desc nulls last`, sql`${candidates.actionUpdatedAt} asc`, ])) .limit(25); return rows.map(row => row.action); }; - const deferRefusedContinuation = (action: typeof issueRecoveryActions.$inferSelect) => db.update(issueRecoveryActions) - .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( - { continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` }) - .where(and( - eq(issueRecoveryActions.companyId, action.companyId), - eq(issueRecoveryActions.id, action.id), - eq(issueRecoveryActions.status, "resolved"), - sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, - )); + const deferRefusedContinuation = (action: typeof issueRecoveryActions.$inferSelect) => recordContinuationDeliveryAttempt(db, action); const agentHasRoom = async (action: typeof issueRecoveryActions.$inferSelect, agentId: string, kind: ContinuationClass) => { if ((admittedByCompany.get(action.companyId) ?? 0) >= companyLimit) return false; + // A few at a time per agent, fresh or not: the rest waits here, durably, + // until a queued recovery run starts or is dropped. + if (await agentRecoveryQueueIsFull(db, action.companyId, agentId)) return false; if (kind === "fresh") return true; const [agent] = await db.select().from(agents).where(and(eq(agents.companyId, action.companyId), eq(agents.id, agentId))); if (!agent) return true; @@ -549,6 +549,8 @@ export async function deliverReconciledExecutions( seenAgents.add(action.returnOwnerAgentId); continue; } + const slot = await reserveAutomaticRunCreation(db, action.companyId, { issueId: task.id, sourceRunId: decision.runId }); + if (slot.heldUntil) { await deferRefusedContinuation(action); continue; } const run = await wake(action.returnOwnerAgentId, { source: "automation", triggerDetail: "system", @@ -569,9 +571,10 @@ export async function deliverReconciledExecutions( ...((action.evidence.freshContinuation as { isolation?: unknown } | undefined)?.isolation === "fresh_isolated_workspace" ? { freshContinuationIsolation: action.id } : {}), }, - }); + }).catch(async error => { await slot.release(); throw error; }); seenAgents.add(action.returnOwnerAgentId); if (!run) { + await slot.release(); await deferRefusedContinuation(action); continue; } diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index 4aa41b431ff..30f41f5c78a 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -9,7 +9,7 @@ import { boundRunIssueIdSql } from "./run-task-binding.js"; import { and, asc, eq, getTableColumns, is, sql } from "drizzle-orm"; import { PgTransaction } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; -import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; +import { heartbeatRunEvents, heartbeatRuns, primeDatabaseWorkTransactionId } from "@paperclipai/db"; import { nativeSha256 } from "./native-runtime/canonical.js"; import { isUniqueViolation } from "../db-errors.js"; @@ -345,9 +345,9 @@ function decodeRunEventRow(raw: Record): typeof heartbeatRunEve /** * A native source event: one lock read, then one statement that finds a * replayed event or reserves the sequence and inserts, and reads how much of - * the source is retained. The lock must stay a statement of its own. A - * statement that waits for the run lock keeps the snapshot it took before - * waiting, so it would not see an event the lock holder had just committed. + * the source is retained. A statement that waits for the run lock keeps the + * snapshot it took before waiting, so it would not see an event the lock + * holder had just committed. */ async function appendNativeSourceEvent( tx: Db, @@ -361,15 +361,26 @@ async function appendNativeSourceEvent( // checkpoints, that are written for the same run while events stream. // Only progress needs the task binding. Other native/recovery events // must not read a potentially large context snapshot to append a log. + const replayed = sql`run_id = ${input.runId} and (source_event_id = ${source.sourceEventId} + or (source_instance_id = ${source.sourceInstanceId} and source_seq = ${source.sourceSeq}))`; + const probes = wakes.reassessment || wakes.progress || wakes.response || wakes.retention; + const columns = sql`"company_id", "agent_id", ("native_source_cursors" ->> ${source.sourceInstanceId})::bigint as "source_cursor"${ + wakes.progress || wakes.response || wakes.reassessment ? sql`, "runtime_mode"` : sql.empty()}${ + wakes.progress ? sql`, ${boundRunIssueIdSql()} as "issue_id"` : sql.empty()}${ + Object.values(wakes).some(Boolean) ? sql`, pg_current_xact_id()::text as "xid"` : sql.empty()}`; const [run] = await tx.execute<{ company_id: string; agent_id: string; source_cursor: string | number | null; - runtime_mode?: string | null; issue_id?: string | null }>( - sql`select "company_id", "agent_id", ("native_source_cursors" ->> ${source.sourceInstanceId})::bigint as "source_cursor"${ - wakes.progress || wakes.response || wakes.reassessment ? sql`, "runtime_mode"` : sql.empty()}${ - wakes.progress ? sql`, ${boundRunIssueIdSql()} as "issue_id"` : sql.empty() - } from "heartbeat_runs" where "id" = ${input.runId} for no key update`); + runtime_mode?: string | null; issue_id?: string | null; xid?: string; probe_current?: boolean; replay_found?: boolean }>(probes + ? sql`with "snapshot_run" as (select "next_event_seq" from "heartbeat_runs" where "id" = ${input.runId}), + "locked_run" as (select ${columns}, "next_event_seq" from "heartbeat_runs" where "id" = ${input.runId} for no key update) + select "locked_run".*, "locked_run"."next_event_seq" = (select "next_event_seq" from "snapshot_run") as "probe_current", + exists (select 1 from heartbeat_run_events where ${replayed}) as "replay_found" + from "locked_run"` + : sql`select ${columns} from "heartbeat_runs" where "id" = ${input.runId} for no key update`); if (!run || run.company_id !== input.companyId || run.agent_id !== input.agentId) { throw new Error("heartbeat_run_event_binding_mismatch"); } + if (run.xid) primeDatabaseWorkTransactionId(tx, run.xid); + const knownNew = run.probe_current === true && run.replay_found === false; // The stored cursor says the run holds every event of this source up to it. // An event at or just past it moves the cursor without counting the run's // events; any other event, or a run without a cursor, recounts them. @@ -380,15 +391,14 @@ async function appendNativeSourceEvent( if (wakes.recovery) await notifyLiveRunRecovery(tx, input); const sourceHash = nativeSha256(source.canonicalPayload); - const replayed = sql`run_id = ${input.runId} and (source_event_id = ${source.sourceEventId} - or (source_instance_id = ${source.sourceInstanceId} and source_seq = ${source.sourceSeq}))`; const native = run.runtime_mode === "native"; const reassessment = wakes.reassessment && native, progress = wakes.progress && native && run.issue_id; const response = wakes.response && native; if (reassessment || progress || response || wakes.retention) { // A wake is registered before the write it announces, and a replayed // event registers none. So these events look for their replay first. - const [existing] = await tx.execute>(sql`select * from heartbeat_run_events where ${replayed} limit 1`); + const [existing] = knownNew ? [] + : await tx.execute>(sql`select * from heartbeat_run_events where ${replayed} limit 1`); if (existing) { const row = decodeRunEventRow(existing); if (row.sourcePayloadSha256 !== sourceHash) throw new HeartbeatRunEventConflictError(); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index ed49232fc19..f382ea499ba 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -12,7 +12,7 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche import { deferredWorkIntentWait } from "../work-intent-outbox.js"; import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHeartbeatTimerPolicy } from "./timer-policy.js"; -import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js"; +import { QUEUED_RUN_AGING_MS, RECOVERY_WAKE_REASONS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js"; import { queuedRunDispatchFacts } from "./queued-run-line.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; @@ -908,6 +908,47 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (claimedRun && isRunAdmissionClosed()) throw new RunAdmissionClosedDuringClaim(); } + const CLOSED_TASK_CONTINUATION = { error: "The task was closed before this continuation started.", + errorCode: "issue_closed_before_start" }; + const RECOVERY_NO_LONGER_NEEDED = { error: "The task no longer needed this recovery run when its turn came.", + errorCode: "recovery_no_longer_needed" }; + + /** + * A system recovery run waits behind people's work, so by its turn the task + * may have moved on. It is still needed only while its agent still owns the + * task and no other run continues it: none started after it was queued, + * none running, and no queued non-recovery run (a person's or an + * assignment's). Of two queued recovery runs for one task, the newer one stays. + */ + async function recoveryRunNoLongerNeeded(run: typeof heartbeatRuns.$inferSelect, issueId: string, + task: { assigneeAgentId: string | null }) { + if (run.wakeupRequestId) { + const [request] = await db.select({ actor: agentWakeupRequests.requestedByActorType }).from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.id, run.wakeupRequestId))); + // A person asked for this one (a board restore): it is theirs to keep. + if (request?.actor === "user") return false; + } + if (task.assigneeAgentId !== run.agentId) return true; + const recoveryReasons = sql.join([...RECOVERY_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `); + // The run it recovers is never newer work, whatever the two rows' timestamps say. + const notOwnRuns = [run.id, run.retryOfRunId, readNonEmptyString(parseObject(run.contextSnapshot).previousRunId)] + .filter((id): id is string => Boolean(id && isUuidLike(id))); + const [other] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, run.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + notInArray(heartbeatRuns.id, notOwnRuns), + or( + // Ran after this one was queued: the task already continued. + gt(heartbeatRuns.startedAt, run.createdAt), + eq(heartbeatRuns.status, "running"), + and(eq(heartbeatRuns.status, "queued"), or( + sql`coalesce(${heartbeatRuns.contextSnapshot}->>'wakeReason', '') not in (${recoveryReasons})`, + gt(heartbeatRuns.createdAt, run.createdAt))), + ), + )).limit(1); + return Boolean(other); + } + /** System continuations that only make sense for an open task. */ const SYSTEM_CONTINUATION_WAKE_REASONS = new Set(["issue_recovery_action_restored", "issue_continuation_needed", "issue_assignment_recovery", ISSUE_BLOCKER_STALLED_WAKE_REASON, "issue_children_completed"]); @@ -925,13 +966,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende * attached after the check changes the run, the cancel matches nothing, and * the check runs again. */ - async function cancelClosedTaskContinuation(run: typeof heartbeatRuns.$inferSelect, issueId: string) { + async function cancelClosedTaskContinuation(run: typeof heartbeatRuns.$inferSelect, issueId: string, + outcome: { error: string; errorCode: string } = CLOSED_TASK_CONTINUATION) { for (let attempt = 0; attempt < 3; attempt += 1) { const decision = await checkClosedTaskContinuation(run, issueId); if (decision.kind !== "cancel") return decision; const write = await setRunStatusFromLive(run.id, "cancelled", ["queued"], { - finishedAt: new Date(), error: "The task was closed before this continuation started.", - errorCode: "issue_closed_before_start", + finishedAt: new Date(), ...outcome, }, undefined, sql`${heartbeatRuns}.xmin::text = ${decision.rowVersion}`); if (write.updated && write.run) return { kind: "cancelled" as const, run: write.run }; if (write.run?.status !== "queued") return { kind: "not_queued" as const }; @@ -992,13 +1033,19 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const continuedIssueId = readNonEmptyString(runContext.issueId); if (systemContinuationReason && SYSTEM_CONTINUATION_WAKE_REASONS.has(systemContinuationReason) && continuedIssueId && runContext.onboardingCompletion !== true) { - const [target] = await db.select({ status: issues.status }).from(issues) + const [target] = await db.select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId }).from(issues) .where(and(eq(issues.companyId, run.companyId), eq(issues.id, continuedIssueId))); - if (target && ["done", "cancelled"].includes(target.status)) { - const closed = await cancelClosedTaskContinuation(run, continuedIssueId); + const closedTask = Boolean(target && ["done", "cancelled"].includes(target.status)); + // Dropped quietly: the run is cancelled before it started and its wake is + // skipped, the same as a continuation whose task closed. Nothing failed. + const outcome = closedTask ? CLOSED_TASK_CONTINUATION + : target && RECOVERY_WAKE_REASONS.has(systemContinuationReason) + && await recoveryRunNoLongerNeeded(run, continuedIssueId, target) ? RECOVERY_NO_LONGER_NEEDED : null; + if (outcome) { + const closed = await cancelClosedTaskContinuation(run, continuedIssueId, outcome); if (closed.kind === "cancelled") { const cancelled = closed.run; - await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: cancelled.finishedAt ?? new Date(), error: "issue_closed_before_start" }); + await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: cancelled.finishedAt ?? new Date(), error: outcome.errorCode }); await releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: true, deferredPostCommitEffects }); } // A person's undelivered message rides on this run: it runs as their @@ -2425,7 +2472,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende agent: typeof agents.$inferSelect, queuedRuns: QueuedRunCandidate[], dependencyReadiness: Map, - issueById: Map, + issueById: Map, ) { const now = new Date(); pruneDeclinedAgedClaims(now); @@ -2531,7 +2579,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }))]; const issueRows = queuedIssueIds.length > 0 ? await db.select({ id: issues.id, status: issues.status, priority: issues.priority, updatedAt: issues.updatedAt, - executionRunId: issues.executionRunId }).from(issues) + executionRunId: issues.executionRunId, createdByUserId: issues.createdByUserId }).from(issues) .where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds))) : []; const issueById = new Map(issueRows.map((row) => [row.id, row])); diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.test.ts b/server/src/services/heartbeat/queued-run-dispatch-order.test.ts index 1abcb6b1818..3ae58093c2d 100644 --- a/server/src/services/heartbeat/queued-run-dispatch-order.test.ts +++ b/server/src/services/heartbeat/queued-run-dispatch-order.test.ts @@ -4,9 +4,12 @@ import { QUEUED_RUN_AGING_MS, RECOVERY_BACKLOG_AGE_MS, createQueuedRunDispatcher, + dispatchCountersFromStarts, + queuedRunStartOf, RECOVERY_DISPATCH_SHARE, isRecoveryWakeReason, orderQueuedRunsForDispatch, + queuedRunDispatchLine, queuedRunDispatchClass, type QueuedRunDispatchFacts, } from "./queued-run-dispatch-order.js"; @@ -58,10 +61,17 @@ describe("queued run dispatch order", () => { expect(ordered.slice(0, 2).map(entry => entry.id).sort()).toEqual([chat.id, assignment.id].sort()); }); - it("keeps draining recovery wakes while people keep sending work", () => { + it("holds recovery wakes while people keep sending work", () => { + const burst = Array.from({ length: 10 }, () => backlogRecovery()); + const { started, pending } = dispatch(burst, { steps: 40, arrivals: () => [personRetry()] }); + expect(started.every(entry => entry.wakeReason === "task_retry")).toBe(true); + expect(pending.filter(entry => entry.wakeReason === "issue_recovery_action_restored")).toHaveLength(burst.length); + }); + + it("keeps draining recovery wakes at their share while agents keep sending work", () => { const burst = Array.from({ length: 10 }, () => backlogRecovery()); const steps = burst.length * RECOVERY_DISPATCH_SHARE; - const { started } = dispatch(burst, { steps, arrivals: () => [personRetry()] }); + const { started } = dispatch(burst, { steps, arrivals: () => [run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) })] }); const recoveryStarts = started.filter(entry => entry.wakeReason === "issue_recovery_action_restored").length; expect(recoveryStarts).toBe(burst.length); for (let window = 0; window + RECOVERY_DISPATCH_SHARE <= started.length; window += 1) { @@ -69,18 +79,22 @@ describe("queued run dispatch order", () => { if (window + RECOVERY_DISPATCH_SHARE <= steps) expect(slice.some(entry => entry.wakeReason === "issue_recovery_action_restored")).toBe(true); } - expect(started.filter(entry => entry.wakeReason === "task_retry").length).toBe(steps - burst.length); + expect(started.filter(entry => entry.wakeReason === "issue_assigned").length).toBe(steps - burst.length); }); - it("keeps recovery's share when one dispatch fills many slots", () => { - const people = Array.from({ length: 12 }, () => personRetry()); + it("keeps recovery's share of agent work when one dispatch fills many slots, and none of a person's", () => { + const work = Array.from({ length: 12 }, () => run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) })); const recovery = Array.from({ length: 6 }, () => backlogRecovery()); - const batch = orderQueuedRunsForDispatch([...recovery, ...people], { now, startsSinceRecovery: 0 }).slice(0, 8); - expect(batch.map(entry => entry.wakeReason === "task_retry" ? "P" : "R").join("")).toBe("PPPRPPPR"); + const batch = orderQueuedRunsForDispatch([...recovery, ...work], { now, startsSinceRecovery: 0 }).slice(0, 8); + expect(batch.map(entry => entry.wakeReason === "issue_assigned" ? "A" : "R").join("")).toBe("AAARAAAR"); + const people = Array.from({ length: 4 }, () => personRetry()); + const mixed = orderQueuedRunsForDispatch([...recovery, ...people, ...work], { now, startsSinceRecovery: 0 }).slice(0, 8); + expect(mixed.map(entry => entry.wakeReason === "task_retry" ? "P" : entry.wakeReason === "issue_assigned" ? "A" : "R").join("")) + .toBe("PPPPRAAA"); }); it("offers the next recovery run when the reserved recovery run cannot be claimed", () => { - const people = Array.from({ length: 4 }, () => personRetry()); + const people = Array.from({ length: 4 }, () => run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) })); const stuck = backlogRecovery(59); const claimable = backlogRecovery(30); const dispatcher = createQueuedRunDispatcher([...people, stuck, claimable], { now, startsSinceRecovery: RECOVERY_DISPATCH_SHARE - 1 }); @@ -89,7 +103,7 @@ describe("queued run dispatch order", () => { const second = dispatcher.next(); expect(second!.id).toBe(claimable.id); dispatcher.claimed(second!); - expect(dispatcher.next()!.wakeReason).toBe("task_retry"); + expect(dispatcher.next()!.wakeReason).toBe("issue_assigned"); }); it("does not put a fresh interruption's continuation behind old-backlog recovery wakes", () => { @@ -125,29 +139,22 @@ describe("queued run dispatch order", () => { expect(ordered.map(entry => entry.id)).toEqual([urgent.id, earlier.id, later.id, inProgress.id, todo.id]); }); - it("starts an old queued continuation within a bounded number of starts while fresh person and system work keeps arriving", () => { + it("starts an old queued continuation within a bounded number of starts while fresh agent and recovery work keeps arriving", () => { const continuation = run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(150), sourceEndedAt: minutesAgo(152) }); const started: QueuedRunDispatchFacts[] = []; let pending: QueuedRunDispatchFacts[] = [continuation]; const steps = 40; for (let step = 0; step < steps; step += 1) { - pending.push(personRetry(), run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }), + pending.push(run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }), run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(1), sourceEndedAt: minutesAgo(2) })); - const recent = started.slice(-RECOVERY_DISPATCH_SHARE).reverse(); - const lastRecovery = recent.findIndex(entry => isRecoveryWakeReason(entry.wakeReason)); - const oldest = pending.reduce((left, right) => left.createdAt <= right.createdAt ? left : right); - const recentForOldest = started.slice(-OLDEST_DISPATCH_SHARE).reverse(); - const lastOldest = recentForOldest.findIndex(entry => entry.createdAt <= oldest.createdAt); - const [next] = orderQueuedRunsForDispatch(pending, { now, - startsSinceRecovery: lastRecovery < 0 ? recent.length : lastRecovery, - startsSinceOldest: lastOldest < 0 ? recentForOldest.length : lastOldest }); + const [next] = orderQueuedRunsForDispatch(pending, { now, ...dispatchCountersFromStarts(pending, [...started].reverse().map(run => queuedRunStartOf(run, now)), now) }); started.push(next!); pending = pending.filter(entry => entry.id !== next!.id); } const position = started.findIndex(entry => entry.id === continuation.id); expect(position).toBeGreaterThanOrEqual(0); - expect(position).toBeLessThan(OLDEST_DISPATCH_SHARE); - expect(started.slice(0, position).every(entry => entry.wakeReason === "task_retry" || entry.wakeReason === "issue_continuation_needed")).toBe(true); + expect(position).toBeLessThan(RECOVERY_DISPATCH_SHARE); + expect(started.slice(0, position).every(entry => entry.wakeReason === "issue_assigned")).toBe(true); }); it("serves the longest-waiting run once every share even when fresher work outranks it", () => { @@ -166,12 +173,11 @@ describe("queued run dispatch order", () => { expect(ordered[0]!.wakeReason).toBe("task_retry"); }); - it("lets an old-backlog recovery run age into the work tier", () => { - const aged = run({ wakeReason: "issue_continuation_needed", createdAt: new Date(now.getTime() - QUEUED_RUN_AGING_MS), - sourceEndedAt: minutesAgo(200) }); + it("never lets a long-waiting recovery run age ahead of agent work or into the oldest-run share", () => { + const aged = run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(400), sourceEndedAt: minutesAgo(402) }); const fresh = run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }); - const ordered = orderQueuedRunsForDispatch([fresh, aged], { now, startsSinceRecovery: 0 }); - expect(ordered.map(entry => entry.id)).toEqual([aged.id, fresh.id]); + const ordered = orderQueuedRunsForDispatch([fresh, aged], { now, startsSinceRecovery: 0, startsSinceOldest: OLDEST_DISPATCH_SHARE - 1 }); + expect(ordered.map(entry => entry.id)).toEqual([fresh.id, aged.id]); }); it("passes the oldest-run share to the next oldest run while the oldest one keeps failing to claim", () => { @@ -194,17 +200,105 @@ describe("queued run dispatch order", () => { for (let step = 0; step < 3 * OLDEST_DISPATCH_SHARE; step += 1) { pending.push(run({ wakeReason: "issue_recovery_action_restored", createdAt: minutesAgo(1), sourceEndedAt: minutesAgo(2) }), run({ wakeReason: "issue_assigned", readinessRank: 1, createdAt: minutesAgo(1) })); - const recent = started.slice(-RECOVERY_DISPATCH_SHARE).reverse(); - const lastRecovery = recent.findIndex(entry => isRecoveryWakeReason(entry.wakeReason)); - const oldest = pending.reduce((left, right) => left.createdAt <= right.createdAt ? left : right); - const recentForOldest = started.slice(-OLDEST_DISPATCH_SHARE).reverse(); - const lastOldest = recentForOldest.findIndex(entry => entry.createdAt <= oldest.createdAt); - const [next] = orderQueuedRunsForDispatch(pending, { now, - startsSinceRecovery: lastRecovery < 0 ? recent.length : lastRecovery, - startsSinceOldest: lastOldest < 0 ? recentForOldest.length : lastOldest }); + const [next] = orderQueuedRunsForDispatch(pending, { now, ...dispatchCountersFromStarts(pending, [...started].reverse().map(run => queuedRunStartOf(run, now)), now) }); started.push(next!); pending = pending.filter(entry => entry.id !== next!.id); } for (const continuation of continuations) expect(started.map(entry => entry.id)).toContain(continuation.id); }); }); + +describe("deep per-agent queue: a recovery burst behind 20 busy slots (paperclip.staging, 2026-11-13)", () => { + // Codie, maxConcurrentRuns 20, every slot busy: each freed slot is one dispatch. + const at = new Date("2026-11-13T14:30:00.000Z"); + const clock = (iso: string) => new Date(`2026-11-13T${iso}:00.000Z`); + const addMinutes = (date: Date, minutes: number) => new Date(date.getTime() + minutes * 60_000); + let id = 0; + const make = (input: Partial & { wakeReason: string; createdAt: Date }): QueuedRunDispatchFacts => + ({ id: `codie-${++id}`, requestedByUser: false, sourceEndedAt: null, readinessRank: 1, priorityRank: 2, ...input }); + const isPerson = (run: QueuedRunDispatchFacts) => run.requestedByUser; + + function todaysQueue() { + // Agent-made assignments queued before the people's tasks. + const assignments = Array.from({ length: 60 }, (_, index) => make({ wakeReason: "issue_assigned", + createdAt: addMinutes(clock("05:00"), index * 2) })); + // Two tasks people created, stalled by a restart at 07:25 and waiting since 07:29 on system wakes: + // a monitor check and the assigned-work sweep's continuation. Neither wake was a person's. + const waitingPeople = [ + make({ wakeReason: "issue_monitor_due", personTask: true, readinessRank: 0, createdAt: clock("07:29") }), + make({ wakeReason: "issue_continuation_needed", personTask: true, readinessRank: 0, sourceEndedAt: clock("07:27"), + createdAt: addMinutes(clock("07:29"), 1) }), + ]; + // Later person-requested work on tasks already in progress, at higher priority. + const laterPeople = Array.from({ length: 30 }, (_, index) => make({ wakeReason: "issue_commented", requestedByUser: true, + readinessRank: 0, priorityRank: 1, createdAt: addMinutes(clock("08:00"), index * 12) })); + // The burst: 139 system recovery wakes between 10:46 and 11:47 for in-progress tasks. + const burst = Array.from({ length: 139 }, (_, index) => make({ wakeReason: "issue_recovery_action_restored", + readinessRank: 0, sourceEndedAt: clock("09:00"), createdAt: addMinutes(clock("10:46"), Math.floor(index * 61 / 139)) })); + return { assignments, waitingPeople, laterPeople, burst }; + } + + function dispatchOneAtATime(queue: QueuedRunDispatchFacts[], steps: number) { + const started: Array<{ run: QueuedRunDispatchFacts; personWaiting: boolean }> = []; + let pending = [...queue]; + for (let step = 0; step < steps; step += 1) { + // People and agents keep adding work while the queue drains. + if (step % 2 === 0) pending.push(make({ wakeReason: "issue_commented", requestedByUser: true, readinessRank: 0, + priorityRank: 1, createdAt: at })); + pending.push(make({ wakeReason: "issue_assigned", createdAt: at })); + const history = started.map(entry => entry.run); + const [next] = orderQueuedRunsForDispatch(pending, { now: at, ...dispatchCountersFromStarts(pending, [...history].reverse().map(run => queuedRunStartOf(run, at)), at) }); + started.push({ run: next!, personWaiting: pending.some(entry => entry !== next && isPerson(entry) && entry.readinessRank < 3) }); + pending = pending.filter(entry => entry.id !== next!.id); + } + return started; + } + + it("starts the oldest run on a task a person is waiting on within the oldest-run share, and never starts recovery while a person waits", () => { + const { assignments, waitingPeople, laterPeople, burst } = todaysQueue(); + const started = dispatchOneAtATime([...burst, ...assignments, ...laterPeople, ...waitingPeople], 40); + const position = started.findIndex(entry => entry.run.id === waitingPeople[0]!.id); + expect(position).toBeGreaterThanOrEqual(0); + expect(position).toBeLessThan(OLDEST_DISPATCH_SHARE); + // The next one starts within the following share. + const second = started.findIndex(entry => entry.run.id === waitingPeople[1]!.id); + expect(second).toBeGreaterThanOrEqual(0); + expect(second).toBeLessThan(2 * OLDEST_DISPATCH_SHARE); + const recoveryAheadOfPeople = started.filter(entry => entry.run.wakeReason === "issue_recovery_action_restored" && entry.personWaiting); + expect(recoveryAheadOfPeople).toHaveLength(0); + }); + + it("does not let older agent work use up the turn of a run on a task a person created", () => { + // No person-requested arrivals: normal order serves the older, higher-priority assignments. + const assignments = Array.from({ length: 60 }, (_, index) => make({ wakeReason: "issue_assigned", priorityRank: 1, + createdAt: addMinutes(clock("05:00"), index * 2) })); + const monitor = make({ wakeReason: "issue_monitor_due", personTask: true, priorityRank: 3, createdAt: clock("07:29") }); + const started: QueuedRunDispatchFacts[] = []; + let pending = [...assignments, monitor]; + for (let step = 0; step < 2 * OLDEST_DISPATCH_SHARE && pending.length; step += 1) { + const [next] = orderQueuedRunsForDispatch(pending, { now: at, + ...dispatchCountersFromStarts(pending, [...started].reverse().map(run => queuedRunStartOf(run, at)), at) }); + started.push(next!); + pending = pending.filter(entry => entry.id !== next!.id); + } + const position = started.findIndex(entry => entry.id === monitor.id); + expect(position).toBeGreaterThanOrEqual(0); + expect(position).toBeLessThan(OLDEST_DISPATCH_SHARE); + // The place-in-line order agrees. + const line = queuedRunDispatchLine([...assignments, monitor], { now: at, startsSinceRecovery: 0, startsSinceOldest: 0 }); + expect(line.findIndex(entry => entry.id === monitor.id)).toBeLessThan(OLDEST_DISPATCH_SHARE); + }); + + it("keeps recovery behind agent assignments once people are served, and still drains it at its share", () => { + const { assignments, burst } = todaysQueue(); + const started: QueuedRunDispatchFacts[] = []; + let pending = [...burst.slice(0, 3), ...assignments]; + for (let step = 0; step < 4 * RECOVERY_DISPATCH_SHARE && pending.length; step += 1) { + const [next] = orderQueuedRunsForDispatch(pending, { now: at, ...dispatchCountersFromStarts(pending, [...started].reverse().map(run => queuedRunStartOf(run, at)), at) }); + started.push(next!); + pending = pending.filter(entry => entry.id !== next!.id); + } + expect(started.filter(entry => entry.wakeReason === "issue_recovery_action_restored")).toHaveLength(3); + expect(started[0]!.wakeReason).toBe("issue_assigned"); + }); +}); diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.ts b/server/src/services/heartbeat/queued-run-dispatch-order.ts index 149d4be515b..0aef1f561c5 100644 --- a/server/src/services/heartbeat/queued-run-dispatch-order.ts +++ b/server/src/services/heartbeat/queued-run-dispatch-order.ts @@ -27,6 +27,8 @@ export type QueuedRunDispatchFacts = { readinessRank: number; priorityRank: number; shareDeferred?: boolean; + /** The task was created by a person, who waits on its outcome whichever system wake moves it. */ + personTask?: boolean; }; export type QueuedRunDispatchClass = "person" | "work" | "recovery" | "recovery_backlog"; @@ -48,8 +50,18 @@ export function queuedRunDispatchClass(run: QueuedRunDispatchFacts, now: Date): return "work"; } -const CLASS_TIER: Record = { person: 0, work: 1, recovery: 1, recovery_backlog: 2 }; +// System recovery sits below all other work and never ages out of it: a +// person's run always goes first, and recovery takes only its share of the +// starts that would otherwise go to agent work. +const CLASS_TIER: Record = { person: 0, work: 1, recovery: 2, recovery_backlog: 3 }; const NOT_READY_RANK = 3; +const NOT_READY_TIER = 4; + +const isReady = (run: QueuedRunDispatchFacts) => run.readinessRank < NOT_READY_RANK; +const isPersonRun = (run: QueuedRunDispatchFacts, now: Date) => queuedRunDispatchClass(run, now) === "person"; +/** A run a person asked for, or any run on a task a person created. */ +const isPersonWaitingRun = (run: QueuedRunDispatchFacts, now: Date) => Boolean(run.personTask) || isPersonRun(run, now); +const isOldestShareEligible = (run: QueuedRunDispatchFacts) => isReady(run) && !run.shareDeferred; export type QueuedRunDispatchInput = { now: Date; startsSinceRecovery: number; startsSinceOldest?: number }; @@ -57,20 +69,102 @@ export function hasQueuedRunAged(run: Pick, return now.getTime() - run.createdAt.getTime() >= QUEUED_RUN_AGING_MS; } -export function oldestReadyQueuedRun(runs: readonly T[]): T | null { +export function oldestReadyQueuedRun(runs: readonly T[], + include: (run: T) => boolean = () => true): T | null { let oldest: T | null = null; for (const run of runs) { - if (run.readinessRank >= NOT_READY_RANK || run.shareDeferred) continue; + if (!isOldestShareEligible(run) || !include(run)) continue; if (!oldest || run.createdAt.getTime() < oldest.createdAt.getTime()) oldest = run; } return oldest; } +/** + * The run the oldest-run share serves next: the longest-waiting run past the + * aging bound that a person asked for or whose task a person created (a + * system continuation of a person's stalled task included), else the + * longest-waiting agent-work run past it. Other system recovery is never + * served by this share; it has its own. + * + * So such a run that has waited past the bound starts within + * OLDEST_DISPATCH_SHARE x (aged person-task runs queued before it + 1) starts, + * however much newer or higher-priority work arrives. Waiting never fails or + * cancels it; it only moves it forward. + */ +export function oldestShareCandidate(runs: readonly T[], now: Date): T | null { + const aged = (run: T) => hasQueuedRunAged(run, now); + return oldestReadyQueuedRun(runs, run => aged(run) && isPersonWaitingRun(run, now)) + ?? oldestReadyQueuedRun(runs, run => aged(run) && !isSystemRecoveryRun(run)); +} + +/** + * One of the agent's recent starts. `personOrigin`: a person asked for it (its + * wake or source chain). `personWaiting`: that, or its task was created by a + * person. + */ +export type QueuedRunStart = { wakeReason: string | null; createdAt: Date; startedAt?: Date | null; personOrigin?: boolean; + personWaiting?: boolean }; + +/** A dispatched run as the start history records it. */ +export function queuedRunStartOf(run: QueuedRunDispatchFacts, now: Date): QueuedRunStart { + return { wakeReason: run.wakeReason, createdAt: run.createdAt, personOrigin: isPersonRun(run, now), + personWaiting: isPersonWaitingRun(run, now) }; +} + +/** + * Whether a start used up the oldest-run share's turn: it was at least as old + * as the candidate, and, while the candidate is a run people wait on, it was + * one too. Older agent work never uses up a person's turn. + */ +function servesOldestShare(started: { createdAt: Date; personWaiting: boolean }, candidate: QueuedRunDispatchFacts, now: Date) { + return started.createdAt.getTime() <= candidate.createdAt.getTime() + && (started.personWaiting || !isPersonWaitingRun(candidate, now)); +} + +/** Where the start history must reach back to for `dispatchCountersFromStarts`, or null when no counter needs it. */ +export function dispatchHistoryStart(runs: readonly T[], now: Date): Date | null { + const recovery = runs.filter(run => isSystemRecoveryRun(run)); + const countsRecovery = recovery.length > 0 && recovery.length < runs.length; + const candidate = runs.length > 1 ? oldestShareCandidate(runs, now) : null; + const at = Math.min(countsRecovery ? Math.min(...recovery.map(run => run.createdAt.getTime())) : Infinity, + candidate?.createdAt.getTime() ?? Infinity); + return Number.isFinite(at) ? new Date(at) : null; +} + +/** + * Derives the share counters from this agent's recent starts, newest first. + * The dispatcher is rebuilt for every pass, so its shares carry over through + * the agent's own start history rather than process memory. A person's + * recovery restart (`personOrigin`) is person work, so it does not use up + * system recovery's share. + */ +export function dispatchCountersFromStarts(runs: readonly T[], + recentStarts: readonly QueuedRunStart[], now: Date): Pick { + const startsSince = (at: number) => recentStarts.filter(start => !start.startedAt || start.startedAt.getTime() >= at); + const recovery = runs.filter(run => isSystemRecoveryRun(run)); + let startsSinceRecovery = 0; + if (recovery.length > 0 && recovery.length < runs.length) { + const starts = startsSince(Math.min(...recovery.map(run => run.createdAt.getTime()))).slice(0, RECOVERY_DISPATCH_SHARE); + const last = starts.findIndex(start => isRecoveryWakeReason(start.wakeReason) && !start.personOrigin); + startsSinceRecovery = last < 0 ? starts.length : last; + } + let startsSinceOldest = 0; + const candidate = runs.length > 1 ? oldestShareCandidate(runs, now) : null; + if (candidate) { + const starts = startsSince(candidate.createdAt.getTime()).slice(0, OLDEST_DISPATCH_SHARE); + const last = starts.findIndex(start => servesOldestShare({ createdAt: start.createdAt, + personWaiting: Boolean(start.personWaiting) }, candidate, now)); + startsSinceOldest = last < 0 ? starts.length : last; + } + return { startsSinceRecovery, startsSinceOldest }; +} + function sortForDispatch(runs: readonly T[], now: Date): T[] { const tier = (run: T) => { - if (run.readinessRank >= NOT_READY_RANK) return 3; - const classTier = CLASS_TIER[queuedRunDispatchClass(run, now)]; - return hasQueuedRunAged(run, now) ? Math.min(classTier, CLASS_TIER.work) : classTier; + if (!isReady(run)) return NOT_READY_TIER; + const runClass = queuedRunDispatchClass(run, now); + // Old backlog that has waited past the aging bound rejoins the other recovery, never more. + return runClass === "recovery_backlog" && hasQueuedRunAged(run, now) ? CLASS_TIER.recovery : CLASS_TIER[runClass]; }; return [...runs].sort((left, right) => tier(left) - tier(right) @@ -89,17 +183,24 @@ export function createQueuedRunDispatcher( return { next(): T | null { if (!remaining.length) return null; - const oldest = sinceOldest >= OLDEST_DISPATCH_SHARE - 1 ? oldestReadyQueuedRun(remaining) : null; - if (oldest && hasQueuedRunAged(oldest, input.now)) return remaining.splice(remaining.indexOf(oldest), 1)[0]!; - const recoveryIndex = sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1 - ? remaining.findIndex(run => run.readinessRank < NOT_READY_RANK && isSystemRecoveryRun(run)) - : -1; + const oldest = sinceOldest >= OLDEST_DISPATCH_SHARE - 1 ? oldestShareCandidate(remaining, input.now) : null; + if (oldest) return remaining.splice(remaining.indexOf(oldest), 1)[0]!; + // Recovery's share comes out of agent work, never out of a person's turn. + const personWaiting = remaining.some(run => isReady(run) && isPersonRun(run, input.now)); + // The share goes to the longest-waiting recovery run once one has waited + // past the aging bound, so fresh recovery cannot keep an old one waiting. + const agedRecovery = oldestReadyQueuedRun(remaining, run => isSystemRecoveryRun(run) && hasQueuedRunAged(run, input.now)); + const recoveryIndex = personWaiting || sinceRecovery < RECOVERY_DISPATCH_SHARE - 1 ? -1 + : agedRecovery ? remaining.indexOf(agedRecovery) + : remaining.findIndex(run => isReady(run) && isSystemRecoveryRun(run)); return remaining.splice(recoveryIndex < 0 ? 0 : recoveryIndex, 1)[0]!; }, claimed(run: T) { sinceRecovery = isSystemRecoveryRun(run) ? 0 : sinceRecovery + 1; - const oldest = oldestReadyQueuedRun(remaining); - sinceOldest = !oldest || oldest.createdAt.getTime() >= run.createdAt.getTime() ? 0 : sinceOldest + 1; + // The same rule dispatchCountersFromStarts reads from the start history. + const oldest = oldestShareCandidate([...remaining, run], input.now); + sinceOldest = !oldest || servesOldestShare({ createdAt: run.createdAt, personWaiting: isPersonWaitingRun(run, input.now) }, + oldest, input.now) ? 0 : sinceOldest + 1; }, }; } @@ -128,50 +229,50 @@ export function queuedRunDispatchLine( runs: readonly T[], input: QueuedRunDispatchInput, ): T[] { - const order = sortForDispatch(runs, input.now); + const now = input.now; + const order = sortForDispatch(runs, now); const removed = new Uint8Array(order.length); - // Eligible for the oldest share, oldest first; ties keep dispatch order, - // as oldestReadyQueuedRun keeps the first of equal ages. - const byAge = order.map((_, index) => index) - .filter(index => order[index]!.readinessRank < NOT_READY_RANK && !order[index]!.shareDeferred) - .sort((left, right) => order[left]!.createdAt.getTime() - order[right]!.createdAt.getTime() || left - right); - const recovery = order.map((_, index) => index) - .filter(index => order[index]!.readinessRank < NOT_READY_RANK && isSystemRecoveryRun(order[index]!)); - let head = 0; - let agePointer = 0; - let recoveryPointer = 0; - const firstOf = (indexes: number[] | null, pointer: number) => { - if (!indexes) { - while (head < order.length && removed[head]) head += 1; - return head < order.length ? head : -1; - } - while (pointer < indexes.length && removed[indexes[pointer]!]) pointer += 1; - return pointer; + const indexes = (include: (run: T) => boolean) => order.map((_, index) => index).filter(index => include(order[index]!)); + // Oldest first; ties keep dispatch order, as oldestReadyQueuedRun keeps the first of equal ages. + const byAge = (list: number[]) => + list.sort((left, right) => order[left]!.createdAt.getTime() - order[right]!.createdAt.getTime() || left - right); + // A list's first entry not yet started; lists stay in their order, so each pointer only moves forward. + const cursor = (list: number[]) => { + let pointer = 0; + return () => { + while (pointer < list.length && removed[list[pointer]!]) pointer += 1; + return pointer < list.length ? list[pointer]! : -1; + }; }; - const oldestIndex = () => { - agePointer = firstOf(byAge, agePointer); - return agePointer < byAge.length ? byAge[agePointer]! : -1; + const agedFirst = (next: () => number) => () => { + const index = next(); + return index >= 0 && hasQueuedRunAged(order[index]!, now) ? index : -1; }; + const oldestPersonWaiting = agedFirst(cursor(byAge(indexes(run => isOldestShareEligible(run) && isPersonWaitingRun(run, now))))); + const oldestWork = agedFirst(cursor(byAge(indexes(run => isOldestShareEligible(run) && !isSystemRecoveryRun(run))))); + const oldestShare = () => { const person = oldestPersonWaiting(); return person >= 0 ? person : oldestWork(); }; + const oldestRecovery = agedFirst(cursor(byAge(indexes(run => isOldestShareEligible(run) && isSystemRecoveryRun(run))))); + const firstRecovery = cursor(indexes(run => isReady(run) && isSystemRecoveryRun(run))); + const head = cursor(order.map((_, index) => index)); + let readyPeople = order.filter(run => isReady(run) && isPersonRun(run, now)).length; let sinceRecovery = input.startsSinceRecovery; let sinceOldest = input.startsSinceOldest ?? 0; const result: T[] = []; while (result.length < order.length) { - let pick = -1; - if (sinceOldest >= OLDEST_DISPATCH_SHARE - 1) { - const oldest = oldestIndex(); - if (oldest >= 0 && hasQueuedRunAged(order[oldest]!, input.now)) pick = oldest; + const candidate = oldestShare(); + let pick = sinceOldest >= OLDEST_DISPATCH_SHARE - 1 ? candidate : -1; + if (pick < 0 && readyPeople === 0 && sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1) { + pick = oldestRecovery(); + if (pick < 0) pick = firstRecovery(); } - if (pick < 0 && sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1) { - recoveryPointer = firstOf(recovery, recoveryPointer); - if (recoveryPointer < recovery.length) pick = recovery[recoveryPointer]!; - } - if (pick < 0) pick = firstOf(null, 0); + if (pick < 0) pick = head(); removed[pick] = 1; const run = order[pick]!; + if (isReady(run) && isPersonRun(run, now)) readyPeople -= 1; result.push(run); sinceRecovery = isSystemRecoveryRun(run) ? 0 : sinceRecovery + 1; - const oldest = oldestIndex(); - sinceOldest = oldest < 0 || order[oldest]!.createdAt.getTime() >= run.createdAt.getTime() ? 0 : sinceOldest + 1; + sinceOldest = candidate < 0 || servesOldestShare({ createdAt: run.createdAt, personWaiting: isPersonWaitingRun(run, now) }, + order[candidate]!, now) ? 0 : sinceOldest + 1; } return result; } diff --git a/server/src/services/heartbeat/queued-run-line.test.ts b/server/src/services/heartbeat/queued-run-line.test.ts index ded2a0c0d8f..c9b47af90b2 100644 --- a/server/src/services/heartbeat/queued-run-line.test.ts +++ b/server/src/services/heartbeat/queued-run-line.test.ts @@ -35,6 +35,7 @@ function queue(seed: number, size: number): QueuedRunDispatchFacts[] { readinessRank: pick([0, 0, 1, 2, 3]), priorityRank: pick([0, 1, 2, 3, 4]), shareDeferred: next() < 0.1, + personTask: next() < 0.3, }; }); } diff --git a/server/src/services/heartbeat/queued-run-line.ts b/server/src/services/heartbeat/queued-run-line.ts index 29d3c6bcd28..9fb3047faba 100644 --- a/server/src/services/heartbeat/queued-run-line.ts +++ b/server/src/services/heartbeat/queued-run-line.ts @@ -1,4 +1,4 @@ -import { and, asc, desc, eq, gte, inArray, isNotNull, sql } from "drizzle-orm"; +import { and, asc, desc, eq, gte, inArray, isNotNull, sql, type SQL } from "drizzle-orm"; import { agentWakeupRequests, agents, heartbeatRuns, issues, type Db } from "@paperclipai/db"; import type { ExecutionAgentQueue } from "@paperclipai/shared"; import { parseObject } from "../../adapters/utils.js"; @@ -6,10 +6,8 @@ import { OLDEST_DISPATCH_SHARE, PERSON_WAKE_REASONS, RECOVERY_DISPATCH_SHARE, - hasQueuedRunAged, - isRecoveryWakeReason, - isSystemRecoveryRun, - oldestReadyQueuedRun, + dispatchCountersFromStarts, + dispatchHistoryStart, queuedRunDispatchLine, type QueuedRunDispatchFacts, type QueuedRunDispatchInput, @@ -40,6 +38,33 @@ export function issueRunPriorityRank(priority: string | null | undefined) { } } +/** + * SQL: whether the run `runId` was asked for by a person, itself or through + * its source chain (`previousRunId`, else `retryOfRunId`), the same rule as + * {@link listPersonOriginSourceRunIds}. For use inside another query. + */ +export function personOriginRunSql(companyId: SQL, runId: SQL) { + const uuidPattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + return sql`exists ( + with recursive origin_chain(run_id, depth) as ( + select ${runId}, 0 + union all + select parent.id, origin_chain.depth + 1 + from origin_chain + join heartbeat_runs origin_child on origin_child.id = origin_chain.run_id + join heartbeat_runs parent on parent.company_id = ${companyId} and parent.id = coalesce( + case when origin_child.context_snapshot->>'previousRunId' ~* ${uuidPattern} + then (origin_child.context_snapshot->>'previousRunId')::uuid end, + origin_child.retry_of_run_id) + where origin_chain.depth < ${PERSON_ORIGIN_CHAIN_DEPTH} + ) + select 1 from origin_chain join heartbeat_runs origin_run on origin_run.id = origin_chain.run_id + where origin_run.context_snapshot->>'wakeReason' in (${sql.join([...PERSON_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `)}) + or origin_run.context_snapshot->>'taskRetryRequestId' is not null + or exists (select 1 from agent_wakeup_requests origin_wake where origin_wake.company_id = ${companyId} + and origin_wake.requested_by_actor_type = 'user' and (origin_wake.run_id = origin_run.id or origin_wake.id = origin_run.wakeup_request_id)))`; +} + export async function listPersonOriginSourceRunIds(db: Db, companyId: string, sourceRunIds: string[]) { const uuidPattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; const result = await db.execute<{ root_id: string }>(sql` @@ -77,7 +102,7 @@ export async function queuedRunDispatchFacts( agent: Pick; queuedRuns: R[]; dependencyReadiness: Map; - issueById: Map; + issueById: Map; shareDeferred?: (runId: string) => boolean; now: Date; }, @@ -135,39 +160,30 @@ export async function queuedRunDispatchFacts( readinessRank: issueId ? (ready ? (issue?.status === "in_progress" ? 0 : 1) : 3) : 2, priorityRank: issueRunPriorityRank(issue?.priority), shareDeferred: input.shareDeferred?.(run.id) ?? false, + personTask: Boolean(issue?.createdByUserId), }; }); - const recoveryRuns = facts.filter(entry => isSystemRecoveryRun(entry)); - const oldestReady = oldestReadyQueuedRun(facts); - const agedOldest = oldestReady && hasQueuedRunAged(oldestReady, now) ? oldestReady : null; - const countsRecovery = recoveryRuns.length > 0 && recoveryRuns.length < facts.length; - let startsSinceRecovery = 0; - let startsSinceOldest = 0; - if (countsRecovery || (agedOldest && facts.length > 1)) { - const oldestRecoveryAt = countsRecovery ? Math.min(...recoveryRuns.map(entry => entry.createdAt.getTime())) : Infinity; - const since = new Date(Math.min(oldestRecoveryAt, agedOldest?.createdAt.getTime() ?? Infinity)); - const recentStarts = await db.select({ id: heartbeatRuns.id, wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, - startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt }) - .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), - isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, since))) - .orderBy(desc(heartbeatRuns.startedAt)) - .limit(Math.max(RECOVERY_DISPATCH_SHARE, OLDEST_DISPATCH_SHARE)); - const startsSince = (at: number) => recentStarts.filter(row => row.startedAt && row.startedAt.getTime() >= at); - if (countsRecovery) { - const starts = startsSince(oldestRecoveryAt).slice(0, RECOVERY_DISPATCH_SHARE); - const recoveryStartIds = starts.flatMap(row => isRecoveryWakeReason(row.wakeReason) ? [row.id] : []); - const personStarts = new Set(recoveryStartIds.length - ? await listPersonOriginSourceRunIds(db, agent.companyId, recoveryStartIds) : []); - const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason) && !personStarts.has(row.id)); - startsSinceRecovery = lastRecovery < 0 ? starts.length : lastRecovery; - } - if (agedOldest && facts.length > 1) { - const starts = startsSince(agedOldest.createdAt.getTime()).slice(0, OLDEST_DISPATCH_SHARE); - const lastOldest = starts.findIndex(row => row.createdAt.getTime() <= agedOldest.createdAt.getTime()); - startsSinceOldest = lastOldest < 0 ? starts.length : lastOldest; - } - } + const since = dispatchHistoryStart(facts, now); + const startRows = since ? await db.select({ id: heartbeatRuns.id, wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, + startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt, + issueId: sql`${heartbeatRuns.contextSnapshot}->>'issueId'` }) + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), + isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, since))) + .orderBy(desc(heartbeatRuns.startedAt)) + .limit(Math.max(RECOVERY_DISPATCH_SHARE, OLDEST_DISPATCH_SHARE)) : []; + // A person's recovery restart is person work, so it does not use up system recovery's share; + // only a start people wait on uses up their oldest-run turn. + const personStarts = new Set(startRows.length + ? await listPersonOriginSourceRunIds(db, agent.companyId, startRows.map(row => row.id)) : []); + const startedTaskIds = [...new Set(startRows.flatMap(row => row.issueId && RUN_ID_PATTERN.test(row.issueId) ? [row.issueId] : []))]; + const personTaskIds = new Set(startedTaskIds.length + ? (await db.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, agent.companyId), + inArray(issues.id, startedTaskIds), isNotNull(issues.createdByUserId)))).map(row => row.id) + : []); + const recentStarts = startRows.map(row => ({ ...row, personOrigin: personStarts.has(row.id), + personWaiting: personStarts.has(row.id) || Boolean(row.issueId && personTaskIds.has(row.issueId)) })); + const { startsSinceRecovery, startsSinceOldest } = dispatchCountersFromStarts(facts, recentStarts, now); return { facts, dispatchInput: { now, startsSinceRecovery, startsSinceOldest } }; } @@ -255,7 +271,7 @@ export async function agentQueuedRunLine( const [dependencyReadiness, issueRows] = issueIds.length ? await Promise.all([ issuesSvc.listDependencyReadiness(agent.companyId, issueIds), - db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues) + db.select({ id: issues.id, status: issues.status, priority: issues.priority, createdByUserId: issues.createdByUserId }).from(issues) .where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, issueIds))), ]) : [new Map(), []]; diff --git a/server/src/services/mcp-http.ts b/server/src/services/mcp-http.ts index 8fbdb7fab90..0a0b1dadbf2 100644 --- a/server/src/services/mcp-http.ts +++ b/server/src/services/mcp-http.ts @@ -36,6 +36,9 @@ const MODERN_ERROR_CODES = new Set([MCP_ERROR_HEADER_MISMATCH, MCP_ERROR_MISSING export const MCP_TASKS_EXTENSION_ID = "io.modelcontextprotocol/tasks"; const MCP_TASK_METHODS = new Set(["tasks/get", "tasks/update", "tasks/cancel"]); +/** MCP OAuth Client Credentials extension; declared only by connections that use machine sign-in. */ +export const MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID = "io.modelcontextprotocol/oauth-client-credentials"; + const META_PROTOCOL_VERSION = "io.modelcontextprotocol/protocolVersion"; const META_CLIENT_CAPABILITIES = "io.modelcontextprotocol/clientCapabilities"; const META_CLIENT_INFO = "io.modelcontextprotocol/clientInfo"; @@ -54,7 +57,7 @@ export function isModernMcpProtocolVersion(version: unknown): version is string * they may then return URL input requests, which Paperclip shows as a link. * Legacy sessions keep the handshake capabilities they have always received. */ -export function mcpClientCapabilities(protocolVersion: string, options: { declareTasks?: boolean; elicitationForm?: boolean } = {}): Record { +export function mcpClientCapabilities(protocolVersion: string, options: { declareTasks?: boolean; elicitationForm?: boolean; declareClientCredentials?: boolean } = {}): Record { const version: string = protocolVersion; // The skills and tasks extensions are defined against stateless MCP only. // Tasks are declared only where Paperclip can wait on one: a provider tool @@ -66,6 +69,7 @@ export function mcpClientCapabilities(protocolVersion: string, options: { declar return { elicitation: options.elicitationForm ? { form: {}, url: {} } : { url: {} }, extensions: { ...MCP_APPS_CLIENT_CAPABILITY, ...MCP_SKILLS_CLIENT_CAPABILITY, ...(options.declareTasks ? { [MCP_TASKS_EXTENSION_ID]: {} } : {}), + ...(options.declareClientCredentials ? { [MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID]: {} } : {}), } }; } // 2025-06-18 knows only form questions (`{}`); 2025-11-25 added URL mode. @@ -209,18 +213,20 @@ export type McpJsonRpcRequest = { id?: string | number; method: string; params?: export function mcpHttpRequestInit( headers: Record | undefined, message: McpJsonRpcRequest, - options: { toolInputSchema?: unknown; declareTasks?: boolean; elicitationForm?: boolean } = {}, + options: { toolInputSchema?: unknown; declareTasks?: boolean; elicitationForm?: boolean; declareClientCredentials?: boolean; progressToken?: string } = {}, ): { method: "POST"; headers: Record; body: string } { const requestHeaders = mcpHttpRequestHeaders(headers); const protocolVersion = mcpProtocolVersionOf(headers); let params = message.params ?? {}; + // Asks the server to stream `notifications/progress` for this request. + if (options.progressToken) params = { ...params, _meta: { ...record(params._meta), progressToken: options.progressToken } }; if (isModernMcpProtocolVersion(protocolVersion)) { params = { ...params, _meta: { ...record(params._meta), [META_PROTOCOL_VERSION]: protocolVersion, - [META_CLIENT_CAPABILITIES]: mcpClientCapabilities(protocolVersion, { declareTasks: options.declareTasks, elicitationForm: options.elicitationForm }), + [META_CLIENT_CAPABILITIES]: mcpClientCapabilities(protocolVersion, { declareTasks: options.declareTasks, elicitationForm: options.elicitationForm, declareClientCredentials: options.declareClientCredentials }), [META_CLIENT_INFO]: MCP_CLIENT_INFO, }, }; @@ -666,6 +672,15 @@ export function parseMcpHttpResponseBody(bodyText: string, contentType: string | throw new SyntaxError("MCP SSE response contained no data events"); } +/** A `notifications/progress` for this request's token, or null for any other message. */ +export function mcpProgressNotificationOf(message: unknown, progressToken: string | undefined): { progress: unknown; total: unknown; message: unknown } | null { + const payload = record(message); + if (!progressToken || !payload || payload.method !== "notifications/progress" || "id" in payload) return null; + const params = record(payload.params); + if (!params || params.progressToken !== progressToken) return null; + return { progress: params.progress, total: params.total, message: params.message }; +} + export type McpHttpResponseProgress = { bytesReceived: number; chunksReceived: number; @@ -683,6 +698,15 @@ export async function readMcpHttpResponse( maxBytes?: number; onRequest?: (message: Record) => Promise; onProgress?: (progress: McpHttpResponseProgress) => void; + /** + * Bound an SSE stream per message instead of in total, so handled + * notifications (such as progress) do not use up the result's limit. Only + * for callers with an overall deadline: a stream of small notifications + * could otherwise never end. + */ + limitPerMessage?: boolean; + /** Notifications sent on this request's stream before its result. Must not block. */ + onNotification?: (message: Record) => void; } = {}, ): Promise { const progress: McpHttpResponseProgress = { bytesReceived: 0, chunksReceived: 0, eventsReceived: 0, matchedResponse: false, ended: false }; @@ -714,6 +738,10 @@ export async function readMcpHttpResponse( return record; } if ("method" in record && "id" in record) await options.onRequest?.(record); + else if ("method" in record) { + // A notification observer never fails or delays the call. + try { options.onNotification?.(record); } catch { /* ignored */ } + } return undefined; }; const event = async (value: string) => { @@ -729,17 +757,22 @@ export async function readMcpHttpResponse( progress.bytesReceived = bytes; if (value) progress.chunksReceived += 1; progress.ended = done; - if (bytes > maxBytes) throw new McpHttpResponseError("too_large", "MCP response exceeded the size limit"); + // A JSON body is one message. An SSE stream is bounded in total unless the + // caller asks for a per-message bound (see limitPerMessage). + const perMessage = isStream && options.limitPerMessage === true; + if (!perMessage && bytes > maxBytes) throw new McpHttpResponseError("too_large", "MCP response exceeded the size limit"); buffer += decoder.decode(value, { stream: !done }); if (isStream) { // Normalize CRLF after concatenating chunks, including split CR/LF pairs. buffer = buffer.replace(/\r\n/g, "\n"); let boundary: number; while ((boundary = buffer.indexOf("\n\n")) >= 0) { + if (perMessage && Buffer.byteLength(buffer.slice(0, boundary), "utf8") > maxBytes) throw new McpHttpResponseError("too_large", "MCP response exceeded the size limit"); const result = await event(buffer.slice(0, boundary)); buffer = buffer.slice(boundary + 2); if (result !== undefined) return result; } + if (perMessage && Buffer.byteLength(buffer, "utf8") > maxBytes) throw new McpHttpResponseError("too_large", "MCP response exceeded the size limit"); } if (done) break; } diff --git a/server/src/services/native-runtime/native-runner-checkpoint-save.test.ts b/server/src/services/native-runtime/native-runner-checkpoint-save.test.ts new file mode 100644 index 00000000000..bd179a05724 --- /dev/null +++ b/server/src/services/native-runtime/native-runner-checkpoint-save.test.ts @@ -0,0 +1,82 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { agents, checkpointDb, closeCheckpointDb, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, + subscribeDatabaseWork } from "@paperclipai/db"; +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +(support.supported ? describe : describe.skip)("native runner checkpoint save", () => { + let temporary: Awaited>; + let db: ReturnType; + beforeAll(async () => { + process.env.PAPERCLIP_SECRETS_MASTER_KEY ??= randomBytes(32).toString("base64"); + temporary = await startEmbeddedPostgresTestDatabase("paperclip-checkpoint-save-"); + db = createDb(temporary.connectionString); + }, 60_000); + afterAll(async () => { + if (db) await closeCheckpointDb(db, 1); + await db?.$client.end({ timeout: 1 }); + await temporary?.cleanup(); + }); + + const fixture = async (leaseExpiresAt: Date) => { + const [company] = await db.insert(companies).values({ name: "Checkpoint save", issuePrefix: `CS${randomUUID().slice(0, 4)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Checkpoint agent", adapterType: "process" }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company!.id, title: "Checkpoint task" }).returning(); + const runId = randomUUID(), leaseOwner = randomUUID(), bootId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: company!.id, agentId: agent!.id, status: "running", + runtimeMode: "native", nativeIssueId: issue!.id, issueId: issue!.id }); + await db.insert(nativeRunFinalizations).values({ runId, companyId: company!.id, issueId: issue!.id, phase: "observed", attempt: 0, + controllerBootId: bootId, controllerPid: process.pid, controllerProcessStartedAt: new Date(), controllerGeneration: 1, + leaseOwner, leaseExpiresAt }); + const write = nativeRunnerCheckpointWriter(db, { companyId: company!.id, issueId: issue!.id, runId, authorityRunId: runId, + bootId, generation: 1, leaseOwner }); + const checkpoint = (changed: Map, reset: boolean) => ({ + runId, records: new Map(changed), changedRecords: changed, removedRecords: [], reset, workCommandIds: [], predecessor: null, + runnerVersion: "test", runnerDigest: `sha256:${"a".repeat(64)}`, + recoveryEvidence: { runnerInstanceId: "test", normalizedSessionId: "test", authenticatedConnectionCount: 1, highestRecordedSourceSeq: 0 }, + restoreSessionDirectory() {}, + }) as unknown as Parameters[0]; + return { runId, write, checkpoint }; + }; + + const sentStatements = async (action: () => Promise) => { + const statements: string[] = []; + const options = checkpointDb(db).$client.options as { debug?: unknown }; + options.debug = (_connection: number, text: string) => { statements.push(text.trim().replace(/\s+/g, " ")); }; + try { await action(); return statements; } finally { options.debug = undefined; } + }; + + it("saves a checkpoint with the owner lock, the write and the commit, and registers its wake before the write", async () => { + const f = await fixture(new Date(Date.now() + 3_600_000)); + await f.write(f.checkpoint(new Map([["header", "{}"], ["cursor", "{\"ackedSourceSeq\":0}"]]), true)); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.nativeProcessSessions, event => { events.push(event); }); + let statements: string[]; + try { + statements = await sentStatements(() => f.write(f.checkpoint(new Map([["event:1", "{}"], ["cursor", "{\"ackedSourceSeq\":1}"]]), false))); + } finally { unsubscribe(); } + expect(statements).toHaveLength(4); + expect(statements[0]).toMatch(/^begin/i); + expect(statements[1]).toMatch(/pg_current_xact_id\(\)/); + expect(statements[2]).toMatch(/^insert into "native_runner_checkpoints"/); + expect(statements[3]).toMatch(/^commit/i); + expect(events).toEqual(["intent", "settled"]); + expect((await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints) + .where(eq(nativeRunnerCheckpoints.runId, f.runId))).map(row => row.part).sort()).toEqual(["cursor", "event:1", "header"]); + }); + + it("refuses a save after the owner's lease expired and writes nothing", async () => { + const f = await fixture(new Date(Date.now() + 3_600_000)); + await f.write(f.checkpoint(new Map([["header", "{}"]]), true)); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + await expect(f.write(f.checkpoint(new Map([["event:1", "{}"]]), false))).rejects.toThrow("native_runner_checkpoint_owner_lost"); + expect((await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints) + .where(eq(nativeRunnerCheckpoints.runId, f.runId))).map(row => row.part)).toEqual(["header"]); + }); +}); diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index d4a9b2be91e..8582d339ed7 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -3,7 +3,7 @@ import { NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { createHash } from "node:crypto"; import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql, type SQL } from "drizzle-orm"; -import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; +import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, primeDatabaseWorkTransactionId, type Db } from "@paperclipai/db"; import { inspectDurablePrpCheckpointHeader, inspectUnauthenticatedBootstrapCheckpoint, inspectUnclaimedBootstrapCheckpoint, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js"; import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret } from "../../secrets/local-encrypted-provider.js"; import { legacyControllerBootId } from "../legacy-controller-lease.js"; @@ -76,9 +76,12 @@ export async function lockNativeRunnerCheckpointOwner( // wall time. Sampling in the locking SELECT can precede its lock wait. // Keep both steps in one statement without another database round trip. const locked = nativeRunnerCheckpointOwnerLock(tx, owner); - const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner, - leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, phase: locked.phase, now: sql`clock_timestamp()` }).from(locked); - if (!owned) throw new Error("native_runner_checkpoint_owner_lost"); + const [row] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner, + leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, phase: locked.phase, now: sql`clock_timestamp()`, + xid: sql`pg_current_xact_id()::text` }).from(locked); + if (!row) throw new Error("native_runner_checkpoint_owner_lost"); + const { xid, ...owned } = row; + primeDatabaseWorkTransactionId(tx, xid); return owned; } diff --git a/server/src/services/native-runtime/runner-api-sensitive-response.ts b/server/src/services/native-runtime/runner-api-sensitive-response.ts index 97410b665cf..c84d81421c9 100644 --- a/server/src/services/native-runtime/runner-api-sensitive-response.ts +++ b/server/src/services/native-runtime/runner-api-sensitive-response.ts @@ -1,6 +1,6 @@ export function runnerApiSensitiveResponse(method: string, path: string): boolean { if (method === "POST" && /^\/api\/chat-endpoints\/\{[^}]+\}\/setup$/.test(path)) return true; - if (/(\/apps\/connect$|\/email\/connections$|setup-secret|\/github\/(setup|identity\/start)|\/slack\/resume|start-authorization|finalize-oauth-access|\/oauth\/|\/registration|\/managed\/authorize|\/install$|\/login|\/setup-token-login-sessions|\/local\/(attempts|check)(\/|$)|\/test-calls(\/|$))/.test(path)) return true; + if (/(\/apps\/connect$|\/email\/connections$|setup-secret|\/github\/(setup|identity\/start)|\/slack\/resume|start-authorization|finalize-oauth-access|\/machine-sign-in|\/oauth\/|\/registration|\/managed\/authorize|\/install$|\/login|\/setup-token-login-sessions|\/local\/(attempts|check)(\/|$)|\/test-calls(\/|$))/.test(path)) return true; if (/\/secrets\/\{[^}]+\}\/value$/.test(path)) return true; return /^\/api\/tool-gateway\/sessions(?:\/|$)/.test(path) || /^\/api\/companies\/\{[^}]+\}\/exports?(?:\/|$)/.test(path) diff --git a/server/src/services/recovery-queue-admission.ts b/server/src/services/recovery-queue-admission.ts new file mode 100644 index 00000000000..148bc4a54d0 --- /dev/null +++ b/server/src/services/recovery-queue-admission.ts @@ -0,0 +1,42 @@ +import { and, eq, sql, type SQL } from "drizzle-orm"; +import { heartbeatRuns, type Db } from "@paperclipai/db"; +import { RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; + +/** + * System recovery joins an agent's queue a few runs at a time. The rest stays + * pending with its producer (a resolved recovery action), which delivers the + * next one when a queued recovery run starts or is dropped. Recovery waits + * behind people's work in the dispatcher, so a deep recovery queue would only + * hold stale runs; pending recovery is re-read when it is admitted instead. + */ +export const QUEUED_RECOVERY_RUNS_PER_AGENT = 3; + +/** + * Evidence keys under which a restored task records the wake it is owed + * (legacy-workspace rehome, released workspace writer). While its assignee's + * recovery queue is full, that wake waits pending here. + */ +export const HELD_RECOVERY_WAKE_KEYS = ["legacyWorkspaceRehome", "releasedWorkspaceWriter"] as const; + +/** Queued recovery runs of `agentId`, as SQL on a `heartbeat_runs` alias. */ +export function queuedRecoveryRunsOf(companyId: SQL | string, agentId: SQL | string) { + const reasons = sql.join([...RECOVERY_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `); + return sql`select 1 from ${heartbeatRuns} queued_recovery + where queued_recovery.company_id = ${companyId} and queued_recovery.agent_id = ${agentId} + and queued_recovery.status = 'queued' and queued_recovery.context_snapshot->>'wakeReason' in (${reasons})`; +} + +/** + * A soft cap, read before the producer's wake: producers that run at the same + * moment can each see room, so the queue can briefly hold one more run per + * concurrent producer (two today: continuation delivery and restored-task + * wakes). That stays "a few at a time"; the dispatcher keeps every recovery + * run behind people's work and re-checks it before it starts. + */ +export async function agentRecoveryQueueIsFull(db: Db, companyId: string, agentId: string) { + const reasons = [...RECOVERY_WAKE_REASONS]; + const [row] = await db.select({ count: sql`count(*)::int` }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' in (${sql.join(reasons.map(reason => sql`${reason}`), sql`, `)})`)); + return Number(row?.count ?? 0) >= QUEUED_RECOVERY_RUNS_PER_AGENT; +} diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 880445c9921..c7ac8ea5525 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -31,10 +31,12 @@ import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; import { workspaceRecoverySourceEligible } from "../workspace-recovery-source.js"; import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js"; -import { OLDEST_DISPATCH_SHARE, QUEUED_RUN_AGING_MS } from "../heartbeat/queued-run-dispatch-order.js"; +import { HELD_RECOVERY_WAKE_KEYS, queuedRecoveryRunsOf } from "../recovery-queue-admission.js"; import { listPersonOriginSourceRunIds } from "../heartbeat/queued-run-line.js"; +import { aliasedRun, queuedRunStalledSql } from "./stalled-queued-run-sql.js"; import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js"; -import { ASSIGNED_WORK_CONTINUED_ACTION, taskAutomaticAttempts } from "./task-automatic-attempts.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION, PAUSED_FOR_AUTOMATIC_WORK, taskAutomaticAttempts } from "./task-automatic-attempts.js"; +import { automaticRunCreationHeldUntil, readAutomaticWorkStatus, reserveAutomaticRunCreation } from "./automatic-run-brake.js"; import { isReattachPhaseActive, reattachPhaseRecheckAt } from "../heartbeat/run-admission.js"; import { ensureFreshContinuation, type FreshContinuationResult } from "../fresh-continuation-fallback.js"; import { canContinueRecoveryRunAutomatically } from "../continuation-cause.js"; @@ -343,11 +345,23 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids), eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, - sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text, + // Recent, or waiting its turn behind the agent's queued recovery runs. + sql`(coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text, ${issueRecoveryActions.updatedAt}::text)::timestamptz - > ${at.toISOString()}::timestamptz - ${`${PENDING_CONTINUATION_BOUND_MS} milliseconds`}::interval`)); + > ${at.toISOString()}::timestamptz - ${`${PENDING_CONTINUATION_BOUND_MS} milliseconds`}::interval + or exists (${queuedRecoveryRunsOf(sql`${issueRecoveryActions.companyId}`, sql`${issueRecoveryActions.returnOwnerAgentId}`)}))`)); + // A restored task's wake that is waiting its turn behind its assignee's + // queued recovery runs (see HELD_RECOVERY_WAKE_KEYS) is its pending path too. + const heldWakes = await database.select({ issueId: issueRecoveryActions.sourceIssueId, agentId: issues.assigneeAgentId }) + .from(issueRecoveryActions) + .innerJoin(issues, and(eq(issues.companyId, issueRecoveryActions.companyId), eq(issues.id, issueRecoveryActions.sourceIssueId))) + .where(and( + eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids), + eq(issueRecoveryActions.status, "resolved"), + or(...HELD_RECOVERY_WAKE_KEYS.map(key => sql`${issueRecoveryActions.evidence} #>> array[${key}::text, 'wake'] = 'pending'`)), + sql`exists (${queuedRecoveryRunsOf(sql`${issues.companyId}`, sql`${issues.assigneeAgentId}`)})`)); const wakeCounts = counts(wakes); - for (const continuation of pendingContinuations) { + for (const continuation of [...pendingContinuations, ...heldWakes]) { const issue = issueRows.find(row => row.id === continuation.issueId); if (!issue || !continuation.agentId || issue.assigneeAgentId !== continuation.agentId) continue; wakeCounts.set(continuation.issueId, (wakeCounts.get(continuation.issueId) ?? 0) + 1); @@ -614,6 +628,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { if (repairTimes.length >= ASSIGNED_WORK_REPAIRS_PER_MINUTE) { return { verdict, outcome: "deferred_rate_limit", retryAt: new Date(repairTimes[0] + 60_000 + Math.floor(Math.random() * 5_000)) }; } + const heldUntil = facts.latestRun + ? await automaticRunCreationHeldUntil(db, issue.companyId, { issueId: issue.id, sourceRunId: facts.latestRun.id }) : null; + if (heldUntil) return { verdict, outcome: "deferred_rate_limit", retryAt: heldUntil }; const claimed = await claimContinuation(issue, at); if (claimed.kind !== "claimed") return { verdict, outcome: claimed.kind === "capped" ? "deferred_cap" : claimed.kind === "changed" ? "changed" : "deferred_rate_limit", retryAt: claimed.kind === "changed" ? null : claimed.retryAt }; @@ -635,6 +652,15 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { await settleContinuationActivity(claimed, claimed.reason, "not_started", { waitingUntil: fresh.retryAt.toISOString() }); return { verdict: claimed.verdict, outcome: fresh.kind === "backoff" ? "deferred_cap" : "waiting_on_recovery", retryAt: fresh.retryAt }; } + const slot = facts.latestRun + ? await reserveAutomaticRunCreation(db, issue.companyId, { issueId: issue.id, sourceRunId: facts.latestRun.id }) : null; + if (slot?.heldUntil) { + if (claimed.restoredTo) await revertRestoredBlock(issue, claimed); + await settleContinuationActivity(claimed, claimed.reason, "not_started", + { waitingUntil: slot.heldUntil.toISOString(), [PAUSED_FOR_AUTOMATIC_WORK]: "true" }); + return { verdict: claimed.verdict, outcome: "deferred_rate_limit", retryAt: slot.heldUntil }; + } + const release = async () => { if (slot && !slot.heldUntil) await slot.release(); }; const run = await deps.enqueueWakeup(issue.assigneeAgentId!, { source: facts.latestRun ? "automation" : "assignment", triggerDetail: "system", reason: facts.latestRun ? "issue_continuation_needed" : "issue_assigned", @@ -650,6 +676,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { // A refused continuation stays a violation; the claim spaces the next try. // A block lifted for it is put back, so the task never sits unblocked // with nothing pending. + if (!run) await release(); if (!run && claimed.restoredTo) await revertRestoredBlock(issue, claimed); await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started", fresh?.kind === "person_owned" ? { waitingFor: fresh.reason } : {}); @@ -667,12 +694,19 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { } async function replaceStalledRun(issue: IssueRow, claimed: Claimed, runId: string): Promise { + const slot = await reserveAutomaticRunCreation(db, issue.companyId, { issueId: issue.id, sourceRunId: runId }); + if (slot.heldUntil) { + await settleContinuationActivity(claimed, claimed.reason, "not_started", + { waitingUntil: slot.heldUntil.toISOString(), [PAUSED_FOR_AUTOMATIC_WORK]: "true" }); + return { verdict: claimed.verdict, outcome: "deferred_rate_limit", retryAt: slot.heldUntil }; + } const cancelled = !claimed.stalledFence ? false : await deps.cancelStalledQueuedRun!({ companyId: issue.companyId, runId, ...claimed.stalledFence }).catch(error => { logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id, runId }, "assigned-work sweep could not cancel a stalled queued run"); return false; }); if (!cancelled) { + await slot.release(); await db.delete(activityLog).where(eq(activityLog.id, claimed.activityId)); return { verdict: claimed.verdict, outcome: "changed", retryAt: null }; } @@ -686,6 +720,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work replacement for a stalled run was not admitted"); return null; }); + if (!run) await slot.release(); await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started"); return { verdict: claimed.verdict, outcome: run ? "replaced_stalled_run" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } @@ -929,7 +964,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const queuedByAgent = new Map(queuedRows.map(row => [row.agentId, Number(row.queuedRuns)])); const slotRows = stalledRows.map(row => ({ agentName: row.agentName, stalledRuns: Number(row.stalledRuns), queuedRuns: queuedByAgent.get(row.agentId) ?? 0 })); - return { ...summary, agentsWithStalledSlots: slotRows }; + return { ...summary, agentsWithStalledSlots: slotRows, automaticWork: await readAutomaticWorkStatus(db, companyId) }; } return { checkIssue, listSuspects, report, loadFacts, flushPassLog: passLog.flush }; @@ -944,24 +979,6 @@ function interactionNotSupersededSql(companyId: SQLWrapper, issueIdText: SQLWrap and retry.created_at > ${createdAt})`; } -type StalledRunColumns = { id: SQLWrapper; companyId: SQLWrapper; agentId: SQLWrapper; status: SQLWrapper; createdAt: SQLWrapper }; - -function queuedRunStalledSql(run: StalledRunColumns, at: Date) { - const agedBefore = new Date(at.getTime() - QUEUED_RUN_AGING_MS).toISOString(); - return sql`(${run.status} = 'queued' and ${run.createdAt} <= ${agedBefore}::timestamptz - and not exists (select 1 from heartbeat_runs older where older.company_id = ${run.companyId} - and older.agent_id = ${run.agentId} and older.status = 'queued' and older.created_at < ${run.createdAt}) - and (select count(*) from (select 1 from heartbeat_runs started where started.company_id = ${run.companyId} - and started.agent_id = ${run.agentId} and started.id <> ${run.id} - and started.started_at > greatest(${run.createdAt} + ${`${QUEUED_RUN_AGING_MS} milliseconds`}::interval, - (select max(coalesce(prior.started_at, prior.finished_at)) from heartbeat_runs prior - where prior.company_id = ${run.companyId} and prior.agent_id = ${run.agentId} and prior.created_at < ${run.createdAt} - and (prior.started_at >= ${run.createdAt} or (prior.started_at is null and prior.finished_at >= ${run.createdAt})))) - limit ${OLDEST_DISPATCH_SHARE + 1}) passed_over) > ${OLDEST_DISPATCH_SHARE})`; -} - -const aliasedRun = (alias: string): StalledRunColumns => ({ id: sql.raw(`${alias}.id`), companyId: sql.raw(`${alias}.company_id`), - agentId: sql.raw(`${alias}.agent_id`), status: sql.raw(`${alias}.status`), createdAt: sql.raw(`${alias}.created_at`) }); /** The status a block replaced, from the activity entry that blocked the task: * the normal update route records it as `_previous.status`, recovery as diff --git a/server/src/services/recovery/automatic-run-brake.ts b/server/src/services/recovery/automatic-run-brake.ts new file mode 100644 index 00000000000..fa345482221 --- /dev/null +++ b/server/src/services/recovery/automatic-run-brake.ts @@ -0,0 +1,565 @@ +import { eq, sql } from "drizzle-orm"; +import { automaticRunBrakes, type Db } from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import { PERSON_WAKE_REASONS, RECOVERY_WAKE_REASONS } from "../heartbeat/queued-run-dispatch-order.js"; +import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js"; + +export const AUTOMATIC_RUN_BRAKE = { + windowMs: 10 * 60_000, + startProgressMs: 2 * 60_000, + minCounted: 12, + failureShare: 0.8, + consecutiveProviderRefusals: 5, + minOpenMs: 2 * 60_000, + quietCloseMs: 15 * 60_000, + probeIntervalMs: 60_000, + probeClaimBoundMs: 10_000, + probeRunningBoundMs: 10 * 60_000, + probeSuccessesToRelease: 2, + releaseIntervalMs: 60_000, + releaseInitialRate: 2, + releaseSteps: 6, + creationIntervalMs: 60_000, + cacheTtlMs: 5_000, + maxCompanies: 256, + maxRuns: 1_024, + rowLimit: 500, + runningLimit: 200, +} as const; + +export type AutomaticRunBrakeState = "closed" | "open" | "half_open"; + +export type AutomaticRunFacts = { + id: string; + companyId: string; + retryOfRunId?: string | null; + scheduledRetryReason?: string | null; + contextSnapshot?: Record | null; +}; + +const EXTRA_AUTOMATIC_WAKE_REASONS: ReadonlySet = new Set(["issue_assignment_recovery", "transient_failure_retry"]); + +export function isAutomaticRun(run: AutomaticRunFacts): boolean { + const context = run.contextSnapshot ?? {}; + const reason = typeof context.wakeReason === "string" ? context.wakeReason : null; + if (context.explicitUserContinuation || context.taskRetryRequestId || (reason && PERSON_WAKE_REASONS.has(reason))) return false; + return Boolean(run.retryOfRunId || run.scheduledRetryReason || context.previousRunId || context.retryReason + || context.freshContinuationIsolation + || (reason && (RECOVERY_WAKE_REASONS.has(reason) || EXTRA_AUTOMATIC_WAKE_REASONS.has(reason)))); +} + +export type AutomaticRunOutcome = "failed_before_starting" | "healthy" | "neutral"; + +export type AutomaticRunOutcomeFacts = { + status: string; + errorCode: string | null; + runtimeMode: string | null; + executionStage: string | null; + startedAt: Date | null; + hasOutput: boolean; + providerWorkStarted: string | null; + nativePreDispatchFailure?: unknown; +}; + +export type AutomaticRunOutcomeRules = { + isPersonOwnedFailureCode: (code: string | null) => boolean; + restartCodes: readonly string[]; + isTransientSetupCode: (code: string | null) => boolean; + hasNativePreDispatchFailureRecord: (resultJson: unknown) => boolean; + listPersonOriginSourceRunIds: (db: Db, companyId: string, runIds: string[]) => Promise; +}; + +let outcomeRules: Promise | null = null; + +export function automaticRunOutcomeRules(): Promise { + outcomeRules ??= Promise.all([import("../fresh-continuation-fallback.js"), import("../heartbeat/retries.js"), + import("../heartbeat/run-failure-cause.js"), import("../heartbeat/queued-run-line.js")]) + .then(([fallback, retries, failureCause, queuedRunLine]) => ({ + isPersonOwnedFailureCode: fallback.isPersonOwnedFailureCode, + restartCodes: fallback.BACKLOG_CONTINUATION_SOURCE_CODES, + isTransientSetupCode: retries.isTransientWorkspacePreparationCode, + hasNativePreDispatchFailureRecord: failureCause.hasNativePreDispatchFailureRecord, + listPersonOriginSourceRunIds: queuedRunLine.listPersonOriginSourceRunIds, + })) + .catch(error => { outcomeRules = null; throw error; }); + return outcomeRules; +} + +export function classifyAutomaticRunOutcome(run: AutomaticRunOutcomeFacts, rules: AutomaticRunOutcomeRules): AutomaticRunOutcome { + if (run.status === "running") return run.executionStage === "dispatching" || run.hasOutput ? "healthy" : "neutral"; + if (!run.startedAt || run.status === "cancelled" || !["succeeded", "failed", "timed_out", "interrupted"].includes(run.status)) return "neutral"; + if (run.status === "succeeded" || rules.isPersonOwnedFailureCode(run.errorCode)) return "healthy"; + if (run.errorCode && rules.restartCodes.includes(run.errorCode)) return "neutral"; + const startPhase = run.providerWorkStarted === "false" || rules.isTransientSetupCode(run.errorCode) + || rules.hasNativePreDispatchFailureRecord({ nativePreDispatchFailure: run.nativePreDispatchFailure }) + || (run.runtimeMode === "legacy" && run.executionStage === "preparing"); + return startPhase ? "failed_before_starting" : "healthy"; +} + +type BrakeRow = typeof automaticRunBrakes.$inferSelect; + +type SignalRow = { + db_now: string | Date; + id: string; + status: string; + error_code: string | null; + runtime_mode: string | null; + execution_stage: string | null; + started_at: string | Date | null; + finished_at: string | Date | null; + created_at: string | Date; + has_output: boolean; + provider_work_started: string | null; + native_pre_dispatch: unknown; + retry_of_run_id: string | null; + scheduled_retry_reason: string | null; + context: Record | null; +}; + +type Evidence = { + counted: number; + failedBeforeStarting: number; + healthy: number; + providerRefusalStreak: boolean; + recoveredSince: boolean; + lastStartAt: number | null; + lastActivityAt: number | null; +}; + +export type AutomaticRunBrakeSnapshot = { + companyId: string; + state: AutomaticRunBrakeState; + pausedSince: Date | null; + stateChangedAt: Date | null; + reason: Record; + nextProbeAt: Date | null; + nextCreationAt: Date | null; + probeRunId: string | null; + probeSuccesses: number; + dbNow: Date; + evidence: Evidence; +}; + +const toDate = (value: string | Date | null | undefined) => value == null ? null : value instanceof Date ? value : new Date(value); + +function boundedMap(max: number) { + const map = new Map(); + return { + get(key: K) { + const value = map.get(key); + if (value !== undefined) { map.delete(key); map.set(key, value); } + return value; + }, + set(key: K, value: V) { + map.delete(key); + map.set(key, value); + while (map.size > max) map.delete(map.keys().next().value as K); + }, + delete(key: K) { map.delete(key); }, + clear() { map.clear(); }, + get size() { return map.size; }, + }; +} + +function rampStep(stateChangedAt: Date, now: Date) { + return Math.max(0, Math.floor((now.getTime() - stateChangedAt.getTime()) / AUTOMATIC_RUN_BRAKE.releaseIntervalMs)); +} + +const outcomeFacts = (row: SignalRow): AutomaticRunOutcomeFacts => ({ status: row.status, errorCode: row.error_code, + runtimeMode: row.runtime_mode, executionStage: row.execution_stage, startedAt: toDate(row.started_at), hasOutput: row.has_output, + providerWorkStarted: row.provider_work_started, nativePreDispatchFailure: row.native_pre_dispatch }); + +const latest = (values: Array) => { + const known = values.filter((value): value is number => typeof value === "number"); + return known.length ? Math.max(...known) : null; +}; + +function evaluate(rows: SignalRow[], state: AutomaticRunBrakeState, lower: number, now: Date, rules: AutomaticRunOutcomeRules): Evidence { + const automatic = rows.filter(row => isAutomaticRun({ id: row.id, companyId: "", retryOfRunId: row.retry_of_run_id, + scheduledRetryReason: row.scheduled_retry_reason, contextSnapshot: row.context })); + const outcomes = automatic.flatMap(row => { + const finishedAt = toDate(row.finished_at); + if (row.status !== "running" && (!finishedAt || finishedAt.getTime() < lower)) return []; + const startedAt = toDate(row.started_at); + if (row.status === "running" && (!startedAt || startedAt.getTime() < lower)) return []; + const outcome = classifyAutomaticRunOutcome(outcomeFacts(row), rules); + return outcome === "neutral" ? [] : [{ outcome, row, at: (finishedAt ?? startedAt!).getTime() }]; + }).sort((left, right) => right.at - left.at); + const failedBeforeStarting = outcomes.filter(entry => entry.outcome === "failed_before_starting").length; + const streak = outcomes.slice(0, AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals); + const lastStartAt = latest(automatic.map(row => toDate(row.started_at)?.getTime())); + const lastActivityAt = latest([lastStartAt, ...automatic.map(row => toDate(row.finished_at)?.getTime())]); + return { + counted: outcomes.length, + failedBeforeStarting, + healthy: outcomes.length - failedBeforeStarting, + providerRefusalStreak: streak.length === AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals + && streak.every(entry => entry.row.error_code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE), + recoveredSince: state === "open" && outcomes.length > 0 && (outcomes[0]!.row.status === "succeeded" + || (outcomes.length >= AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease + && outcomes.slice(0, AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease).every(entry => entry.outcome === "healthy"))), + lastStartAt, + lastActivityAt, + }; +} + +function trips(evidence: Evidence) { + return evidence.providerRefusalStreak || (evidence.counted >= AUTOMATIC_RUN_BRAKE.minCounted + && evidence.failedBeforeStarting / evidence.counted >= AUTOMATIC_RUN_BRAKE.failureShare); +} + +export type AutomaticWork = { issueId: string; sourceRunId: string | null }; + +export type AutomaticRunCreationSlot = { heldUntil: Date } | { heldUntil: null; release: () => Promise }; + +const NO_RESERVATION = async () => undefined; + +type CacheEntry = { snapshot: AutomaticRunBrakeSnapshot; readAt: number; skewMs: number; heldUntil: number }; + +export function createAutomaticRunBrake(options: { loadRules?: () => Promise } = {}) { + const loadRules = options.loadRules ?? automaticRunOutcomeRules; + const companies = boundedMap(AUTOMATIC_RUN_BRAKE.maxCompanies); + const personOrigin = boundedMap(AUTOMATIC_RUN_BRAKE.maxRuns); + let rulesFailureLogged = false; + + async function rulesOrNull() { + try { + return await loadRules(); + } catch (error) { + if (!rulesFailureLogged) { + rulesFailureLogged = true; + logger.error({ err: error, event: "automatic_run_brake.rules_unavailable" }, + "Automatic work pause could not load its run rules; automatic work is not paused"); + } + return null; + } + } + + async function readRow(db: Db, companyId: string): Promise { + const [row] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId)); + return row ?? null; + } + + async function readSignal(db: Db, companyId: string, row: BrakeRow | null) { + const state = row?.state ?? "closed"; + const changedAt = row?.stateChangedAt ?? null; + const lower = state === "open" + ? sql`coalesce(${changedAt?.toISOString() ?? null}::timestamptz, now() - ${`${AUTOMATIC_RUN_BRAKE.windowMs} milliseconds`}::interval)` + : sql`greatest(now() - ${`${AUTOMATIC_RUN_BRAKE.windowMs} milliseconds`}::interval, ${changedAt?.toISOString() ?? null}::timestamptz)`; + const columns = sql`r.id, r.status, r.error_code, r.runtime_mode, r.execution_stage, r.started_at, r.finished_at, r.created_at, + r.last_output_at is not null as has_output, r.result_json #>> '{executionRecovery,providerWorkStarted}' as provider_work_started, + r.result_json -> 'nativePreDispatchFailure' as native_pre_dispatch, r.retry_of_run_id, r.scheduled_retry_reason, + jsonb_build_object('wakeReason', r.context_snapshot->'wakeReason', 'previousRunId', r.context_snapshot->'previousRunId', + 'retryReason', r.context_snapshot->'retryReason', 'freshContinuationIsolation', r.context_snapshot->'freshContinuationIsolation', + 'explicitUserContinuation', r.context_snapshot->'explicitUserContinuation', + 'taskRetryRequestId', r.context_snapshot->'taskRetryRequestId') as context`; + const rows = [...await db.execute(sql` + with clock as (select now() as db_now, ${lower} as lower_at) + select clock.db_now, clock.lower_at, runs.* from clock left join lateral ( + (select ${columns} from heartbeat_runs r + where r.company_id = ${companyId} and r.finished_at is not null and r.finished_at >= clock.lower_at + order by r.finished_at desc limit ${AUTOMATIC_RUN_BRAKE.rowLimit}) + union all + (select ${columns} from heartbeat_runs r + where r.company_id = ${companyId} and r.status = 'running' + order by r.started_at desc nulls last limit ${AUTOMATIC_RUN_BRAKE.runningLimit}) + ) runs on true`)]; + return { rows: rows.filter(row => row.id), now: toDate(rows[0]!.db_now)!, lower: toDate(rows[0]!.lower_at)!.getTime() }; + } + + async function transition(db: Db, companyId: string, row: BrakeRow | null, to: AutomaticRunBrakeState, + reason: Record): Promise { + const from = row?.state ?? "closed"; + const values = { + state: to, + reason, + stateChangedAt: sql`date_trunc('milliseconds', now())`, + openedAt: to === "open" ? (from === "half_open" && row?.openedAt ? row.openedAt : sql`date_trunc('milliseconds', now())`) : to === "closed" ? null : row?.openedAt ?? null, + nextProbeAt: null, + probeRunId: null, + probeSuccesses: 0, + releaseWindowStartedAt: null, + releaseWindowStarts: 0, + nextCreationAt: null, + updatedAt: sql`now()`, + }; + const expectedChangedAt = row?.stateChangedAt?.toISOString() ?? null; + const [updated] = await db.insert(automaticRunBrakes).values({ companyId, ...values }) + .onConflictDoUpdate({ + target: automaticRunBrakes.companyId, + set: values, + setWhere: sql`${automaticRunBrakes.state} = ${from} + and date_trunc('milliseconds', ${automaticRunBrakes.stateChangedAt}) is not distinct from ${expectedChangedAt}::timestamptz`, + }).returning(); + if (!updated) return readRow(db, companyId); + const event = to === "open" ? (from === "half_open" ? "reopened" : "opened") : to; + logger[to === "open" ? "warn" : "info"]({ event: `automatic_run_brake.${event}`, companyId, from, to, ...reason }, + to === "open" ? "Automatic work paused for a company while its runs fail before starting" + : to === "half_open" ? "Automatic work for a company is resuming gradually" : "Automatic work for a company is running normally again"); + return updated; + } + + async function settleProbe(db: Db, row: BrakeRow, rows: SignalRow[], rules: AutomaticRunOutcomeRules): Promise { + if (row.state !== "open" || !row.probeRunId) return row; + const known = rows.find(candidate => candidate.id === row.probeRunId); + const [probe] = known ? [known] : [...await db.execute(sql` + select r.id, r.status, r.error_code, r.runtime_mode, r.execution_stage, r.started_at, r.finished_at, r.created_at, + r.last_output_at is not null as has_output, r.result_json #>> '{executionRecovery,providerWorkStarted}' as provider_work_started, + r.result_json -> 'nativePreDispatchFailure' as native_pre_dispatch + from heartbeat_runs r where r.company_id = ${row.companyId} and r.id = ${row.probeRunId}`)]; + if (probe && (probe.status === "queued" || probe.status === "scheduled_retry")) return row; + const outcome = probe ? classifyAutomaticRunOutcome(outcomeFacts(probe), rules) : "neutral"; + if (probe?.status === "running" && outcome === "neutral") return row; + const completed = probe?.status === "succeeded"; + const successes = outcome === "healthy" ? row.probeSuccesses + (completed ? AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease : 1) + : outcome === "failed_before_starting" ? 0 : row.probeSuccesses; + const [updated] = await db.update(automaticRunBrakes).set({ + probeRunId: null, probeSuccesses: successes, updatedAt: sql`now()`, + ...(outcome === "healthy" ? { nextProbeAt: sql`now()` } : {}), + }).where(sql`${automaticRunBrakes.companyId} = ${row.companyId} and ${automaticRunBrakes.state} = 'open' + and ${automaticRunBrakes.probeRunId} = ${row.probeRunId}`).returning(); + return updated ?? (await readRow(db, row.companyId)) ?? row; + } + + async function refresh(db: Db, companyId: string): Promise { + const rules = await rulesOrNull(); + if (!rules) { + const now = new Date(); + const entry: CacheEntry = { readAt: now.getTime(), skewMs: 0, heldUntil: 0, snapshot: { companyId, state: "closed", pausedSince: null, + stateChangedAt: null, reason: {}, nextProbeAt: null, nextCreationAt: null, probeRunId: null, probeSuccesses: 0, dbNow: now, + evidence: { counted: 0, failedBeforeStarting: 0, healthy: 0, providerRefusalStreak: false, recoveredSince: false, + lastStartAt: null, lastActivityAt: null } } }; + companies.set(companyId, entry); + return entry; + } + let row = await readRow(db, companyId); + let signal = await readSignal(db, companyId, row); + if (row) row = await settleProbe(db, row, signal.rows, rules); + let state: AutomaticRunBrakeState = row?.state ?? "closed"; + let evidence = evaluate(signal.rows, state, signal.lower, signal.now, rules); + const numbers = { windowMinutes: AUTOMATIC_RUN_BRAKE.windowMs / 60_000, recentAutomaticRuns: evidence.counted, + failedBeforeStarting: evidence.failedBeforeStarting, healthy: evidence.healthy }; + const changedAt = row?.stateChangedAt?.getTime() ?? 0; + const elapsed = signal.now.getTime() - changedAt; + let target: { to: AutomaticRunBrakeState; reason: Record } | null = null; + if (state === "closed" && trips(evidence)) { + target = { to: "open", reason: { ...numbers, trigger: evidence.providerRefusalStreak ? "sandbox_refusals" : "failed_before_starting" } }; + } else if (state === "half_open" && trips(evidence)) { + target = { to: "open", reason: { ...numbers, trigger: evidence.providerRefusalStreak ? "sandbox_refusals" : "failed_before_starting" } }; + } else if (state === "half_open" && rampStep(row!.stateChangedAt, signal.now) >= AUTOMATIC_RUN_BRAKE.releaseSteps) { + target = { to: "closed", reason: { ...numbers, trigger: "released" } }; + } else if (state === "open" && elapsed >= AUTOMATIC_RUN_BRAKE.minOpenMs + && (evidence.recoveredSince || (row?.probeSuccesses ?? 0) >= AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease)) { + target = { to: "half_open", reason: { ...numbers, trigger: "runs_starting_again" } }; + } else if (state === "open" && elapsed >= AUTOMATIC_RUN_BRAKE.quietCloseMs + && (evidence.lastActivityAt ?? 0) < signal.now.getTime() - AUTOMATIC_RUN_BRAKE.quietCloseMs) { + target = { to: "closed", reason: { ...numbers, trigger: "no_automatic_work" } }; + } + if (target) { + row = await transition(db, companyId, row, target.to, target.reason); + state = row?.state ?? "closed"; + if (state !== "closed") { + signal = await readSignal(db, companyId, row); + evidence = evaluate(signal.rows, state, signal.lower, signal.now, rules); + } + } + const snapshot: AutomaticRunBrakeSnapshot = { + companyId, state, pausedSince: row?.openedAt ?? null, stateChangedAt: row?.stateChangedAt ?? null, + reason: row?.reason ?? {}, nextProbeAt: row?.nextProbeAt ?? null, nextCreationAt: row?.nextCreationAt ?? null, + probeRunId: row?.probeRunId ?? null, probeSuccesses: row?.probeSuccesses ?? 0, dbNow: signal.now, evidence, + }; + const readAt = Date.now(); + const entry = { snapshot, readAt, skewMs: signal.now.getTime() - readAt, heldUntil: companies.get(companyId)?.heldUntil ?? 0 }; + companies.set(companyId, entry); + return entry; + } + + async function current(db: Db, companyId: string): Promise { + const entry = companies.get(companyId); + if (entry && Date.now() - entry.readAt < AUTOMATIC_RUN_BRAKE.cacheTtlMs) return entry; + return refresh(db, companyId); + } + + async function read(db: Db, companyId: string): Promise { + return (await current(db, companyId)).snapshot; + } + + async function isPersonWork(db: Db, companyId: string, work: AutomaticWork) { + if (!work.sourceRunId) return true; + const personReasons = sql.join([...PERSON_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `); + const [asked] = [...await db.execute<{ asked: boolean }>(sql` + with source as (select max(coalesce(r.finished_at, r.created_at)) as finished_at from heartbeat_runs r + where r.company_id = ${companyId} and r.id = ${work.sourceRunId}::uuid) + select exists (select 1 from agent_wakeup_requests w, source + where w.company_id = ${companyId} and w.requested_by_actor_type = 'user' and w.payload->>'issueId' = ${work.issueId} + and ((w.run_id is null and w.status in ('queued', 'deferred_issue_execution', 'claimed')) + or w.requested_at > coalesce(source.finished_at, '-infinity'::timestamptz))) + or exists (select 1 from heartbeat_runs h, source + where h.company_id = ${companyId} and h.context_snapshot->>'issueId' = ${work.issueId} + and h.created_at > coalesce(source.finished_at, '-infinity'::timestamptz) + and (h.context_snapshot->>'wakeReason' in (${personReasons}) or h.context_snapshot ? 'taskRetryRequestId' + or jsonb_typeof(h.context_snapshot->'explicitUserContinuation') = 'object')) as asked`)]; + if (asked?.asked) return true; + const rules = await rulesOrNull(); + return !rules || (await rules.listPersonOriginSourceRunIds(db, companyId, [work.sourceRunId])).includes(work.sourceRunId); + } + + async function creationHeldUntil(db: Db, companyId: string, work: AutomaticWork): Promise { + const entry = await current(db, companyId); + if (entry.snapshot.state !== "open") return null; + const now = Date.now(); + const lastStartAt = entry.snapshot.evidence.lastStartAt; + const startHold = lastStartAt !== null && lastStartAt - entry.skewMs > now - AUTOMATIC_RUN_BRAKE.startProgressMs + ? now + AUTOMATIC_RUN_BRAKE.creationIntervalMs : 0; + const slotHold = entry.snapshot.nextCreationAt ? entry.snapshot.nextCreationAt.getTime() - entry.skewMs : 0; + const until = Math.max(startHold, slotHold); + if (until <= now) return null; + return await isPersonWork(db, companyId, work) ? null : new Date(until); + } + + async function reserveCreation(db: Db, companyId: string, work: AutomaticWork): Promise { + const entry = await current(db, companyId); + if (entry.snapshot.state !== "open") return { heldUntil: null, release: NO_RESERVATION }; + if (await isPersonWork(db, companyId, work)) return { heldUntil: null, release: NO_RESERVATION }; + const recheck = new Date(Date.now() + AUTOMATIC_RUN_BRAKE.creationIntervalMs); + const lastStartAt = entry.snapshot.evidence.lastStartAt; + if (lastStartAt !== null && lastStartAt - entry.skewMs > Date.now() - AUTOMATIC_RUN_BRAKE.startProgressMs) return { heldUntil: recheck }; + const [seeded] = await db.update(automaticRunBrakes).set({ + nextCreationAt: sql`date_trunc('milliseconds', now()) + ${`${AUTOMATIC_RUN_BRAKE.creationIntervalMs} milliseconds`}::interval`, + updatedAt: sql`now()`, + }).where(sql`${automaticRunBrakes.companyId} = ${companyId} and ${automaticRunBrakes.state} = 'open' + and (${automaticRunBrakes.nextCreationAt} is null or ${automaticRunBrakes.nextCreationAt} <= now())`) + .returning({ nextCreationAt: automaticRunBrakes.nextCreationAt }); + if (seeded?.nextCreationAt) { + const token = seeded.nextCreationAt.toISOString(); + entry.snapshot = { ...entry.snapshot, nextCreationAt: seeded.nextCreationAt }; + return { heldUntil: null, release: async () => { + await db.update(automaticRunBrakes).set({ nextCreationAt: null, updatedAt: sql`now()` }) + .where(sql`${automaticRunBrakes.companyId} = ${companyId} and ${automaticRunBrakes.nextCreationAt} = ${token}::timestamptz + and not exists (select 1 from heartbeat_runs r where r.company_id = ${companyId} + and r.context_snapshot->>'issueId' = ${work.issueId} + and r.created_at >= ${token}::timestamptz - ${`${AUTOMATIC_RUN_BRAKE.creationIntervalMs} milliseconds`}::interval)`); + companies.delete(companyId); + } }; + } + const row = await readRow(db, companyId); + if (row?.state !== "open") { companies.delete(companyId); return { heldUntil: null, release: NO_RESERVATION }; } + const at = row.nextCreationAt ? row.nextCreationAt.getTime() - entry.skewMs : recheck.getTime(); + return { heldUntil: new Date(Math.max(Date.now(), at)) }; + } + + async function isPersonOrigin(db: Db, run: AutomaticRunFacts) { + const known = personOrigin.get(run.id); + if (known !== undefined) return known; + const rules = await rulesOrNull(); + if (!rules) return true; + const origin = (await rules.listPersonOriginSourceRunIds(db, run.companyId, [run.id])).includes(run.id); + personOrigin.set(run.id, origin); + return origin; + } + + async function grantProbe(db: Db, entry: CacheEntry, run: AutomaticRunFacts) { + const [granted] = await db.update(automaticRunBrakes).set({ + probeRunId: run.id, updatedAt: sql`now()`, + nextProbeAt: sql`now() + ${`${AUTOMATIC_RUN_BRAKE.probeIntervalMs} milliseconds`}::interval`, + }).where(sql`${automaticRunBrakes.companyId} = ${run.companyId} and ${automaticRunBrakes.state} = 'open' + and (((${automaticRunBrakes.nextProbeAt} is null or ${automaticRunBrakes.nextProbeAt} <= now()) + and (${automaticRunBrakes.probeRunId} is null or not exists (select 1 from heartbeat_runs p + where p.company_id = ${automaticRunBrakes.companyId} and p.id = ${automaticRunBrakes.probeRunId} and p.status = 'running' + and p.started_at > now() - ${`${AUTOMATIC_RUN_BRAKE.probeRunningBoundMs} milliseconds`}::interval))) + or (${automaticRunBrakes.probeRunId} is not null + and ${automaticRunBrakes.nextProbeAt} <= now() + ${`${AUTOMATIC_RUN_BRAKE.probeIntervalMs - AUTOMATIC_RUN_BRAKE.probeClaimBoundMs} milliseconds`}::interval + and not exists (select 1 from heartbeat_runs p where p.company_id = ${automaticRunBrakes.companyId} + and p.id = ${automaticRunBrakes.probeRunId} and p.started_at is not null)))`) + .returning({ nextProbeAt: automaticRunBrakes.nextProbeAt }); + const now = Date.now(); + if (granted) { + entry.heldUntil = now + AUTOMATIC_RUN_BRAKE.probeClaimBoundMs; + entry.snapshot = { ...entry.snapshot, probeRunId: run.id, nextProbeAt: granted.nextProbeAt }; + return true; + } + const nextProbeAt = entry.snapshot.nextProbeAt ? entry.snapshot.nextProbeAt.getTime() - entry.skewMs : 0; + entry.heldUntil = Math.max(now + AUTOMATIC_RUN_BRAKE.cacheTtlMs, nextProbeAt); + return false; + } + + async function grantRelease(db: Db, entry: CacheEntry, companyId: string) { + const [granted] = [...await db.execute<{ starts: number; window_ends_at: string | Date }>(sql` + with ramp as ( + select b.company_id, b.state_changed_at + floor(extract(epoch from now() - b.state_changed_at) * 1000 + / ${AUTOMATIC_RUN_BRAKE.releaseIntervalMs}) * ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval as window_start, + ${AUTOMATIC_RUN_BRAKE.releaseInitialRate} * power(2, floor(extract(epoch from now() - b.state_changed_at) * 1000 + / ${AUTOMATIC_RUN_BRAKE.releaseIntervalMs}))::int as rate + from automatic_run_brakes b where b.company_id = ${companyId} and b.state = 'half_open' + ) + update automatic_run_brakes b set + release_window_starts = case when b.release_window_started_at = ramp.window_start then b.release_window_starts + 1 else 1 end, + release_window_started_at = ramp.window_start, updated_at = now() + from ramp + where b.company_id = ramp.company_id and b.state = 'half_open' + and (b.release_window_started_at is distinct from ramp.window_start or b.release_window_starts < ramp.rate) + returning b.release_window_starts as starts, ramp.window_start + ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval as window_ends_at`)]; + if (granted) return true; + const step = rampStep(entry.snapshot.stateChangedAt ?? entry.snapshot.dbNow, new Date(Date.now() + entry.skewMs)); + const windowEnds = (entry.snapshot.stateChangedAt?.getTime() ?? 0) + (step + 1) * AUTOMATIC_RUN_BRAKE.releaseIntervalMs; + entry.heldUntil = Math.max(Date.now() + 1_000, windowEnds - entry.skewMs); + return false; + } + + async function holdAutomaticRunStart(db: Db, run: AutomaticRunFacts, options: { personOrigin?: boolean } = {}): Promise { + if (!isAutomaticRun(run)) return false; + const cached = companies.get(run.companyId); + if (cached && cached.snapshot.state === "closed" && Date.now() - cached.readAt < AUTOMATIC_RUN_BRAKE.cacheTtlMs) return false; + if (options.personOrigin ?? await isPersonOrigin(db, run)) return false; + if (cached && cached.heldUntil > Date.now()) return true; + const entry = await current(db, run.companyId); + if (entry.snapshot.state === "closed") return false; + if (entry.heldUntil > Date.now()) return true; + const allowed = entry.snapshot.state === "open" ? await grantProbe(db, entry, run) : await grantRelease(db, entry, run.companyId); + return !allowed; + } + + return { + read, + refresh: async (db: Db, companyId: string) => (await refresh(db, companyId)).snapshot, + creationHeldUntil, + reserveCreation, + holdAutomaticRunStart, + cacheSizes: () => ({ companies: companies.size, runs: personOrigin.size }), + reset: () => { companies.clear(); personOrigin.clear(); }, + }; +} + +export const automaticRunBrake = createAutomaticRunBrake(); + +export function automaticRunCreationHeldUntil(db: Db, companyId: string, work: AutomaticWork) { + return automaticRunBrake.creationHeldUntil(db, companyId, work); +} + +export function reserveAutomaticRunCreation(db: Db, companyId: string, work: AutomaticWork) { + return automaticRunBrake.reserveCreation(db, companyId, work); +} + +export function automaticWorkStatusText(snapshot: AutomaticRunBrakeSnapshot) { + if (snapshot.state === "closed") return "Automatic work is running normally."; + const since = (snapshot.pausedSince ?? snapshot.stateChangedAt)?.toISOString() ?? "recently"; + if (snapshot.state === "half_open") return `Automatic work is resuming gradually since ${snapshot.stateChangedAt?.toISOString() ?? since} because recent runs started normally again.`; + const reason = snapshot.reason as { trigger?: string; recentAutomaticRuns?: number; failedBeforeStarting?: number }; + const because = reason.trigger === "sandbox_refusals" + ? `the last ${AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals} automatic runs could not get a sandbox` + : `${reason.failedBeforeStarting ?? 0} of ${reason.recentAutomaticRuns ?? 0} recent automatic runs failed before starting`; + const next = snapshot.nextProbeAt ?? snapshot.nextCreationAt; + return `Automatic work is paused for this company since ${since} because ${because}${next ? `; next try at ${next.toISOString()}` : ""}.`; +} + +export async function readAutomaticWorkStatus(db: Db, companyId: string) { + const snapshot = await automaticRunBrake.read(db, companyId); + return { + state: snapshot.state, + pausedSince: snapshot.pausedSince, + nextTryAt: snapshot.state === "open" ? snapshot.nextProbeAt ?? snapshot.nextCreationAt : null, + recentAutomaticRuns: snapshot.evidence.counted, + failedBeforeStarting: snapshot.evidence.failedBeforeStarting, + summary: automaticWorkStatusText(snapshot), + }; +} diff --git a/server/src/services/recovery/continuation-delivery-attempt.ts b/server/src/services/recovery/continuation-delivery-attempt.ts new file mode 100644 index 00000000000..ce3b7c2d221 --- /dev/null +++ b/server/src/services/recovery/continuation-delivery-attempt.ts @@ -0,0 +1,15 @@ +import { and, eq, sql } from "drizzle-orm"; +import { issueRecoveryActions, type Db } from "@paperclipai/db"; + +/** Records a delivery attempt for a pending continuation, so the next pass tries the records not yet tried first. */ +export function recordContinuationDeliveryAttempt(db: Db, action: Pick) { + return db.update(issueRecoveryActions) + .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( + { continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` }) + .where(and( + eq(issueRecoveryActions.companyId, action.companyId), + eq(issueRecoveryActions.id, action.id), + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + )); +} diff --git a/server/src/services/recovery/legacy-workspace-rehome.ts b/server/src/services/recovery/legacy-workspace-rehome.ts index 733e425e791..12248901393 100644 --- a/server/src/services/recovery/legacy-workspace-rehome.ts +++ b/server/src/services/recovery/legacy-workspace-rehome.ts @@ -6,6 +6,7 @@ import { releaseAncestorWorkspaceBinding } from "../execution-workspaces.js"; import { issueService } from "../issues.js"; import { REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE } from "../remote-workspace-writer-hold.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { agentRecoveryQueueIsFull } from "../recovery-queue-admission.js"; export const LEGACY_WORKSPACE_REHOME_SCHEMA = "paperclip.legacy-workspace-rehome.v1"; export const LEGACY_WORKSPACE_REHOME_OUTCOME = "legacy_shared_workspace_rehomed"; @@ -14,6 +15,7 @@ const BOARD_ESCALATION_POLICY = "board_escalation_no_takeover_v1"; const PAGE_SIZE = 25; const REHOME_WAKE_RETRY_BASE_MS = 5 * 60_000; const REHOME_WAKE_RETRY_MAX_MS = 60 * 60_000; +const RECOVERY_QUEUE_TURN_MS = 60_000; // A block set by the stranded escalation lands right after its failed run. const BLOCK_OWNERSHIP_SLACK_MS = 60_000; @@ -123,6 +125,15 @@ export async function deliverPendingRecoveryWakes(db: Db, wake: RehomeWakeup, sc eq(agentWakeupRequests.companyId, row.companyId), sql`${agentWakeupRequests.payload}->>'issueId' = ${row.issueId}`, sql`${agentWakeupRequests.payload}->>'recoveryActionId' = ${row.actionId}`, isNotNull(agentWakeupRequests.runId))).limit(1) : []; let accepted = !owed || Boolean(delivered); + // A few recovery runs per agent at a time: this one waits its turn, without + // counting as a refusal, and is offered again on the next pass. + if (!accepted && await agentRecoveryQueueIsFull(db, row.companyId, row.agentId!)) { + await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + array[${kind.key}::text, 'wakeRetryAt'], ${JSON.stringify(Date.now() + RECOVERY_QUEUE_TURN_MS)}::jsonb)` }) + .where(and(eq(issueRecoveryActions.companyId, row.companyId), eq(issueRecoveryActions.id, row.actionId), + sql`${wakeField(kind.key, "wake")} = 'pending'`)); + continue; + } try { // Only a created run settles the wake; a refusal (for example a paused // company or disabled on-demand wakes) returns null and is retried. diff --git a/server/src/services/recovery/stalled-queued-run-sql.ts b/server/src/services/recovery/stalled-queued-run-sql.ts new file mode 100644 index 00000000000..13e5552cfba --- /dev/null +++ b/server/src/services/recovery/stalled-queued-run-sql.ts @@ -0,0 +1,46 @@ +import { sql, type SQLWrapper } from "drizzle-orm"; +import { OLDEST_DISPATCH_SHARE, QUEUED_RUN_AGING_MS } from "../heartbeat/queued-run-dispatch-order.js"; +import { personOriginRunSql } from "../heartbeat/queued-run-line.js"; + +/** + * The assigned-work sweep's test for a queued run the dispatcher keeps passing + * over: its agent's oldest queued run, past the aging bound, with more than + * OLDEST_DISPATCH_SHARE later starts that the dispatcher ranked no higher. + */ +export type StalledRunColumns = { id: SQLWrapper; companyId: SQLWrapper; agentId: SQLWrapper; status: SQLWrapper; createdAt: SQLWrapper }; + +const UUID_TEXT = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + +/** + * A run people wait on, as the dispatcher ranks it: a person asked for it + * (itself or through its source chain, the dispatcher's own rule) or its task + * was created by a person. `alias` names a heartbeat_runs row in the enclosing query. + */ +function personWaitingRunSql(alias: string) { + const run = (column: string) => sql.raw(`${alias}.${column}`); + return sql`(${personOriginRunSql(sql`${run("company_id")}`, sql`${run("id")}`)} + or exists (select 1 from issues person_task where person_task.company_id = ${run("company_id")} + and person_task.id = case when ${run("context_snapshot")}->>'issueId' ~* ${UUID_TEXT} + then (${run("context_snapshot")}->>'issueId')::uuid end + and person_task.created_by_user_id is not null))`; +} + +export function queuedRunStalledSql(run: StalledRunColumns, at: Date) { + const agedBefore = new Date(at.getTime() - QUEUED_RUN_AGING_MS).toISOString(); + return sql`(${run.status} = 'queued' and ${run.createdAt} <= ${agedBefore}::timestamptz + and not exists (select 1 from heartbeat_runs older where older.company_id = ${run.companyId} + and older.agent_id = ${run.agentId} and older.status = 'queued' and older.created_at < ${run.createdAt}) + and (select count(*) from (select 1 from heartbeat_runs started where started.company_id = ${run.companyId} + and started.agent_id = ${run.agentId} and started.id <> ${run.id} + -- Work people wait on goes ahead of a run they do not by design; it does not pass that run over. + and not (${personWaitingRunSql("started")} and not exists (select 1 from heartbeat_runs waiting + where waiting.id = ${run.id} and ${personWaitingRunSql("waiting")})) + and started.started_at > greatest(${run.createdAt} + ${`${QUEUED_RUN_AGING_MS} milliseconds`}::interval, + (select max(coalesce(prior.started_at, prior.finished_at)) from heartbeat_runs prior + where prior.company_id = ${run.companyId} and prior.agent_id = ${run.agentId} and prior.created_at < ${run.createdAt} + and (prior.started_at >= ${run.createdAt} or (prior.started_at is null and prior.finished_at >= ${run.createdAt})))) + limit ${OLDEST_DISPATCH_SHARE + 1}) passed_over) > ${OLDEST_DISPATCH_SHARE})`; +} + +export const aliasedRun = (alias: string): StalledRunColumns => ({ id: sql.raw(`${alias}.id`), companyId: sql.raw(`${alias}.company_id`), + agentId: sql.raw(`${alias}.agent_id`), status: sql.raw(`${alias}.status`), createdAt: sql.raw(`${alias}.created_at`) }); diff --git a/server/src/services/recovery/task-automatic-attempts.ts b/server/src/services/recovery/task-automatic-attempts.ts index 20a2aa266f7..e1e47d17673 100644 --- a/server/src/services/recovery/task-automatic-attempts.ts +++ b/server/src/services/recovery/task-automatic-attempts.ts @@ -4,13 +4,15 @@ import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRAN import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js"; export { ASSIGNED_WORK_CONTINUED_ACTION, SCHEDULER_TRANSIENT_RETRY_REASON }; +export const PAUSED_FOR_AUTOMATIC_WORK = "pausedForAutomaticWork"; export async function taskAutomaticAttempts(db: Db, companyId: string, issueId: string, at: Date) { const since = new Date(at.getTime() - CONTINUATION_CAP_WINDOW_MS); const [continuations, transientRetries] = await Promise.all([ db.select({ createdAt: activityLog.createdAt, outcome: sql`${activityLog.details}->>'outcome'` }).from(activityLog).where(and( eq(activityLog.companyId, companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), - eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since))) + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since), + sql`coalesce(${activityLog.details}->>${PAUSED_FOR_AUTOMATIC_WORK}, '') <> 'true'`)) .orderBy(desc(activityLog.createdAt)).limit(64), db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, diff --git a/server/src/services/saved-work-recovery-overdue.test.ts b/server/src/services/saved-work-recovery-overdue.test.ts index eb583d68320..fd3f17941e2 100644 --- a/server/src/services/saved-work-recovery-overdue.test.ts +++ b/server/src/services/saved-work-recovery-overdue.test.ts @@ -1,6 +1,6 @@ import { and, eq, sql } from "drizzle-orm"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { activityLog, automaticRunBrakes, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { registerIssueRecoveryWork } from "./recovery/work.js"; import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; @@ -11,6 +11,7 @@ import { heartbeatService } from "./heartbeat.js"; import { deliverReconciledExecutions } from "./execution-recovery-resolution.js"; import { assignedWorkSweep, ASSIGNED_WORK_CONTINUED_ACTION } from "./recovery/assigned-work-sweep.js"; import { SAVED_WORK_RECOVERY_START_BOUND_MS } from "./recovery/assigned-work-invariant.js"; +import { automaticRunBrake } from "./recovery/automatic-run-brake.js"; import { action, connectionString, db, failedRemotePreparation, useWorkspaceRecoveryFixture } from "./__tests__/workspace-recovery-fixture.js"; const targets = vi.hoisted(() => new Map()); @@ -90,6 +91,17 @@ describe("a saved-work recovery the queue has not reached", () => { expect(successor).toBeDefined(); }, 90_000); + it("starts the overdue recovery while automatic work is paused, without using the pause's creation slot", async () => { + const f = await stagingShape(45 * 60_000); + await db.insert(automaticRunBrakes).values({ companyId: f.companyId, state: "open", openedAt: new Date() }); + automaticRunBrake.reset(); + expect(await f.sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId })) + .toMatchObject({ verdict: { reason: "saved_work_recovery_overdue" }, outcome: "started_saved_work_recovery" }); + expect((await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, f.companyId)))[0]) + .toMatchObject({ state: "open", nextCreationAt: null }); + automaticRunBrake.reset(); + }); + it("counts a recovery the queue has time to start as the task's pending path, and looks again when its bound ends", async () => { let clock = Date.now(); const f = await stagingShape(2 * 60_000, { now: () => new Date(clock) }); diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index cd3f5dd5b1e..f11b63d9c2b 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -45,6 +45,7 @@ import { managedConnectorProfile } from "./tool-access/oauth/client-settings.js" import { createOAuthClientRegistration } from "./tool-access/oauth/client-registration.js"; import { createOAuthRefreshLeases } from "./tool-access/oauth/refresh-leases.js"; import { createLegacyOAuthRefresh } from "./tool-access/oauth/connection-refresh.js"; +import { createClientCredentialsService } from "./tool-access/oauth/client-credentials-service.js"; import { createOAuthGrantRefresh } from "./tool-access/oauth/grant-refresh.js"; import { createGitHubGrantContinuity } from "./tool-access/github-grant-continuity.js"; import { createToolRuntimeHealth } from "./tool-access/runtime/health.js"; @@ -67,7 +68,7 @@ import { createOAuthClientProtocol } from "./tool-access/oauth/client.js"; import type { OAuthClientRegistrationSource } from "./tool-access/oauth/types.js"; export { OAUTH_CLIENT_ID_METADATA_DOCUMENT_PATH, resolveOAuthClientIdMetadataDocumentUrl, oauthClientIdMetadataDocument } from "./tool-access/oauth/client-metadata.js"; -import { connectionMethodFor, connectionMethodForConnection, credentialFieldsFor, normalizeConnectionMethodConfig, projectedConnectionToolInputSchema } from "./tool-access/connection-methods.js"; +import { connectionMethodFor, connectionMethodForConnection, credentialFieldsFor, normalizeConnectionMethodConfig, projectedConnectionHeaders, projectedConnectionToolInputSchema } from "./tool-access/connection-methods.js"; export { normalizeConnectionMethodConfig, projectedConnectionHeaders, @@ -788,8 +789,19 @@ export function toolAccessService( const { syncCredentialBindings, annotateSavedOAuthClientSecret, oauthClientForConnection, createOrRotateOAuthSecret, getOAuthGrantRow, resolveOAuthGrantSecret } = credentialStore; const ensureOAuthClient = createOAuthClientRegistration(db, { now, options, store: credentialStore, getConnectionRow, requestOAuthClientRegistration }); const refreshLeases = createOAuthRefreshLeases(db, now, getConnectionRow, getOAuthGrantRow); - const maybeRefreshOAuthCredentials = createLegacyOAuthRefresh(db, { now, secrets, store: credentialStore, leases: refreshLeases, getConnectionRow, exchangeOAuthToken }); - const refreshOAuthGrantCredentials = createOAuthGrantRefresh(db, { now, store: credentialStore, leases: refreshLeases, currentCloudConnector, getConnectionRow, exchangeOAuthToken }); + // Later dependencies are reached through closures: they are only called at request time. + const clientCredentials = createClientCredentialsService(db, { + now, secrets, store: credentialStore, exchangeOAuthToken, getConnectionRow, fetchRemoteHttpUrl, assertOAuthEndpointUrl, + assertRemoteConnectionEndpointsAllowed, ensureDefaultOrganizationGrant, nextAvailableConnectionName, + checkConnectionHealth: (connectionId, actor) => checkConnectionHealth(connectionId, actor), + refreshCatalog: (connectionId, actor, refreshOptions) => refreshCatalog(connectionId, actor, refreshOptions), + probeServerWithToken: (connection, authorization, actor) => catalogDiscovery.remoteTools( + connection, { ...projectedConnectionHeaders(connection), Authorization: authorization }, actor, undefined, [], { live: true }, + ), + }); + const { requestClientCredentialsToken } = clientCredentials; + const maybeRefreshOAuthCredentials = createLegacyOAuthRefresh(db, { now, secrets, store: credentialStore, leases: refreshLeases, getConnectionRow, exchangeOAuthToken, requestClientCredentialsToken }); + const refreshOAuthGrantCredentials = createOAuthGrantRefresh(db, { now, store: credentialStore, leases: refreshLeases, currentCloudConnector, getConnectionRow, exchangeOAuthToken, requestClientCredentialsToken }); const { refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt, githubConnectionContinuitySchedule } = createGitHubGrantContinuity(db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }); const vercelCredentials = createVercelCredentialService(db, now, vercelConnect); const { resolveVercelCredentialHeaders, vercelGrantForConnection } = vercelCredentials; @@ -803,7 +815,7 @@ export function toolAccessService( const defaultCatalogAccess = createCatalogDefaultAccess(db); const { enableCatalogEntriesByDefault, upsertAskFirstPolicies } = defaultCatalogAccess; const catalogDiscovery = createToolCatalogDiscovery(db, { now, audit, secrets, credentials: credentialAccess, vercel: vercelCredentials, vercelConnect, setupPolicy, requestRemoteHttpEndpoint, resolveOAuthGrantSecret, discoverOAuthEndpoints, localTools }); - const connectionHealth = createConnectionHealth(db, { now, audit, getConnectionRow, discovery: catalogDiscovery, setupPolicy, resolveCredentialHeaders, refreshManagedGitHubGrantAccess, ensureRuntimeSlot, stdioTemplateId }); + const connectionHealth = createConnectionHealth(db, { now, audit, getConnectionRow, discovery: catalogDiscovery, setupPolicy, resolveCredentialHeaders, vaultGrantForConnection, refreshManagedGitHubGrantAccess, ensureRuntimeSlot, stdioTemplateId }); const { updateConnectionHealth, checkConnectionHealth, sweepConnectionHealth } = connectionHealth; const refreshCatalog = createToolCatalogPublication(db, { now, audit, getConnectionRow, discoverTools: catalogDiscovery.discoverTools, discoverSkills: catalogDiscovery.discoverSkills, assertConnectorConnectionSetupEnabled, updateConnectionHealth, vaultGrantForConnection, ensureRuntimeSlot, enableCatalogEntriesByDefault }); const catalogReader = createToolCatalogReader(db, options, now, getConnectionRow, refreshCatalog, toCatalogEntryForConnection); @@ -1065,8 +1077,10 @@ export function toolAccessService( const eligible = grants.filter( (grant) => grant.connectionId === connection.id && - (connection.credentialPolicy !== "per_user" || - (grant.kind === "user" && grant.subjectUserId === userId)), + (connection.credentialPolicy === "per_agent" + ? grant.kind === "agent" + : connection.credentialPolicy !== "per_user" || + (grant.kind === "user" && grant.subjectUserId === userId)), ); connection.requiresReauthorization = !eligible.some( (grant) => grant.status === "active", @@ -3893,6 +3907,12 @@ export function toolAccessService( finishGalleryAppConnection, + createMachineSignIn: clientCredentials.createMachineSignIn, + testMachineSignIn: clientCredentials.testMachineSignIn, + rotateMachineSignIn: clientCredentials.rotateMachineSignIn, + completeMachineSignInRotation: clientCredentials.completeMachineSignInRotation, + getMachineSignIn: clientCredentials.getMachineSignIn, + reconnectGalleryApp, storeConnectorOAuthSecret: createOrRotateOAuthSecret, diff --git a/server/src/services/tool-access/catalog/discovery.ts b/server/src/services/tool-access/catalog/discovery.ts index b7e21d06eb0..70c5145e0e7 100644 --- a/server/src/services/tool-access/catalog/discovery.ts +++ b/server/src/services/tool-access/catalog/discovery.ts @@ -1,4 +1,5 @@ import { assertSupportedConnection } from "../connections/setup-policy.js"; +import { usesClientCredentials } from "../oauth/client-credentials.js"; import { vercelCredentialFor, vercelConnectResourcesFor } from "../vercel-credentials.js"; import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js"; import type { ActorInfo } from "../types.js"; @@ -168,7 +169,9 @@ export function createToolCatalogDiscovery(db: Db, { now, audit, secrets, creden const sendRemote = (init: RequestInit) => withMcpConnectionFailure(() => requestRemoteHttpEndpoint(new URL(endpoint), init)); const sendToolsList = (requestHeaders: Record, cursor?: string) => { sessionHeaders = requestHeaders; - return sendRemote(mcpHttpRequestInit(requestHeaders, { id: listRequestId, method: "tools/list", params: cursor ? { cursor } : {} })); + return sendRemote(mcpHttpRequestInit(requestHeaders, { id: listRequestId, method: "tools/list", params: cursor ? { cursor } : {} }, { + declareClientCredentials: usesClientCredentials(connection), + })); }; const connectionConfig = asRecord(connection.config); const sourceTemplateKey = typeof connectionConfig?.sourceTemplateKey === "string" @@ -333,7 +336,10 @@ export function createToolCatalogDiscovery(db: Db, { now, audit, secrets, creden if ( response.status === 401 && connection.authKind === "oauth" && - connection.credentialSource === "paperclip_vault" + connection.credentialSource === "paperclip_vault" && + // A machine sign-in probe with candidate credentials must fail on its own + // answer, not succeed by falling back to the stored ones. + !(credentialHeaders && usesClientCredentials(connection)) ) { headers = { ...projectedConnectionHeaders(connection), @@ -386,6 +392,15 @@ export function createToolCatalogDiscovery(db: Db, { now, audit, secrets, creden connection.credentialSecretRefs.length === 0 && Object.keys(headers).length === 0 && !urlCarriesCredential(sentUrl); + if (response.status === 401 && usesClientCredentials(connection)) { + // Machine sign-in is pinned to the sign-in service recorded at setup. + // Never adopt whatever the server's challenge now advertises. + await response.body?.cancel().catch(() => undefined); + throw unprocessable( + "The server didn't accept this connection's machine sign-in. Test machine sign-in again from the connection's settings.", + { code: "client_credentials_rejected", status: response.status, setupUrl: connectionSetupUrl(connection) }, + ); + } if ( response.status === 401 && (/bearer|oauth|authorization/i.test(authenticate) || diff --git a/server/src/services/tool-access/catalog/failures.ts b/server/src/services/tool-access/catalog/failures.ts index 106a3323755..ee7d72bbb5a 100644 --- a/server/src/services/tool-access/catalog/failures.ts +++ b/server/src/services/tool-access/catalog/failures.ts @@ -20,6 +20,12 @@ export function isOAuthEndpointRejection(error: unknown): boolean { return typeof code === "string" && code.endsWith("_endpoint_rejected"); } +/** A caller without an agent grant has not tested the connection's credential. */ +export function isCallerAuthorizationFailure(error: unknown, actor?: ActorInfo): boolean { + return actor?.actorType === "agent" && error instanceof HttpError + && asRecord(error.details).code === "agent_authorization_required"; +} + export function healthFailureHttpStatus(failure: { status: ToolConnectionHealthStatus; code: string; @@ -31,6 +37,7 @@ export function healthFailureHttpStatus(failure: { if (failure.code === "oauth_insufficient_scope") return 422; if (failure.code === "slack_mcp_access_disabled") return 422; if (failure.code === "user_authorization_required") return 422; + if (failure.code === "agent_authorization_required" || failure.code === "agent_authorization_ambiguous") return 422; if (failure.code === "github_installation_required") return 422; if (failure.code === "composio_broker_retired") return 422; if (failure.code === "tool_connection_transport_unsupported") return 422; @@ -62,7 +69,8 @@ export function sanitizeHttpFailure(error: unknown): { if (code === "slack_mcp_access_disabled" || code === "google_workspace_services_unavailable") { return { status: "error", message: error.message, code }; } - if (code === "user_authorization_required" || code === "github_installation_required") { + if (code === "user_authorization_required" || code === "agent_authorization_required" + || code === "agent_authorization_ambiguous" || code === "github_installation_required") { return { status: "error", message: error.message, code }; } if (code === "tool_connection_transport_unsupported") { diff --git a/server/src/services/tool-access/catalog/health.ts b/server/src/services/tool-access/catalog/health.ts index c343d2daf00..2716415199e 100644 --- a/server/src/services/tool-access/catalog/health.ts +++ b/server/src/services/tool-access/catalog/health.ts @@ -21,7 +21,7 @@ import type { ConnectionCredentialAccess } from "../credential-access.js"; import type { ConnectionSetupPolicy } from "../connections/setup-policy.js"; import { toConnection } from "../connections/projection.js"; import { isAgentMailConnection, type ToolCatalogDiscovery } from "./discovery.js"; -import { healthFailureHttpStatus, sanitizeHttpFailure, unreachableServerRemediation, unsupportedToolConnectionTransport } from "./failures.js"; +import { healthFailureHttpStatus, isCallerAuthorizationFailure, sanitizeHttpFailure, unreachableServerRemediation, unsupportedToolConnectionTransport } from "./failures.js"; function asRecord(value: unknown): Record { if (value && typeof value === "object" && !Array.isArray(value)) return value as Record; @@ -35,12 +35,13 @@ type HealthDependencies = { discovery: ToolCatalogDiscovery; setupPolicy: ConnectionSetupPolicy; resolveCredentialHeaders: ConnectionCredentialAccess["resolveCredentialHeaders"]; + vaultGrantForConnection: ConnectionCredentialAccess["vaultGrantForConnection"]; refreshManagedGitHubGrantAccess: ReturnType["refreshManagedGitHubGrantAccess"]; ensureRuntimeSlot: ReturnType["ensureRuntimeSlot"]; stdioTemplateId(companyId: string, config: Record): Promise; }; -export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, discovery, setupPolicy, resolveCredentialHeaders, refreshManagedGitHubGrantAccess, ensureRuntimeSlot, stdioTemplateId }: HealthDependencies) { +export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, discovery, setupPolicy, resolveCredentialHeaders, vaultGrantForConnection, refreshManagedGitHubGrantAccess, ensureRuntimeSlot, stdioTemplateId }: HealthDependencies) { const { discoverTools, remoteTools, validateAgentMailConnection, validateCogneeConnection } = discovery; const { assertConnectorConnectionSetupEnabled } = setupPolicy; async function updateConnectionHealth( @@ -94,7 +95,9 @@ export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, d config.sourceTemplateKey === "github" && oauth.connectorProfile === "github.code" ) { - const activeGrants = await db + const activeGrants = connection.credentialPolicy === "per_agent" + ? [await vaultGrantForConnection(connection, actor)].filter(grant => grant !== null) + : await db .select() .from(connectionGrants) .where( @@ -187,6 +190,7 @@ export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, d }); return { connection: toConnection(updated), runtimeSlot }; } catch (error) { + if (isCallerAuthorizationFailure(error, actor)) throw error; if ( error instanceof HttpError && asRecord(error.details).code === "github_access_changed" diff --git a/server/src/services/tool-access/catalog/publication.ts b/server/src/services/tool-access/catalog/publication.ts index e5e0ab241d5..fb5a2b0911d 100644 --- a/server/src/services/tool-access/catalog/publication.ts +++ b/server/src/services/tool-access/catalog/publication.ts @@ -29,7 +29,7 @@ import { isAgentMailConnection, type ToolCatalogDiscovery } from "./discovery.js import { classifyRisk, shouldQuarantineNewEntries, isGoogleWorkspaceToolAllowed, type McpToolDescriptor } from "./descriptors.js"; import { toConnection } from "../connections/projection.js"; import { toCatalogEntry } from "./projection.js"; -import { healthFailureHttpStatus, sanitizeHttpFailure, unreachableServerRemediation } from "./failures.js"; +import { healthFailureHttpStatus, isCallerAuthorizationFailure, sanitizeHttpFailure, unreachableServerRemediation } from "./failures.js"; function asRecord(value: unknown): Record { if (value && typeof value === "object" && !Array.isArray(value)) return value as Record; @@ -94,6 +94,7 @@ export function createToolCatalogPublication(db: Db, { now, audit, getConnection unavailableServices, ); } catch (error) { + if (isCallerAuthorizationFailure(error, actor)) throw error; if ( error instanceof HttpError && asRecord(error.details).code === "github_access_changed" diff --git a/server/src/services/tool-access/credential-access.ts b/server/src/services/tool-access/credential-access.ts index 65551c7aa30..5fdf4f70901 100644 --- a/server/src/services/tool-access/credential-access.ts +++ b/server/src/services/tool-access/credential-access.ts @@ -39,6 +39,33 @@ export function createConnectionCredentialAccess(db: Db, { secrets, audit, verce connection: typeof toolConnections.$inferSelect, actor?: ActorInfo, ): Promise { + if (connection.credentialPolicy === "per_agent") { + const agentId = actor?.actorType === "agent" ? actor.actorId : null; + if (actor?.actorType === "agent" && !agentId) { + throw unprocessable("This connection needs the agent's authorization", { + code: "agent_authorization_required", + }); + } + // Discovery has no run identity in background/operator work. Use only + // an unambiguous active agent grant; an agent caller must use its own. + const grants = await db.select().from(connectionGrants).where(and( + eq(connectionGrants.companyId, connection.companyId), + eq(connectionGrants.connectionId, connection.id), + eq(connectionGrants.kind, "agent"), + eq(connectionGrants.status, "active"), + ...(agentId ? [eq(connectionGrants.subjectAgentId, agentId)] : []), + )).limit(2); + if (grants.length !== 1) { + throw unprocessable(grants.length > 1 + ? "This connection has multiple agent accounts. Use a separate connection for each agent." + : "This connection needs the agent's authorization", { + code: grants.length > 1 ? "agent_authorization_ambiguous" : "agent_authorization_required", + setupUrl: connectionSetupUrl(connection), + reconnectUrl: connectionReconnectUrl(connection), + }); + } + return grants[0]!; + } const actorUserId = connectionActorUserId(actor); if (actorUserId) { diff --git a/server/src/services/tool-access/oauth/client-credentials-service.ts b/server/src/services/tool-access/oauth/client-credentials-service.ts new file mode 100644 index 00000000000..3ef09066467 --- /dev/null +++ b/server/src/services/tool-access/oauth/client-credentials-service.ts @@ -0,0 +1,753 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, ne, sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { connectionGrants, toolApplications, toolConnections } from "@paperclipai/db"; +import type { + ConnectToolAppResult, + ToolCatalogRefreshResult, + ToolConnectionHealthCheckResult, + CreateMachineSignIn, + MachineSignInPublicKey, + MachineSignInSetupResult, + MachineSignInStatus, + RotateMachineSignIn, + TestMachineSignIn, + ToolCredentialSecretRef, +} from "@paperclipai/shared"; +import { badRequest, conflict, HttpError, unprocessable } from "../../../errors.js"; +import { writeConnectionCredential } from "../../connection-credentials.js"; +import { emitConnectionCreated, emitConnectionUpdated } from "../../connector-telemetry.js"; +import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js"; +import type { secretService } from "../../secrets.js"; +import { splitRemoteUrlCredential } from "../../remote-url-credentials.js"; +import { connectionActorUserId } from "../actor.js"; +import { toApplication } from "../applications/projection.js"; +import { groupedActions } from "../catalog/action-summary.js"; +import type { createOrganizationGrantStore } from "../connections/organization-grant.js"; +import { toConnection } from "../connections/projection.js"; +import { connectionUid } from "../identifiers.js"; +import type { ActorInfo } from "../types.js"; +import type { createOAuthClientProtocol } from "./client.js"; +import { + CLIENT_PRIVATE_KEY_NEXT_PATH, + CLIENT_PRIVATE_KEY_PATH, + CLIENT_SECRET_PATH, + chooseAssertionAlgorithm, + chooseSecretAuthMethod, + clientCredentialsSettings, + generateClientKeyPair, + publicKeyForPrivateKey, + signClientAssertion, + type ClientCredentialsAuthMethod, + type ClientCredentialsSettings, + type ClientPublicKey, +} from "./client-credentials.js"; +import { actorForSecret, type OAuthCredentialStore } from "./credential-store.js"; +import { + asRecord, + canonicalResourceIndicator, + challengeOAuthHints, + normalizeOauthScopes, + oauthConfig, + protectedResourceMetadataUrls, + sameOAuthIssuer, + wellKnownMetadataUrls, +} from "./metadata.js"; +import { normalizeOAuthProviderError } from "./provider-errors.js"; +import type { OAuthTransport } from "./types.js"; + +type Dependencies = OAuthTransport & { + now: () => Date; + secrets: ReturnType; + store: Pick; + exchangeOAuthToken: ReturnType["exchangeOAuthToken"]; + getConnectionRow(idOrUid: string, companyId?: string): Promise; + assertRemoteConnectionEndpointsAllowed(config: Record): Promise; + ensureDefaultOrganizationGrant: ReturnType; + checkConnectionHealth(connectionId: string, actor?: ActorInfo): Promise; + refreshCatalog(connectionId: string, actor?: ActorInfo, options?: { enableAllByDefault?: boolean }): Promise; + /** List the server's tools with exactly these credentials, never the stored ones. */ + probeServerWithToken(connection: typeof toolConnections.$inferSelect, authorization: string, actor?: ActorInfo): Promise; + nextAvailableConnectionName(requestedName: string, existingNames: readonly string[]): string; +}; + +type DiscoveredSignInService = { + issuer: string | null; + tokenUrl: string; + authMethods: string[]; + signingAlgorithms: string[]; +}; + +type TokenOverrides = { + clientId?: string | null; + clientSecret?: string | null; + scopes?: string[]; + /** Sign with the staged replacement key instead of the current one. */ + usePendingKey?: boolean; + /** Sign with exactly this key (already read from the vault), so the key tested is the key promoted. */ + signingKey?: { privateKeyPem: string; publicKey: ClientPublicKey }; + authMethod?: ClientCredentialsAuthMethod; +}; + +const PERSONAL_REFUSAL = + "Machine sign-in is only for connections your organization shares. Personal connections sign in as you."; + +function issuerChangedError(advertised: string[]) { + const hosts = advertised.map((value) => { + try { return new URL(value).host; } catch { return value; } + }); + return unprocessable( + `This server now sends sign-in to a different service${hosts.length ? ` (${hosts.join(", ")})` : ""}. Paperclip won't send this connection's credentials there. Set up machine sign-in again to trust the new service.`, + { code: "client_credentials_issuer_changed" }, + ); +} + +function machineSignInChangedError() { + return conflict("This connection's machine sign-in changed or was removed while this was running. Reload it and try again.", { + code: "client_credentials_changed", + }); +} + +function rotationChangedError() { + return conflict("The new key changed while the rotation was finishing. Register the key shown now, then try again.", { + code: "client_credentials_rotation_changed", + }); +} + +function toPublicKey(key: ClientPublicKey | null): MachineSignInPublicKey | null { + if (!key) return null; + return { + keyId: key.keyId, + algorithm: key.algorithm, + pem: key.publicKeyPem, + jwk: key.publicJwk as Record, + createdAt: key.createdAt, + }; +} + +export function machineSignInStatus(connection: typeof toolConnections.$inferSelect): MachineSignInStatus { + const settings = clientCredentialsSettings(connection); + if (!settings) throw unprocessable("This connection doesn't use machine sign-in", { code: "client_credentials_not_configured" }); + const oauth = oauthConfig(connection); + return { + connectionId: connection.id, + method: settings.authMethod === "private_key_jwt" ? "paperclip_key" : "client_secret", + clientId: settings.clientId, + issuer: settings.issuer, + tokenUrl: settings.tokenUrl, + scopes: settings.scopes, + publicKey: toPublicKey(settings.key), + pendingPublicKey: toPublicKey(settings.pendingKey), + hasClientSecret: connection.credentialSecretRefs.some((ref) => ref.configPath === CLIENT_SECRET_PATH), + lastTokenAt: typeof oauth.refreshedAt === "string" ? oauth.refreshedAt + : typeof oauth.connectedAt === "string" ? oauth.connectedAt : null, + }; +} + +function scopesFrom(value: string | undefined): string[] | undefined { + return value === undefined ? undefined : normalizeOauthScopes(value.split(/\s+/)); +} + +export function requireClientCredentials( + request: ClientCredentialsService["requestClientCredentialsToken"] | undefined, +): ClientCredentialsService["requestClientCredentialsToken"] { + if (!request) throw new Error("Machine sign-in token requests are not configured for this service"); + return request; +} + +export function createClientCredentialsService(db: Db, deps: Dependencies) { + const { now, secrets, store, exchangeOAuthToken, getConnectionRow, fetchRemoteHttpUrl, assertOAuthEndpointUrl } = deps; + + async function fetchJson(url: string): Promise | null> { + try { + const response = await fetchRemoteHttpUrl(assertOAuthEndpointUrl("metadata", url), { headers: { accept: "application/json" } }); + if (!response.ok) { + await response.body?.cancel().catch(() => undefined); + return null; + } + const parsed = asRecord(await response.json().catch(() => null)); + return Object.keys(parsed).length ? parsed : null; + } catch { + return null; + } + } + + /** RFC 8414 / OIDC metadata for one issuer, accepted only when it names that same issuer. */ + async function signInServiceMetadata(issuer: string): Promise { + for (const url of wellKnownMetadataUrls(issuer)) { + const metadata = await fetchJson(url); + if (!metadata || typeof metadata.token_endpoint !== "string") continue; + if (typeof metadata.issuer !== "string" || !sameOAuthIssuer(metadata.issuer, issuer)) continue; + const grants = normalizeOauthScopes(metadata.grant_types_supported); + if (grants.length > 0 && !grants.includes("client_credentials")) { + throw unprocessable("This sign-in service doesn't offer machine sign-in. Check the provider's docs for a service account option.", { + code: "client_credentials_unsupported", + }); + } + return { + issuer: metadata.issuer.trim(), + tokenUrl: assertOAuthEndpointUrl("token", metadata.token_endpoint), + authMethods: normalizeOauthScopes(metadata.token_endpoint_auth_methods_supported), + signingAlgorithms: normalizeOauthScopes(metadata.token_endpoint_auth_signing_alg_values_supported), + }; + } + return null; + } + + /** The `resource_metadata` address from the server's answer to an unauthenticated request, if any. */ + async function challengedMetadataUrl(serverUrl: string): Promise { + try { + const response = await fetchRemoteHttpUrl(serverUrl, { + method: "POST", + headers: { "content-type": "application/json", accept: "application/json, text/event-stream" }, + body: JSON.stringify({ jsonrpc: "2.0", id: "paperclip-sign-in-probe", method: "server/discover", params: {} }), + }); + await response.body?.cancel().catch(() => undefined); + if (response.status !== 401) return null; + return challengeOAuthHints(response.headers.get("www-authenticate") ?? "").metadataUrl; + } catch { + return null; + } + } + + /** The sign-in services an MCP server's protected-resource metadata lists, or null when it publishes none. */ + async function advertisedSignInServices(serverUrl: string): Promise { + let endpoint: URL; + try { endpoint = new URL(serverUrl); } catch { return null; } + // The server's own challenge may name where its metadata lives; follow + // that (through the same URL checks) before the well-known locations. + const hinted = await challengedMetadataUrl(serverUrl); + for (const url of [...new Set([...(hinted ? [hinted] : []), ...protectedResourceMetadataUrls(endpoint)])]) { + const metadata = await fetchJson(url); + if (!metadata) continue; + const servers = Array.isArray(metadata.authorization_servers) + ? metadata.authorization_servers.filter((value): value is string => typeof value === "string" && value.trim().length > 0) + : []; + if (servers.length > 0) return servers.map((value) => value.trim()); + } + return null; + } + + async function discoverSignInService(serverUrl: string, authorizationServer?: string): Promise { + if (authorizationServer) { + const asIssuer = await signInServiceMetadata(authorizationServer); + if (asIssuer) return asIssuer; + // No metadata at that address: the admin gave the token URL itself. + return { issuer: null, tokenUrl: assertOAuthEndpointUrl("token", authorizationServer), authMethods: [], signingAlgorithms: [] }; + } + for (const issuer of (await advertisedSignInServices(serverUrl)) ?? []) { + const found = await signInServiceMetadata(issuer); + if (found) return found; + } + // A server may be its own sign-in service. + const self = await signInServiceMetadata(new URL(serverUrl).origin); + if (self) return self; + throw unprocessable( + "Paperclip couldn't find this server's sign-in service. Enter the sign-in service address or token URL from the provider's docs.", + { code: "client_credentials_server_not_found" }, + ); + } + + /** + * The issuer pin. A connection's credentials are bound to the sign-in service + * recorded at setup. If the MCP server now advertises other services and not + * that one, refuse rather than follow it. + */ + async function assertPinnedSignInService(connection: typeof toolConnections.$inferSelect, settings: ClientCredentialsSettings) { + if (!settings.issuer) return; + const serverUrl = typeof connection.config.url === "string" ? connection.config.url : null; + if (!serverUrl) return; + const advertised = await advertisedSignInServices(serverUrl); + if (advertised && !advertised.some((issuer) => sameOAuthIssuer(issuer, settings.issuer))) { + throw issuerChangedError(advertised); + } + } + + async function vaultValue(connection: typeof toolConnections.$inferSelect, configPath: string, actor?: ActorInfo) { + const ref = connection.credentialSecretRefs.find((candidate) => candidate.configPath === configPath); + if (!ref) return null; + return secrets.resolveSecretValue(connection.companyId, ref.secretId, ref.versionSelector ?? "latest", { + consumerType: "tool_connection", + consumerId: connection.id, + configPath, + actorType: actor?.actorType ?? "system", + actorId: actor?.actorId ?? null, + }); + } + + /** + * One token request with `grant_type=client_credentials`, authenticated the + * way this connection was set up. Credentials go only to the pinned token + * endpoint; there is no fallback to deployment-wide OAuth clients. + */ + async function requestClientCredentialsToken( + connection: typeof toolConnections.$inferSelect, + actor?: ActorInfo, + overrides: TokenOverrides = {}, + ) { + const settings = clientCredentialsSettings(connection); + if (!settings) throw unprocessable("This connection doesn't use machine sign-in", { code: "client_credentials_not_configured" }); + const clientId = overrides.clientId ?? settings.clientId; + if (!clientId) { + throw unprocessable("Enter the client ID the provider gave you for Paperclip.", { code: "client_credentials_client_id_missing" }); + } + const authMethod = overrides.authMethod ?? settings.authMethod; + const scopes = overrides.scopes ?? settings.scopes; + let clientAssertion: string | null = null; + let clientSecret: string | null = null; + if (authMethod === "private_key_jwt") { + const key = overrides.signingKey?.publicKey ?? (overrides.usePendingKey ? settings.pendingKey : settings.key); + const privateKeyPem = overrides.signingKey?.privateKeyPem + ?? await vaultValue(connection, overrides.usePendingKey ? CLIENT_PRIVATE_KEY_NEXT_PATH : CLIENT_PRIVATE_KEY_PATH, actor); + if (!key || !privateKeyPem) { + throw unprocessable("This connection's Paperclip key is missing. Set up machine sign-in again.", { code: "client_credentials_key_missing" }); + } + // The key ID comes from the private key actually read, never from config + // alone: a refresh that overlaps a rotation may read the new key while + // holding the old settings, and its signature and kid must still agree. + const signing = publicKeyForPrivateKey(privateKeyPem, key.algorithm, new Date(key.createdAt)); + clientAssertion = signClientAssertion({ + privateKeyPem, + algorithm: key.algorithm, + keyId: signing.keyId, + clientId, + audience: settings.tokenUrl, + now: now(), + }); + } else { + clientSecret = overrides.clientSecret ?? await vaultValue(connection, CLIENT_SECRET_PATH, actor); + if (!clientSecret) { + throw unprocessable("Enter the client secret the provider gave you for Paperclip.", { code: "client_credentials_secret_missing" }); + } + } + try { + return await exchangeOAuthToken({ + tokenUrl: settings.tokenUrl, + clientId, + clientSecret, + tokenEndpointAuthMethod: authMethod === "private_key_jwt" ? "none" : authMethod, + clientAssertion, + grantType: "client_credentials", + scopes, + resource: settings.resource, + }); + } catch (error) { + if (!(error instanceof HttpError)) throw error; + const providerError = normalizeOAuthProviderError(asRecord(error.details).providerError); + if (providerError === "invalid_client" || providerError === "unauthorized_client") { + throw unprocessable( + authMethod === "private_key_jwt" + ? "The provider didn't accept Paperclip's key. Check the client ID and that the public key shown here is registered for it." + : "The provider didn't accept the client ID and secret. Check both and try again.", + { code: "client_credentials_rejected", providerError }, + ); + } + if (providerError === "invalid_scope") { + throw unprocessable("The provider refused the requested scopes. Check the scopes in the provider's docs.", { + code: "client_credentials_scope_rejected", + providerError, + }); + } + throw error; + } + } + + function assertOrganizationConnection(connection: typeof toolConnections.$inferSelect) { + if (connection.credentialPolicy !== "shared") throw unprocessable(PERSONAL_REFUSAL, { code: "client_credentials_personal_refused" }); + } + + async function machineSignInConnection(companyId: string, connectionId: string) { + const connection = await getConnectionRow(connectionId, companyId); + if (connection.status === "archived") { + throw unprocessable("This connection was removed. Set up machine sign-in again to reconnect.", { + code: "client_credentials_connection_removed", + }); + } + assertOrganizationConnection(connection); + if (!clientCredentialsSettings(connection)) { + throw unprocessable("This connection doesn't use machine sign-in", { code: "client_credentials_not_configured" }); + } + return connection; + } + + async function writeVaultCredential( + connection: { companyId: string; name: string }, + configPath: string, + label: string, + value: string, + actor: ActorInfo | undefined, + existingRef?: ToolCredentialSecretRef, + ): Promise { + const { secret } = await db.transaction((tx) => writeConnectionCredential(tx, { + companyId: connection.companyId, + connectionName: connection.name, + configPath, + label, + value, + existingRef, + actor: actorForSecret(actor), + })); + return { secretId: secret.id, versionSelector: "latest", configPath, required: false, label }; + } + + async function saveConnection( + previous: typeof toolConnections.$inferSelect, + patch: Partial, + ) { + // Commit only if nothing that decides how this connection signs in changed + // since it was read (a rotation or another test), and it was not removed. + const clientAuth = asRecord(oauthConfig(previous).clientAuth); + const [updated] = await db.update(toolConnections) + .set({ ...patch, ...(patch.config ? { transportConfig: patch.config } : {}), updatedAt: now() }) + .where(and( + eq(toolConnections.id, previous.id), + eq(toolConnections.companyId, previous.companyId), + ne(toolConnections.status, "archived"), + sql`coalesce(${toolConnections.config} -> 'oauth' -> 'clientAuth', '{}'::jsonb) = ${JSON.stringify(clientAuth)}::jsonb`, + sql`${toolConnections.credentialSecretRefs} = ${JSON.stringify(previous.credentialSecretRefs)}::jsonb`, + )) + .returning(); + if (!updated) throw machineSignInChangedError(); + await store.syncCredentialBindings(updated); + if (updated.status === "active") await deps.ensureDefaultOrganizationGrant(updated); + emitConnectionUpdated(updated, previous, "gallery"); + return updated; + } + + async function createMachineSignIn(companyId: string, input: CreateMachineSignIn, actor?: ActorInfo): Promise { + await actor?.managementContext?.reauthorize(); + const split = splitRemoteUrlCredential(input.link); + if (split?.secretUrl) { + throw badRequest("Remove the key from the server address. Machine sign-in sends credentials only to the sign-in service."); + } + const serverUrl = split?.publicUrl ?? input.link; + const config: Record = { url: serverUrl, quarantineNewEntries: false, unverifiedServer: true }; + await deps.assertRemoteConnectionEndpointsAllowed(config); + const service = await discoverSignInService(serverUrl, input.authorizationServer); + const createdAt = now(); + let authMethod: ClientCredentialsAuthMethod; + let generated: Awaited> | null = null; + if (input.method === "paperclip_key") { + if (service.authMethods.length > 0 && !service.authMethods.includes("private_key_jwt")) { + throw unprocessable("This provider doesn't accept Paperclip keys. Choose Client secret instead.", { + code: "client_credentials_key_unsupported", + }); + } + authMethod = "private_key_jwt"; + generated = await generateClientKeyPair(chooseAssertionAlgorithm(service.signingAlgorithms), createdAt); + } else { + if (service.authMethods.length > 0 && !service.authMethods.some((method) => method === "client_secret_basic" || method === "client_secret_post")) { + throw unprocessable("This provider doesn't accept client secrets. Choose Paperclip key instead.", { + code: "client_credentials_secret_unsupported", + }); + } + authMethod = chooseSecretAuthMethod(service.authMethods); + } + config.oauth = { + provider: "client_credentials", + grantType: "client_credentials", + tokenUrl: service.tokenUrl, + issuer: service.issuer, + resource: canonicalResourceIndicator(serverUrl), + scopes: scopesFrom(input.scopes) ?? [], + clientId: input.clientId ?? null, + tokenEndpointAuthMethodsSupported: service.authMethods, + clientAuth: { method: authMethod, ...(generated ? { key: generated.publicKey } : {}), configuredAt: createdAt.toISOString() }, + }; + + const [applicationNames, connectionNames] = await Promise.all([ + db.select({ name: toolApplications.name }).from(toolApplications).where(eq(toolApplications.companyId, companyId)), + db.select({ name: toolConnections.name }).from(toolConnections).where(eq(toolConnections.companyId, companyId)), + ]); + const requestedName = input.name ?? new URL(serverUrl).host.replace(/^www\./, ""); + const name = deps.nextAvailableConnectionName(requestedName, [...applicationNames, ...connectionNames].map((row) => row.name)); + + const credentialSecretRefs: ToolCredentialSecretRef[] = []; + try { + if (generated) { + credentialSecretRefs.push(await writeVaultCredential({ companyId, name }, CLIENT_PRIVATE_KEY_PATH, "Machine sign-in key", generated.privateKeyPem, actor)); + } else if (input.clientSecret) { + credentialSecretRefs.push(await writeVaultCredential({ companyId, name }, CLIENT_SECRET_PATH, "Machine sign-in client secret", input.clientSecret, actor)); + } + const connection = await db.transaction(async (tx) => { + let application: typeof toolApplications.$inferSelect | undefined; + if (input.applicationId) { + [application] = await tx.select().from(toolApplications) + .where(and(eq(toolApplications.id, input.applicationId), eq(toolApplications.companyId, companyId))); + if (!application) throw badRequest("The app for this connection was not found"); + } else { + [application] = await tx.insert(toolApplications).values({ + companyId, + applicationKey: `app-gallery:link:${randomUUID()}`, + name, + description: `Connected app at ${serverUrl}`, + type: "mcp_http", + status: "draft", + metadata: { source: "link" }, + }).returning(); + } + const connectionId = randomUUID(); + const [row] = await tx.insert(toolConnections).values({ + id: connectionId, + companyId, + applicationId: application!.id, + name, + uid: connectionUid(application!.applicationKey ?? application!.name, name, connectionId), + connectionKind: "managed", + ownership: "customer", + authKind: "oauth", + credentialSource: "paperclip_vault", + transport: "mcp_remote", + status: "draft", + enabled: false, + config, + transportConfig: config, + credentialRefs: [], + credentialSecretRefs, + credentialPolicy: "shared", + createdByAgentId: actor?.actorType === "agent" ? (actor.actorId ?? null) : null, + createdByUserId: connectionActorUserId(actor), + }).returning(); + return row!; + }); + await store.syncCredentialBindings(connection); + emitConnectionCreated(connection, "gallery"); + return { connectionId: connection.id, connection: toConnection(connection), machineSignIn: machineSignInStatus(connection) }; + } catch (error) { + // Nothing references these yet; never leave an orphaned key or secret behind. + for (const ref of credentialSecretRefs) await secrets.remove(ref.secretId).catch(() => undefined); + throw error; + } + } + + /** + * Test the settings against the real sign-in service and server, then turn + * the connection on. A failed test leaves the draft (and its key) in place so + * the admin can fix the provider side and try again. + */ + async function testMachineSignIn( + companyId: string, + connectionId: string, + input: TestMachineSignIn, + actor?: ActorInfo, + ): Promise { + await actor?.managementContext?.reauthorize(); + let connection = await machineSignInConnection(companyId, connectionId); + const settings = clientCredentialsSettings(connection)!; + await assertPinnedSignInService(connection, settings); + const scopes = scopesFrom(input.scopes); + const token = await requestClientCredentialsToken(connection, actor, { + clientId: input.clientId, + clientSecret: input.clientSecret, + scopes, + }); + // Prove the server accepts the new client before anything is saved: a client + // the sign-in service accepts may still lack access to this server, and a + // failed retest must leave a working connection's credentials untouched. + await deps.probeServerWithToken(connection, `Bearer ${token.accessToken}`, actor); + // The settings worked: keep them. + const credentialSecretRefs = [...connection.credentialSecretRefs]; + if (input.clientSecret) { + const existing = credentialSecretRefs.find((ref) => ref.configPath === CLIENT_SECRET_PATH); + const ref = await writeVaultCredential(connection, CLIENT_SECRET_PATH, "Machine sign-in client secret", input.clientSecret, actor, existing); + if (!existing) credentialSecretRefs.push(ref); + } + const accessRef = await store.createOrRotateOAuthSecret({ + companyId, + connection, + configPath: "oauth.access_token", + label: "OAuth access token", + value: token.accessToken, + actor, + }); + const at = now(); + const expiresAt = token.expiresIn ? new Date(at.getTime() + token.expiresIn * 1000).toISOString() : null; + const oauth = oauthConfig(connection); + const nextConfig = { + ...connection.config, + oauth: { + ...oauth, + clientId: input.clientId ?? settings.clientId, + scopes: scopes ?? settings.scopes, + expiresAt, + scope: token.scope, + tokenType: token.tokenType, + connectedAt: oauth.connectedAt ?? at.toISOString(), + refreshedAt: at.toISOString(), + }, + }; + connection = await saveConnection(connection, { + config: nextConfig, + credentialSecretRefs: [...credentialSecretRefs.filter((ref) => ref.configPath !== "oauth.access_token"), accessRef], + credentialRefs: [ + ...connection.credentialRefs.filter((ref) => ref.name !== "oauth.access_token"), + { name: "oauth.access_token", secretId: accessRef.secretId, version: "latest", placement: "header", key: "Authorization", prefix: "Bearer " }, + ], + }); + // The organization grant is what tool calls run on; give it the token now, + // with this token's expiry (the grant's own expiry wins over the connection's). + await deps.ensureDefaultOrganizationGrant(connection); + await db.update(connectionGrants).set({ + providerTenant: sql`jsonb_set(coalesce(${connectionGrants.providerTenant}, '{}'::jsonb), '{oauth}', (coalesce(${connectionGrants.providerTenant}->'oauth', '{}'::jsonb) || ${JSON.stringify({ accessTokenExpiresAt: expiresAt, tokenType: token.tokenType, refreshedAt: at.toISOString() })}::jsonb) - 'refreshLease')`, + updatedAt: now(), + }).where(and( + eq(connectionGrants.companyId, companyId), + eq(connectionGrants.connectionId, connection.id), + eq(connectionGrants.kind, "organization"), + eq(connectionGrants.isDefault, true), + )); + await deps.checkConnectionHealth(connection.id, actor); + const before = connection; + connection = await db.transaction(async (tx) => { + await notifyToolConnectionHealth(tx, before); + const [active] = await tx.update(toolConnections) + .set({ status: "active", enabled: true, updatedAt: now() }) + .where(and(eq(toolConnections.id, before.id), eq(toolConnections.companyId, companyId), ne(toolConnections.status, "archived"))) + .returning(); + // Removed while the test ran: never bring it back. + if (!active) throw machineSignInChangedError(); + await tx.update(toolApplications) + .set({ status: "active", updatedAt: now() }) + .where(and(eq(toolApplications.id, before.applicationId), eq(toolApplications.companyId, companyId))); + return active!; + }); + emitConnectionUpdated(connection, before, "gallery"); + const refresh = await deps.refreshCatalog(connection.id, actor, { enableAllByDefault: false }); + const [application] = await db.select().from(toolApplications).where(eq(toolApplications.id, connection.applicationId)); + return { + connectionId: refresh.connection.id, + application: toApplication(application!), + connection: refresh.connection, + catalog: refresh.catalog, + actions: groupedActions(refresh.catalog), + suggestedDefaults: { access: "all_agents", askFirstRiskLevels: [] }, + auth: null, + }; + } + + /** + * Replace the key or secret without downtime. A new secret is tried before it + * replaces the old one. A new key is staged: the current key keeps working + * until the admin registers the new public key and completes the rotation. + */ + async function rotateMachineSignIn( + companyId: string, + connectionId: string, + input: RotateMachineSignIn, + actor?: ActorInfo, + ): Promise { + await actor?.managementContext?.reauthorize(); + const connection = await machineSignInConnection(companyId, connectionId); + const settings = clientCredentialsSettings(connection)!; + const oauth = oauthConfig(connection); + const clientAuth = asRecord(oauth.clientAuth); + if (input.method === "client_secret") { + const authMethod = settings.authMethod === "private_key_jwt" + ? chooseSecretAuthMethod(normalizeOauthScopes(oauth.tokenEndpointAuthMethodsSupported)) + : settings.authMethod; + await assertPinnedSignInService(connection, settings); + await requestClientCredentialsToken(connection, actor, { clientSecret: input.clientSecret, authMethod }); + const existing = connection.credentialSecretRefs.find((ref) => ref.configPath === CLIENT_SECRET_PATH); + const ref = await writeVaultCredential(connection, CLIENT_SECRET_PATH, "Machine sign-in client secret", input.clientSecret, actor, existing); + const retired = connection.credentialSecretRefs.filter((candidate) => + candidate.configPath === CLIENT_PRIVATE_KEY_PATH || candidate.configPath === CLIENT_PRIVATE_KEY_NEXT_PATH); + const updated = await saveConnection(connection, { + config: { ...connection.config, oauth: { ...oauth, clientAuth: { method: authMethod, configuredAt: now().toISOString() } } }, + credentialSecretRefs: [ + ...connection.credentialSecretRefs.filter((candidate) => candidate.configPath !== CLIENT_SECRET_PATH && !retired.includes(candidate)), + ref, + ], + }); + for (const old of retired) await secrets.remove(old.secretId).catch(() => undefined); + return { connectionId, connection: toConnection(updated), machineSignIn: machineSignInStatus(updated), completed: true }; + } + const generated = await generateClientKeyPair(settings.key?.algorithm ?? "RS256", now()); + const existing = connection.credentialSecretRefs.find((ref) => ref.configPath === CLIENT_PRIVATE_KEY_NEXT_PATH); + const ref = await writeVaultCredential(connection, CLIENT_PRIVATE_KEY_NEXT_PATH, "Machine sign-in replacement key", generated.privateKeyPem, actor, existing); + const updated = await saveConnection(connection, { + config: { ...connection.config, oauth: { ...oauth, clientAuth: { ...clientAuth, pendingKey: generated.publicKey } } }, + credentialSecretRefs: [...connection.credentialSecretRefs.filter((candidate) => candidate.configPath !== CLIENT_PRIVATE_KEY_NEXT_PATH), ref], + }); + return { connectionId, connection: toConnection(updated), machineSignIn: machineSignInStatus(updated), completed: false }; + } + + /** Switch to the staged key once the provider accepts it. */ + async function completeMachineSignInRotation( + companyId: string, + connectionId: string, + actor?: ActorInfo, + ): Promise { + await actor?.managementContext?.reauthorize(); + const connection = await machineSignInConnection(companyId, connectionId); + const settings = clientCredentialsSettings(connection)!; + if (!settings.pendingKey) throw unprocessable("There's no new key waiting. Start a key rotation first.", { code: "client_credentials_no_pending_key" }); + await assertPinnedSignInService(connection, settings); + // Read the staged key once and derive its public half from that exact value: + // the key that passes the test is the key that gets promoted, even if + // another rotation replaces the staged key meanwhile. + const pendingValue = await vaultValue(connection, CLIENT_PRIVATE_KEY_NEXT_PATH, actor); + if (!pendingValue) throw unprocessable("The new key is missing. Start the rotation again.", { code: "client_credentials_key_missing" }); + const promoted = publicKeyForPrivateKey(pendingValue, settings.pendingKey.algorithm, new Date(settings.pendingKey.createdAt)); + if (promoted.keyId !== settings.pendingKey.keyId) throw rotationChangedError(); + await requestClientCredentialsToken(connection, actor, { + authMethod: "private_key_jwt", + signingKey: { privateKeyPem: pendingValue, publicKey: promoted }, + }); + const pendingRef = connection.credentialSecretRefs.find((ref) => ref.configPath === CLIENT_PRIVATE_KEY_NEXT_PATH); + // One locked step: the row still stages this key, then the vault and config change together. + const updated = await db.transaction(async (tx) => { + const [locked] = await tx.select().from(toolConnections) + .where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId))) + .for("update"); + const lockedSettings = locked ? clientCredentialsSettings(locked) : null; + if (!locked || lockedSettings?.pendingKey?.keyId !== promoted.keyId) throw rotationChangedError(); + const current = locked.credentialSecretRefs.find((ref) => ref.configPath === CLIENT_PRIVATE_KEY_PATH); + const { secret } = await writeConnectionCredential(tx, { + companyId, connectionName: locked.name, configPath: CLIENT_PRIVATE_KEY_PATH, label: "Machine sign-in key", + value: pendingValue, existingRef: current, actor: actorForSecret(actor), + }); + const keyRef: ToolCredentialSecretRef = { + secretId: secret.id, versionSelector: "latest", configPath: CLIENT_PRIVATE_KEY_PATH, required: false, label: "Machine sign-in key", + }; + const lockedOauth = oauthConfig(locked); + const nextConfig = { + ...locked.config, + oauth: { ...lockedOauth, clientAuth: { method: "private_key_jwt", key: promoted, configuredAt: now().toISOString() } }, + }; + const [row] = await tx.update(toolConnections).set({ + config: nextConfig, + transportConfig: nextConfig, + credentialSecretRefs: [ + ...locked.credentialSecretRefs.filter((ref) => + ref.configPath !== CLIENT_PRIVATE_KEY_PATH && ref.configPath !== CLIENT_PRIVATE_KEY_NEXT_PATH && ref.configPath !== CLIENT_SECRET_PATH), + keyRef, + ], + updatedAt: now(), + }).where(eq(toolConnections.id, locked.id)).returning(); + return row!; + }); + await store.syncCredentialBindings(updated); + if (updated.status === "active") await deps.ensureDefaultOrganizationGrant(updated); + emitConnectionUpdated(updated, connection, "gallery"); + const retired = connection.credentialSecretRefs.filter((ref) => + (ref === pendingRef) || (ref.configPath === CLIENT_SECRET_PATH)); + for (const old of retired) await secrets.remove(old.secretId).catch(() => undefined); + return { connectionId, connection: toConnection(updated), machineSignIn: machineSignInStatus(updated) }; + } + + async function getMachineSignIn(companyId: string, connectionId: string): Promise { + return machineSignInStatus(await machineSignInConnection(companyId, connectionId)); + } + + return { + requestClientCredentialsToken, + assertPinnedSignInService, + createMachineSignIn, + testMachineSignIn, + rotateMachineSignIn, + completeMachineSignInRotation, + getMachineSignIn, + }; +} + +export type ClientCredentialsService = ReturnType; diff --git a/server/src/services/tool-access/oauth/client-credentials.ts b/server/src/services/tool-access/oauth/client-credentials.ts new file mode 100644 index 00000000000..802238d5bcd --- /dev/null +++ b/server/src/services/tool-access/oauth/client-credentials.ts @@ -0,0 +1,188 @@ +import { createHash, createPublicKey, generateKeyPair, randomUUID, sign, type JsonWebKey, type KeyObject } from "node:crypto"; +import type { toolConnections } from "@paperclipai/db"; +import { asRecord, normalizeOauthScopes, oauthConfig } from "./metadata.js"; + +/** + * MCP OAuth Client Credentials extension (2026-07-28): machine-to-machine sign-in + * for organization-owned connections. No person signs in; Paperclip presents a + * client it was registered as, either with a signed assertion (RFC 7523, + * `private_key_jwt`, recommended) or a client secret. + * + * This module is the protocol half: key material, assertions, and how the + * connection records its pinned authorization server. Secret values never live + * here; they are read from the company vault by the caller just before a token + * request and never stored in config. + */ +export const CLIENT_ASSERTION_TYPE = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"; + +/** Vault paths. The private key and client secret are only ever sent to the pinned token endpoint. */ +export const CLIENT_PRIVATE_KEY_PATH = "oauth.client_private_key"; +export const CLIENT_PRIVATE_KEY_NEXT_PATH = "oauth.client_private_key_next"; +export const CLIENT_SECRET_PATH = "oauth.client_secret"; + +export type ClientCredentialsAuthMethod = "private_key_jwt" | "client_secret_basic" | "client_secret_post"; +export type ClientAssertionAlgorithm = "RS256" | "ES256"; + +/** Public half of a Paperclip-held key, safe to show and to store in config. */ +export type ClientPublicKey = { + keyId: string; + algorithm: ClientAssertionAlgorithm; + publicKeyPem: string; + publicJwk: JsonWebKey; + createdAt: string; +}; + +export type ClientCredentialsSettings = { + clientId: string | null; + tokenUrl: string; + /** Pinned authorization-server issuer. Null when the admin entered a token URL directly. */ + issuer: string | null; + scopes: string[]; + resource: string | null; + authMethod: ClientCredentialsAuthMethod; + key: ClientPublicKey | null; + pendingKey: ClientPublicKey | null; +}; + +function isAuthMethod(value: unknown): value is ClientCredentialsAuthMethod { + return value === "private_key_jwt" || value === "client_secret_basic" || value === "client_secret_post"; +} + +function publicKeyFrom(value: unknown): ClientPublicKey | null { + const record = asRecord(value); + if (typeof record.keyId !== "string" || typeof record.publicKeyPem !== "string") return null; + if (record.algorithm !== "RS256" && record.algorithm !== "ES256") return null; + return { + keyId: record.keyId, + algorithm: record.algorithm, + publicKeyPem: record.publicKeyPem, + publicJwk: asRecord(record.publicJwk) as JsonWebKey, + createdAt: typeof record.createdAt === "string" ? record.createdAt : new Date(0).toISOString(), + }; +} + +/** + * The machine sign-in settings a connection was set up with, or null when the + * connection does not use them. Older env-configured client-credentials rows + * (no `clientAuth`) keep their original path. + */ +export function clientCredentialsSettings( + connection: Pick, +): ClientCredentialsSettings | null { + const oauth = oauthConfig(connection); + if (oauth.grantType !== "client_credentials") return null; + const clientAuth = asRecord(oauth.clientAuth); + if (!isAuthMethod(clientAuth.method) || typeof oauth.tokenUrl !== "string") return null; + return { + clientId: typeof oauth.clientId === "string" && oauth.clientId.trim() ? oauth.clientId.trim() : null, + tokenUrl: oauth.tokenUrl, + issuer: typeof oauth.issuer === "string" && oauth.issuer.trim() ? oauth.issuer.trim() : null, + scopes: normalizeOauthScopes(oauth.scopes), + resource: typeof oauth.resource === "string" && oauth.resource ? oauth.resource : null, + authMethod: clientAuth.method, + key: publicKeyFrom(clientAuth.key), + pendingKey: publicKeyFrom(clientAuth.pendingKey), + }; +} + +export function usesClientCredentials(connection: Pick): boolean { + return clientCredentialsSettings(connection) !== null; +} + +/** Prefer RS256 (the most widely accepted); use ES256 only when the server asks for it. */ +export function chooseAssertionAlgorithm(supported: string[] | undefined): ClientAssertionAlgorithm { + if (!supported?.length || supported.includes("RS256")) return "RS256"; + if (supported.includes("ES256")) return "ES256"; + return "RS256"; +} + +function base64Url(value: Buffer | string): string { + return Buffer.from(value).toString("base64url"); +} + +/** RFC 7638 JWK thumbprint, used as the key ID so it is stable and verifiable. */ +export function jwkThumbprint(jwk: JsonWebKey): string { + const canonical = jwk.kty === "EC" + ? { crv: jwk.crv, kty: jwk.kty, x: jwk.x, y: jwk.y } + : { e: jwk.e, kty: jwk.kty, n: jwk.n }; + return base64Url(createHash("sha256").update(JSON.stringify(canonical)).digest()); +} + +export function publicKeyForPrivateKey(privateKeyPem: string, algorithm: ClientAssertionAlgorithm, createdAt: Date): ClientPublicKey { + const publicKey = createPublicKey(privateKeyPem); + const exported = publicKey.export({ format: "jwk" }); + const keyId = jwkThumbprint(exported); + return { + keyId, + algorithm, + publicKeyPem: publicKey.export({ format: "pem", type: "spki" }).toString(), + publicJwk: { ...exported, kid: keyId, alg: algorithm, use: "sig" }, + createdAt: createdAt.toISOString(), + }; +} + +function generatePrivateKey(algorithm: ClientAssertionAlgorithm): Promise { + return new Promise((resolve, reject) => { + const done = (error: Error | null, _publicKey: KeyObject, privateKey: KeyObject) => + error ? reject(error) : resolve(privateKey); + if (algorithm === "ES256") generateKeyPair("ec", { namedCurve: "P-256" }, done); + else generateKeyPair("rsa", { modulusLength: 2048 }, done); + }); +} + +/** + * A fresh signing key. The private half goes straight to the vault; only the + * public half is kept in config. Generated off the event loop: RSA key + * generation takes long enough to stall unrelated requests if done inline. + */ +export async function generateClientKeyPair(algorithm: ClientAssertionAlgorithm, now = new Date()): Promise<{ + privateKeyPem: string; + publicKey: ClientPublicKey; +}> { + const privateKey = await generatePrivateKey(algorithm); + const privateKeyPem = privateKey.export({ format: "pem", type: "pkcs8" }).toString(); + return { privateKeyPem, publicKey: publicKeyForPrivateKey(privateKeyPem, algorithm, now) }; +} + +/** Assertions are single-use and short-lived: a captured one is useless a minute later. */ +export const CLIENT_ASSERTION_LIFETIME_SECONDS = 60; + +/** + * RFC 7523 §3 client assertion: `iss` = `sub` = client ID, `aud` = the token + * endpoint it is presented to, a short `exp`, and a unique `jti`. + */ +export function signClientAssertion(input: { + privateKeyPem: string; + algorithm: ClientAssertionAlgorithm; + keyId: string; + clientId: string; + audience: string; + now?: Date; +}): string { + const issuedAt = Math.floor((input.now ?? new Date()).getTime() / 1000); + const header = { alg: input.algorithm, typ: "JWT", kid: input.keyId }; + const claims = { + iss: input.clientId, + sub: input.clientId, + aud: input.audience, + iat: issuedAt, + exp: issuedAt + CLIENT_ASSERTION_LIFETIME_SECONDS, + jti: randomUUID(), + }; + const signingInput = `${base64Url(JSON.stringify(header))}.${base64Url(JSON.stringify(claims))}`; + const signature = sign( + "sha256", + Buffer.from(signingInput), + input.algorithm === "ES256" + ? { key: input.privateKeyPem, dsaEncoding: "ieee-p1363" } + : input.privateKeyPem, + ); + return `${signingInput}.${base64Url(signature)}`; +} + +/** Secret authentication method the server accepts, preferring Basic as the spec requires servers to support it. */ +export function chooseSecretAuthMethod(supported: string[] | undefined): "client_secret_basic" | "client_secret_post" { + if (!supported?.length || supported.includes("client_secret_basic")) return "client_secret_basic"; + if (supported.includes("client_secret_post")) return "client_secret_post"; + return "client_secret_basic"; +} diff --git a/server/src/services/tool-access/oauth/client.ts b/server/src/services/tool-access/oauth/client.ts index 71d1eadea52..d5a1d898e9b 100644 --- a/server/src/services/tool-access/oauth/client.ts +++ b/server/src/services/tool-access/oauth/client.ts @@ -3,6 +3,7 @@ import { asRecord, originOf } from "./metadata.js"; import { normalizeOAuthProviderError, oauthProviderErrorMessage } from "./provider-errors.js"; import type { OAuthProviderEndpoints, OAuthTokenEndpointAuthMethod, OAuthTransport } from "./types.js"; import { oauthApplicationType } from "./client-metadata.js"; +import { CLIENT_ASSERTION_TYPE } from "./client-credentials.js"; const MAX_OAUTH_DCR_CLIENT_ID_LENGTH = 4_096; @@ -309,6 +310,12 @@ export function createOAuthClientProtocol({ fetchRemoteHttpUrl, assertOAuthEndpo refreshToken?: string | null; /** RFC 8707 resource indicator: the MCP server this token is for. */ resource?: string | null; + /** + * RFC 7523 signed client assertion (`private_key_jwt`). When present it is + * the only client authentication sent: the client is identified by the + * assertion's `sub`, so no client_id or secret goes in the request. + */ + clientAssertion?: string | null; }) { const body = new URLSearchParams(); if (input.grantType === "client_credentials") { @@ -324,10 +331,14 @@ export function createOAuthClientProtocol({ fetchRemoteHttpUrl, assertOAuthEndpo body.set("redirect_uri", input.redirectUri ?? ""); body.set("code_verifier", input.codeVerifier ?? ""); } - const tokenEndpointAuthMethod = - input.tokenEndpointAuthMethod ?? - (input.clientSecret ? "client_secret_post" : "none"); - if (tokenEndpointAuthMethod !== "client_secret_basic") + const tokenEndpointAuthMethod = input.clientAssertion + ? "private_key_jwt" + : input.tokenEndpointAuthMethod ?? + (input.clientSecret ? "client_secret_post" : "none"); + if (input.clientAssertion) { + body.set("client_assertion_type", CLIENT_ASSERTION_TYPE); + body.set("client_assertion", input.clientAssertion); + } else if (tokenEndpointAuthMethod !== "client_secret_basic") body.set("client_id", input.clientId); if (tokenEndpointAuthMethod === "client_secret_post") { if (!input.clientSecret) { diff --git a/server/src/services/tool-access/oauth/connection-refresh.ts b/server/src/services/tool-access/oauth/connection-refresh.ts index 4cdf1e8a8e5..c67de010179 100644 --- a/server/src/services/tool-access/oauth/connection-refresh.ts +++ b/server/src/services/tool-access/oauth/connection-refresh.ts @@ -14,6 +14,8 @@ import { withoutOAuthRefreshLease, type OAuthRefreshLeases } from "./refresh-lea import { oauthExpiresAtMs, oauthSecretRef } from "./client-settings.js"; import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js"; import { singleFlight } from "../single-flight.js"; +import { usesClientCredentials } from "./client-credentials.js"; +import { requireClientCredentials, type ClientCredentialsService } from "./client-credentials-service.js"; function asRecord(value: unknown): Record { if (value && typeof value === "object" && !Array.isArray(value)) return value as Record; @@ -27,9 +29,11 @@ type LegacyRefreshDependencies = { leases: Pick; getConnectionRow(idOrUid: string, companyId?: string): Promise; exchangeOAuthToken: ReturnType["exchangeOAuthToken"]; + /** Absent only in narrow test harnesses that never use machine sign-in. */ + requestClientCredentialsToken?: ClientCredentialsService["requestClientCredentialsToken"]; }; -export function createLegacyOAuthRefresh(db: Db, { now, secrets, store, leases, getConnectionRow, exchangeOAuthToken }: LegacyRefreshDependencies) { +export function createLegacyOAuthRefresh(db: Db, { now, secrets, store, leases, getConnectionRow, exchangeOAuthToken, requestClientCredentialsToken }: LegacyRefreshDependencies) { // These maps remove duplicate work inside one service instance. OAuth also // uses the database refresh lease below as its cross-process boundary. const oauthRefreshFlights = new Map>(); @@ -138,12 +142,15 @@ const oauthRefreshFlights = new Map>(); }, ); } - const client = await oauthClientForConnection( + // Machine sign-in uses only the connection's own vault credentials and its + // pinned token endpoint, never a deployment-wide client. + const machineSignIn = usesClientCredentials(connection); + const client = machineSignIn ? null : await oauthClientForConnection( connection, oauth.provider, actor, ); - if (!client.clientId) + if (client && !client.clientId) throw unprocessable( `OAuth client id is not configured for ${oauth.provider}`, ); @@ -180,9 +187,9 @@ const oauthRefreshFlights = new Map>(); : null; let token: Awaited>; try { - token = await exchangeOAuthToken({ + token = machineSignIn || !client ? await requireClientCredentials(requestClientCredentialsToken)(connection, actor) : await exchangeOAuthToken({ tokenUrl: oauth.tokenUrl, - clientId: client.clientId, + clientId: client.clientId!, clientSecret: client.clientSecret, tokenEndpointAuthMethod: storedOAuthTokenEndpointAuthMethod( oauth, diff --git a/server/src/services/tool-access/oauth/grant-refresh.ts b/server/src/services/tool-access/oauth/grant-refresh.ts index 87b2dfc5618..e3ee3c79d95 100644 --- a/server/src/services/tool-access/oauth/grant-refresh.ts +++ b/server/src/services/tool-access/oauth/grant-refresh.ts @@ -18,6 +18,8 @@ import { withoutOAuthRefreshLease, type OAuthRefreshLeases } from "./refresh-lea import { managedConnectorProfile, oauthGrantConfig, oauthGrantExpiresAtMs } from "./client-settings.js"; import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js"; import { singleFlight } from "../single-flight.js"; +import { usesClientCredentials } from "./client-credentials.js"; +import { requireClientCredentials, type ClientCredentialsService } from "./client-credentials-service.js"; function asRecord(value: unknown): Record { if (value && typeof value === "object" && !Array.isArray(value)) return value as Record; @@ -31,9 +33,11 @@ type GrantRefreshDependencies = { currentCloudConnector: () => PaperclipCloudConnector | null; getConnectionRow(idOrUid: string, companyId?: string): Promise; exchangeOAuthToken: ReturnType["exchangeOAuthToken"]; + /** Absent only in narrow test harnesses that never use machine sign-in. */ + requestClientCredentialsToken?: ClientCredentialsService["requestClientCredentialsToken"]; }; -export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentCloudConnector, getConnectionRow, exchangeOAuthToken }: GrantRefreshDependencies) { +export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentCloudConnector, getConnectionRow, exchangeOAuthToken, requestClientCredentialsToken }: GrantRefreshDependencies) { const oauthGrantRefreshFlights = new Map>(); const { getOAuthGrantRow, resolveOAuthGrantSecret, createOrRotateOAuthSecret, oauthClientForConnection, syncCredentialBindings } = store; const { acquireOAuthGrantRefreshLease, clearOAuthGrantRefreshLease } = leases; @@ -272,10 +276,13 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo if (!lease.leaseId) return lease.grant; try { const grant = lease.grant; - const refreshRef = grant.credentialSecretRefs.find( + // Machine sign-in has no refresh token: it simply asks for a new access + // token with the connection's own client credentials. + const machineSignIn = usesClientCredentials(connection); + const refreshRef = machineSignIn ? undefined : grant.credentialSecretRefs.find( (ref) => ref.configPath === "oauth.refresh_token", ); - if (!refreshRef) { + if (!machineSignIn && !refreshRef) { throw unprocessable( "OAuth credentials have expired and no refresh token is available", { @@ -285,29 +292,32 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo }, ); } - const refreshSecret = await resolveOAuthGrantSecret( + const refreshSecret = refreshRef ? await resolveOAuthGrantSecret( connection, grant, refreshRef, input.actor, input, - ); + ) : null; const credentialActor: ActorInfo | undefined = grant.kind === "user" && grant.subjectUserId ? { actorType: "user", actorId: grant.subjectUserId } : input.actor; - const client = await oauthClientForConnection( - connection, - oauthProvider, - credentialActor, - ); - if (!client.clientId) - throw unprocessable( - `OAuth client id is not configured for ${oauthProvider}`, - ); const grantOauth = oauthGrantConfig(grant); let token: Awaited>; try { + if (machineSignIn) { + token = await requireClientCredentials(requestClientCredentialsToken)(connection, input.actor); + } else { + const client = await oauthClientForConnection( + connection, + oauthProvider, + credentialActor, + ); + if (!client.clientId) + throw unprocessable( + `OAuth client id is not configured for ${oauthProvider}`, + ); token = await exchangeOAuthToken({ tokenUrl: oauthTokenUrl, clientId: client.clientId, @@ -323,14 +333,16 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo : normalizeOauthScopes(oauth.scopes).length > 0 ? normalizeOauthScopes(oauth.scopes) : normalizeOauthScopes(oauth.scope), - refreshToken: refreshSecret.value, + refreshToken: refreshSecret!.value, resource: typeof oauth.resource === "string" && oauth.resource ? oauth.resource : null, }); + } } catch (error) { if ( + refreshRef && refreshSecret && error instanceof HttpError && asRecord(error.details).code === "oauth_reauthorization_required" ) { diff --git a/server/src/services/tool-call-progress.ts b/server/src/services/tool-call-progress.ts new file mode 100644 index 00000000000..429142b781e --- /dev/null +++ b/server/src/services/tool-call-progress.ts @@ -0,0 +1,195 @@ +import { randomUUID } from "node:crypto"; +import { and, desc, eq, gte, isNotNull, isNull } from "drizzle-orm"; +import { toolInvocations, type Db } from "@paperclipai/db"; +import { + normalizeToolCallProgress, + type LiveEventType, + type ToolCallProgress, + type ToolCallProgressSnapshot, +} from "@paperclipai/shared"; +import { logger } from "../middleware/logger.js"; +import { publishLiveEvent } from "./live-events.js"; +import { assignedToolName } from "./native-runtime/assigned-mcp-tools.js"; + +/** + * Live progress for a connected-app tool call that an agent run made. + * + * The provider streams MCP `notifications/progress` on the call's response. + * Each update is published to the run's viewers at most every + * PUBLISH_INTERVAL_MS (the latest one wins) and saved on the invocation at + * most every SAVE_INTERVAL_MS, so a reload mid-call shows the latest state. + * Nothing here is awaited by the call: the result is never delayed. + */ +export const TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS = 250; +export const TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS = 1_000; +export const TOOL_CALL_PROGRESS_EVENT: LiveEventType = "heartbeat.run.tool_progress"; +/** Progress older than this is not shown after a reload (the call is long gone). */ +const SNAPSHOT_MAX_AGE_MS = 24 * 60 * 60_000; + +export interface ToolCallProgressReporter { + /** Sent as the request's `_meta.progressToken`. */ + readonly token: string; + /** Records a provider update. Never throws, never blocks. */ + report(update: { progress: unknown; total?: unknown; message?: unknown }): void; + /** The call ended: stop timers and tell viewers to drop the progress. */ + finish(): void; +} + +type Publish = (input: { companyId: string; type: LiveEventType; payload: Record }) => void; +type Save = (progress: ToolCallProgress) => Promise; + +/** Leading-edge throttle that keeps only the latest value for its trailing run. */ +function latestThrottle(intervalMs: number, run: (value: T) => void) { + let lastAt = -Infinity; + let pending: { value: T } | null = null; + let timer: ReturnType | null = null; + const flush = () => { + timer = null; + if (!pending) return; + const { value } = pending; + pending = null; + lastAt = Date.now(); + run(value); + }; + return { + push(value: T) { + pending = { value }; + if (timer) return; + const wait = lastAt + intervalMs - Date.now(); + if (wait <= 0) flush(); + else { timer = setTimeout(flush, wait); timer.unref?.(); } + }, + cancel() { + if (timer) clearTimeout(timer); + timer = null; + pending = null; + }, + }; +} + +export function createToolCallProgressReporter(input: { + companyId: string; + runId: string; + issueId: string | null; + agentId: string | null; + invocationId: string; + toolName: string; + startedAt?: Date; + publish?: Publish; + save?: Save; +}): ToolCallProgressReporter { + const publish: Publish = input.publish ?? ((event) => { publishLiveEvent(event); }); + const startedAt = (input.startedAt ?? new Date()).toISOString(); + const token = `paperclip-progress-${randomUUID()}`; + let finished = false; + let last: ToolCallProgress | null = null; + const snapshot = (progress: ToolCallProgress, done: boolean): ToolCallProgressSnapshot => ({ + ...progress, invocationId: input.invocationId, runId: input.runId, toolName: input.toolName, + nativeToolName: assignedToolName(input.toolName), startedAt, done, + }); + const send = (value: ToolCallProgressSnapshot) => { + try { + publish({ + companyId: input.companyId, + type: TOOL_CALL_PROGRESS_EVENT, + payload: { ...value, issueId: input.issueId, agentId: input.agentId }, + }); + } catch (err) { + logger.debug({ err, invocationId: input.invocationId }, "tool call progress publish failed"); + } + }; + const live = latestThrottle(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS, (progress) => send(snapshot(progress, false))); + // At most one save in flight; while it runs only the newest update waits. + let saving = false; + let queuedSave: ToolCallProgress | null = null; + const save = (progress: ToolCallProgress) => { + if (!input.save || finished) return; + if (saving) { queuedSave = progress; return; } + saving = true; + input.save(progress) + .catch((err) => logger.debug({ err, invocationId: input.invocationId }, "tool call progress save failed")) + .finally(() => { + saving = false; + const next = queuedSave; + queuedSave = null; + if (next) save(next); + }); + }; + const saved = latestThrottle(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS, save); + return { + token, + report(update) { + if (finished) return; + const progress = normalizeToolCallProgress(update); + // Progress must increase; a lower value is out of order and is dropped. + if (!progress || (last && progress.progress < last.progress)) return; + last = progress; + live.push(progress); + if (input.save) saved.push(progress); + }, + finish() { + if (finished) return; + finished = true; + live.cancel(); + saved.cancel(); + queuedSave = null; + if (last) send(snapshot(last, true)); + }, + }; +} + +/** Saves a running call's latest progress. A finished call is never touched. */ +export function saveToolCallProgress(db: Db, companyId: string, invocationId: string): Save { + return async (progress) => { + await db.update(toolInvocations) + .set({ progress }) + .where(and(eq(toolInvocations.id, invocationId), eq(toolInvocations.companyId, companyId), isNull(toolInvocations.completedAt))); + }; +} + +/** Finished calls are reported for this long, so a chat that missed their end learns it. */ +const FINISHED_REPORT_MS = 60 * 60_000; + +/** + * The latest progress of a task's running tool calls, for a reload mid-call, + * plus its recently finished calls marked `done`. A chat drops a call only on + * `done`: a running call may not have saved its first update yet. + */ +export async function listToolCallProgress(db: Db, companyId: string, issueId: string, now = new Date()): Promise { + const columns = { + id: toolInvocations.id, + runId: toolInvocations.runId, + toolName: toolInvocations.toolName, + progress: toolInvocations.progress, + startedAt: toolInvocations.startedAt, + createdAt: toolInvocations.createdAt, + completedAt: toolInvocations.completedAt, + }; + const task = and(eq(toolInvocations.companyId, companyId), eq(toolInvocations.issueId, issueId), isNotNull(toolInvocations.runId)); + const [running, finished] = await Promise.all([ + db.select(columns).from(toolInvocations).where(and(task, + isNull(toolInvocations.completedAt), + isNotNull(toolInvocations.progress), + gte(toolInvocations.createdAt, new Date(now.getTime() - SNAPSHOT_MAX_AGE_MS)), + )).orderBy(desc(toolInvocations.createdAt)).limit(50), + db.select(columns).from(toolInvocations).where(and(task, + gte(toolInvocations.completedAt, new Date(now.getTime() - FINISHED_REPORT_MS)), + )).orderBy(desc(toolInvocations.completedAt)).limit(50), + ]); + return [...running, ...finished].flatMap((row) => { + const done = row.completedAt !== null; + const savedAt = row.progress ? Date.parse(String(row.progress.updatedAt)) : NaN; + const at = done ? row.completedAt! : Number.isFinite(savedAt) ? new Date(savedAt) : now; + const progress = normalizeToolCallProgress(row.progress ?? (done ? { progress: 0 } : {}), at); + if (!progress || !row.runId) return []; + return [{ + ...progress, + invocationId: row.id, + runId: row.runId, + toolName: row.toolName, + nativeToolName: assignedToolName(row.toolName), + startedAt: (row.startedAt ?? row.createdAt).toISOString(), + done, + }]; + }); +} diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index 90ac0b99ff1..9a2adc2011b 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -30,6 +30,7 @@ import { isInsufficientConnectionScope, INSUFFICIENT_CONNECTION_SCOPE_MESSAGE } import { githubAccessDeniedInstructions, githubAccessError } from "./tool-access/github-access-errors.js"; import { githubDefaultOwnerApplies, githubDefaultOwnerArguments } from "./tool-access/github-default-owner.js"; import { toolSourceTemplateKeySql } from "./tool-access/source-template-key.js"; +import { usesClientCredentials } from "./tool-access/oauth/client-credentials.js"; import { boundedMcpToolName } from "./mcp-tool-names.js"; import { browserUseService } from "./browser-use.js"; import { isBrowserUseConnection } from "./browser-use-client.js"; @@ -203,7 +204,9 @@ import { type McpInputRequired, mcpHttpRequestHeaders, parseMcpHttpResponseBody, + mcpProgressNotificationOf, } from "./mcp-http.js"; +import { createToolCallProgressReporter, saveToolCallProgress, type ToolCallProgressReporter } from "./tool-call-progress.js"; import { buildElicitationQuestionSet, elicitationQuestionHash, @@ -5505,6 +5508,8 @@ export function createToolGatewayService( protocolMethod?: string; protocolParams?: Record; timeoutMs: number; + /** Receives the call's `notifications/progress`; its token is sent with tools/call. */ + progressReporter?: ToolCallProgressReporter | null; }, /** One restart and one deadline are shared by every protocol recovery in a call. */ recovery: { restarted: boolean; deadline: number } = { restarted: false, deadline: Date.now() + input.timeoutMs }, @@ -5585,6 +5590,8 @@ export function createToolGatewayService( try { const message = JSON.parse(line) as Record; const id = typeof message.id === "number" ? message.id : null; + const progressUpdate = mcpProgressNotificationOf(message, input.progressReporter?.token); + if (progressUpdate) input.progressReporter!.report(progressUpdate); if (id !== null && pending.has(id)) { const waiter = pending.get(id)!; pending.delete(id); @@ -5760,6 +5767,7 @@ export function createToolGatewayService( withMeta(protocolVersion, input.protocolParams ?? { name: input.entry?.toolName, arguments: input.parameters ?? {}, + ...(input.progressReporter ? { _meta: { progressToken: input.progressReporter.token } } : {}), }), ); } catch (error) { @@ -7816,6 +7824,7 @@ export function createToolGatewayService( const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), ms); timer.unref?.(); + const progressReporter = toolCallProgressReporterFor(session, tool, invocationId); // The cached MCP session this call used, so a failure resets only this // identity's session and leaves other agents on the connection alone. let mcpSession: { scope: string; headers: Record; sessionId: string } | undefined; @@ -8026,8 +8035,10 @@ export function createToolGatewayService( toolInputSchema: entry.inputSchema ?? undefined, // Paperclip can wait on a task here, durably (see waitForMcpTask). declareTasks: tasksOn, + progressToken: progressReporter?.token, // A person can answer the app's form questions in the task chat. elicitationForm: Boolean(elicitationCall), + declareClientCredentials: usesClientCredentials(connection), }), redirect: "manual", signal: controller.signal, @@ -8245,7 +8256,13 @@ export function createToolGatewayService( const body = response.ok ? JSON.stringify(await readMcpHttpResponse(response, callRequestId, { maxBytes: MAX_REMOTE_MCP_RESPONSE_BYTES, + // The call has a deadline (controller), so progress may outlast the total limit. + limitPerMessage: true, onProgress: (progress) => { execution.stream = progress; }, + onNotification: (message) => { + const update = mcpProgressNotificationOf(message, progressReporter?.token); + if (update) progressReporter!.report(update); + }, onRequest: async (message) => { const pending = extractRemoteMcpPending(message); if (pending) { @@ -8508,15 +8525,31 @@ export function createToolGatewayService( ); } finally { clearTimeout(timer); + progressReporter?.finish(); } } + /** Progress is shown to the run's viewers, so only a call made by a run reports it. */ + function toolCallProgressReporterFor(session: ToolGatewaySession, tool: ToolGatewayDescriptor, invocationId: string | undefined) { + if (!session.runId || !invocationId) return null; + return createToolCallProgressReporter({ + companyId: session.companyId, + runId: session.runId, + issueId: session.issueId, + agentId: session.agentId, + invocationId, + toolName: tool.name, + save: saveToolCallProgress(db, session.companyId, invocationId), + }); + } + async function executeLocalStdioTool( session: ToolGatewaySession, tool: ToolGatewayDescriptor, parameters: unknown, ms: number, useProviderDefaultTimeout = false, + invocationId?: string, ): Promise { const { entry, connection } = await resolveConnectedLocalStdioTool( session, @@ -8530,8 +8563,10 @@ export function createToolGatewayService( template, grant, ); - const invoke = () => callLocalStdioMcp({ connection, entry, template, env, parameters, - timeoutMs: useProviderDefaultTimeout && template.templateId === "paperclip.cognee-cloud" ? 60_000 : ms }); + const progressReporter = template.templateId === "paperclip.cognee-cloud" ? null : toolCallProgressReporterFor(session, tool, invocationId); + const invoke = () => callLocalStdioMcp({ connection, entry, template, env, parameters, progressReporter, + timeoutMs: useProviderDefaultTimeout && template.templateId === "paperclip.cognee-cloud" ? 60_000 : ms }) + .finally(() => progressReporter?.finish()); // Cognee is a bundled HTTP client. Provider failures are tool failures, not // crashed local processes, and must never consume slots or restart budgets. const result = template.templateId === "paperclip.cognee-cloud" @@ -9579,6 +9614,7 @@ export function createToolGatewayService( args.parameters, executionTimeoutMs, args.timeoutMs === undefined, + args.invocationId, ) : null; if (!connectedMcpExecution) { @@ -10456,6 +10492,8 @@ export function createToolGatewayService( tool, parameters, executionTimeoutMs, + false, + invocation.id, ) : null; const result = connectedMcpExecution @@ -13432,6 +13470,7 @@ export function createToolGatewayService( effectiveParameters, executionTimeoutMs, input.timeoutMs === undefined, + invocationId, ) : null; const result = recoveredGitHubPublication ?? (connectedMcpExecution diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index ddb291fc18b..46250ab43be 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -229,7 +229,7 @@ "packages/shared/src/connector-availability.ts": "82aec0dfb2e9b8cb04e21c4bcd9aab74b087c76a", "packages/shared/src/connector-identity.test.ts": "d58357707faefa7c98bb906cdaeb4db39d31081a", "packages/shared/src/connector-identity.ts": "5ede83c9c5225dbe8af9cf3a49c74e477ef5b955", - "packages/shared/src/constants.ts": "3149ee1cc5ffa34b50ecd493276b3a70fc46adc2", + "packages/shared/src/constants.ts": "47973de810c0dfac21625e2687af80b37e8da00c", "packages/shared/src/crash-point-registry.ts": "3cbf2d290bc111061376b43020848623e493737c", "packages/shared/src/crash-points.test.ts": "3564a1f62d22f54a3c2fb0a72b3a6cde17c957d2", "packages/shared/src/crash-points.ts": "c15259f39610e352eac5423a491548ec1bae94fe", @@ -268,7 +268,7 @@ "packages/shared/src/home-paths.ts": "fa4f1c440653b76b3dbb7a51026d557287deb45d", "packages/shared/src/humanize-connection.test.ts": "ba9e90a44c63452b0bf9f733adb90241921e8eca", "packages/shared/src/humanize-connection.ts": "fbd414476a0ef7106a7c80d8572208781b091699", - "packages/shared/src/index.ts": "2d2b5ba044c9c7a22ff99022c0fac1e0da5e4a94", + "packages/shared/src/index.ts": "efab7ad18d89a60ddd3f961e35060b68424ef40b", "packages/shared/src/issue-activity-identity.test.ts": "3dc294640f54c2be67943e3e0149cf4eff2c95ca", "packages/shared/src/issue-activity-identity.ts": "1b0f07ec3fe0506aaaf80a4fa9893f3baf8f0075", "packages/shared/src/issue-attribution.test.ts": "84563685e7f3c48029243cf1cc24a3a09ee3cde9", @@ -366,6 +366,7 @@ "packages/shared/src/telemetry/server-started-event.test.ts": "3d0e0e2289badb41c92ba0d69e3668d00b3b5fdb", "packages/shared/src/telemetry/state.ts": "a060af26a8e9a82a8afb0e2fccbfe6a59e9d0a8a", "packages/shared/src/telemetry/types.ts": "083d4341a72ac47bc1a3bbff163965de8b06f90a", + "packages/shared/src/tool-call-progress.ts": "89655767bfd0180e99ac517754d72e25f753eb33", "packages/shared/src/trust-policy.ts": "88001a8d629a2af66dcd8d75b7b4d36298455ef6", "packages/shared/src/types/access.ts": "015d51792da07acaebe59e0e395698d207811d0d", "packages/shared/src/types/activity.ts": "6f5f27417a0bb26ec4daba6ec16a04ce31e484f2", @@ -407,7 +408,7 @@ "packages/shared/src/types/inbox-agent-policy.ts": "cb9d41d9681e80b834889268ad96015a39df92f6", "packages/shared/src/types/inbox-dismissal.ts": "58fb7013847396ac9971eb5799dd0bf0486e0371", "packages/shared/src/types/inbox-mine.ts": "1d302b531d34842189704abd2f75f8743d09490d", - "packages/shared/src/types/index.ts": "f63f67250d6d5a0b737478c634244ae3aba0328a", + "packages/shared/src/types/index.ts": "8abf1f4ed91e67f8e620a1a0b127799c793554ff", "packages/shared/src/types/instance.ts": "ddabd7cc0d885b5fdf45b33a7937b7b704f8fb7d", "packages/shared/src/types/issue-tree-control.ts": "75489fffd491b8350b05caf57d7796419797d9df", "packages/shared/src/types/issue.ts": "1a59939658d0b28807b8018bacf83a3ded4fef74", @@ -430,7 +431,7 @@ "packages/shared/src/types/tailscale.ts": "d02a917945d7779009a2dea4c4b535e9478e0680", "packages/shared/src/types/task-workspace.ts": "a23afa13d2cb58dc2eb72692ed28c16ed6fb84fb", "packages/shared/src/types/teams-catalog.ts": "632fe75f8e5a4cfc455843c2a1a2daff28a022be", - "packages/shared/src/types/tool-access.ts": "4c05d176f9bb1d30593fd904c7c4fc3e91b431ca", + "packages/shared/src/types/tool-access.ts": "1d2c29fb7c0c22ae30b53ff18fe44545c18b4835", "packages/shared/src/types/user-profile.ts": "1ac26e033a79fab21dc267fd983959814c8b680f", "packages/shared/src/types/voice-sessions.ts": "95e3b62cdd51f62290e68536728ddd7297bc0459", "packages/shared/src/types/work-product.ts": "26f9129f606a207ce7a218ccef40f96747283086", @@ -484,7 +485,7 @@ "packages/shared/src/validators/folder.ts": "b486444ab09ec3d5a9afec7dbd2e71c1bc2beb67", "packages/shared/src/validators/goal.ts": "602724134915e1e83c50115abba36b288c0abd3d", "packages/shared/src/validators/inbox-agent-policy.ts": "29cef157f4d9b564fd813868038aa8c813df4a7b", - "packages/shared/src/validators/index.ts": "a62024f564b768d87e9b8a2869c0b8b9ce92dc09", + "packages/shared/src/validators/index.ts": "b0bc5449eb5c21fc89064d9d5243223ab33cf5e0", "packages/shared/src/validators/instance.test.ts": "3faa9593a9a05f74429d9b1d2251906a03e9a903", "packages/shared/src/validators/instance.ts": "d801db219659707f0c9a8c1fbad06e862be4892d", "packages/shared/src/validators/issue-chat-bootstrap.test.ts": "e1ebb9aa94da22c825bc3ddd32ce2cfc98bfe645", @@ -519,7 +520,7 @@ "packages/shared/src/validators/teams-catalog.ts": "7bddf40c091a52aaa54e79de784bea5ae496f9aa", "packages/shared/src/validators/text.ts": "74c430fe17cdcfd4b56dd08a4c57cd3d567670a8", "packages/shared/src/validators/tool-access.test.ts": "b5838320115946a936b256919a5bd3a2294c6070", - "packages/shared/src/validators/tool-access.ts": "cd28549a930e0b222904bb1121db17659bf12b05", + "packages/shared/src/validators/tool-access.ts": "f6ada4ebdc2d318c0bd50a82dac0692082e03468", "packages/shared/src/validators/trust-policy.test.ts": "1fc4818984eb0e14f034d56ad147600224496d4e", "packages/shared/src/validators/trust-policy.ts": "f16b002db940b06bd882d51df8b5712c576e1d7b", "packages/shared/src/validators/voice-sessions.ts": "018a71f9ac544645041ac3cc9919e59c95fbe998", diff --git a/ui/src/api/mcpApps.ts b/ui/src/api/mcpApps.ts index 45d0b835b1e..b5b7fa0f746 100644 --- a/ui/src/api/mcpApps.ts +++ b/ui/src/api/mcpApps.ts @@ -1,4 +1,4 @@ -import type { McpAppCallToolResult, McpAppModelContext, McpAppViewDetail, McpAppViewSummary } from "@paperclipai/shared"; +import type { McpAppCallToolResult, McpAppModelContext, McpAppViewDetail, McpAppViewSummary, ToolCallProgressSnapshot } from "@paperclipai/shared"; import { api } from "./client"; /** MCP Apps views recorded for a task's tool calls. */ @@ -17,4 +17,7 @@ export const mcpAppsApi = { /** Records that the viewer sent a view's proposed message (sent with this client request id). */ messageSent: (issueId: string, viewId: string, clientRequestId: string) => api.post<{ commentId: string }>(`/issues/${issueId}/mcp-app-views/${viewId}/messages`, { clientRequestId }), + /** The latest progress of the task's running app tool calls (for a reload mid-call). */ + toolCallProgress: (issueId: string) => + api.get(`/issues/${issueId}/tool-call-progress`, { cache: "no-store" }), }; diff --git a/ui/src/api/tools.ts b/ui/src/api/tools.ts index 9334522137c..2903ec92f1a 100644 --- a/ui/src/api/tools.ts +++ b/ui/src/api/tools.ts @@ -306,6 +306,16 @@ export const toolsApi = { }, connectApp: (companyId: string, input: ConnectToolApp) => api.post(`/companies/${companyId}/tools/apps/connect`, input), + createMachineSignIn: (companyId: string, input: import("@paperclipai/shared").CreateMachineSignIn) => + api.post(`/companies/${companyId}/tools/machine-sign-in`, input), + getMachineSignIn: (connectionId: string) => + api.get(`/tool-connections/${connectionId}/machine-sign-in`), + testMachineSignIn: (connectionId: string, input: import("@paperclipai/shared").TestMachineSignIn) => + api.post(`/tool-connections/${connectionId}/machine-sign-in/test`, input), + rotateMachineSignIn: (connectionId: string, input: import("@paperclipai/shared").RotateMachineSignIn) => + api.post(`/tool-connections/${connectionId}/machine-sign-in/rotate`, input), + completeMachineSignInRotation: (connectionId: string) => + api.post(`/tool-connections/${connectionId}/machine-sign-in/rotate/complete`, {}), startOAuth: ( connectionId: string, input: { diff --git a/ui/src/components/TaskChatThread.tsx b/ui/src/components/TaskChatThread.tsx index 7594e3307a4..4c5286945fe 100644 --- a/ui/src/components/TaskChatThread.tsx +++ b/ui/src/components/TaskChatThread.tsx @@ -47,6 +47,7 @@ import { TaskChatLiveTail } from "@/components/task-chat/TaskChatLiveTail"; import { McpAppIssueProvider } from "@/components/mcp-app/McpAppIssueContext"; import { ProviderToolIconsProvider } from "@/components/ProviderToolIcon"; import { McpAppTurnViews, mcpAppRunRefreshKey } from "@/components/mcp-app/McpAppTurnViews"; +import { ToolCallProgressProvider } from "@/components/task-chat/tool-call-progress"; import { TaskChatRunnerHeader, TaskChatRunnerTurn } from "@/components/task-chat/TaskChatRunnerTurn"; import { partitionRunHealthItems } from "@/components/task-chat/TaskChatRunHealthBanner"; import { @@ -3372,6 +3373,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { + + diff --git a/ui/src/components/task-chat/TaskChatLiveRunPill.test.tsx b/ui/src/components/task-chat/TaskChatLiveRunPill.test.tsx index 70559fab815..869f33a7f7f 100644 --- a/ui/src/components/task-chat/TaskChatLiveRunPill.test.tsx +++ b/ui/src/components/task-chat/TaskChatLiveRunPill.test.tsx @@ -120,6 +120,20 @@ describe("TaskChatLiveRunPill", () => { expect(container.querySelector("a")?.getAttribute("href")).toBe("/agents/codie/advanced"); }); + it("says a queued run with no named hold is waiting to start once the startup grace passes", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-13T17:46:13Z")); + const queuedAt = Date.now() - 5_000; + const queued = { phase: "queued", label: "Queued", cause: null } as ExecutionProjection; + act(() => root.render()); + expect(container.textContent).toContain("Working"); + act(() => vi.advanceTimersByTime(26_000)); + expect(container.textContent).toContain("Waiting to start"); + expect(container.textContent).not.toContain("Working"); + expect(container.querySelector(".animate-spin")).toBeNull(); + expect(container.querySelector(".shimmer-text")).toBeNull(); + }); + it("names another server hold on a queued run instead of Working", () => { const execution = { phase: "queued", label: "Waiting for maintenance", cause: "task_drain" } as ExecutionProjection; act(() => root.render()); diff --git a/ui/src/components/task-chat/TaskChatLiveRunPill.tsx b/ui/src/components/task-chat/TaskChatLiveRunPill.tsx index e0e4194a44d..df953d50a7b 100644 --- a/ui/src/components/task-chat/TaskChatLiveRunPill.tsx +++ b/ui/src/components/task-chat/TaskChatLiveRunPill.tsx @@ -1,5 +1,5 @@ import type { ExecutionProjection } from "@paperclipai/shared"; -import { Loader2 } from "lucide-react"; +import { Hourglass, Loader2 } from "lucide-react"; import type { TranscriptEntry } from "../../adapters"; import { cn } from "@/lib/utils"; import { useSecondTick } from "@/hooks/useSecondTick"; @@ -9,6 +9,7 @@ import { isTerminalRunStatus } from "@/components/task-chat/transcript-adapter"; import { useTaskChatShimmer } from "./TaskChatShimmer"; import { AgentQueueWait } from "@/components/AgentQueueWait"; import { agentQueueWaitOf } from "@/lib/agent-queue-wait"; +import { WAITING_TO_START_LABEL, useWaitingToStart } from "./useWaitingToStart"; /** * "ran N commands, called M tools" for the live tail's status pill, counted off @@ -66,6 +67,8 @@ export function TaskChatLiveRunPill({ const agentQueue = status === "queued" ? agentQueueWaitOf(execution) : null; // A specific server hold (maintenance, agent setup) names the wait instead of "Working". const holdLabel = status === "queued" && execution?.cause ? execution.label : null; + // Past the startup grace period with no named hold, the turn says it waits. + const waitingToStart = useWaitingToStart(status === "queued" && !holdLabel && !agentQueue, startedAtMs); // One shared page-wide ticker drives the live elapsed readout, matching the // default view's `useLiveElapsed`. useSecondTick(active && startedAtMs != null && !agentQueue); @@ -76,7 +79,7 @@ export function TaskChatLiveRunPill({ ? formatDurationWords(elapsedMs) : null; const failed = ["failed", "timed_out", "cancelled", "interrupted"].includes(status); - const verb = active ? holdLabel ?? "Working" : failed ? "Stopped" : "Worked"; + const verb = active ? holdLabel ?? (waitingToStart ? WAITING_TO_START_LABEL : "Working") : failed ? "Stopped" : "Worked"; const suffix = elapsed ? `for ${elapsed}` : null; if (agentQueue) { @@ -93,7 +96,11 @@ export function TaskChatLiveRunPill({ data-testid="task-chat-live-run-pill" > - {active ? ( + {active && waitingToStart ? ( + + + ) : active ? ( @@ -102,7 +109,7 @@ export function TaskChatLiveRunPill({ )} - {active ? {verb} : verb} + {active && !waitingToStart ? {verb} : verb} {suffix ? {suffix} : null} {toolSummary ? · {toolSummary} : null} diff --git a/ui/src/components/task-chat/TaskChatRunnerTurn.test.tsx b/ui/src/components/task-chat/TaskChatRunnerTurn.test.tsx index 1c050084bbb..0864dfd42a7 100644 --- a/ui/src/components/task-chat/TaskChatRunnerTurn.test.tsx +++ b/ui/src/components/task-chat/TaskChatRunnerTurn.test.tsx @@ -111,6 +111,54 @@ describe("TaskChatRunnerTurn", () => { expect(container.textContent).not.toContain("Thinking"); }); + it("keeps startup feedback briefly, then says a queued turn with no named hold is waiting to start", () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date("2026-11-13T17:39:05Z")); + render([], "queued"); + expect(container.querySelector('[data-testid="task-chat-turn-status-header"]')?.textContent).toContain("Working"); + expect(container.querySelector('[data-testid="task-chat-current-activity-label"]')?.textContent).toBe("Thinking"); + // The fixture's run was queued 2 s before render: the grace ends at +28 s. + act(() => { vi.advanceTimersByTime(27_000); }); + expect(container.textContent).not.toContain("Waiting to start"); + act(() => { vi.advanceTimersByTime(2_000); }); + expect(container.querySelector('[data-testid="task-chat-turn-status-header"]')?.textContent).toContain("Waiting to start"); + expect(container.querySelector('[data-testid="task-chat-current-activity-label"]')?.textContent).toBe("Waiting to start"); + expect(container.textContent).not.toContain("Working"); + } finally { + vi.useRealTimers(); + } + }); + + it("does not repeat the agent's waiting line in the turn body", () => { + // The shared header (rendered by the thread) shows the waiting line; the body adds no second line. + act(() => root.render( + + + , + )); + expect(container.textContent).not.toContain("Waiting for Codie"); + expect(container.querySelector('[data-testid="task-chat-current-activity"]')).toBeNull(); + }); + + it("does not call a running turn waiting, however long it runs", () => { + vi.useFakeTimers(); + try { + render([], "running"); + act(() => { vi.advanceTimersByTime(10 * 60_000); }); + expect(container.textContent).not.toContain("Waiting to start"); + expect(container.querySelector('[data-testid="task-chat-turn-status-header"]')?.textContent).toContain("Working"); + } finally { + vi.useRealTimers(); + } + }); + const resolvedQuestion = ( id: string, prompt: string, diff --git a/ui/src/components/task-chat/TaskChatRunnerTurn.tsx b/ui/src/components/task-chat/TaskChatRunnerTurn.tsx index 2416c719ca5..3b1b6780087 100644 --- a/ui/src/components/task-chat/TaskChatRunnerTurn.tsx +++ b/ui/src/components/task-chat/TaskChatRunnerTurn.tsx @@ -1,7 +1,9 @@ import { groupImageResources } from "./image-resources"; import { AgentQueueWait } from "@/components/AgentQueueWait"; import { agentQueueWaitOf } from "@/lib/agent-queue-wait"; +import { WAITING_TO_START_LABEL, useWaitingToStart } from "./useWaitingToStart"; import { McpAppTurnViews, mcpAppRunRefreshKey } from "@/components/mcp-app/McpAppTurnViews"; +import { ToolCallProgressRunProvider } from "./tool-call-progress"; import { useRef, type ReactNode } from "react"; import type { ExecutionProjection } from "@paperclipai/shared"; import { useSecondTick } from "@/hooks/useSecondTick"; @@ -134,11 +136,17 @@ interface RunnerHeaderProps { continuedAfterSteering?: boolean; } -function runnerWaitingLabel({ runId, status, execution, preparationLabel }: RunnerHeaderProps) { +function runnerWaitingLabel({ runId, status, execution, preparationLabel }: RunnerHeaderProps, waitingToStart = false) { if (isTerminalRunStatus(status)) return null; if (execution?.phase === "waiting_for_provider") return execution.label; if (!runId && preparationLabel) return preparationLabel; - return status === "queued" && execution?.cause ? execution.label : null; + if (status === "queued" && execution?.cause) return execution.label; + return waitingToStart ? WAITING_TO_START_LABEL : null; +} + +/** A run's queued turn with no named hold, past the startup grace period. */ +function useRunnerWaitingToStart({ runId, status, execution, startedAtMs }: Pick) { + return useWaitingToStart(status === "queued" && !!runId && !execution?.cause, startedAtMs); } function agentQueueWaitFor({ status, execution }: Pick) { @@ -147,10 +155,11 @@ function agentQueueWaitFor({ status, execution }: Pick {props.agentName ? : null} {statusContent ?? (agentQueueWaitFor(props) ? : null) ?? } + continuedAfterSteering={props.continuedAfterSteering} waitingLabel={runnerWaitingLabel(props, waitingToStart)} />} ; } @@ -206,7 +215,8 @@ export function TaskChatRunnerTurn({ }) { const { failures: mcpStartupFailures, rest: turnItems } = partitionRunHealthItems(items); const terminal = isTerminalRunStatus(status); - const waitingLabel = runnerWaitingLabel({ runId, status, execution, preparationLabel, startedAtMs }); + const waitingToStart = useRunnerWaitingToStart({ runId, status, execution, startedAtMs }); + const waitingLabel = runnerWaitingLabel({ runId, status, execution, preparationLabel, startedAtMs }, waitingToStart); const yielded = turnItems.some( (item) => item.kind === "protocol" && @@ -258,7 +268,7 @@ export function TaskChatRunnerTurn({ )); return ( -
) : null} - {!final && currentActivityItems.length === 0 ? : null} -
+ {/* The header's waiting line already names the agent's queue; no second line. */} + {!final && currentActivityItems.length === 0 && !agentQueueWaitFor({ status, execution }) + ? : null} + ); } diff --git a/ui/src/components/task-chat/TaskChatToolCard.tsx b/ui/src/components/task-chat/TaskChatToolCard.tsx index a344f482a53..7e79f3b4cd5 100644 --- a/ui/src/components/task-chat/TaskChatToolCard.tsx +++ b/ui/src/components/task-chat/TaskChatToolCard.tsx @@ -14,6 +14,8 @@ import { useTaskChatShimmer } from "./TaskChatShimmer"; import { TaskChatDiffCounts, TaskChatDiffView } from "./TaskChatDiffView"; import { useTaskReferenceLabel, withTaskReference } from "./task-reference"; import { ProviderToolIcon, useProviderToolIcons } from "@/components/ProviderToolIcon"; +import type { ToolCallProgressSnapshot } from "@paperclipai/shared"; +import { TaskChatToolProgress, useToolCallProgress } from "./tool-call-progress"; const STATUS_ICON = { pending: { Icon: Loader2, spin: false, tone: "text-muted-foreground" }, @@ -48,8 +50,15 @@ function TaskReferenceToolTarget({ taskRefId, target }: { taskRefId: string; tar * inset). An edit row carries its `+N −M` counts and opens to a compact, * height-capped diff of that one edit. */ -export function TaskChatToolCard({ item }: { item: TaskChatToolItem }) { +export function TaskChatToolCard({ item, progress: givenProgress }: { + item: TaskChatToolItem; + /** A fixed progress state (stories); otherwise a running call's live progress is shown. */ + progress?: ToolCallProgressSnapshot | null; +}) { const { Icon, spin, tone } = STATUS_ICON[item.status]; + const liveProgress = useToolCallProgress(item); + // Progress belongs to the running call only; a finished row drops it. + const progress = item.status === "in_progress" ? (givenProgress !== undefined ? givenProgress : liveProgress) : null; const shimmer = useTaskChatShimmer(item.id); const RowIcon = toolTaxonomy(item.rawName ?? item.name).icon; const providerIcons = useProviderToolIcons(item.rawName ?? item.name); @@ -130,6 +139,7 @@ export function TaskChatToolCard({ item }: { item: TaskChatToolItem }) { ) : null} + {progress ? : null} {item.target && showDetail && !item.diff?.lines?.length ? (
({ list: vi.fn() })); +vi.mock("@/api/mcpApps", () => ({ mcpAppsApi: { toolCallProgress: savedProgress.list } })); + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const snapshot = (fields: Partial = {}): ToolCallProgressSnapshot => ({ + invocationId: "call-1", runId: "run-1", toolName: "kv_demo__slow_with_progress", + startedAt: "2026-11-13T10:00:00.000Z", updatedAt: "2026-11-13T10:00:05.000Z", + progress: 2, total: 5, message: "Exporting rows", done: false, ...fields, +}); +const tool = (fields: Partial = {}): TaskChatToolItem => ({ + id: "toolu_1", kind: "tool", name: "slow_with_progress", rawName: "mcp__paperclip__kv_demo__slow_with_progress", + status: "in_progress", ...fields, +}); + +describe("tool call progress matching", () => { + it("matches the gateway tool name inside the agent's tool name, and nothing else", () => { + expect(toolCallNameMatches("mcp__paperclip__kv_demo__slow_with_progress", "kv_demo__slow_with_progress")).toBe(true); + expect(toolCallNameMatches("kv_demo__slow_with_progress", "kv_demo__slow_with_progress")).toBe(true); + expect(toolCallNameMatches("mcp__paperclip__kv_demo__slow_with_progressive", "kv_demo__slow_with_progress")).toBe(false); + expect(toolCallNameMatches("mcp__paperclip__other_slow_with_progress", "kv_demo__slow_with_progress")).toBe(false); + expect(toolCallNameMatches(undefined, "x")).toBe(false); + // Gateway names carry "." and ":", which agents may rewrite. + expect(toolCallNameMatches("mcp__paperclip__mcp_kv-demo-1a2b3c4d_slow-with-progress", "mcp.kv-demo-1a2b3c4d:slow-with-progress")).toBe(true); + expect(toolCallNameMatches("mcp__paperclip__mcp.kv-demo-1a2b3c4d:slow-with-progress", "mcp.kv-demo-1a2b3c4d:slow-with-progress")).toBe(true); + expect(toolCallNameMatches("mcp__paperclip__mcp.kv-demo-99999999:slow-with-progress", "mcp.kv-demo-1a2b3c4d:slow-with-progress")).toBe(false); + }); + + it("finds a native runner's row by the shortened, hashed name it gives the tool", () => { + const gatewayName = "mcp.kv-demo-1a2b3c4d:slow-with-progress"; + const nativeName = "app_mcp_kv_demo_1a2b3c4d_slow_with_progress_889fb9a8d98e"; + const entry = snapshot({ toolName: gatewayName, nativeToolName: nativeName }); + expect(pickToolCallProgress([entry], tool({ rawName: nativeName, name: nativeName }), "run-1")).toBe(entry); + expect(pickToolCallProgress([entry], tool({ rawName: `mcp__paperclip__${nativeName}` }), "run-1")).toBe(entry); + // Another tool's hash never matches. + expect(pickToolCallProgress([entry], tool({ rawName: "app_mcp_kv_demo_1a2b3c4d_slow_with_progress_000000000000" }), "run-1")).toBeNull(); + expect(readToolCallProgressEvent({ ...entry })?.nativeToolName).toBe(nativeName); + }); + + it("finds a wrapper call's row (OpenCode's paperclip_call_assigned_tool) by the tool it runs", () => { + const gatewayName = "mcp.kv-demo-1a2b3c4d:slow-with-progress"; + const nativeName = "app_mcp_kv_demo_1a2b3c4d_slow_with_progress_889fb9a8d98e"; + const items = transcriptToTaskChatItems([{ + kind: "tool_call", ts: "2026-11-13T10:00:00.000Z", toolUseId: "call-1", + name: "paperclip_call_assigned_tool", input: { name: nativeName, arguments: { steps: 10 } }, + }], { runId: "run-1", running: true }); + const row = items.find((item): item is TaskChatToolItem => item.kind === "tool")!; + expect(row.appToolName).toBe(nativeName); + const entry = snapshot({ toolName: gatewayName, nativeToolName: nativeName }); + expect(pickToolCallProgress([entry], row, "run-1")).toBe(entry); + // Any other call, or a wrapper without a name, carries none. + expect(wrappedAppToolName("Bash", { name: nativeName })).toBeUndefined(); + expect(wrappedAppToolName("mcp__paperclip__paperclip_call_assigned_tool", { name: nativeName })).toBe(nativeName); + expect(wrappedAppToolName("paperclip_call_assigned_tool", { arguments: {} })).toBeUndefined(); + }); + + it("picks the newest unfinished call of the row's tool in the row's run", () => { + const entries = [ + snapshot({ invocationId: "old", startedAt: "2026-11-13T09:00:00.000Z" }), + snapshot({ invocationId: "new", startedAt: "2026-11-13T10:00:00.000Z" }), + snapshot({ invocationId: "finished", startedAt: "2026-11-13T11:00:00.000Z", done: true }), + snapshot({ invocationId: "other-run", runId: "run-2", startedAt: "2026-11-13T12:00:00.000Z" }), + snapshot({ invocationId: "other-tool", toolName: "kv_demo__get", startedAt: "2026-11-13T13:00:00.000Z" }), + ]; + expect(pickToolCallProgress(entries, tool(), "run-1")?.invocationId).toBe("new"); + expect(pickToolCallProgress(entries, tool(), null)?.invocationId).toBe("other-run"); + expect(pickToolCallProgress(entries, tool({ rawName: "Bash", name: "Bash" }), "run-1")).toBeNull(); + }); + + it("keeps the newest state of each call, a finished call stays finished, and memory is bounded", () => { + const store = new ToolCallProgressStore(); + store.put(snapshot({ progress: 3, updatedAt: "2026-11-13T10:00:06.000Z" })); + store.put(snapshot({ progress: 2, updatedAt: "2026-11-13T10:00:05.000Z" })); + expect([...store.entries()].map((entry) => entry.progress)).toEqual([3]); + store.put(snapshot({ progress: 3, done: true, updatedAt: "2026-11-13T10:00:06.000Z" })); + store.put(snapshot({ progress: 4, updatedAt: "2026-11-13T10:00:09.000Z" })); + expect([...store.entries()]).toEqual([]); + for (let i = 0; i < 80; i++) store.put(snapshot({ invocationId: `call-${i + 2}` })); + expect([...store.entries()]).toHaveLength(50); + }); + + it("a fresh read ends calls that finished unseen, and never drops a call it does not list", () => { + const store = new ToolCallProgressStore(); + store.put(snapshot({ invocationId: "finished-while-hidden", updatedAt: "2026-11-13T10:00:05.000Z" })); + store.put(snapshot({ invocationId: "not-saved-yet", updatedAt: "2026-11-13T10:00:06.000Z" })); + // The read reports the finished call as done and does not list the other one yet. + for (const entry of [snapshot({ invocationId: "finished-while-hidden", done: true, updatedAt: "2026-11-13T10:00:09.000Z" })]) store.put(entry); + expect(pickToolCallProgress(store.entries(), tool(), "run-1")?.invocationId).toBe("not-saved-yet"); + expect([...store.entries()].map((entry) => entry.invocationId)).toEqual(["not-saved-yet"]); + }); + + it("keeps running calls when a read reports many finished ones", () => { + const store = new ToolCallProgressStore(); + for (let i = 0; i < 50; i++) store.put(snapshot({ invocationId: `running-${i}` })); + for (let i = 0; i < 50; i++) store.put(snapshot({ invocationId: `history-${i}`, done: true })); + expect([...store.entries()]).toHaveLength(50); + expect([...store.entries()].every((entry) => entry.invocationId.startsWith("running-") && !entry.done)).toBe(true); + }); + + it("a call's live end before a delayed read listing it as running keeps it ended", () => { + const store = new ToolCallProgressStore(); + // Opened near the end: the live end arrives first, then the read made just before it. + store.put(snapshot({ invocationId: "ending", done: true, updatedAt: "2026-11-13T10:00:10.000Z" })); + store.put(snapshot({ invocationId: "ending", progress: 9, total: 10, updatedAt: "2026-11-13T10:00:09.000Z" })); + expect([...store.entries()]).toEqual([]); + expect(pickToolCallProgress(store.entries(), tool(), "run-1")).toBeNull(); + }); + + it("reads live events defensively", () => { + expect(readToolCallProgressEvent({ ...snapshot(), message: "x".repeat(500) })?.message).toHaveLength(200); + expect(readToolCallProgressEvent({ ...snapshot(), runId: undefined })).toBeNull(); + expect(readToolCallProgressEvent({ ...snapshot(), progress: "2" })).toBeNull(); + expect(readToolCallProgressEvent({ ...snapshot(), total: -1 })?.total).toBeNull(); + }); +}); + +describe("tool row progress states", () => { + let container: HTMLDivElement; + let root: Root; + let queryClient: QueryClient; + beforeEach(() => { + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + savedProgress.list.mockReset(); + savedProgress.list.mockResolvedValue([]); + }); + afterEach(() => { + act(() => root.unmount()); + container.remove(); + }); + const render = (node: React.ReactNode) => act(() => root.render( + + {node} + , + )); + const settle = () => act(async () => { for (let i = 0; i < 5; i++) await Promise.resolve(); }); + + it("running with a total: a filling bar, the message, percent and elapsed time", () => { + act(() => root.render()); + const bar = container.querySelector('[role="progressbar"]')!; + expect(bar.getAttribute("aria-valuenow")).toBe("40"); + expect(container.querySelector('[data-testid="task-chat-tool-progress-fill"]')!.style.width).toBe("40%"); + expect(container.textContent).toContain("Exporting rows"); + expect(container.textContent).toContain("40%"); + expect(container.querySelector('[data-testid="task-chat-tool-progress-elapsed"]')!.textContent).toBe("1m 5s"); + }); + + it("running without a total: an indeterminate sweep and no percent", () => { + act(() => root.render()); + expect(container.querySelector('[data-testid="task-chat-tool-progress-indeterminate"]')).not.toBeNull(); + expect(container.querySelector('[role="progressbar"]')!.hasAttribute("aria-valuenow")).toBe(false); + expect(container.textContent).not.toContain("%"); + expect(container.querySelector('[data-testid="task-chat-tool-progress-message"]')).toBeNull(); + expect(container.textContent).toContain("12s"); + }); + + it("renders provider text as plain text, never markup", () => { + act(() => root.render( done' })} nowMs={Date.parse("2026-11-13T10:00:01.000Z")} />)); + expect(container.querySelector("img")).toBeNull(); + expect(container.textContent).toContain(' done'); + }); + + it("drops the progress when the call completes", () => { + render(); + expect(container.querySelector('[data-testid="task-chat-tool-progress"]')).not.toBeNull(); + render(); + expect(container.querySelector('[data-testid="task-chat-tool-progress"]')).toBeNull(); + }); + + it("restores the saved progress of a running call after a reload", async () => { + savedProgress.list.mockResolvedValue([snapshot({ progress: 3, message: "Writing file" })]); + render(); + await vi.waitFor(async () => { + await settle(); + expect(container.querySelector('[data-testid="task-chat-tool-progress-message"]')?.textContent).toBe("Writing file"); + }); + expect(savedProgress.list).toHaveBeenCalledWith("issue-1"); + }); + + it("reads fresh saved progress when someone leaves the task and comes back mid-call", async () => { + savedProgress.list.mockResolvedValue([snapshot({ progress: 1, message: "Step 1" })]); + render(); + await vi.waitFor(async () => { + await settle(); + expect(container.querySelector('[data-testid="task-chat-tool-progress-message"]')?.textContent).toBe("Step 1"); + }); + act(() => root.render(
)); + savedProgress.list.mockResolvedValue([snapshot({ progress: 4, message: "Step 4", updatedAt: "2026-11-13T10:00:09.000Z" })]); + render(); + await vi.waitFor(async () => { + await settle(); + expect(container.querySelector('[data-testid="task-chat-tool-progress-message"]')?.textContent).toBe("Step 4"); + }); + expect(savedProgress.list).toHaveBeenCalledTimes(2); + }); + + it("never asks for progress for a finished row", async () => { + render(); + await settle(); + expect(savedProgress.list).not.toHaveBeenCalled(); + expect(container.querySelector('[data-testid="task-chat-tool-progress"]')).toBeNull(); + }); +}); diff --git a/ui/src/components/task-chat/tool-call-progress.tsx b/ui/src/components/task-chat/tool-call-progress.tsx new file mode 100644 index 00000000000..0d71a7d3112 --- /dev/null +++ b/ui/src/components/task-chat/tool-call-progress.tsx @@ -0,0 +1,221 @@ +import { createContext, useContext, useEffect, useMemo, useState, useSyncExternalStore, type ReactNode } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { + boundToolCallProgressMessage, + normalizeToolCallProgress, + toolCallProgressFraction, + type ToolCallProgressSnapshot, +} from "@paperclipai/shared"; +import { mcpAppsApi } from "@/api/mcpApps"; +import { useSecondTick } from "@/hooks/useSecondTick"; +import { useCompanyLiveEvent } from "@/context/LiveUpdatesProvider"; +import { queryKeys } from "@/lib/queryKeys"; +import { cn, formatDurationMs } from "@/lib/utils"; +import type { TaskChatToolItem } from "./task-chat-model"; + +/** + * Live progress for a running connected-app tool call. The provider streams + * MCP progress to Paperclip, which publishes it on the run's live events and + * saves the latest on the call (read once, for a reload mid-call). + */ + +/** The run whose rows are showing, when known; progress from other runs is ignored. */ +const ToolCallProgressRunContext = createContext(null); + +export function ToolCallProgressRunProvider({ runId, children }: { runId: string | null | undefined; children: ReactNode }) { + return {children}; +} + +/** Lowercase words joined by "_", so "mcp.kv-demo-1a:slow" and "mcp_kv_demo_1a_slow" compare equal. */ +function comparableToolName(name: string) { + return name.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, ""); +} + +/** + * Whether a transcript row's tool name ("mcp__paperclip__") is + * the gateway tool a call reported. Agents may rewrite characters their tool + * names cannot hold, so names are compared word by word, as a whole suffix. + */ +export function toolCallNameMatches(rawName: string | undefined, toolName: string): boolean { + if (!rawName || !toolName) return false; + const row = comparableToolName(rawName); + const tool = comparableToolName(toolName); + return Boolean(tool) && (row === tool || row.endsWith(`_${tool}`)); +} + +/** A live event payload as a snapshot, or null when it is not one. Values are checked again here. */ +export function readToolCallProgressEvent(payload: Record): ToolCallProgressSnapshot | null { + const invocationId = typeof payload.invocationId === "string" ? payload.invocationId : null; + const runId = typeof payload.runId === "string" ? payload.runId : null; + const toolName = typeof payload.toolName === "string" ? payload.toolName : null; + const nativeToolName = typeof payload.nativeToolName === "string" ? payload.nativeToolName : null; + const startedAt = typeof payload.startedAt === "string" && Number.isFinite(Date.parse(payload.startedAt)) ? payload.startedAt : null; + if (!invocationId || !runId || !toolName || !startedAt) return null; + const updatedAt = typeof payload.updatedAt === "string" && Number.isFinite(Date.parse(payload.updatedAt)) ? new Date(payload.updatedAt) : new Date(); + const progress = normalizeToolCallProgress(payload, updatedAt); + if (!progress) return null; + return { ...progress, invocationId, runId, toolName, nativeToolName, startedAt, done: payload.done === true }; +} + +/** Whether a row is a call of this tool, under its gateway name or the native runner's name for it. */ +export function rowShowsTool(item: Pick, entry: Pick): boolean { + // A wrapper call (OpenCode's paperclip_call_assigned_tool) is matched by the tool it runs. + return [item.appToolName, item.rawName ?? item.name].some((rowName) => Boolean(rowName) && ( + toolCallNameMatches(rowName, entry.toolName) + || (Boolean(entry.nativeToolName) && toolCallNameMatches(rowName, entry.nativeToolName!)))); +} + +/** The progress for a row: the latest unfinished call of the same tool, in the same run when known. */ +export function pickToolCallProgress( + entries: Iterable, + item: Pick, + runId: string | null, +): ToolCallProgressSnapshot | null { + let best: ToolCallProgressSnapshot | null = null; + for (const entry of entries) { + if (entry.done || (runId && entry.runId !== runId)) continue; + if (!rowShowsTool(item, entry)) continue; + if (!best || entry.startedAt > best.startedAt) best = entry; + } + return best; +} + +/** Running calls kept per chat; the oldest are forgotten first. */ +const MAX_TRACKED_CALLS = 50; +/** Finished call ids remembered per chat, so a delayed read cannot revive one. */ +const MAX_REMEMBERED_FINISHED = 500; + +/** + * The chat's known tool call progress. Rows subscribe only while running, so a + * burst of updates re-renders the running row, never the whole chat. + */ +export class ToolCallProgressStore { + #entries = new Map(); + #finished = new Set(); + #listeners = new Set<() => void>(); + #onWanted: () => void; + constructor(onWanted: () => void = () => undefined) { this.#onWanted = onWanted; } + readonly subscribe = (listener: () => void) => { + this.#listeners.add(listener); + return () => { this.#listeners.delete(listener); }; + }; + /** A running row asks once for the saved progress (a reload mid-call). */ + want() { this.#onWanted(); } + entries() { return this.#entries.values(); } + /** + * Keeps the newer update of each running call. A finished call is removed + * and remembered (bounded, apart from shown progress), so a later "running" + * snapshot of it, from a delayed read, never brings its progress back. + */ + put(snapshot: ToolCallProgressSnapshot) { + const id = snapshot.invocationId; + if (this.#finished.has(id)) return; + if (snapshot.done) { + this.#finished.add(id); + while (this.#finished.size > MAX_REMEMBERED_FINISHED) this.#finished.delete(this.#finished.values().next().value!); + if (!this.#entries.delete(id)) return; + } else { + const previous = this.#entries.get(id); + if (previous && previous.updatedAt > snapshot.updatedAt) return; + this.#entries.delete(id); + this.#entries.set(id, snapshot); + while (this.#entries.size > MAX_TRACKED_CALLS) this.#entries.delete(this.#entries.keys().next().value!); + } + for (const listener of this.#listeners) listener(); + } +} + +const ToolCallProgressStoreContext = createContext(null); + +/** + * One per task chat: listens for the task's tool call progress on the shared + * live socket, and reads the saved progress once, only when a call is running. + */ +export function ToolCallProgressProvider({ issueId, children }: { issueId: string | null | undefined; children: ReactNode }) { + const [wantedFor, setWantedFor] = useState(null); + // A new store (and a fresh read) per task and per mount: updates sent while the + // chat was closed are only in the saved snapshot. + const store = useMemo(() => new ToolCallProgressStore(() => setWantedFor(issueId ?? null)), [issueId]); + const saved = useQuery({ + queryKey: queryKeys.toolCallProgress(issueId ?? ""), + queryFn: () => mcpAppsApi.toolCallProgress(issueId!), + enabled: Boolean(issueId) && wantedFor === issueId, + // Read again whenever a chat with a running call mounts; live events cover the rest. + staleTime: 0, + refetchOnMount: "always", + // Live events are not handled while the tab is hidden; read again on return. + refetchOnWindowFocus: true, + }); + useEffect(() => { + // Finished calls arrive as done; a call missing from the read is never dropped, + // since a running call may not have saved its first update yet. + for (const entry of saved.data ?? []) store.put(entry); + }, [saved.data, store]); + useCompanyLiveEvent((event) => { + if (event.type !== "heartbeat.run.tool_progress" || !issueId || event.payload?.issueId !== issueId) return; + const snapshot = readToolCallProgressEvent(event.payload); + if (snapshot) store.put(snapshot); + }); + return {children}; +} + +const noSubscription = () => () => undefined; + +/** Progress for a running tool row; null for any other row, which never re-renders for progress. */ +export function useToolCallProgress(item: TaskChatToolItem): ToolCallProgressSnapshot | null { + const store = useContext(ToolCallProgressStoreContext); + const runId = useContext(ToolCallProgressRunContext); + const running = item.status === "in_progress" && store !== null; + useEffect(() => { if (running) store!.want(); }, [running, store]); + return useSyncExternalStore( + running ? store!.subscribe : noSubscription, + () => (running ? pickToolCallProgress(store!.entries(), item, runId) : null), + ); +} + +/** + * The running row's progress: a bar that fills when the total is known (a + * quiet sweep otherwise), the provider's latest message as plain text, and + * elapsed time. + */ +export function TaskChatToolProgress({ progress, nowMs }: { progress: ToolCallProgressSnapshot; nowMs?: number }) { + // The shared one-second ticker, only while no fixed time is given. + useSecondTick(nowMs === undefined); + const now = nowMs ?? Date.now(); + const fraction = toolCallProgressFraction(progress); + const percent = fraction === null ? null : Math.round(fraction * 100); + // Rendered as text only: React escapes it, and it is bounded again here. + const message = boundToolCallProgressMessage(progress.message); + const elapsed = formatDurationMs(Math.max(0, now - Date.parse(progress.startedAt))); + return ( +
+
+
+
+
+ {message ? ( + {message} + ) : } + {percent !== null ? {percent}% : null} + {elapsed} +
+
+ ); +} diff --git a/ui/src/components/task-chat/transcript-adapter.ts b/ui/src/components/task-chat/transcript-adapter.ts index 964bfd91b65..65f29ebfc4f 100644 --- a/ui/src/components/task-chat/transcript-adapter.ts +++ b/ui/src/components/task-chat/transcript-adapter.ts @@ -197,6 +197,17 @@ function isOpaqueId(value: unknown): boolean { * The task a tool call is about, when its input names one by UUID. The row * shows that task's identifier (resolved at render time) instead of the id. */ +/** Runners without direct app tools call them through this wrapper, naming the tool in its input. */ +const WRAPPED_APP_TOOL = "paperclip_call_assigned_tool"; + +/** The app tool a wrapper call runs (its input's `name`), or undefined for any other call. */ +export function wrappedAppToolName(rawName: string | null | undefined, input: unknown): string | undefined { + if (!rawName || !(rawName === WRAPPED_APP_TOOL || rawName.endsWith(`_${WRAPPED_APP_TOOL}`) || rawName.endsWith(`.${WRAPPED_APP_TOOL}`))) return undefined; + if (input == null || typeof input !== "object" || Array.isArray(input)) return undefined; + const name = (input as Record).name; + return typeof name === "string" && name.trim() ? name.trim() : undefined; +} + export function toolInputTaskRef(input: unknown): string | undefined { if (input == null || typeof input !== "object" || Array.isArray(input)) return undefined; const record = input as Record; @@ -908,6 +919,7 @@ export function transcriptToTaskChatItems( } } if (entry.detail) applyToolDetail(existing, entry.detail); + existing.appToolName ??= wrappedAppToolName(existing.rawName, entry.input); lastToolIndex = existingIndex!; } else { const tool: TaskChatToolItem = { @@ -917,6 +929,7 @@ export function transcriptToTaskChatItems( rawName: entry.name ?? undefined, target: summarizeToolInput(entry.input), ...(toolInputTaskRef(entry.input) ? { taskRefId: toolInputTaskRef(entry.input) } : {}), + ...(wrappedAppToolName(entry.name, entry.input) ? { appToolName: wrappedAppToolName(entry.name, entry.input) } : {}), status: "in_progress", ...(entry.parentToolUseId ? { parentToolUseId: entry.parentToolUseId } : {}), }; diff --git a/ui/src/components/task-chat/useWaitingToStart.ts b/ui/src/components/task-chat/useWaitingToStart.ts new file mode 100644 index 00000000000..68ce4ba4640 --- /dev/null +++ b/ui/src/components/task-chat/useWaitingToStart.ts @@ -0,0 +1,29 @@ +import { useEffect, useReducer } from "react"; + +/** + * How long a queued turn keeps the startup feedback ("Working" / "Thinking") + * before it says it is waiting. DESIGN.md keeps that feedback from message + * submission through admission so a normal start does not flicker. Admission + * normally claims a run within a few seconds; a run still queued after 30 s + * with no named hold is not starting, and saying "Working" would be untrue. + */ +export const QUEUED_START_GRACE_MS = 30_000; +export const WAITING_TO_START_LABEL = "Waiting to start"; + +/** + * True once an eligible queued turn has waited past the grace period since + * `sinceMs`. One timer at the threshold; no per-second ticking. + */ +export function useWaitingToStart(eligible: boolean, sinceMs: number | null): boolean { + const [, rerender] = useReducer((count: number) => count + 1, 0); + const remainingMs = eligible && sinceMs != null ? QUEUED_START_GRACE_MS - (Date.now() - sinceMs) : null; + const pending = remainingMs != null && remainingMs > 0; + useEffect(() => { + if (!eligible || sinceMs == null) return; + const delay = QUEUED_START_GRACE_MS - (Date.now() - sinceMs); + if (delay <= 0) return; + const timer = setTimeout(rerender, delay + 50); + return () => clearTimeout(timer); + }, [eligible, sinceMs, pending]); + return remainingMs != null && remainingMs <= 0; +} diff --git a/ui/src/context/LiveUpdatesProvider.tsx b/ui/src/context/LiveUpdatesProvider.tsx index da10b1800a3..da33c16bbe1 100644 --- a/ui/src/context/LiveUpdatesProvider.tsx +++ b/ui/src/context/LiveUpdatesProvider.tsx @@ -2155,6 +2155,9 @@ function handleLiveEvent( const nameOf = (id: string) => resolveAgentName(queryClient, expectedCompanyId, id); + // Tool call progress is read by the tool row itself (useToolCallProgress); it + // changes no cached run state and must not invalidate queries several times a second. + if (event.type === "heartbeat.run.tool_progress") return; const payload = event.payload ?? {}; // Resolve membership before terminal lifecycle patches remove live-run rows. const suppressRunToast = diff --git a/ui/src/features/connections/ConnectionSetupFlow.tsx b/ui/src/features/connections/ConnectionSetupFlow.tsx index e3ba1e0123d..aa311babc4b 100644 --- a/ui/src/features/connections/ConnectionSetupFlow.tsx +++ b/ui/src/features/connections/ConnectionSetupFlow.tsx @@ -1465,6 +1465,19 @@ function StandardConnectionSetupFlow({ }; const effectiveGrantKind = fixedGrantKind ?? grantKind; + // A connected result (from the link probe or machine sign-in) goes straight to + // review with every discovered action enabled by default. + function acceptConnectedResult(result: ConnectToolAppResult) { + setLinkGuidance(null); + setConnectResult(result); + const defaults: Record = {}; + for (const a of result.actions.readOnly) defaults[a.catalogEntryId] = true; + for (const a of result.actions.canMakeChanges) defaults[a.catalogEntryId] = true; + setEnabled(defaults); + finishMutation.mutate({ result, enabled: defaults }); + } + const acceptMachineSignInResult = acceptConnectedResult; + const connectMutation = useMutation({ mutationFn: async (entryOverride?: AppDefinition) => { if (requiredAgentId && !resumeConnectionId && !reconnectConnectionId @@ -1590,13 +1603,7 @@ function StandardConnectionSetupFlow({ } oauthPopupRef.current?.close(); oauthPopupRef.current = null; - setLinkGuidance(null); - setConnectResult(result); - const defaults: Record = {}; - for (const a of result.actions.readOnly) defaults[a.catalogEntryId] = true; - for (const a of result.actions.canMakeChanges) defaults[a.catalogEntryId] = true; - setEnabled(defaults); - finishMutation.mutate({ result, enabled: defaults }); + acceptConnectedResult(result); }, onError: (error) => { oauthPopupRef.current?.close(); @@ -2826,6 +2833,9 @@ function StandardConnectionSetupFlow({ setLinkGuidance(null); connectApp(); }} + machineSignIn={effectiveGrantKind === "organization" && selectedCompanyId && !reconnectConnectionId + ? { companyId: selectedCompanyId, onConnected: acceptMachineSignInResult } + : null} /> )} diff --git a/ui/src/features/connections/GenericMcpConnectStep.machine.test.tsx b/ui/src/features/connections/GenericMcpConnectStep.machine.test.tsx new file mode 100644 index 00000000000..3304705a9fd --- /dev/null +++ b/ui/src/features/connections/GenericMcpConnectStep.machine.test.tsx @@ -0,0 +1,69 @@ +// @vitest-environment jsdom +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { newCustomHeaderRow } from "@/pages/apps/generic-mcp-connect"; +import { GenericMcpConnectStep } from "./GenericMcpConnectStep"; + +vi.mock("@/api/tools", () => ({ toolsApi: {} })); +vi.mock("@/pages/apps/AppLogo", () => ({ AppLogo: () => null })); + +describe("Machine sign-in in the custom server step", () => { + let root: Root; + let container: HTMLDivElement; + + beforeEach(() => { + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + }); + afterEach(() => { + act(() => root.unmount()); + container.remove(); + }); + + async function render(machineSignIn: { companyId: string; onConnected: () => void } | null) { + await act(async () => { + root.render( + undefined} + authMode="auto" onAuthModeChange={() => undefined} headers={[newCustomHeaderRow()]} onHeadersChange={() => undefined} + oauthClientId="" onOAuthClientIdChange={() => undefined} oauthClientSecret="" onOAuthClientSecretChange={() => undefined} + advancedOpen onAdvancedOpenChange={() => undefined} guidance={null} submitting={false} + onBack={() => undefined} onConnect={() => undefined} machineSignIn={machineSignIn} + defaults={

Connects for every agent. Change

} + />, + ); + }); + } + const buttons = () => [...container.querySelectorAll("button")].map((button) => button.textContent ?? ""); + + it("is offered for organization connections and replaces Check link with its own setup", async () => { + await render({ companyId: "company-1", onConnected: () => undefined }); + const option = [...container.querySelectorAll('[role="radio"]')].find((item) => item.textContent?.includes("Machine sign-in")); + expect(option).toBeDefined(); + expect(buttons()).toContain("Check link"); + await act(async () => { (option as HTMLElement).click(); }); + expect(container.querySelector('[data-testid="machine-sign-in-setup"]')).not.toBeNull(); + expect(buttons()).not.toContain("Check link"); + expect(buttons()).toContain("Create key"); + // Who gets access stays visible and changeable while machine setup is open. + expect(container.querySelector('[data-testid="access-defaults"]')).not.toBeNull(); + }); + + it("keeps machine sign-in open when arrow keys move its own method choice", async () => { + await render({ companyId: "company-1", onConnected: () => undefined }); + const option = [...container.querySelectorAll('[role="radio"]')].find((item) => item.textContent?.includes("Machine sign-in")); + await act(async () => { (option as HTMLElement).click(); }); + const inner = [...container.querySelectorAll('[role="radio"]')].find((item) => item.textContent === "Paperclip key (recommended)") as HTMLElement; + await act(async () => { inner.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true })); }); + expect(container.querySelector('[data-testid="machine-sign-in-setup"]')).not.toBeNull(); + const secret = [...container.querySelectorAll('[role="radio"]')].find((item) => item.textContent === "Client secret"); + expect(secret?.getAttribute("aria-checked")).toBe("true"); + }); + + it("is not offered for personal connections", async () => { + await render(null); + expect(container.textContent).not.toContain("Machine sign-in"); + }); +}); diff --git a/ui/src/features/connections/GenericMcpConnectStep.tsx b/ui/src/features/connections/GenericMcpConnectStep.tsx index 40e4b14759a..34dc87c8c9d 100644 --- a/ui/src/features/connections/GenericMcpConnectStep.tsx +++ b/ui/src/features/connections/GenericMcpConnectStep.tsx @@ -1,6 +1,6 @@ import { useState, type KeyboardEvent, type ReactNode } from "react"; import { Loader2, Plus, X } from "lucide-react"; -import type { AppDefinition } from "@paperclipai/shared"; +import type { AppDefinition, ConnectToolAppResult } from "@paperclipai/shared"; import { InlineBanner } from "@/components/InlineBanner"; import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; @@ -8,6 +8,7 @@ import { Input } from "@/components/ui/input"; import { RadioCard } from "@/components/ui/radio-card"; import { redactUrlSecrets } from "@/lib/redact-url-secrets"; import { AppLogo } from "@/pages/apps/AppLogo"; +import { MachineSignInSetup } from "./MachineSignInSetup"; import { canSubmitGenericConnect, customHeaderError, @@ -112,6 +113,7 @@ export function GenericMcpConnectStep({ defaults, onBack, onConnect, + machineSignIn, }: { link: string; name: string; @@ -138,8 +140,13 @@ export function GenericMcpConnectStep({ defaults?: ReactNode; onBack: () => void; onConnect: () => void; + /** Offered only for organization connections: personal connections sign in as the person. */ + machineSignIn?: { companyId: string; onConnected: (result: ConnectToolAppResult) => void } | null; }) { const [clientFieldsOpen, setClientFieldsOpen] = useState(false); + // Machine sign-in runs its own setup and test, so it is a choice beside the modes, not one of them. + const [machineSelected, setMachineSelected] = useState(false); + const machine = Boolean(machineSelected && machineSignIn); const headerError = authMode === "custom_headers" ? customHeaderError(headers) : null; const canSubmit = canSubmitGenericConnect({ link, @@ -156,7 +163,7 @@ export function GenericMcpConnectStep({ // Stay open while the chosen option still needs something typed into it, so // a disabled primary action always has its reason on screen. const choiceNeedsInput = (authMode === "bearer" || authMode === "custom_headers") && !canSubmit; - const expanded = advancedOpen || choiceNeedsInput; + const expanded = advancedOpen || choiceNeedsInput || machine; const showClientFields = clientFieldsOpen || oauthClientId.trim().length > 0 || oauthClientSecret.trim().length > 0 @@ -169,6 +176,8 @@ export function GenericMcpConnectStep({ const moveSelection = (event: KeyboardEvent) => { // Arrow keys inside an option's own fields belong to that field. if ((event.target as HTMLElement).getAttribute("role") !== "radio") return; + // Only this group's own options: a nested choice (machine sign-in's) handles its own keys. + if ((event.target as HTMLElement).closest('[role="radiogroup"]') !== event.currentTarget) return; const step = event.key === "ArrowDown" || event.key === "ArrowRight" ? 1 : event.key === "ArrowUp" || event.key === "ArrowLeft" @@ -176,9 +185,14 @@ export function GenericMcpConnectStep({ : 0; if (step === 0) return; event.preventDefault(); - const index = AUTH_OPTIONS.findIndex((option) => option.mode === authMode); - const next = (index + step + AUTH_OPTIONS.length) % AUTH_OPTIONS.length; - onAuthModeChange(AUTH_OPTIONS[next]!.mode); + const count = AUTH_OPTIONS.length + (machineSignIn ? 1 : 0); + const index = machine ? AUTH_OPTIONS.length : AUTH_OPTIONS.findIndex((option) => option.mode === authMode); + const next = (index + step + count) % count; + if (next === AUTH_OPTIONS.length) setMachineSelected(true); + else { + setMachineSelected(false); + onAuthModeChange(AUTH_OPTIONS[next]!.mode); + } event.currentTarget.querySelectorAll('[role="radio"]')[next]?.focus(); }; @@ -353,7 +367,7 @@ export function GenericMcpConnectStep({ type="button" variant="link" className="h-auto p-0 text-xs font-semibold underline underline-offset-2" - disabled={submitting || choiceNeedsInput} + disabled={submitting || choiceNeedsInput || machine} data-testid="generic-mcp-sign-in-toggle" > {expanded ? "Done" : "Change"} @@ -368,7 +382,7 @@ export function GenericMcpConnectStep({ onKeyDown={moveSelection} > {AUTH_OPTIONS.map((option) => { - const selected = option.mode === authMode; + const selected = !machine && option.mode === authMode; const fields = selected ? optionFields(option.mode) : null; return (
@@ -379,27 +393,60 @@ export function GenericMcpConnectStep({ tabIndex={selected ? 0 : -1} disabled={submitting} className="bg-background" - onClick={() => onAuthModeChange(option.mode)} + onClick={() => { + setMachineSelected(false); + onAuthModeChange(option.mode); + }} /> {fields ?
{fields}
: null}
); })} + {machineSignIn ? ( +
+ setMachineSelected(true)} + /> + {machine ? ( +
+ +
+ ) : null} +
+ ) : null}
- {defaults} + {/* Machine sign-in owns its footer (Test connection, not Check link) and shows the access line above it. */} + {machine ? null : ( + <> + {defaults} -
- - -
+
+ + +
+ + )}
); } diff --git a/ui/src/features/connections/MachineSignInSetup.test.tsx b/ui/src/features/connections/MachineSignInSetup.test.tsx new file mode 100644 index 00000000000..3dd8b219370 --- /dev/null +++ b/ui/src/features/connections/MachineSignInSetup.test.tsx @@ -0,0 +1,137 @@ +// @vitest-environment jsdom +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import type { ConnectToolAppResult, MachineSignInSetupResult } from "@paperclipai/shared"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MachineSignInSetup, type MachineSignInClient } from "./MachineSignInSetup"; + +vi.mock("@/api/tools", () => ({ toolsApi: {} })); + +const draft = { + connectionId: "connection-1", + connection: {}, + machineSignIn: { + connectionId: "connection-1", method: "paperclip_key", clientId: null, issuer: "https://auth.example", tokenUrl: "https://auth.example/token", + scopes: [], hasClientSecret: false, lastTokenAt: null, pendingPublicKey: null, + publicKey: { keyId: "kid-1", algorithm: "RS256", pem: "-----BEGIN PUBLIC KEY-----\nabc\n-----END PUBLIC KEY-----\n", jwk: { kty: "RSA", kid: "kid-1" }, createdAt: "2026-11-13T00:00:00Z" }, + }, +} as unknown as MachineSignInSetupResult; + +describe("Machine sign-in setup", () => { + let root: Root; + let container: HTMLDivElement; + const flush = async () => { await act(async () => { await new Promise((resolve) => setTimeout(resolve, 0)); }); }; + const button = (name: string) => [...container.querySelectorAll("button")].find((item) => item.textContent === name); + const click = async (name: string) => { await act(async () => { button(name)!.click(); }); await flush(); }; + const type = async (id: string, value: string) => { + const input = container.querySelector(`#${id}`)!; + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(input, value); + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + }; + + beforeEach(() => { + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + }); + afterEach(() => { + act(() => root.unmount()); + container.remove(); + }); + + it("creates a key, shows the public key to register, then tests with the client ID", async () => { + const client: MachineSignInClient = { + create: vi.fn(async () => draft), + test: vi.fn(async () => ({ connectionId: "connection-1" }) as ConnectToolAppResult), + status: vi.fn(), + }; + const onConnected = vi.fn(); + await act(async () => { + root.render( undefined} onConnected={onConnected} />); + }); + expect(container.textContent).toContain("Paperclip key (recommended)"); + await click("Create key"); + expect(client.create).toHaveBeenCalledWith("company-1", { link: "https://mcp.example/mcp", method: "paperclip_key" }); + expect(container.textContent).toContain("Register this public key"); + expect(container.querySelector("textarea")?.value).toContain("BEGIN PUBLIC KEY"); + expect(button("Test connection")?.disabled).toBe(true); + await type("machine-sign-in-client-id-confirm", "paperclip-automation"); + await click("Test connection"); + expect(client.test).toHaveBeenCalledWith("connection-1", { clientId: "paperclip-automation", scopes: "" }); + expect(onConnected).toHaveBeenCalled(); + }); + + it("keeps the draft and shows the provider's answer when the test fails", async () => { + const client: MachineSignInClient = { + create: vi.fn(async () => draft), + test: vi.fn(async () => { throw new Error("The provider didn't accept Paperclip's key."); }), + status: vi.fn(), + }; + await act(async () => { + root.render( undefined} onConnected={() => undefined} />); + }); + await type("machine-sign-in-client-id-confirm", "wrong"); + await click("Test connection"); + expect(container.textContent).toContain("Machine sign-in didn’t work"); + expect(container.textContent).toContain("The provider didn't accept Paperclip's key."); + expect(container.textContent).toContain("Register this public key"); + }); + + it("asks for the client ID and secret together for a client secret", async () => { + const client: MachineSignInClient = { + create: vi.fn(async () => ({ ...draft, machineSignIn: { ...draft.machineSignIn, method: "client_secret" as const, publicKey: null } })), + test: vi.fn(async () => ({}) as ConnectToolAppResult), + status: vi.fn(), + }; + await act(async () => { + root.render( undefined} onConnected={() => undefined} />); + }); + expect(button("Test connection")?.disabled).toBe(true); + await type("machine-sign-in-client-id", "id-1"); + await type("machine-sign-in-client-secret", "secret-1"); + await click("Test connection"); + expect(client.create).toHaveBeenCalledWith("company-1", expect.objectContaining({ method: "client_secret", clientId: "id-1", clientSecret: "secret-1" })); + expect(client.test).toHaveBeenCalledWith("connection-1", { clientId: "id-1", scopes: "" }); + }); + + it("lets a wrong secret be replaced after a failed test", async () => { + const secretDraft = { ...draft, machineSignIn: { ...draft.machineSignIn, method: "client_secret" as const, publicKey: null, clientId: "id-1", hasClientSecret: true, scopes: ["a.read"] } }; + const client: MachineSignInClient = { + create: vi.fn(), + test: vi.fn(async () => { throw new Error("The provider didn't accept the client ID and secret."); }), + status: vi.fn(), + }; + await act(async () => { + root.render( undefined} onConnected={() => undefined} />); + }); + await click("Test connection"); + expect(container.querySelector("#machine-sign-in-client-secret-replace")).not.toBeNull(); + await type("machine-sign-in-client-secret-replace", "fixed-secret"); + await type("machine-sign-in-scopes-confirm", ""); + await click("Test connection"); + expect(client.test).toHaveBeenLastCalledWith("connection-1", { clientId: "id-1", scopes: "", clientSecret: "fixed-secret" }); + }); + + it("resumes the key draft this tab started instead of creating another", async () => { + sessionStorage.setItem("paperclip.machine-sign-in-draft:company-1:https://mcp.example/mcp", "connection-1"); + let finish!: (value: typeof draft.machineSignIn) => void; + const client: MachineSignInClient = { + create: vi.fn(), + test: vi.fn(), + status: vi.fn(() => new Promise((resolve) => { finish = resolve; })), + }; + await act(async () => { + root.render( undefined} onConnected={() => undefined} />); + }); + // No second draft can be started while the saved one loads. + expect(button("Create key")?.disabled).toBe(true); + await act(async () => { finish(draft.machineSignIn); }); + await flush(); + expect(client.status).toHaveBeenCalledWith("connection-1"); + expect(container.textContent).toContain("Register this public key"); + expect(client.create).not.toHaveBeenCalled(); + sessionStorage.clear(); + }); +}); diff --git a/ui/src/features/connections/MachineSignInSetup.tsx b/ui/src/features/connections/MachineSignInSetup.tsx new file mode 100644 index 00000000000..98f29980b62 --- /dev/null +++ b/ui/src/features/connections/MachineSignInSetup.tsx @@ -0,0 +1,324 @@ +import { useEffect, useState } from "react"; +import { Loader2 } from "lucide-react"; +import type { + ConnectToolAppResult, + CreateMachineSignIn, + MachineSignInMethod, + MachineSignInSetupResult, + MachineSignInStatus, + TestMachineSignIn, +} from "@paperclipai/shared"; +import { toolsApi } from "@/api/tools"; +import { describeError } from "@/api/errors"; +import { InlineBanner } from "@/components/InlineBanner"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { cn } from "@/lib/utils"; +import { copyTextToClipboard } from "@/lib/clipboard"; + +/** The calls this setup makes. Stories pass in-memory fakes. */ +export interface MachineSignInClient { + create(companyId: string, input: CreateMachineSignIn): Promise; + test(connectionId: string, input: TestMachineSignIn): Promise; + status(connectionId: string): Promise; +} + +export const machineSignInApiClient: MachineSignInClient = { + create: (companyId, input) => toolsApi.createMachineSignIn(companyId, input), + test: (connectionId, input) => toolsApi.testMachineSignIn(connectionId, input), + status: (connectionId) => toolsApi.getMachineSignIn(connectionId), +}; + +type MachineSignInDraft = Pick; + +/** + * The draft this browser tab started for a server, so switching options or + * reloading resumes it (and its key) instead of creating another. + */ +function draftStorageKey(companyId: string, link: string) { + return `paperclip.machine-sign-in-draft:${companyId}:${link}`; +} +function readDraftId(key: string): string | null { + try { return sessionStorage.getItem(key); } catch { return null; } +} +function writeDraftId(key: string, connectionId: string | null) { + try { + if (connectionId) sessionStorage.setItem(key, connectionId); + else sessionStorage.removeItem(key); + } catch { /* storage unavailable: the draft is still listed in Apps */ } +} + +const METHOD_OPTIONS: Array<{ method: MachineSignInMethod; label: string; hint: string }> = [ + { + method: "paperclip_key", + label: "Paperclip key (recommended)", + hint: "Paperclip creates a key and keeps its private half. You register the public key with the provider; nothing secret is copied.", + }, + { + method: "client_secret", + label: "Client secret", + hint: "For providers that only issue a secret. Paperclip stores it securely and never shows it again.", + }, +]; + +export interface MachineSignInSetupProps { + companyId: string; + link: string; + name?: string; + client?: MachineSignInClient; + /** Resume at "register the key" with a key Paperclip already created. */ + initialSetup?: MachineSignInSetupResult | null; + initialMethod?: MachineSignInMethod; + initialError?: string | null; + onBack: () => void; + onConnected: (result: ConnectToolAppResult) => void; + /** Shown above the footer, such as who gets access to the connection. */ + beforeFooter?: React.ReactNode; +} + +/** + * Machine sign-in for an organization connection: the server's provider issues + * Paperclip a client of its own, so no person signs in. With a Paperclip key the + * admin registers the public key we show; with a client secret they paste it. + * Either way the last step is a real test against the provider and the server. + */ +export function MachineSignInSetup({ + companyId, + link, + name, + client = machineSignInApiClient, + initialSetup = null, + initialMethod = "paperclip_key", + initialError = null, + onBack, + onConnected, + beforeFooter, +}: MachineSignInSetupProps) { + const [method, setMethod] = useState(initialSetup?.machineSignIn.method ?? initialMethod); + const [signInService, setSignInService] = useState(""); + const [clientId, setClientId] = useState(initialSetup?.machineSignIn.clientId ?? ""); + const [clientSecret, setClientSecret] = useState(""); + const [scopes, setScopes] = useState(initialSetup?.machineSignIn.scopes.join(" ") ?? ""); + const [setup, setSetup] = useState(initialSetup); + const [error, setError] = useState(initialError); + const [busy, setBusy] = useState<"creating" | "testing" | null>(null); + const storageKey = draftStorageKey(companyId, link); + // While a saved draft loads, creating another would race it; wait for it. + const [restoring, setRestoring] = useState(() => !initialSetup && Boolean(readDraftId(storageKey))); + + useEffect(() => { + if (initialSetup) return; + const draftId = readDraftId(storageKey); + if (!draftId) return; + let cancelled = false; + setRestoring(true); + client.status(draftId).then((status) => { + if (cancelled) return; + setSetup({ connectionId: draftId, machineSignIn: status }); + setMethod(status.method); + setClientId((current) => current || (status.clientId ?? "")); + setScopes((current) => current || status.scopes.join(" ")); + }, () => writeDraftId(storageKey, null)).finally(() => { + if (!cancelled) setRestoring(false); + }); + return () => { cancelled = true; }; + }, [client, initialSetup, storageKey]); + + const trimmedScopes = scopes.trim(); + const canCreate = method === "paperclip_key" || (clientId.trim().length > 0 && clientSecret.length > 0); + const canTest = clientId.trim().length > 0; + + async function createDraft(): Promise { + setBusy("creating"); + setError(null); + try { + const created = await client.create(companyId, { + link, + ...(name?.trim() ? { name: name.trim() } : {}), + ...(signInService.trim() ? { authorizationServer: signInService.trim() } : {}), + method, + ...(clientId.trim() ? { clientId: clientId.trim() } : {}), + ...(method === "client_secret" && clientSecret ? { clientSecret } : {}), + ...(trimmedScopes ? { scopes: trimmedScopes } : {}), + }); + setSetup(created); + writeDraftId(storageKey, created.connectionId); + // The secret is saved with the draft; a later test only sends a replacement. + setClientSecret(""); + return created; + } catch (caught) { + setError(describeError(caught).body); + return null; + } finally { + setBusy(null); + } + } + + /** `replacementSecret` is sent only when the admin typed a new one after the draft was saved. */ + async function testConnection(target: MachineSignInDraft, replacementSecret: string) { + setBusy("testing"); + setError(null); + try { + const result = await client.test(target.connectionId, { + clientId: clientId.trim(), + // Always sent, so clearing the field clears the saved scopes. + scopes: trimmedScopes, + ...(target.machineSignIn.method === "client_secret" && replacementSecret ? { clientSecret: replacementSecret } : {}), + }); + writeDraftId(storageKey, null); + onConnected(result); + } catch (caught) { + setError(describeError(caught).body); + } finally { + setBusy(null); + } + } + + const host = (() => { + try { return new URL(link).host; } catch { return link; } + })(); + const publicKey = setup?.machineSignIn.publicKey ?? null; + + return ( +
+

+ For automation your organization owns. Agents use this connection as the provider client you set up for + Paperclip, not as any person. Only shared connections can use machine sign-in. +

+ + {error ? {error} : null} + + {!setup ? ( +
+
{ + const step = event.key === "ArrowRight" || event.key === "ArrowDown" ? 1 + : event.key === "ArrowLeft" || event.key === "ArrowUp" ? -1 : 0; + if (step === 0) return; + event.preventDefault(); + event.stopPropagation(); + const index = METHOD_OPTIONS.findIndex((option) => option.method === method); + const next = (index + step + METHOD_OPTIONS.length) % METHOD_OPTIONS.length; + setMethod(METHOD_OPTIONS[next]!.method); + event.currentTarget.querySelectorAll('[role="radio"]')[next]?.focus(); + }} + > + {METHOD_OPTIONS.map((option) => ( + + ))} +
+

{METHOD_OPTIONS.find((option) => option.method === method)?.hint}

+ + setSignInService(event.target.value)} placeholder="Found automatically" autoComplete="off" className="h-11 font-mono" /> + + + setClientId(event.target.value)} autoComplete="off" className="h-11 font-mono" /> + + {method === "client_secret" ? ( + + setClientSecret(event.target.value)} autoComplete="off" className="h-11 font-mono" /> + + ) : null} + + setScopes(event.target.value)} placeholder="Optional" autoComplete="off" className="h-11 font-mono" /> + +
+ ) : ( +
+ {publicKey ? ( +
+

+ Register this public key with the provider for the Paperclip client. It isn’t secret, so you can paste it + into the provider’s console or send it to whoever manages it. +

+ + +
+ ) : null} + + setClientId(event.target.value)} autoComplete="off" className="h-11 font-mono" /> + + {setup.machineSignIn.method === "client_secret" ? ( + + setClientSecret(event.target.value)} placeholder={setup.machineSignIn.hasClientSecret ? "••••••••" : ""} autoComplete="off" className="h-11 font-mono" /> + + ) : null} + + setScopes(event.target.value)} autoComplete="off" className="h-11 font-mono" /> + +
+ )} + + {beforeFooter} + +
+ + {!setup && method === "paperclip_key" ? ( + + ) : ( + + )} +
+
+ ); +} + +function Field({ id, label, hint, children }: { id: string; label: string; hint?: string; children: React.ReactNode }) { + return ( +
+ +
{children}
+ {hint ?

{hint}

: null} +
+ ); +} + +function PublicKeyBlock({ label, value }: { label: string; value: string }) { + const [copied, setCopied] = useState(false); + return ( +
+
+ {label} + +
+