mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
Merge master and keep the current MCP approval fixture
Resolve the overlap with the temporary demo test correction by keeping master's newer fixture. It reads the option ID from the generated card and submits the answer through the interaction service. The GitHub bridge implementation and its regression tests are unchanged. Verify the resolved demo flow and the focused GitHub broker contracts. Run the server typecheck, then require fresh CI and Greptile review. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
843cd9945e
139 files changed
+11293
-374
No files matched your search
@@ -0,0 +1,41 @@
|
||||
name: Merge queue checks
|
||||
|
||||
# The master ruleset requires `ci / verify` and `ci / e2e`. pr.yml reports them
|
||||
# for a pull request, and a pull request enters the merge queue only after both
|
||||
# pass there. The queue then waits for the same two names on its merge-group
|
||||
# commit, which no pull_request workflow can report. These jobs report them, so
|
||||
# the queue does not stall.
|
||||
#
|
||||
# They run no tests on purpose: a queued merge gets the protection a direct
|
||||
# merge has today, and the exact-source gate still tests every master commit
|
||||
# before its image is published. To test the combined tree before it lands,
|
||||
# replace these jobs with real ones that keep the same names.
|
||||
#
|
||||
# Nothing here reads the merge-group commit: no checkout, no secrets, no token
|
||||
# permissions, GitHub-hosted runners.
|
||||
#
|
||||
# Inert until the merge queue is enabled for master: no merge_group event fires
|
||||
# before then.
|
||||
on:
|
||||
merge_group:
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: merge-queue-checks-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
name: ci / verify
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
steps:
|
||||
- run: echo "ci / verify passed on every pull request in this merge group before it was queued."
|
||||
|
||||
e2e:
|
||||
name: ci / e2e
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
steps:
|
||||
- run: echo "ci / e2e passed on every pull request in this merge group before it was queued."
|
||||
@@ -251,6 +251,16 @@ jobs:
|
||||
- name: Test release verify workflow wiring
|
||||
run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/cloud-readiness-runner-loss.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs
|
||||
|
||||
- name: Test merge-queue workflow wiring
|
||||
# This workflow loads from master and tests the caller's checkout. A
|
||||
# rerun of an older pull request has no copy of these tests.
|
||||
run: |
|
||||
tests=()
|
||||
for test in ./scripts/merge-group-image.test.mjs ./scripts/__tests__/merge-queue-checks-workflow.test.mjs; do
|
||||
if [[ -f "$test" ]]; then tests+=("$test"); fi
|
||||
done
|
||||
if (( ${#tests[@]} > 0 )); then node "$CI_NODE_TEST" "${tests[@]}"; fi
|
||||
|
||||
- name: Test standalone package build concurrency
|
||||
run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs
|
||||
|
||||
|
||||
@@ -149,7 +149,10 @@ Runtime discovery must not replace the turn with a generic spinner or a
|
||||
|
||||
A queued turn shows the same `Working` header and shimmering `Thinking` activity
|
||||
as a running turn, starting at message submission. Keep specific server holds,
|
||||
such as `Waiting for maintenance` or agent setup, visible during admission.
|
||||
such as `Waiting for maintenance`, agent setup, or another open task of the same
|
||||
routine, visible during admission. Admission normally starts a run within seconds,
|
||||
so this startup feedback lasts at most 30 seconds: a turn still queued after that
|
||||
with no named hold says `Waiting to start`, without spinner or shimmer.
|
||||
A run that waits for one of its agent's concurrency slots is a hold too: show
|
||||
`Waiting for <agent> · N slots max · P of Q in line · Configure` (the shared
|
||||
`AgentQueueWait` component) with no spinner or elapsed timer, link Configure to the
|
||||
|
||||
@@ -388,6 +388,22 @@ Without a matching successful merge-group
|
||||
run, or on any mismatch or error, the push builds from source. The PID 1 probe,
|
||||
exact-source gate and signing apply on both paths.
|
||||
|
||||
The `master` ruleset requires `ci / verify` and `ci / e2e`, which `pr.yml`
|
||||
reports only for pull requests.
|
||||
[`merge-queue-checks.yml`](../.github/workflows/merge-queue-checks.yml) reports
|
||||
both names on each merge-group commit so the queue does not stall. It runs no
|
||||
tests: a pull request enters the queue only after both checks pass on it, and
|
||||
the exact-source gate still tests every `master` commit before publication.
|
||||
The queue can therefore land a commit while its image is still building. The
|
||||
`master` push waits up to 15 minutes for that build before it builds from
|
||||
source. Publication waits for the exact-source gate in any case.
|
||||
|
||||
To turn the queue on, add **Require merge queue** to the `master` ruleset with
|
||||
merge method Squash, build concurrency 5, group size 1 to 5 and a 60 minute
|
||||
status check timeout. After the first queued merge, confirm that the `master`
|
||||
commit SHA equals the SHA of its merge-group runs; promotion depends on it.
|
||||
To turn it off, remove the rule. `master` then builds from source again.
|
||||
|
||||
Canary dispatches first look up the full-SHA master image. Reuse requires a
|
||||
valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching
|
||||
registry digest, and signed provenance from this repository's `docker.yml`
|
||||
|
||||
@@ -181,6 +181,16 @@ refs and the same ownership checks. A personal resolver must not fall back to a
|
||||
company credential. Ownership mismatch is `grant_credential_invalid` and tells
|
||||
the owner to reconnect.
|
||||
|
||||
Dedicated agent connections keep credentials on their agent grant. Background
|
||||
catalog and health work may use a single active grant for that connection and
|
||||
company. An agent caller must use its own grant. Missing, revoked, or ambiguous
|
||||
agent authorizations fail before provider requests; they never fall back to a
|
||||
personal, organization, or connection-owned credential. Renaming or pausing the
|
||||
connection must preserve the grant and its secret bindings. A later successful
|
||||
catalog refresh clears an earlier connection-health error without new consent.
|
||||
An agent caller denied for lacking its own grant must not change shared health
|
||||
or the catalog, including when that agent has permission to manage the connection.
|
||||
|
||||
Existing personal connections with company-scoped invocation credentials require
|
||||
their owner to reconnect and enter a fresh key or secret URL. Reconnect creates
|
||||
a user-owned credential and updates the grant and declarations while preserving
|
||||
|
||||
@@ -91,7 +91,7 @@ this as a setup error and keeps the connection available for retry. See
|
||||
|
||||
The connect screen states the sign-in choice in one line, **Paperclip asks the
|
||||
server how to sign in.**, with a **Change** link, the same shape as the access
|
||||
line below it. **Change** lists five options, each with its fields directly
|
||||
line below it. **Change** lists five options (six for organization connections), each with its fields directly
|
||||
beneath it when selected:
|
||||
|
||||
- **Let Paperclip check** — the default; Paperclip probes and walks you through it.
|
||||
@@ -101,6 +101,10 @@ beneath it when selected:
|
||||
Paperclip first.
|
||||
- **Custom headers** — for servers that name their own headers.
|
||||
- **No sign-in needed** — the server is open to anyone with the address.
|
||||
- **Machine sign-in** — organization connections only. Paperclip signs in as a
|
||||
client you registered with the provider, using a Paperclip key or a client
|
||||
secret, with no person involved. It runs its own setup and test in place of
|
||||
**Check link**. See [Machine sign-in](./MCP-CLIENT-CREDENTIALS.md).
|
||||
|
||||
**Done** collapses the list back to a one-line summary of the choice. It stays
|
||||
open while the chosen option still needs a value.
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# Machine sign-in (MCP OAuth client credentials)
|
||||
|
||||
Machine sign-in connects an **organization-owned** remote MCP server where no
|
||||
person signs in: service accounts, CI-style integrations, internal automation.
|
||||
Paperclip signs in as a client the provider registered for it, using the
|
||||
[MCP OAuth Client Credentials extension](https://modelcontextprotocol.io/extensions/auth/oauth-client-credentials)
|
||||
(`io.modelcontextprotocol/oauth-client-credentials`, revision 2026-07-28),
|
||||
which builds on [RFC 6749 §4.4](https://datatracker.ietf.org/doc/html/rfc6749#section-4.4)
|
||||
and [RFC 7523](https://datatracker.ietf.org/doc/html/rfc7523).
|
||||
|
||||
It works the same on Paperclip Cloud and self-hosted instances.
|
||||
|
||||
## Setting it up
|
||||
|
||||
**Apps → Connect an app → Connect your own MCP server**, paste the address,
|
||||
press **Change** beside the sign-in line and choose **Machine sign-in**. The option
|
||||
appears only when the connection is shared with the organization; personal
|
||||
connections sign in as the person.
|
||||
|
||||
1. **Sign-in service.** Leave it blank and Paperclip asks the server where it
|
||||
signs in (its protected-resource metadata, then that service's metadata).
|
||||
Otherwise enter the sign-in service's address, or its token URL when the
|
||||
provider publishes no metadata.
|
||||
2. **How Paperclip proves who it is.**
|
||||
- **Paperclip key (recommended).** Press **Create key**. Paperclip creates a
|
||||
signing key, keeps the private half in the company vault, and shows the
|
||||
public key as PEM and as JSON. Register it with the provider for the
|
||||
Paperclip client. The public key isn't secret.
|
||||
- **Client secret.** For providers that only issue secrets. The secret goes
|
||||
straight into the vault and is never shown again.
|
||||
3. **Client ID and scopes.** Enter the client ID the provider shows for the
|
||||
Paperclip client, and any scopes its docs list (space separated).
|
||||
4. **Test connection.** Paperclip asks the sign-in service for a token, then
|
||||
lists the server's tools with it. Success turns the connection on and goes to
|
||||
the normal review step. A failed test keeps the draft and its key, so you
|
||||
can fix the provider side and test again.
|
||||
|
||||
Errors say what to change: an unaccepted key or secret, refused scopes, a
|
||||
provider that doesn't accept keys (choose Client secret) or secrets (choose
|
||||
Paperclip key), or a sign-in service Paperclip couldn't find.
|
||||
|
||||
## How tokens are requested
|
||||
|
||||
- `grant_type=client_credentials`, with `scope` and the RFC 8707 `resource`
|
||||
(the MCP server address) on every request.
|
||||
- **Paperclip key:** `private_key_jwt`. Each request carries a fresh assertion
|
||||
signed with RS256 (or ES256 when the service accepts only that):
|
||||
`iss` = `sub` = client ID, `aud` = the token endpoint, `exp` 60 seconds after
|
||||
`iat`, a unique `jti`, and `kid` = the key's RFC 7638 thumbprint. Per the
|
||||
extension, no `client_id` or secret is sent alongside it.
|
||||
- **Client secret:** `client_secret_basic`, or `client_secret_post` when that
|
||||
is all the service advertises.
|
||||
- Requests that carry a 2026-07-28 client capability object declare
|
||||
`io.modelcontextprotocol/oauth-client-credentials` for these connections only.
|
||||
|
||||
Tokens live in the vault on the connection's organization grant, like any
|
||||
other OAuth token, and are reused until a minute before they expire. There is
|
||||
no refresh token: Paperclip asks for a new access token. The existing grant
|
||||
refresh lease and in-process single flight mean concurrent agents, across
|
||||
server processes, share one token request. A `401` from the server requests a
|
||||
new token once; a second `401` reports the problem without touching the
|
||||
pinned settings. Agent boot and time to first token do not change: a live
|
||||
token is read the same way as for browser sign-in connections.
|
||||
|
||||
## The sign-in service is pinned
|
||||
|
||||
The connection records the sign-in service's issuer and token endpoint at
|
||||
setup. Credentials and assertions go only to that token endpoint, never to a
|
||||
deployment-wide OAuth client and never to a different service the server later
|
||||
advertises:
|
||||
|
||||
- Test and rotation check the server's protected-resource metadata; if it now
|
||||
lists other sign-in services and not the pinned one, Paperclip refuses and
|
||||
asks you to set up machine sign-in again.
|
||||
- A rejected token at runtime never re-runs discovery or rewrites the
|
||||
connection's endpoints.
|
||||
|
||||
When you enter a token URL directly there is no issuer to compare; the token
|
||||
URL itself is the pin.
|
||||
|
||||
## Rotation without downtime
|
||||
|
||||
- **New client secret:** enter it; Paperclip tests it against the sign-in
|
||||
service first and only then replaces the stored secret (a new version of the
|
||||
same vault entry). A wrong secret changes nothing.
|
||||
- **New key:** start a key rotation; Paperclip creates a replacement key and
|
||||
shows its public key while the current key keeps working. Register the new
|
||||
public key with the provider, then complete the rotation: Paperclip tests the
|
||||
new key and only then switches to it and deletes the old one.
|
||||
|
||||
API: `POST /api/tool-connections/{id}/machine-sign-in/rotate` with
|
||||
`{ "method": "client_secret", "clientSecret": "…" }` or
|
||||
`{ "method": "paperclip_key" }`, then
|
||||
`POST /api/tool-connections/{id}/machine-sign-in/rotate/complete` for a key.
|
||||
|
||||
## Security
|
||||
|
||||
- Private keys and client secrets exist only in the company vault
|
||||
(`oauth.client_private_key`, `oauth.client_private_key_next`,
|
||||
`oauth.client_secret`). Config, API responses, activity and logs carry only
|
||||
the public key, key ID, issuer, method and scope count.
|
||||
- Company scoped. Only connection managers can set it up, and only for
|
||||
organization connections; the service refuses personal connections.
|
||||
- Activity: `tool_connection.machine_sign_in_created`, `…_verified`,
|
||||
`…_rotation_started`, `…_rotated`, and removal as `tool_connection.archived`
|
||||
with `machineSignIn: true`. Removal destroys the vault entries the connection
|
||||
owned.
|
||||
|
||||
## API
|
||||
|
||||
| Route | Use |
|
||||
| --- | --- |
|
||||
| `POST /api/companies/{companyId}/tools/machine-sign-in` | Create the draft (finds and pins the sign-in service, creates the key or stores the secret). |
|
||||
| `GET /api/tool-connections/{id}/machine-sign-in` | Settings and public keys. |
|
||||
| `POST /api/tool-connections/{id}/machine-sign-in/test` | Test with an optional client ID, scopes or secret; turns the connection on. |
|
||||
| `POST /api/tool-connections/{id}/machine-sign-in/rotate` | Rotate the secret, or stage a new key. |
|
||||
| `POST /api/tool-connections/{id}/machine-sign-in/rotate/complete` | Switch to the staged key. |
|
||||
|
||||
Code: `server/src/services/tool-access/oauth/client-credentials.ts` (keys and
|
||||
assertions), `client-credentials-service.ts` (setup, pinning, token requests),
|
||||
the machine sign-in branch in `grant-refresh.ts`, and
|
||||
`ui/src/features/connections/MachineSignInSetup.tsx`. Tests:
|
||||
`server/src/__tests__/tool-access-client-credentials.test.ts`.
|
||||
|
||||
## Not yet covered
|
||||
|
||||
- A settings-page control for rotation (rotation is API-only today).
|
||||
- Enterprise single sign-on and enterprise-managed authorization (ID-JAG) are
|
||||
later phases.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Progress for long MCP tool calls
|
||||
|
||||
When an agent run calls a connected app's tool and the app reports progress,
|
||||
the task chat shows it on the tool's row while the call runs: a bar that fills
|
||||
when the app knows the total (a quiet sweep when it does not), the app's latest
|
||||
status line, and elapsed time. The row drops it when the call finishes.
|
||||
|
||||
Spec: [MCP 2026-07-28, Progress](https://modelcontextprotocol.io/specification/2026-07-28/basic/utilities/progress).
|
||||
|
||||
## How it works
|
||||
|
||||
1. **Request.** For a `tools/call` made by an agent run, the gateway adds a
|
||||
per-call `_meta.progressToken` (`paperclip-progress-<uuid>`). This applies to
|
||||
remote (Streamable HTTP) and local stdio connections, under every protocol
|
||||
version. Calls not made by a run (board test calls, MCP Apps views) send no
|
||||
token.
|
||||
2. **Response stream.** The app sends `notifications/progress` (`progress`,
|
||||
optional `total`, optional `message`) on the call's own response stream: SSE
|
||||
events on the POST response for HTTP, stdout lines for stdio. Under
|
||||
2026-07-28, request-scoped notifications always stay on that stream.
|
||||
`readMcpHttpResponse` hands each notification to an observer before it
|
||||
reaches the result, and the observer can never fail or delay the call.
|
||||
3. **Reporter** (`server/src/services/tool-call-progress.ts`). Each update is
|
||||
checked (finite, non-negative, increasing; `total` kept only when positive and
|
||||
at least `progress`) and its message reduced to plain single-line text of at
|
||||
most 200 characters. The reporter then:
|
||||
- publishes a `heartbeat.run.tool_progress` live event at most every 250 ms
|
||||
(the latest update wins);
|
||||
- saves the latest update on the `tool_invocations.progress` column at most
|
||||
every second, and never onto a finished call;
|
||||
- publishes a final event with `done: true` when the call ends.
|
||||
Nothing the reporter does is awaited by the call.
|
||||
4. **Delivery.** The event goes to the run's company only. The live socket
|
||||
treats it like transcript output: a client receives it only for runs it
|
||||
asked to follow, after the same run read check.
|
||||
5. **Task chat.** `ToolCallProgressProvider` (one per chat) keeps the latest
|
||||
update per call. A running tool row subscribes to it and finds its call by
|
||||
run and tool name. Each update carries both the gateway name and the name
|
||||
native runners give the tool (`assignedToolName`: shortened, with a hash),
|
||||
since native transcripts show the latter. When a running row first mounts, the chat reads
|
||||
`GET /api/issues/:issueId/tool-call-progress` (again on mount and on tab
|
||||
focus), so a reload mid-call shows the latest saved state. The read also
|
||||
marks recently finished calls `done`; a call is never dropped just because
|
||||
the read does not list it, since its first save may still be in flight. Finished rows never fetch or re-render for
|
||||
progress.
|
||||
|
||||
## Speed
|
||||
|
||||
The token adds a few bytes to the request. Parsing happens on the stream that
|
||||
was already being read. Publishing and saving are throttled, fire-and-forget,
|
||||
and stop when the call ends. Agent boot and time to first token are untouched:
|
||||
the code runs only while a tool call is in flight.
|
||||
|
||||
## Trying it
|
||||
|
||||
`packages/kv-demo-mcp-server` has a `slow_with_progress` tool
|
||||
(`steps`, `stepMs`, `withTotal`). Connect the demo server, then ask an agent to
|
||||
call it with, say, `{ "steps": 20, "stepMs": 1000 }`; the tool row fills over
|
||||
20 seconds. `withTotal: false` shows the indeterminate state.
|
||||
|
||||
## Not covered
|
||||
|
||||
- Matching uses the run and the tool name. If one run calls the same tool
|
||||
twice at once, both rows show the newer call's progress.
|
||||
- Progress from a call that became an MCP Task (see `MCP-TASKS.md`) is not shown.
|
||||
- Progress is not forwarded to the agent's own MCP client.
|
||||
@@ -169,5 +169,5 @@ Tracked as follow-ups:
|
||||
agent to retry.
|
||||
- `subscriptions/listen` change notifications (cache invalidation relies on
|
||||
`ttlMs`).
|
||||
- The Tasks extension, Skills over MCP, progress and cancellation, icons, and
|
||||
enterprise-managed authorization.
|
||||
- Cancellation, icons, and enterprise-managed authorization. (Progress is
|
||||
supported: see [MCP-PROGRESS.md](MCP-PROGRESS.md).)
|
||||
@@ -20,9 +20,14 @@ MCP protocol versions (2026-07-28 stateless MCP and older handshake servers):
|
||||
MCP extensions: [MCP Apps](./MCP-APPS.md) (interactive views in the task chat),
|
||||
[MCP Skills](./MCP-SKILLS.md) (skills that connected apps publish, reviewed
|
||||
by the board and delivered to agents), [MCP Tasks](./MCP-TASKS.md) (long
|
||||
provider calls that survive timeouts, restarts, and deploys), and
|
||||
provider calls that survive timeouts, restarts, and deploys),
|
||||
[MCP elicitation](./MCP-ELICITATION.md) (an app's question to the person,
|
||||
answered in the task chat).
|
||||
answered in the task chat), and [Machine sign-in](./MCP-CLIENT-CREDENTIALS.md)
|
||||
(OAuth client credentials for organization connections where no person signs
|
||||
in).
|
||||
|
||||
[Machine sign-in](./MCP-CLIENT-CREDENTIALS.md) (OAuth client credentials for
|
||||
organization connections where no person signs in).
|
||||
|
||||
Long-term memory: [Experimental memory connectors](./MEMORY.md).
|
||||
|
||||
|
||||
@@ -374,6 +374,33 @@ GET /api/attachments/{attachmentId}/content
|
||||
DELETE /api/attachments/{attachmentId}
|
||||
```
|
||||
|
||||
## Tool Call Progress
|
||||
|
||||
```
|
||||
GET /api/issues/{issueId}/tool-call-progress
|
||||
```
|
||||
|
||||
Returns the latest progress of the task's connected-app tool calls that are still running, for a page that loads mid-call, and the calls that finished in the last hour, marked `done`. A client should drop a call's progress only on `done`: a running call may not have saved its first update yet. Live updates arrive as `heartbeat.run.tool_progress` events on the company live-events socket.
|
||||
|
||||
- Board callers only. The caller needs company access and read access to the task; otherwise the response is `404`.
|
||||
- Lists unfinished calls made by a run in the last 24 hours that reported progress (at most 50), then calls finished in the last hour (at most 50).
|
||||
|
||||
Each item:
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `invocationId` | string | The tool call. |
|
||||
| `runId` | string | The run that made the call. |
|
||||
| `toolName` | string | The tool name the agent called. |
|
||||
| `startedAt` | string (ISO) | When the call started. |
|
||||
| `updatedAt` | string (ISO) | When the app last reported progress. |
|
||||
| `progress` | number | Progress so far. |
|
||||
| `total` | number or null | Known end value; `null` when the app does not know it. |
|
||||
| `message` | string or null | The app's latest status line: plain text, at most 200 characters. |
|
||||
| `done` | boolean | `true` once the call has finished. |
|
||||
|
||||
See `doc/connections/MCP-PROGRESS.md`.
|
||||
|
||||
## Issue Lifecycle
|
||||
|
||||
```
|
||||
|
||||
@@ -691,6 +691,7 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([
|
||||
"0470_heartbeat_runs_queued_created_index.sql",
|
||||
"0472_runner_api_capture_pending_deletes.sql",
|
||||
"0476_wakeup_request_run_index.sql",
|
||||
"0486_heartbeat_runs_company_finished_index.sql",
|
||||
"0469_zippy_photon.sql",
|
||||
|
||||
|
||||
|
||||
@@ -54,5 +54,5 @@ export { issueRelations } from "./schema/issue_relations.js";
|
||||
export { issueReferenceMentions } from "./schema/issue_reference_mentions.js";
|
||||
export * from "./schema/index.js";
|
||||
|
||||
export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js";
|
||||
export { databaseWorkScopeKey, primeDatabaseWorkTransactionId, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js";
|
||||
export { withDatabaseOperation } from "./response-capture.js";
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE "tool_invocations" ADD COLUMN "progress" jsonb;
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE "automatic_run_brakes" (
|
||||
"company_id" uuid PRIMARY KEY NOT NULL,
|
||||
"state" text DEFAULT 'closed' NOT NULL,
|
||||
"reason" jsonb DEFAULT '{}'::jsonb NOT NULL,
|
||||
"state_changed_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"opened_at" timestamp with time zone,
|
||||
"next_probe_at" timestamp with time zone,
|
||||
"probe_run_id" uuid,
|
||||
"probe_successes" integer DEFAULT 0 NOT NULL,
|
||||
"release_window_started_at" timestamp with time zone,
|
||||
"release_window_starts" integer DEFAULT 0 NOT NULL,
|
||||
"next_creation_at" timestamp with time zone,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "automatic_run_brakes" ADD CONSTRAINT "automatic_run_brakes_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action;
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "heartbeat_runs_company_finished_idx" ON "heartbeat_runs" USING btree ("company_id","finished_at" DESC NULLS LAST) WHERE "heartbeat_runs"."finished_at" is not null;
|
||||
+285
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "41eacc6a-6eab-4f01-965a-62aa0cd6778f",
|
||||
"prevId": "55a8c37c-453c-4be4-b657-b703932836a0",
|
||||
"id": "9389124c-379a-4b64-9c20-c7b37538e7c8",
|
||||
"prevId": "ceaa38c4-681b-430e-9578-0854650d85b3",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -18092,6 +18092,12 @@
|
||||
"notNull": true,
|
||||
"default": "'company'"
|
||||
},
|
||||
"system_purpose": {
|
||||
"name": "system_purpose",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"owner_user_id": {
|
||||
"name": "owner_user_id",
|
||||
"type": "text",
|
||||
@@ -59601,6 +59607,12 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"progress": {
|
||||
"name": "progress",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
@@ -60107,6 +60119,277 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.tool_mcp_elicitations": {
|
||||
"name": "tool_mcp_elicitations",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"default": "gen_random_uuid()"
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"issue_id": {
|
||||
"name": "issue_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"agent_id": {
|
||||
"name": "agent_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"run_id": {
|
||||
"name": "run_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"invocation_id": {
|
||||
"name": "invocation_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"connection_id": {
|
||||
"name": "connection_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"grant_id": {
|
||||
"name": "grant_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"catalog_tool_name": {
|
||||
"name": "catalog_tool_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"call_identity": {
|
||||
"name": "call_identity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"era": {
|
||||
"name": "era",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"sealed_request_state": {
|
||||
"name": "sealed_request_state",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"questions": {
|
||||
"name": "questions",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"arguments_identity": {
|
||||
"name": "arguments_identity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"dedupe_key": {
|
||||
"name": "dedupe_key",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"claimed_until": {
|
||||
"name": "claimed_until",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"expires_at": {
|
||||
"name": "expires_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"sent_at": {
|
||||
"name": "sent_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"tool_mcp_elicitations_call_idx": {
|
||||
"name": "tool_mcp_elicitations_call_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "connection_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "call_identity",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "status",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_mcp_elicitations_invocation_idx": {
|
||||
"name": "tool_mcp_elicitations_invocation_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "invocation_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_mcp_elicitations_waiting_uq": {
|
||||
"name": "tool_mcp_elicitations_waiting_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "dedupe_key",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"tool_mcp_elicitations_company_id_companies_id_fk": {
|
||||
"name": "tool_mcp_elicitations_company_id_companies_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "companies",
|
||||
"columnsFrom": [
|
||||
"company_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": {
|
||||
"name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "tool_invocations",
|
||||
"columnsFrom": [
|
||||
"invocation_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"tool_mcp_elicitations_connection_id_tool_connections_id_fk": {
|
||||
"name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "tool_connections",
|
||||
"columnsFrom": [
|
||||
"connection_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.tool_mcp_gateway_tokens": {
|
||||
"name": "tool_mcp_gateway_tokens",
|
||||
"schema": "",
|
||||
+402
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "55a8c37c-453c-4be4-b657-b703932836a0",
|
||||
"prevId": "f5f852bd-0568-4b2d-bc35-0dac5d9c8c5b",
|
||||
"id": "78d10bc7-b440-4c00-a02e-90111c50071d",
|
||||
"prevId": "9389124c-379a-4b64-9c20-c7b37538e7c8",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -6582,6 +6582,111 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.automatic_run_brakes": {
|
||||
"name": "automatic_run_brakes",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"state": {
|
||||
"name": "state",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'closed'"
|
||||
},
|
||||
"reason": {
|
||||
"name": "reason",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'{}'::jsonb"
|
||||
},
|
||||
"state_changed_at": {
|
||||
"name": "state_changed_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"opened_at": {
|
||||
"name": "opened_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"next_probe_at": {
|
||||
"name": "next_probe_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"probe_run_id": {
|
||||
"name": "probe_run_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"probe_successes": {
|
||||
"name": "probe_successes",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": 0
|
||||
},
|
||||
"release_window_started_at": {
|
||||
"name": "release_window_started_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"release_window_starts": {
|
||||
"name": "release_window_starts",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": 0
|
||||
},
|
||||
"next_creation_at": {
|
||||
"name": "next_creation_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {
|
||||
"automatic_run_brakes_company_id_companies_id_fk": {
|
||||
"name": "automatic_run_brakes_company_id_companies_id_fk",
|
||||
"tableFrom": "automatic_run_brakes",
|
||||
"tableTo": "companies",
|
||||
"columnsFrom": [
|
||||
"company_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.board_api_keys": {
|
||||
"name": "board_api_keys",
|
||||
"schema": "",
|
||||
@@ -18092,6 +18197,12 @@
|
||||
"notNull": true,
|
||||
"default": "'company'"
|
||||
},
|
||||
"system_purpose": {
|
||||
"name": "system_purpose",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"owner_user_id": {
|
||||
"name": "owner_user_id",
|
||||
"type": "text",
|
||||
@@ -55989,6 +56100,18 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"credential_version": {
|
||||
"name": "credential_version",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"credential_hash": {
|
||||
"name": "credential_hash",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_by_user_id": {
|
||||
"name": "created_by_user_id",
|
||||
"type": "text",
|
||||
@@ -59589,6 +59712,12 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"progress": {
|
||||
"name": "progress",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
@@ -60095,6 +60224,277 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.tool_mcp_elicitations": {
|
||||
"name": "tool_mcp_elicitations",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"default": "gen_random_uuid()"
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"issue_id": {
|
||||
"name": "issue_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"agent_id": {
|
||||
"name": "agent_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"run_id": {
|
||||
"name": "run_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"invocation_id": {
|
||||
"name": "invocation_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"connection_id": {
|
||||
"name": "connection_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"grant_id": {
|
||||
"name": "grant_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"catalog_tool_name": {
|
||||
"name": "catalog_tool_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"call_identity": {
|
||||
"name": "call_identity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"era": {
|
||||
"name": "era",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"sealed_request_state": {
|
||||
"name": "sealed_request_state",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"questions": {
|
||||
"name": "questions",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"arguments_identity": {
|
||||
"name": "arguments_identity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"dedupe_key": {
|
||||
"name": "dedupe_key",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"claimed_until": {
|
||||
"name": "claimed_until",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"expires_at": {
|
||||
"name": "expires_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"sent_at": {
|
||||
"name": "sent_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"tool_mcp_elicitations_call_idx": {
|
||||
"name": "tool_mcp_elicitations_call_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "connection_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "call_identity",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "status",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_mcp_elicitations_invocation_idx": {
|
||||
"name": "tool_mcp_elicitations_invocation_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "invocation_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_mcp_elicitations_waiting_uq": {
|
||||
"name": "tool_mcp_elicitations_waiting_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "dedupe_key",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"tool_mcp_elicitations_company_id_companies_id_fk": {
|
||||
"name": "tool_mcp_elicitations_company_id_companies_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "companies",
|
||||
"columnsFrom": [
|
||||
"company_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": {
|
||||
"name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "tool_invocations",
|
||||
"columnsFrom": [
|
||||
"invocation_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"tool_mcp_elicitations_connection_id_tool_connections_id_fk": {
|
||||
"name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "tool_connections",
|
||||
"columnsFrom": [
|
||||
"connection_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.tool_mcp_gateway_tokens": {
|
||||
"name": "tool_mcp_gateway_tokens",
|
||||
"schema": "",
|
||||
+559
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "f5f852bd-0568-4b2d-bc35-0dac5d9c8c5b",
|
||||
"prevId": "349cdd89-db04-4340-a5d3-3b877bf7227d",
|
||||
"id": "0c3e74be-7936-4c97-9798-505daf53c16b",
|
||||
"prevId": "78d10bc7-b440-4c00-a02e-90111c50071d",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -6582,6 +6582,111 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.automatic_run_brakes": {
|
||||
"name": "automatic_run_brakes",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"state": {
|
||||
"name": "state",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'closed'"
|
||||
},
|
||||
"reason": {
|
||||
"name": "reason",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'{}'::jsonb"
|
||||
},
|
||||
"state_changed_at": {
|
||||
"name": "state_changed_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"opened_at": {
|
||||
"name": "opened_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"next_probe_at": {
|
||||
"name": "next_probe_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"probe_run_id": {
|
||||
"name": "probe_run_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"probe_successes": {
|
||||
"name": "probe_successes",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": 0
|
||||
},
|
||||
"release_window_started_at": {
|
||||
"name": "release_window_started_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"release_window_starts": {
|
||||
"name": "release_window_starts",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": 0
|
||||
},
|
||||
"next_creation_at": {
|
||||
"name": "next_creation_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {
|
||||
"automatic_run_brakes_company_id_companies_id_fk": {
|
||||
"name": "automatic_run_brakes_company_id_companies_id_fk",
|
||||
"tableFrom": "automatic_run_brakes",
|
||||
"tableTo": "companies",
|
||||
"columnsFrom": [
|
||||
"company_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.board_api_keys": {
|
||||
"name": "board_api_keys",
|
||||
"schema": "",
|
||||
@@ -18092,6 +18197,12 @@
|
||||
"notNull": true,
|
||||
"default": "'company'"
|
||||
},
|
||||
"system_purpose": {
|
||||
"name": "system_purpose",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"owner_user_id": {
|
||||
"name": "owner_user_id",
|
||||
"type": "text",
|
||||
@@ -36023,6 +36134,28 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"heartbeat_runs_company_finished_idx": {
|
||||
"name": "heartbeat_runs_company_finished_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "finished_at",
|
||||
"isExpression": false,
|
||||
"asc": false,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"heartbeat_runs\".\"finished_at\" is not null",
|
||||
"concurrently": true,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"heartbeat_runs_company_created_at_desc_idx": {
|
||||
"name": "heartbeat_runs_company_created_at_desc_idx",
|
||||
"columns": [
|
||||
@@ -55953,6 +56086,153 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.tailscale_connections": {
|
||||
"name": "tailscale_connections",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"default": "gen_random_uuid()"
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"name": {
|
||||
"name": "name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'pending'"
|
||||
},
|
||||
"secret_id": {
|
||||
"name": "secret_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"credential_version": {
|
||||
"name": "credential_version",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"credential_hash": {
|
||||
"name": "credential_hash",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_by_user_id": {
|
||||
"name": "created_by_user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"oauth_state_hash": {
|
||||
"name": "oauth_state_hash",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"oauth_expires_at": {
|
||||
"name": "oauth_expires_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"tailscale_connections_company_idx": {
|
||||
"name": "tailscale_connections_company_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tailscale_connections_oauth_state_idx": {
|
||||
"name": "tailscale_connections_oauth_state_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "oauth_state_hash",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"tailscale_connections_company_id_companies_id_fk": {
|
||||
"name": "tailscale_connections_company_id_companies_id_fk",
|
||||
"tableFrom": "tailscale_connections",
|
||||
"tableTo": "companies",
|
||||
"columnsFrom": [
|
||||
"company_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"tailscale_connections_secret_id_company_secrets_id_fk": {
|
||||
"name": "tailscale_connections_secret_id_company_secrets_id_fk",
|
||||
"tableFrom": "tailscale_connections",
|
||||
"tableTo": "company_secrets",
|
||||
"columnsFrom": [
|
||||
"secret_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.tool_access_audit_events": {
|
||||
"name": "tool_access_audit_events",
|
||||
"schema": "",
|
||||
@@ -59454,6 +59734,12 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"progress": {
|
||||
"name": "progress",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
@@ -59960,6 +60246,277 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.tool_mcp_elicitations": {
|
||||
"name": "tool_mcp_elicitations",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"default": "gen_random_uuid()"
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"issue_id": {
|
||||
"name": "issue_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"agent_id": {
|
||||
"name": "agent_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"run_id": {
|
||||
"name": "run_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"invocation_id": {
|
||||
"name": "invocation_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"connection_id": {
|
||||
"name": "connection_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"grant_id": {
|
||||
"name": "grant_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"catalog_tool_name": {
|
||||
"name": "catalog_tool_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"call_identity": {
|
||||
"name": "call_identity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"era": {
|
||||
"name": "era",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"sealed_request_state": {
|
||||
"name": "sealed_request_state",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"questions": {
|
||||
"name": "questions",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"arguments_identity": {
|
||||
"name": "arguments_identity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"dedupe_key": {
|
||||
"name": "dedupe_key",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"claimed_until": {
|
||||
"name": "claimed_until",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"expires_at": {
|
||||
"name": "expires_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"sent_at": {
|
||||
"name": "sent_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"tool_mcp_elicitations_call_idx": {
|
||||
"name": "tool_mcp_elicitations_call_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "connection_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "call_identity",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "status",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_mcp_elicitations_invocation_idx": {
|
||||
"name": "tool_mcp_elicitations_invocation_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "invocation_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"tool_mcp_elicitations_waiting_uq": {
|
||||
"name": "tool_mcp_elicitations_waiting_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "dedupe_key",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"tool_mcp_elicitations_company_id_companies_id_fk": {
|
||||
"name": "tool_mcp_elicitations_company_id_companies_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "companies",
|
||||
"columnsFrom": [
|
||||
"company_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": {
|
||||
"name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "tool_invocations",
|
||||
"columnsFrom": [
|
||||
"invocation_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"tool_mcp_elicitations_connection_id_tool_connections_id_fk": {
|
||||
"name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk",
|
||||
"tableFrom": "tool_mcp_elicitations",
|
||||
"tableTo": "tool_connections",
|
||||
"columnsFrom": [
|
||||
"connection_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.tool_mcp_gateway_tokens": {
|
||||
"name": "tool_mcp_gateway_tokens",
|
||||
"schema": "",
|
||||
+21
@@ -3375,6 +3375,27 @@
|
||||
"when": 1794611818460,
|
||||
"tag": "0483_busy_tombstone",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 484,
|
||||
"version": "7",
|
||||
"when": 1794621009918,
|
||||
"tag": "0484_certain_zemo",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 485,
|
||||
"version": "7",
|
||||
"when": 1794623155354,
|
||||
"tag": "0485_moaning_ultimatum",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 486,
|
||||
"version": "7",
|
||||
"when": 1794623177597,
|
||||
"tag": "0486_heartbeat_runs_company_finished_index",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { integer, jsonb, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core";
|
||||
import { companies } from "./companies.js";
|
||||
|
||||
export const automaticRunBrakes = pgTable("automatic_run_brakes", {
|
||||
companyId: uuid("company_id").primaryKey().references(() => companies.id, { onDelete: "cascade" }),
|
||||
state: text("state").$type<"closed" | "open" | "half_open">().notNull().default("closed"),
|
||||
reason: jsonb("reason").$type<Record<string, unknown>>().notNull().default({}),
|
||||
stateChangedAt: timestamp("state_changed_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
openedAt: timestamp("opened_at", { withTimezone: true }),
|
||||
nextProbeAt: timestamp("next_probe_at", { withTimezone: true }),
|
||||
probeRunId: uuid("probe_run_id"),
|
||||
probeSuccesses: integer("probe_successes").notNull().default(0),
|
||||
releaseWindowStartedAt: timestamp("release_window_started_at", { withTimezone: true }),
|
||||
releaseWindowStarts: integer("release_window_starts").notNull().default(0),
|
||||
nextCreationAt: timestamp("next_creation_at", { withTimezone: true }),
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
});
|
||||
@@ -248,6 +248,8 @@ export const heartbeatRuns = pgTable(
|
||||
table.status,
|
||||
table.processStartedAt,
|
||||
),
|
||||
companyFinishedIdx: index("heartbeat_runs_company_finished_idx").on(table.companyId, table.finishedAt.desc())
|
||||
.where(sql`${table.finishedAt} is not null`).concurrently(),
|
||||
companyCreatedAtDescIdx: index("heartbeat_runs_company_created_at_desc_idx").on(
|
||||
table.companyId,
|
||||
table.createdAt.desc(),
|
||||
|
||||
@@ -26,6 +26,7 @@ export { budgetResumeClaims } from "./budget_resume_claims.js";
|
||||
export { agentConfigRevisions } from "./agent_config_revisions.js";
|
||||
export { agentApiKeys } from "./agent_api_keys.js";
|
||||
export { agentRuntimeState } from "./agent_runtime_state.js";
|
||||
export { automaticRunBrakes } from "./automatic_run_brakes.js";
|
||||
export { agentTaskSessions, agentSessionGoalActions } from "./agent_task_sessions.js";
|
||||
export { agentWakeupRequests } from "./agent_wakeup_requests.js";
|
||||
export { projects } from "./projects.js";
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
uuid,
|
||||
} from "drizzle-orm/pg-core";
|
||||
import type {
|
||||
ToolCallProgress,
|
||||
ConnectionAgentInstructions,
|
||||
ConnectionTokenIssuanceOutcome,
|
||||
ConnectionTokenIssuancePath,
|
||||
@@ -804,6 +805,8 @@ export const toolInvocations = pgTable(
|
||||
errorMessage: text("error_message"),
|
||||
startedAt: timestamp("started_at", { withTimezone: true }),
|
||||
completedAt: timestamp("completed_at", { withTimezone: true }),
|
||||
/** The provider's latest progress while the call runs (MCP `notifications/progress`), bounded. */
|
||||
progress: jsonb("progress").$type<ToolCallProgress>(),
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
|
||||
@@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { createDb, withDedicatedDbConnection } from "./client.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js";
|
||||
import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js";
|
||||
import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("database work signals", () => {
|
||||
@@ -145,6 +145,22 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
} finally { release(); await serverDone?.catch(() => {}); unsubscribe(); }
|
||||
});
|
||||
|
||||
it("resolves an uncertain transaction with a transaction id that a statement returned", async () => {
|
||||
await expect(db.transaction(async tx => {
|
||||
const [{ xid }] = await tx.execute(sql`select pg_current_xact_id()::text as xid`);
|
||||
primeDatabaseWorkTransactionId(tx, xid as string);
|
||||
await signalDatabaseWork(tx, "primed-disconnect");
|
||||
await tx.execute(sql`insert into work_signal_test values ('primed-disconnect')`);
|
||||
const [{ pid }] = await tx.execute(sql`select pg_backend_pid() as pid`);
|
||||
await db.execute(sql`select pg_terminate_backend(${pid as number})`);
|
||||
await tx.execute(sql`select 1`);
|
||||
})).rejects.toThrow();
|
||||
expect(databaseWorkPending(db, "primed-disconnect")).toBe(true);
|
||||
await reconcileDatabaseWork(db, "primed-disconnect");
|
||||
expect(databaseWorkPending(db, "primed-disconnect")).toBe(false);
|
||||
expect(await db.execute(sql`select * from work_signal_test where id = 'primed-disconnect'`)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("recovers after a real backend disconnect without replaying the queue write", async () => {
|
||||
await expect(db.transaction(async tx => {
|
||||
await signalDatabaseWork(tx, "disconnect");
|
||||
@@ -196,6 +212,17 @@ describe("work signal query boundaries", () => {
|
||||
expect(write).not.toHaveBeenCalled();
|
||||
expect(databaseWorkPending(connection, "failed")).toBe(false);
|
||||
});
|
||||
it("reads no transaction id when a statement of the transaction already returned it", async () => {
|
||||
const execute = vi.fn();
|
||||
const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) });
|
||||
primeDatabaseWorkTransactionId(connection, "outside");
|
||||
await connection.transaction(async (tx: any) => {
|
||||
primeDatabaseWorkTransactionId(tx, "42");
|
||||
await signalDatabaseWork(tx, "primed");
|
||||
await signalDatabaseWork(tx, "primed-again");
|
||||
});
|
||||
expect(execute).not.toHaveBeenCalled();
|
||||
});
|
||||
it("rejects registration on an autocommit root", async () => {
|
||||
const connection = installDatabaseWorkSignals({ execute: vi.fn(), transaction: vi.fn() });
|
||||
await expect(signalDatabaseWork(connection, "bad")).rejects.toThrow("inside a transaction");
|
||||
|
||||
@@ -88,6 +88,11 @@ export async function signalDatabaseWork(connection: object, topic: string, hint
|
||||
if (context.scope.notify) notifyAtCommit(context, notification(context.scope, topic));
|
||||
}
|
||||
|
||||
export function primeDatabaseWorkTransactionId(connection: object, xid: string): void {
|
||||
const state = contexts.get(connection)?.transaction;
|
||||
if (state && !state.xid) state.xid = Promise.resolve(xid);
|
||||
}
|
||||
|
||||
function notifyAtCommit(context: Context, payload: string): void {
|
||||
if (context.ended) throw new Error("Database work was signalled after its transaction or savepoint ended");
|
||||
const key = `${DATABASE_WORK_CHANNEL}\u0000${payload}`;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# KV Demo MCP Server
|
||||
|
||||
A standalone, self-contained MCP server for demos. One process exposes four
|
||||
key/value MCP tools **and** a tiny web UI that renders the values those tools
|
||||
key/value MCP tools (plus a progress demo tool) **and** a tiny web UI that renders the values those tools
|
||||
mutate — so you can call a tool from a Paperclip agent and watch the value
|
||||
appear in a browser tab.
|
||||
|
||||
@@ -24,6 +24,10 @@ recorded walkthrough in [doc/MCP-DEMO-SCRIPT.md](../../doc/MCP-DEMO-SCRIPT.md).
|
||||
- `kv_set` (write) — set one key to a string value.
|
||||
- `kv_delete` (destructive) — delete a key. Carries `destructiveHint: true`
|
||||
so Paperclip's catalog quarantines it on first sight.
|
||||
- `slow_with_progress` (read) — waits `steps` × `stepMs` and sends MCP
|
||||
`notifications/progress` after each step (`withTotal: false` leaves the total
|
||||
out). For the task chat's tool progress; see
|
||||
[doc/connections/MCP-PROGRESS.md](../../doc/connections/MCP-PROGRESS.md).
|
||||
- A Values UI at `/` — an auto-refreshing HTML table over the same store.
|
||||
- A JSON state route at `/api/state` — what the UI polls.
|
||||
|
||||
|
||||
@@ -83,4 +83,26 @@ describe("kv demo HTTP server", () => {
|
||||
expect(state.count).toBe(1);
|
||||
await client.close();
|
||||
});
|
||||
|
||||
it("streams progress for slow_with_progress on the call's response, before the result", async () => {
|
||||
const { base } = await startServer();
|
||||
const client = await mcpClient(base);
|
||||
const updates: Array<{ progress: number; total?: number; message?: string }> = [];
|
||||
const result = await client.callTool(
|
||||
{ name: "slow_with_progress", arguments: { steps: 3, stepMs: 5 } },
|
||||
undefined,
|
||||
{ onprogress: (update) => { updates.push(update); } },
|
||||
);
|
||||
expect(updates).toEqual([
|
||||
{ progress: 1, total: 3, message: "Finished step 1 of 3" },
|
||||
{ progress: 2, total: 3, message: "Finished step 2 of 3" },
|
||||
{ progress: 3, total: 3, message: "Finished step 3 of 3" },
|
||||
]);
|
||||
expect(JSON.stringify(result.content)).toContain("completed");
|
||||
const indeterminate: Array<{ total?: number }> = [];
|
||||
await client.callTool({ name: "slow_with_progress", arguments: { steps: 1, stepMs: 0, withTotal: false } }, undefined,
|
||||
{ onprogress: (update) => { indeterminate.push(update); } });
|
||||
expect(indeterminate[0]?.total).toBeUndefined();
|
||||
await client.close();
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
||||
import { KvStore } from "./store.js";
|
||||
import { createToolDefinitions } from "./tools.js";
|
||||
import { registerSlowWithProgressTool } from "./slow-progress.js";
|
||||
|
||||
export interface CreateKvDemoMcpServerResult {
|
||||
server: McpServer;
|
||||
@@ -29,6 +30,7 @@ export function createKvDemoMcpServer(store: KvStore = new KvStore()): CreateKvD
|
||||
tool.execute,
|
||||
);
|
||||
}
|
||||
registerSlowWithProgressTool(server);
|
||||
|
||||
return { server, store };
|
||||
}
|
||||
@@ -36,6 +38,7 @@ export function createKvDemoMcpServer(store: KvStore = new KvStore()): CreateKvD
|
||||
export { KvStore } from "./store.js";
|
||||
export type { KvEntry, KvStateSnapshot } from "./store.js";
|
||||
export { createToolDefinitions } from "./tools.js";
|
||||
export { SLOW_WITH_PROGRESS_TOOL, registerSlowWithProgressTool } from "./slow-progress.js";
|
||||
export { createKvDemoHttpServer } from "./http.js";
|
||||
export type { KvDemoHttpServer, KvDemoHttpOptions } from "./http.js";
|
||||
export { readConfigFromEnv } from "./config.js";
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
||||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* A demo tool that takes a while and reports MCP progress as it goes, so a
|
||||
* client can show it. Each step waits `stepMs`, then sends
|
||||
* `notifications/progress` for the caller's `progressToken` (when it sent
|
||||
* one). `withTotal: false` leaves `total` out, for an indeterminate display.
|
||||
*/
|
||||
export const SLOW_WITH_PROGRESS_TOOL = "slow_with_progress";
|
||||
|
||||
const schema = z.object({
|
||||
steps: z.number().int().min(1).max(100).default(5),
|
||||
stepMs: z.number().int().min(0).max(10_000).default(1_000),
|
||||
withTotal: z.boolean().default(true),
|
||||
});
|
||||
|
||||
export function registerSlowWithProgressTool(server: McpServer): void {
|
||||
server.registerTool(
|
||||
SLOW_WITH_PROGRESS_TOOL,
|
||||
{
|
||||
description: "Wait for a few steps, reporting progress after each one. For progress demos.",
|
||||
inputSchema: schema.shape,
|
||||
annotations: { title: "Slow task with progress", readOnlyHint: true, openWorldHint: false },
|
||||
},
|
||||
async (input, extra) => {
|
||||
const { steps, stepMs, withTotal } = schema.parse(input);
|
||||
const progressToken = extra._meta?.progressToken;
|
||||
for (let step = 1; step <= steps; step++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, stepMs));
|
||||
if (extra.signal.aborted) break;
|
||||
if (progressToken !== undefined) {
|
||||
await extra.sendNotification({
|
||||
method: "notifications/progress",
|
||||
params: {
|
||||
progressToken,
|
||||
progress: step,
|
||||
...(withTotal ? { total: steps } : {}),
|
||||
message: `Finished step ${step} of ${steps}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
return { content: [{ type: "text", text: JSON.stringify({ steps, completed: true }) }] };
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -19,10 +19,10 @@ async function connectedClient() {
|
||||
}
|
||||
|
||||
describe("kv demo MCP tools", () => {
|
||||
it("exposes exactly kv_set, kv_get, kv_list, kv_delete", async () => {
|
||||
it("exposes exactly kv_set, kv_get, kv_list, kv_delete and the progress demo tool", async () => {
|
||||
const { client } = await connectedClient();
|
||||
const { tools } = await client.listTools();
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(["kv_delete", "kv_get", "kv_list", "kv_set"]);
|
||||
expect(tools.map((t) => t.name).sort()).toEqual(["kv_delete", "kv_get", "kv_list", "kv_set", "slow_with_progress"]);
|
||||
});
|
||||
|
||||
it("reflects kv_set writes in the shared store", async () => {
|
||||
|
||||
@@ -216,6 +216,55 @@ identities. The report's shape is `$defs.backgroundTasks` in
|
||||
generated from it and tested against
|
||||
`protocol/fixtures/provider-boundary/acpx-background-tasks.json`.
|
||||
|
||||
The Claude sidecar reports `{"live": N}` with N above zero only while N shell
|
||||
tasks are running and nothing has made the count unknown; finished tasks do
|
||||
not count. It reports `{"live": 0}` only while its provider process has never
|
||||
had a background task and Claude has not acted between prompts. Every other
|
||||
case is unknown and the field is omitted: once a background task has finished
|
||||
or a provider-started cycle has run and no shell is running, and permanently
|
||||
after a dropped, declined, failed or cancelled cycle, a task that is not a
|
||||
shell, a malformed task update, a scheduling tool, a full ledger, or idle
|
||||
activity that was not adopted. The ACP stream does not say which task a Claude
|
||||
cycle reports on, so the sidecar does not guess: after a provider-started turn,
|
||||
Runner keeps waiting only while a background command is still running, and a
|
||||
command that finishes while Claude is already reporting on an earlier one gets
|
||||
no turn of its own. An origin-tagged usage update that directly follows the
|
||||
previous one with the same context size and cost and zero usage is an empty
|
||||
result, not a cycle, and is ignored.
|
||||
|
||||
A controller opts in per turn with `providerStartedTurns: true` on
|
||||
`turn.start`. Only then, when that completed Claude turn reported no accepted
|
||||
`paperclip_finish` or `paperclip_block` result for the run, its terminal reports
|
||||
`backgroundTasks: {"live": N}` with N above zero, and the same restart
|
||||
conditions hold, Runner sends the sidecar `session.provider_turn` with
|
||||
`armed: true` for that terminal. Only if the sidecar replies `armed: true` does
|
||||
Runner keep it and await a provider-started turn: the PRP `turn.completed`
|
||||
carries `awaitingProviderTurn: true` and no `run.terminal` follows. Without the
|
||||
opt-in, with a result, or when arming is refused, every event and the turn-end
|
||||
restart are exactly as before.
|
||||
|
||||
The sidecar adopts Claude's own cycle only while armed. A cycle that starts
|
||||
after the terminal and before the arming command is held, with nothing lost,
|
||||
for at most 5 s; arming adopts it and anything else drops it. When armed, the
|
||||
sidecar emits `runtime.turn_started` with `{"origin": "provider"}` and a
|
||||
sidecar-minted turn id. Runner admits it only while awaiting and only for
|
||||
Claude, with the same settled-identity and receipt rules as a controller turn,
|
||||
and publishes PRP `turn.started` with `origin: "provider"`. Any other
|
||||
provider-started turn fails the session closed. A controller `turn.start` while
|
||||
awaiting disarms and replaces the sidecar first. If the provider exits while armed, the
|
||||
sidecar reports `runtime.diagnostic` with code `ACPX_PROVIDER_PROCESS_LOST`; that,
|
||||
or a failed sidecar transport, ends the wait and Runner publishes the
|
||||
`run.terminal` it held back for the completed turn. Park is refused while awaiting.
|
||||
Runner waits at most 5 s for the sidecar to answer an arm or a disarm. No answer
|
||||
is handled as a lost sidecar, and Runner still publishes the held `run.terminal`.
|
||||
When a `run.attach` from a different run succeeds, the wait ends and Runner
|
||||
discards the held `run.terminal`. If that attach fails, the wait and its held
|
||||
`run.terminal` stay with the old run. A `run.attach` from the same run publishes
|
||||
the held `run.terminal`.
|
||||
Runner saves the wait with its held `run.terminal` in the provider state; a
|
||||
Runner restart during the wait publishes that `run.terminal` once and resumes
|
||||
in a fresh process.
|
||||
|
||||
The check also inspects the live provider
|
||||
after that drain, so a late active-goal notification cannot be hidden by the
|
||||
older executor journal. This hook does not detach a controller, stop a process,
|
||||
|
||||
@@ -26,10 +26,11 @@ impl AcpxCommandExecutor {
|
||||
|
||||
pub(super) fn restart_session_after_turn(&mut self) -> Result<(), DurableRunnerError> {
|
||||
self.ensure_persistence_healthy()?;
|
||||
if self
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(AcpxProviderSession::continues_in_process)
|
||||
if self.awaiting_provider_turn()
|
||||
|| self
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(AcpxProviderSession::continues_in_process)
|
||||
|| !self.generation_restartable(AcpxProviderSession::can_restart_after_turn)
|
||||
{
|
||||
return Ok(());
|
||||
|
||||
@@ -43,6 +43,8 @@ use crate::qualified_launch::{verify_executable_launch_artifact, verify_launch_a
|
||||
|
||||
#[path = "acpx_park_preparation.rs"]
|
||||
mod park_preparation;
|
||||
#[path = "acpx_provider_turn_wait.rs"]
|
||||
mod provider_turn_wait;
|
||||
|
||||
fn bounded_turn_timing(response: &Value, elapsed: Duration) -> Value {
|
||||
let mut receipt = serde_json::Map::new();
|
||||
@@ -639,6 +641,8 @@ struct AcpxDurableState {
|
||||
/// clears it and resumes the saved session. Written only while set.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
turn_end_restart_failed_run: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
provider_turn_wait: Option<provider_turn_wait::AcpxProviderTurnWait>,
|
||||
}
|
||||
|
||||
/// The saved launch names the same executable and sidecar files as this
|
||||
@@ -719,6 +723,7 @@ impl AcpxDurableState {
|
||||
open_semantic_calls: BTreeSet::new(),
|
||||
retired_semantic_calls: BTreeSet::new(),
|
||||
turn_end_restart_failed_run: None,
|
||||
provider_turn_wait: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -806,6 +811,10 @@ impl AcpxDurableState {
|
||||
.turn_end_restart_failed_run
|
||||
.as_ref()
|
||||
.is_some_and(|run_id| *run_id != self.descriptor.run_id)
|
||||
|| self
|
||||
.provider_turn_wait
|
||||
.as_ref()
|
||||
.is_some_and(|wait| !wait.is_consistent(self))
|
||||
{
|
||||
return Err(DurableRunnerError::invalid(
|
||||
"ACPX durable provider state is malformed or inconsistent",
|
||||
@@ -1030,6 +1039,7 @@ pub struct AcpxCommandExecutor {
|
||||
failed_turn_end_restart: Option<String>,
|
||||
/// The failed turn-end restart's cause, until the Runner records it.
|
||||
unreported_turn_end_restart_failure: Option<String>,
|
||||
turn_accepts_provider_turns: bool,
|
||||
}
|
||||
|
||||
// The durable command's turn and the live provider callback have separate IDs.
|
||||
@@ -1071,6 +1081,7 @@ impl AcpxCommandExecutor {
|
||||
recovery_pending_effects: None,
|
||||
failed_turn_end_restart: None,
|
||||
unreported_turn_end_restart_failure: None,
|
||||
turn_accepts_provider_turns: false,
|
||||
restore_error: None,
|
||||
persistence_error: None,
|
||||
#[cfg(test)]
|
||||
@@ -1284,6 +1295,14 @@ impl AcpxCommandExecutor {
|
||||
if self.session.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
if self.awaiting_provider_turn() {
|
||||
let same_run = self
|
||||
.state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.descriptor.run_id == self.context.run_id);
|
||||
self.end_provider_turn_wait(same_run)?;
|
||||
self.save_state()?;
|
||||
}
|
||||
let Some(state) = self.state.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
@@ -1594,6 +1613,9 @@ impl AcpxCommandExecutor {
|
||||
|
||||
fn save_state(&mut self) -> Result<(), DurableRunnerError> {
|
||||
self.ensure_persistence_healthy()?;
|
||||
if let Some(state) = self.state.as_mut() {
|
||||
state.leave_finished_provider_turn_wait()?;
|
||||
}
|
||||
let result = self
|
||||
.state
|
||||
.as_ref()
|
||||
@@ -1865,6 +1887,7 @@ impl AcpxCommandExecutor {
|
||||
"run.attach requires the same settled ACPX provider profile and session",
|
||||
));
|
||||
}
|
||||
let different_run = state.descriptor.run_id != attachment_context.run_id;
|
||||
if !self.rebind_prepared_session(&descriptor, &tool_set, next_candidate.is_some()) {
|
||||
if let Some(session) = self.session.as_mut() {
|
||||
session
|
||||
@@ -1895,6 +1918,11 @@ impl AcpxCommandExecutor {
|
||||
state.semantic_result = None;
|
||||
state.pending_events.clear();
|
||||
state.lifecycle = "suspended".to_owned();
|
||||
let ended_wait = if different_run {
|
||||
state.provider_turn_wait.take()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// A new run starts a fresh provider generation; the previous run's
|
||||
// failed turn-end restart no longer applies to it. A same-run attach
|
||||
// keeps the restriction.
|
||||
@@ -1914,6 +1942,11 @@ impl AcpxCommandExecutor {
|
||||
if saved.is_ok() && clears_turn_end_restart_failure {
|
||||
self.failed_turn_end_restart = None;
|
||||
}
|
||||
if let (true, Some(wait), Some(session)) =
|
||||
(saved.is_ok(), ended_wait.as_ref(), self.session.as_mut())
|
||||
{
|
||||
session.disarm_provider_turn(&wait.turn_id);
|
||||
}
|
||||
saved
|
||||
}
|
||||
|
||||
@@ -2239,6 +2272,8 @@ impl AcpxCommandExecutor {
|
||||
.map(|state| PathBuf::from(&state.descriptor.cwd))
|
||||
.expect("ACPX state exists before turn start");
|
||||
let measure = payload.get("performance").and_then(Value::as_bool) == Some(true);
|
||||
self.turn_accepts_provider_turns =
|
||||
payload.get("providerStartedTurns").and_then(Value::as_bool) == Some(true);
|
||||
let timing_started = std::time::Instant::now();
|
||||
let start_result = self
|
||||
.session
|
||||
@@ -2475,10 +2510,7 @@ impl AcpxCommandExecutor {
|
||||
.as_ref()
|
||||
.and_then(|state| state.active_turn_id.clone());
|
||||
let Some(turn_id) = turn_id else {
|
||||
return Ok(CommandExecution::result(json!({
|
||||
"status": "already_settled",
|
||||
"reason": reason,
|
||||
})));
|
||||
return self.settled_without_turn(reason);
|
||||
};
|
||||
self.session
|
||||
.as_mut()
|
||||
@@ -2509,10 +2541,7 @@ impl AcpxCommandExecutor {
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.descriptor.agent == "pi");
|
||||
if turn_id.is_none() && !stop_idle_pi {
|
||||
return Ok(CommandExecution::result(json!({
|
||||
"status": "already_settled",
|
||||
"reason": reason,
|
||||
})));
|
||||
return self.settled_without_turn(reason);
|
||||
}
|
||||
let provider_lifetime_fence_candidates = {
|
||||
let session = self
|
||||
@@ -2879,6 +2908,9 @@ impl AcpxCommandExecutor {
|
||||
self.state
|
||||
.as_ref()
|
||||
.and_then(|state| state.goal_source_revision),
|
||||
self.state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.provider_turn_wait.is_some()),
|
||||
);
|
||||
let events = match events {
|
||||
Ok(events) => events,
|
||||
@@ -2913,6 +2945,25 @@ impl AcpxCommandExecutor {
|
||||
};
|
||||
let mut provider_turn_settled = false;
|
||||
for event in events {
|
||||
if let crate::acpx_provider_state::AcpxProviderStateEvent::ProviderTurnStarted {
|
||||
turn_id,
|
||||
} = &event
|
||||
{
|
||||
let state = self
|
||||
.state
|
||||
.as_mut()
|
||||
.expect("ACPX state remains available while polling");
|
||||
if state.lifecycle != "session_open" || state.active_turn_id.is_some() {
|
||||
return Err(DurableRunnerError::invalid(
|
||||
"ACPX provider turn started outside an open idle session",
|
||||
));
|
||||
}
|
||||
state.active_turn_id = Some(turn_id.clone());
|
||||
state.attempted_turn_controls.clear();
|
||||
state.semantic_result = None;
|
||||
state.lifecycle = "turn_active".to_owned();
|
||||
self.context.provider_turn_id = Some(turn_id.clone());
|
||||
}
|
||||
let provider_process_lost = matches!(
|
||||
&event,
|
||||
crate::acpx_provider_state::AcpxProviderStateEvent::TurnTerminal {
|
||||
@@ -2947,10 +2998,16 @@ impl AcpxCommandExecutor {
|
||||
})?;
|
||||
}
|
||||
}
|
||||
let provider_lost = matches!(
|
||||
&event,
|
||||
crate::acpx_provider_state::AcpxProviderStateEvent::Diagnostic { code, .. }
|
||||
if code == crate::generated_acpx_sidecar_contract::GENERATED_ACPX_PROVIDER_TURN_LOST_CODE
|
||||
);
|
||||
let state = self
|
||||
.state
|
||||
.as_mut()
|
||||
.expect("ACPX state remains available while polling");
|
||||
let mut accepted = Vec::with_capacity(normalized.len());
|
||||
for mut event in normalized {
|
||||
if matches!(
|
||||
event.event_type.as_str(),
|
||||
@@ -2973,8 +3030,10 @@ impl AcpxCommandExecutor {
|
||||
if event.event_type == "run.result.proposed" {
|
||||
state.semantic_result = Some(event.payload.clone());
|
||||
}
|
||||
state.push(event)?;
|
||||
accepted.push(event);
|
||||
}
|
||||
let turn_ended = terminal.is_some();
|
||||
let mut run_terminal = None;
|
||||
if let Some(event_type) = terminal {
|
||||
state.active_turn_id = None;
|
||||
state.provider_exit_unconfirmed |= provider_process_lost;
|
||||
@@ -2992,15 +3051,12 @@ impl AcpxCommandExecutor {
|
||||
provider_turn_settled = true;
|
||||
// An ACP goal has session lifetime, not prompt lifetime.
|
||||
// Out-of-prompt goal updates remain observable after quiescence.
|
||||
if !provider_process_lost
|
||||
let goal_continues = !provider_process_lost
|
||||
&& state
|
||||
.goal_projection
|
||||
.pointer("/goal/status")
|
||||
.and_then(Value::as_str)
|
||||
== Some("active")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
== Some("active");
|
||||
let (turn_terminal_state, status) = match event_type.as_str() {
|
||||
"turn.completed" => ("completed", "succeeded"),
|
||||
"turn.cancelled" => ("cancelled", "cancelled"),
|
||||
@@ -3019,18 +3075,20 @@ impl AcpxCommandExecutor {
|
||||
.and_then(Value::as_str),
|
||||
status == "succeeded",
|
||||
);
|
||||
state.push(NormalizedProviderEvent {
|
||||
event_type: "run.terminal".to_owned(),
|
||||
priority: EventPriority::P0,
|
||||
payload: json!({
|
||||
run_terminal = (!goal_continues).then(|| {
|
||||
json!({
|
||||
"schema": "paperclip.prp.terminal.v1",
|
||||
"status": status,
|
||||
"turnTerminalState": turn_terminal_state,
|
||||
"runTerminalState": status,
|
||||
"reportedWorkDisposition": disposition,
|
||||
"provider": "acpx",
|
||||
}),
|
||||
})?;
|
||||
})
|
||||
});
|
||||
}
|
||||
self.settle_turn_events(accepted, turn_ended, run_terminal)?;
|
||||
if provider_lost {
|
||||
self.end_provider_turn_wait(true)?;
|
||||
}
|
||||
}
|
||||
if provider_turn_settled {
|
||||
@@ -3088,6 +3146,7 @@ impl CommandExecutor for AcpxCommandExecutor {
|
||||
|| !state.pending_runtime_requests.is_empty()
|
||||
|| (!state.goal_projection.is_null() && !goal_is_dormant(&state.goal_projection))
|
||||
|| !state.descriptor.names_qualified_release()
|
||||
|| state.provider_turn_wait.is_some()
|
||||
|| self.candidate_admission().is_some()
|
||||
|| !acpx_release_profile(&state.descriptor.agent)
|
||||
.is_some_and(|profile| !profile.qualification_pending)
|
||||
@@ -3529,7 +3588,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn descriptor(agent: &str) -> Value {
|
||||
pub(super) fn descriptor(agent: &str) -> Value {
|
||||
let profile = acpx_release_profile(agent).unwrap();
|
||||
let mut value = json!({
|
||||
"kind": "acpx",
|
||||
|
||||
@@ -24,6 +24,8 @@ use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS, SHORT_STABLE
|
||||
|
||||
#[path = "acpx_provider_generation.rs"]
|
||||
mod generation;
|
||||
#[path = "acpx_provider_turn.rs"]
|
||||
mod provider_turn;
|
||||
|
||||
const MAX_ID_CHARS: usize = 240;
|
||||
const MAX_MODEL_CHARS: usize = 240;
|
||||
@@ -364,6 +366,8 @@ pub struct AcpxProviderSession {
|
||||
quiet_turn_observed: bool,
|
||||
continues_in_process: bool,
|
||||
turn_start_replaced_provider: bool,
|
||||
provider_turn_offer: Option<String>,
|
||||
run_result_reported: Option<String>,
|
||||
generation_has_run_turn: bool,
|
||||
restarted_after_turn_from: Option<u32>,
|
||||
prepared_generation: Option<String>,
|
||||
@@ -429,6 +433,8 @@ impl AcpxProviderSession {
|
||||
quiet_turn_observed: false,
|
||||
continues_in_process: false,
|
||||
turn_start_replaced_provider: false,
|
||||
provider_turn_offer: None,
|
||||
run_result_reported: None,
|
||||
generation_has_run_turn: false,
|
||||
restarted_after_turn_from: None,
|
||||
prepared_generation: None,
|
||||
@@ -684,6 +690,7 @@ impl AcpxProviderSession {
|
||||
&& self.background_work_is_quiet();
|
||||
self.continues_in_process = false;
|
||||
self.quiet_turn_observed = false;
|
||||
self.provider_turn_offer = None;
|
||||
let restart_now = self.generation_has_run_turn && !continue_in_process;
|
||||
if restart_now {
|
||||
if let Err(error) = self.restart_idle_provider(None) {
|
||||
@@ -857,7 +864,16 @@ impl AcpxProviderSession {
|
||||
timeout: Duration,
|
||||
) -> Result<Option<Vec<AcpxProviderStateEvent>>, LocalRunnerError> {
|
||||
Ok(self
|
||||
.poll_output(timeout, false, None)?
|
||||
.poll_output(timeout, false, None, false)?
|
||||
.map(|(events, _)| events))
|
||||
}
|
||||
|
||||
pub fn poll_awaited_event(
|
||||
&mut self,
|
||||
timeout: Duration,
|
||||
) -> Result<Option<Vec<AcpxProviderStateEvent>>, LocalRunnerError> {
|
||||
Ok(self
|
||||
.poll_output(timeout, false, None, true)?
|
||||
.map(|(events, _)| events))
|
||||
}
|
||||
|
||||
@@ -866,6 +882,7 @@ impl AcpxProviderSession {
|
||||
timeout: Duration,
|
||||
disconnected: bool,
|
||||
last_goal_revision: Option<u64>,
|
||||
awaited: bool,
|
||||
) -> Result<Option<(Vec<AcpxProviderStateEvent>, Option<Value>)>, LocalRunnerError> {
|
||||
self.ensure_open()?;
|
||||
let event = match self.transport.poll_event(timeout) {
|
||||
@@ -873,6 +890,11 @@ impl AcpxProviderSession {
|
||||
Err(error) => return Err(self.fail_closed(error)),
|
||||
};
|
||||
let Some(mut event) = event else {
|
||||
if awaited {
|
||||
if let Some(error) = self.transport.exited_error("provider turn wait") {
|
||||
return Err(self.fail_closed(error));
|
||||
}
|
||||
}
|
||||
return Ok(None);
|
||||
};
|
||||
let mut terminal_goal = None;
|
||||
@@ -921,8 +943,18 @@ impl AcpxProviderSession {
|
||||
payload.insert("origin".to_owned(), origin);
|
||||
}
|
||||
}
|
||||
if event.event_type == GeneratedAcpxSidecarEventType::RuntimeTurnStarted {
|
||||
return match self.admit_provider_turn(&event, awaited) {
|
||||
Ok(started) => Ok(Some((vec![started], terminal_goal))),
|
||||
Err(error) => Err(self.fail_closed(error)),
|
||||
};
|
||||
}
|
||||
let terminal_quiet = event.event_type == GeneratedAcpxSidecarEventType::RuntimeTurnTerminal
|
||||
&& generation::reports_no_live_background_tasks(&event.payload);
|
||||
let terminal_live = (event.event_type
|
||||
== GeneratedAcpxSidecarEventType::RuntimeTurnTerminal)
|
||||
.then(|| provider_turn::reported_live_background_tasks(&event.payload))
|
||||
.flatten();
|
||||
let mut next_state = self.state.clone();
|
||||
let events = match next_state.accept_event(&event) {
|
||||
Ok(events) => events,
|
||||
@@ -976,6 +1008,9 @@ impl AcpxProviderSession {
|
||||
if let Err(error) = validate_reserved_terminal_result(&next_state, result) {
|
||||
return Err(self.fail_closed(error));
|
||||
}
|
||||
if result.ok {
|
||||
self.record_run_result();
|
||||
}
|
||||
if let Err(error) =
|
||||
next_reserved_bridge.apply_result(crate::provider_bridge::ToolResult {
|
||||
call_id: result.call_id.clone(),
|
||||
@@ -1082,9 +1117,17 @@ impl AcpxProviderSession {
|
||||
&& self.cancelled_turn_id.as_deref() != Some(turn_id.as_str());
|
||||
}
|
||||
}
|
||||
let terminal_completed = self.completed_turn_observed;
|
||||
let terminal_turn = reconciled_events.iter().find_map(|event| match event {
|
||||
AcpxProviderStateEvent::TurnTerminal { turn_id, .. } => Some(turn_id.clone()),
|
||||
_ => None,
|
||||
});
|
||||
self.state = next_state;
|
||||
self.tool_bridge = next_bridge;
|
||||
self.reserved_tool_bridge = next_reserved_bridge;
|
||||
if let Some(turn_id) = terminal_turn {
|
||||
self.offer_provider_turn(&turn_id, terminal_completed, terminal_live);
|
||||
}
|
||||
Ok(Some((reconciled_events, terminal_goal)))
|
||||
}
|
||||
|
||||
@@ -1103,12 +1146,17 @@ impl AcpxProviderSession {
|
||||
};
|
||||
let duplicate = bridge.has_completed_call(&result.call_id);
|
||||
let detached = bridge.turn_closed();
|
||||
let reported_run_result =
|
||||
is_reserved_terminal_operation(&result.operation_id) && !result.is_error;
|
||||
bridge.apply_result(result.clone()).map_err(|error| {
|
||||
LocalRunnerError::invalid(format!(
|
||||
"ACPX tool result for call {} operation {} is invalid: {error}",
|
||||
result.call_id, result.operation_id
|
||||
))
|
||||
})?;
|
||||
if reported_run_result {
|
||||
self.record_run_result();
|
||||
}
|
||||
if duplicate {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1406,6 +1454,7 @@ impl AcpxProviderSession {
|
||||
self.generation_has_run_turn = false;
|
||||
self.quiet_turn_observed = false;
|
||||
self.continues_in_process = false;
|
||||
self.provider_turn_offer = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2314,6 +2363,8 @@ mod mode_tests {
|
||||
quiet_turn_observed: false,
|
||||
continues_in_process: false,
|
||||
turn_start_replaced_provider: false,
|
||||
provider_turn_offer: None,
|
||||
run_result_reported: None,
|
||||
generation_has_run_turn: false,
|
||||
restarted_after_turn_from: None,
|
||||
prepared_generation: None,
|
||||
|
||||
@@ -80,6 +80,9 @@ pub enum AcpxProviderStateEvent {
|
||||
status: AcpxTurnStatus,
|
||||
error: Option<Value>,
|
||||
},
|
||||
ProviderTurnStarted {
|
||||
turn_id: String,
|
||||
},
|
||||
Process(Value),
|
||||
Goal(Value),
|
||||
Diagnostic {
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use super::AcpxProviderSession;
|
||||
use crate::acpx_provider_state::AcpxProviderStateEvent;
|
||||
use crate::acpx_sidecar_transport::AcpxSidecarEvent;
|
||||
use crate::generated_acpx_sidecar_contract::{
|
||||
read_generated_acpx_background_tasks, GeneratedAcpxSidecarCommand,
|
||||
GENERATED_ACPX_PROVIDER_TURN_ORIGIN,
|
||||
};
|
||||
use crate::local_runner::LocalRunnerError;
|
||||
|
||||
pub(super) fn reported_live_background_tasks(payload: &Value) -> Option<u64> {
|
||||
payload
|
||||
.get("backgroundTasks")
|
||||
.and_then(read_generated_acpx_background_tasks)
|
||||
}
|
||||
|
||||
impl AcpxProviderSession {
|
||||
pub fn run_result_reported(&self) -> bool {
|
||||
self.run_result_reported.as_deref() == Some(self.state.run_id())
|
||||
}
|
||||
|
||||
pub(super) fn record_run_result(&mut self) {
|
||||
self.run_result_reported = Some(self.state.run_id().to_owned());
|
||||
}
|
||||
|
||||
pub fn take_provider_turn_offer(&mut self) -> Option<String> {
|
||||
self.provider_turn_offer.take()
|
||||
}
|
||||
|
||||
pub fn arm_provider_turn(&mut self, turn_id: &str) -> bool {
|
||||
self.request_provider_turn(turn_id, true)
|
||||
.is_some_and(|response| response.get("armed").and_then(Value::as_bool) == Some(true))
|
||||
}
|
||||
|
||||
pub fn disarm_provider_turn(&mut self, turn_id: &str) {
|
||||
if !self.transport_terminated {
|
||||
self.request_provider_turn(turn_id, false);
|
||||
}
|
||||
}
|
||||
|
||||
fn request_provider_turn(&mut self, turn_id: &str, armed: bool) -> Option<Value> {
|
||||
let deadline = Instant::now()
|
||||
+ super::ACPX_TURN_CANCEL_ACK_TIMEOUT.min(self.config.transport.request_timeout);
|
||||
match self.transport.request_by(
|
||||
GeneratedAcpxSidecarCommand::SessionProviderTurn,
|
||||
json!({"turnId": turn_id, "armed": armed}),
|
||||
deadline,
|
||||
) {
|
||||
Ok(Ok(response)) => Some(response),
|
||||
Ok(Err(_)) => None,
|
||||
Err(error) => {
|
||||
self.fail_closed(error);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn offer_provider_turn(
|
||||
&mut self,
|
||||
turn_id: &str,
|
||||
completed: bool,
|
||||
live: Option<u64>,
|
||||
) {
|
||||
let eligible = completed
|
||||
&& live.is_some_and(|live| live > 0)
|
||||
&& !self.run_result_reported()
|
||||
&& self.config.agent == "claude"
|
||||
&& self.config.candidate_admission.is_none()
|
||||
&& !self.runtime_retired
|
||||
&& self.failure.is_none()
|
||||
&& !self.state.has_pending_requests()
|
||||
&& !self.state.has_pending_tools()
|
||||
&& !self.state.settled_turn_identity_capacity_reached()
|
||||
&& !self.tool_bridge.replay_history_blocks_admission()
|
||||
&& !self.reserved_tool_bridge.replay_history_blocks_admission();
|
||||
self.provider_turn_offer = eligible.then(|| turn_id.to_owned());
|
||||
}
|
||||
|
||||
pub(super) fn admit_provider_turn(
|
||||
&mut self,
|
||||
event: &AcpxSidecarEvent,
|
||||
awaited: bool,
|
||||
) -> Result<AcpxProviderStateEvent, LocalRunnerError> {
|
||||
if !awaited
|
||||
|| self.config.agent != "claude"
|
||||
|| self.state.active_turn_id().is_some()
|
||||
|| self.state.has_pending_tools()
|
||||
{
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"ACPX sidecar started a provider turn outside an awaited continuation",
|
||||
));
|
||||
}
|
||||
if event.payload != json!({"origin": GENERATED_ACPX_PROVIDER_TURN_ORIGIN})
|
||||
|| event.run_id.as_deref() != Some(self.state.run_id())
|
||||
{
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"ACPX provider turn start violated its bounded contract",
|
||||
));
|
||||
}
|
||||
let turn_id = event.turn_id.clone().ok_or_else(|| {
|
||||
LocalRunnerError::invalid("ACPX provider turn start omitted its turn binding")
|
||||
})?;
|
||||
self.state.validate_new_turn_identity(&turn_id)?;
|
||||
let mut next_tool_bridge = self.tool_bridge.clone();
|
||||
next_tool_bridge.prepare_turn().map_err(|error| {
|
||||
LocalRunnerError::invalid(format!(
|
||||
"ACPX dynamic tool receipt rotation failed: {error}"
|
||||
))
|
||||
})?;
|
||||
let mut next_reserved_tool_bridge = self.reserved_tool_bridge.clone();
|
||||
next_reserved_tool_bridge.prepare_turn().map_err(|error| {
|
||||
LocalRunnerError::invalid(format!(
|
||||
"ACPX reserved tool receipt rotation failed: {error}"
|
||||
))
|
||||
})?;
|
||||
let mut next_state = self.state.clone();
|
||||
next_state.begin_turn(turn_id.clone())?;
|
||||
self.state = next_state;
|
||||
self.tool_bridge = next_tool_bridge;
|
||||
self.reserved_tool_bridge = next_reserved_tool_bridge;
|
||||
self.provider_turn_offer = None;
|
||||
self.completed_turn_observed = false;
|
||||
self.quiet_turn_observed = false;
|
||||
self.continues_in_process = false;
|
||||
self.cancelled_turn_id = None;
|
||||
Ok(AcpxProviderStateEvent::ProviderTurnStarted { turn_id })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use super::{AcpxCommandExecutor, AcpxDurableState};
|
||||
use crate::durable::{CommandExecution, DurableRunnerError, EventPriority};
|
||||
use crate::provider_events::NormalizedProviderEvent;
|
||||
use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS};
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub(super) struct AcpxProviderTurnWait {
|
||||
pub(super) turn_id: String,
|
||||
pub(super) held_run_terminal: Value,
|
||||
}
|
||||
|
||||
impl AcpxProviderTurnWait {
|
||||
pub(super) fn is_consistent(&self, state: &AcpxDurableState) -> bool {
|
||||
is_stable_id(&self.turn_id, DURABLE_STABLE_ID_CHARS)
|
||||
&& self.held_run_terminal.is_object()
|
||||
&& state.lifecycle == "session_open"
|
||||
&& state.active_turn_id.is_none()
|
||||
&& state.identity.is_some()
|
||||
}
|
||||
|
||||
pub(super) fn into_run_terminal(self) -> NormalizedProviderEvent {
|
||||
run_terminal_event(self.held_run_terminal)
|
||||
}
|
||||
}
|
||||
|
||||
fn run_terminal_event(payload: Value) -> NormalizedProviderEvent {
|
||||
NormalizedProviderEvent {
|
||||
event_type: "run.terminal".to_owned(),
|
||||
priority: EventPriority::P0,
|
||||
payload,
|
||||
}
|
||||
}
|
||||
|
||||
impl AcpxDurableState {
|
||||
pub(super) fn leave_finished_provider_turn_wait(&mut self) -> Result<(), DurableRunnerError> {
|
||||
if self
|
||||
.provider_turn_wait
|
||||
.as_ref()
|
||||
.is_none_or(|wait| wait.is_consistent(self))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let wait = self
|
||||
.provider_turn_wait
|
||||
.take()
|
||||
.expect("checked provider turn wait");
|
||||
if matches!(self.lifecycle.as_str(), "turn_starting" | "turn_active") {
|
||||
return Ok(());
|
||||
}
|
||||
self.push(wait.into_run_terminal())
|
||||
}
|
||||
}
|
||||
|
||||
impl AcpxCommandExecutor {
|
||||
pub(super) fn awaiting_provider_turn(&self) -> bool {
|
||||
self.state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.provider_turn_wait.is_some())
|
||||
}
|
||||
|
||||
pub(super) fn settle_turn_events(
|
||||
&mut self,
|
||||
events: Vec<NormalizedProviderEvent>,
|
||||
turn_ended: bool,
|
||||
run_terminal: Option<Value>,
|
||||
) -> Result<(), DurableRunnerError> {
|
||||
let offer = turn_ended
|
||||
.then(|| {
|
||||
self.session
|
||||
.as_mut()
|
||||
.and_then(|session| session.take_provider_turn_offer())
|
||||
})
|
||||
.flatten();
|
||||
let wait = match (offer, run_terminal) {
|
||||
(Some(turn_id), Some(held_run_terminal))
|
||||
if self.turn_accepts_provider_turns
|
||||
&& self
|
||||
.session
|
||||
.as_mut()
|
||||
.is_some_and(|session| session.arm_provider_turn(&turn_id)) =>
|
||||
{
|
||||
Ok(AcpxProviderTurnWait {
|
||||
turn_id,
|
||||
held_run_terminal,
|
||||
})
|
||||
}
|
||||
(_, run_terminal) => Err(run_terminal),
|
||||
};
|
||||
let state = self
|
||||
.state
|
||||
.as_mut()
|
||||
.expect("ACPX state remains available while polling");
|
||||
for mut event in events {
|
||||
if wait.is_ok() && event.event_type == "turn.completed" {
|
||||
event.payload["awaitingProviderTurn"] = json!(true);
|
||||
}
|
||||
state.push(event)?;
|
||||
}
|
||||
match wait {
|
||||
Ok(wait) => state.provider_turn_wait = Some(wait),
|
||||
Err(Some(payload)) => state.push(run_terminal_event(payload))?,
|
||||
Err(None) => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn end_provider_turn_wait(
|
||||
&mut self,
|
||||
publish: bool,
|
||||
) -> Result<(), DurableRunnerError> {
|
||||
let Some(wait) = self
|
||||
.state
|
||||
.as_mut()
|
||||
.and_then(|state| state.provider_turn_wait.take())
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(session) = self.session.as_mut() {
|
||||
session.disarm_provider_turn(&wait.turn_id);
|
||||
}
|
||||
if publish {
|
||||
self.state
|
||||
.as_mut()
|
||||
.expect("ACPX state exists with a provider turn wait")
|
||||
.push(wait.into_run_terminal())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn settled_without_turn(
|
||||
&mut self,
|
||||
reason: &str,
|
||||
) -> Result<CommandExecution, DurableRunnerError> {
|
||||
if self.awaiting_provider_turn() {
|
||||
self.end_provider_turn_wait(true)?;
|
||||
self.save_state()?;
|
||||
}
|
||||
Ok(CommandExecution {
|
||||
result: json!({"status": "already_settled", "reason": reason}),
|
||||
events: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::super::tests::descriptor;
|
||||
use super::super::AcpxDurableState;
|
||||
use super::AcpxProviderTurnWait;
|
||||
use crate::provider_bridge::{authorized_tool_catalog_digest, AuthorizedToolSet};
|
||||
|
||||
fn state() -> AcpxDurableState {
|
||||
let operations = Vec::new();
|
||||
AcpxDurableState::new(
|
||||
serde_json::from_value(descriptor("claude")).unwrap(),
|
||||
AuthorizedToolSet {
|
||||
schema: "paperclip.runner.authorized-tools.v1".into(),
|
||||
schema_version: 1,
|
||||
catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(),
|
||||
operations,
|
||||
},
|
||||
"test".into(),
|
||||
)
|
||||
}
|
||||
|
||||
fn wait() -> AcpxProviderTurnWait {
|
||||
AcpxProviderTurnWait {
|
||||
turn_id: "turn-1".into(),
|
||||
held_run_terminal: json!({"schema":"paperclip.prp.terminal.v1","turnTerminalState":"completed"}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_outside_a_wait_keeps_the_previous_release_shape_in_both_directions() {
|
||||
let saved = serde_json::to_value(state()).unwrap();
|
||||
assert!(saved.get("providerTurnWait").is_none());
|
||||
let resumed: AcpxDurableState = serde_json::from_value(saved).unwrap();
|
||||
assert_eq!(resumed.provider_turn_wait, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wait_round_trips_and_an_unknown_field_is_refused_like_a_previous_release_refuses_it() {
|
||||
let mut waiting = state();
|
||||
waiting.provider_turn_wait = Some(wait());
|
||||
let saved = serde_json::to_value(&waiting).unwrap();
|
||||
assert_eq!(saved["providerTurnWait"]["turnId"], json!("turn-1"));
|
||||
let resumed: AcpxDurableState = serde_json::from_value(saved.clone()).unwrap();
|
||||
assert_eq!(resumed.provider_turn_wait, Some(wait()));
|
||||
let mut future = saved;
|
||||
future["providerTurnWaitFromAFutureRelease"] = json!({});
|
||||
let refused = serde_json::from_value::<AcpxDurableState>(future).unwrap_err();
|
||||
assert!(refused.to_string().contains("unknown field"), "{refused}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaving_the_open_idle_session_ends_the_wait_by_the_next_lifecycle() {
|
||||
for (lifecycle, active, published) in [
|
||||
("closed", None, true),
|
||||
("suspended", None, true),
|
||||
("prepared", None, true),
|
||||
("turn_starting", Some("turn-2"), false),
|
||||
("turn_active", Some("provider-1"), false),
|
||||
] {
|
||||
let mut left = state();
|
||||
left.provider_turn_wait = Some(wait());
|
||||
left.lifecycle = lifecycle.into();
|
||||
left.active_turn_id = active.map(str::to_owned);
|
||||
left.leave_finished_provider_turn_wait().unwrap();
|
||||
assert_eq!(left.provider_turn_wait, None, "{lifecycle}");
|
||||
let terminals = left
|
||||
.pending_events
|
||||
.iter()
|
||||
.filter(|event| event.event_type == "run.terminal")
|
||||
.count();
|
||||
assert_eq!(terminals, usize::from(published), "{lifecycle}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -484,6 +484,22 @@ impl AcpxSidecarTransport {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn request_by(
|
||||
&mut self,
|
||||
command: GeneratedAcpxSidecarCommand,
|
||||
params: Value,
|
||||
deadline: Instant,
|
||||
) -> Result<Result<Value, LocalRunnerError>, LocalRunnerError> {
|
||||
let bound = self.request_deadline;
|
||||
self.request_deadline = Some(bound.map_or(deadline, |bound| bound.min(deadline)));
|
||||
let outcome = self.request_outcome(command, params);
|
||||
self.request_deadline = bound;
|
||||
match outcome? {
|
||||
CommandOutcome::Success(value) => Ok(Ok(value)),
|
||||
CommandOutcome::Rejected { error, .. } => Ok(Err(error)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn request_turn_start(
|
||||
&mut self,
|
||||
params: Value,
|
||||
@@ -941,6 +957,17 @@ impl AcpxSidecarTransport {
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn exited_error(&mut self, stage: &str) -> Option<LocalRunnerError> {
|
||||
match self.process.try_wait() {
|
||||
Ok(None) => None,
|
||||
Ok(Some(_)) | Err(_) => {
|
||||
let error = self.closed_error(stage);
|
||||
self.poison();
|
||||
Some(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn closed_error(&mut self, stage: &str) -> LocalRunnerError {
|
||||
self.drain_diagnostics(Duration::from_millis(20));
|
||||
let suffix = self.diagnostic_suffix();
|
||||
|
||||
@@ -123,6 +123,8 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut provider_lost = false;
|
||||
let mut session_identity = Value::Null;
|
||||
let mut source_run = String::new();
|
||||
let mut pending_provider_run: Option<String> = None;
|
||||
let mut provider_rearmable = false;
|
||||
let mut source_pending_completion: Option<String> = None;
|
||||
let mut source_goal_revision = 0_u64;
|
||||
let mut runtime_directory: Option<std::path::PathBuf> = None;
|
||||
@@ -217,6 +219,94 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if command == "turn.cancel"
|
||||
&& provider_turn_tokens()
|
||||
.iter()
|
||||
.any(|token| token == "provider-turn-open")
|
||||
&& request["params"]["turnId"]
|
||||
.as_str()
|
||||
.is_some_and(|turn| turn.starts_with("provider-"))
|
||||
{
|
||||
let turn = request["params"]["turnId"].as_str().unwrap().to_owned();
|
||||
write_json(
|
||||
&mut stdout,
|
||||
&json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"id":id,"ok":true,
|
||||
"result":{"cancelled":true,"sessionClosed":true}}),
|
||||
)?;
|
||||
write_json(
|
||||
&mut stdout,
|
||||
&json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":next_sequence,
|
||||
"eventType":"runtime.turn_terminal","runId":"run-1","turnId":turn,
|
||||
"payload":{"status":"cancelled","stopReason":"cancelled"}}),
|
||||
)?;
|
||||
next_sequence += 1;
|
||||
continue;
|
||||
}
|
||||
if command == "session.provider_turn" {
|
||||
if let Some(journal) = source_journal.as_mut() {
|
||||
write_json(journal, &request)?;
|
||||
}
|
||||
let tokens = provider_turn_tokens();
|
||||
let lost = if request["params"]["armed"] == json!(true) {
|
||||
"provider-arm-reply-lost"
|
||||
} else {
|
||||
"provider-disarm-reply-lost"
|
||||
};
|
||||
if tokens.iter().any(|token| token == lost) {
|
||||
continue;
|
||||
}
|
||||
let armed = request["params"]["armed"] == json!(true)
|
||||
&& (pending_provider_run.is_some() || provider_rearmable)
|
||||
&& !tokens.iter().any(|token| token == "provider-arm-reject");
|
||||
write_json(
|
||||
&mut stdout,
|
||||
&json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"id":id,"ok":true,
|
||||
"result":{"armed":armed}}),
|
||||
)?;
|
||||
if request["params"]["armed"] == json!(false)
|
||||
&& tokens
|
||||
.iter()
|
||||
.any(|token| token == "provider-turn-after-disarm")
|
||||
{
|
||||
if let Some(run) = pending_provider_run.take() {
|
||||
next_sequence = write_provider_turn(&mut stdout, next_sequence, &run)?;
|
||||
}
|
||||
}
|
||||
if armed
|
||||
&& tokens
|
||||
.iter()
|
||||
.any(|token| token == "provider-sidecar-exit-after-arm")
|
||||
{
|
||||
std::thread::spawn(|| {
|
||||
std::thread::sleep(Duration::from_millis(300));
|
||||
std::process::exit(0);
|
||||
});
|
||||
}
|
||||
if armed
|
||||
&& tokens
|
||||
.iter()
|
||||
.any(|token| token == "provider-lost-after-arm")
|
||||
{
|
||||
write_json(
|
||||
&mut stdout,
|
||||
&json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":next_sequence,
|
||||
"eventType":"runtime.diagnostic","runId":null,"turnId":null,
|
||||
"payload":{"code":"ACPX_PROVIDER_PROCESS_LOST","message":"The provider exited."}}),
|
||||
)?;
|
||||
next_sequence += 1;
|
||||
}
|
||||
if armed && tokens.iter().any(|token| token == "provider-turn-hold") {
|
||||
continue;
|
||||
}
|
||||
if let (true, Some(run)) = (armed, pending_provider_run.take()) {
|
||||
next_sequence = write_provider_turn(&mut stdout, next_sequence, &run)?;
|
||||
provider_rearmable = tokens.iter().any(|token| token == "provider-terminal-live");
|
||||
} else {
|
||||
provider_rearmable = false;
|
||||
}
|
||||
pending_provider_run = None;
|
||||
continue;
|
||||
}
|
||||
if let Some(journal) = source_journal.as_mut() {
|
||||
write_json(journal, &request)?;
|
||||
if command == "run.attach" {
|
||||
@@ -452,6 +542,12 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
json!({"status":status}),
|
||||
)?;
|
||||
next_sequence += 1;
|
||||
next_sequence = offer_provider_turn(
|
||||
&mut stdout,
|
||||
next_sequence,
|
||||
&source_run,
|
||||
&mut pending_provider_run,
|
||||
)?;
|
||||
}
|
||||
if command == "tool.resolve" {
|
||||
let turn = source_pending_completion
|
||||
@@ -974,6 +1070,22 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
)?;
|
||||
next_sequence += 1;
|
||||
}
|
||||
if command == "turn.start"
|
||||
&& matches!(
|
||||
mode,
|
||||
"turns-tool-terminal"
|
||||
| "turns-reused-tool-id-terminal"
|
||||
| "turns-tool-result-terminal"
|
||||
| "turns-reserved-result-terminal"
|
||||
)
|
||||
{
|
||||
next_sequence = offer_provider_turn(
|
||||
&mut stdout,
|
||||
next_sequence,
|
||||
"run-1",
|
||||
&mut pending_provider_run,
|
||||
)?;
|
||||
}
|
||||
if command == "turn.start" && mode == "turns-multiple-tool-results-terminal" {
|
||||
for index in 1..=2 {
|
||||
write_turn_event(
|
||||
@@ -1448,6 +1560,134 @@ fn bootstrap_success(
|
||||
})
|
||||
}
|
||||
|
||||
fn provider_turn_tokens() -> Vec<String> {
|
||||
BACKGROUND_TASKS
|
||||
.get()
|
||||
.and_then(Option::as_ref)
|
||||
.and_then(|path| std::fs::read_to_string(path).ok())
|
||||
.map(|modes| modes.split_whitespace().map(str::to_owned).collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn offer_provider_turn(
|
||||
output: &mut impl Write,
|
||||
sequence: u64,
|
||||
default_run: &str,
|
||||
pending: &mut Option<String>,
|
||||
) -> io::Result<u64> {
|
||||
let tokens = provider_turn_tokens();
|
||||
if !tokens.iter().any(|token| token == "provider-turn") {
|
||||
return Ok(sequence);
|
||||
}
|
||||
if tokens.iter().any(|token| token == "provider-turn-unarmed") {
|
||||
return write_provider_turn(output, sequence, default_run);
|
||||
}
|
||||
*pending = Some(default_run.to_owned());
|
||||
Ok(sequence)
|
||||
}
|
||||
|
||||
fn write_provider_turn(
|
||||
output: &mut impl Write,
|
||||
mut sequence: u64,
|
||||
default_run: &str,
|
||||
) -> io::Result<u64> {
|
||||
let tokens = provider_turn_tokens();
|
||||
if !tokens.iter().any(|token| token == "provider-turn") {
|
||||
return Ok(sequence);
|
||||
}
|
||||
let value = |prefix: &str, fallback: &str| {
|
||||
tokens
|
||||
.iter()
|
||||
.find_map(|token| token.strip_prefix(prefix).map(str::to_owned))
|
||||
.unwrap_or_else(|| fallback.to_owned())
|
||||
};
|
||||
let turn_id = value("provider-turn-id=", "provider-1");
|
||||
let run_id = value("provider-run=", default_run);
|
||||
let frame = |sequence: u64, event_type: &str, payload: Value| {
|
||||
json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"sequence":sequence,
|
||||
"eventType":event_type,"runId":run_id,"turnId":turn_id,"payload":payload})
|
||||
};
|
||||
write_json(
|
||||
output,
|
||||
&frame(
|
||||
sequence,
|
||||
"runtime.turn_started",
|
||||
json!({"origin":"provider"}),
|
||||
),
|
||||
)?;
|
||||
sequence += 1;
|
||||
if let Some(call_id) = tokens
|
||||
.iter()
|
||||
.find_map(|token| token.strip_prefix("provider-reuse-call="))
|
||||
{
|
||||
write_json(
|
||||
output,
|
||||
&frame(
|
||||
sequence,
|
||||
"runtime.tool_called",
|
||||
json!({"callId":call_id,"operationId":"issues.read","input":{"id":"issue-1"}}),
|
||||
),
|
||||
)?;
|
||||
sequence += 1;
|
||||
}
|
||||
if tokens.iter().any(|token| token == "provider-result") {
|
||||
let result = valid_reserved_completion_result();
|
||||
write_json(
|
||||
output,
|
||||
&frame(
|
||||
sequence,
|
||||
"runtime.tool_called",
|
||||
json!({"callId":"call-provider-finish","operationId":"paperclip_finish","input":result.clone()}),
|
||||
),
|
||||
)?;
|
||||
write_json(
|
||||
output,
|
||||
&frame(
|
||||
sequence + 1,
|
||||
"runtime.event",
|
||||
json!({"type":"semantic_result","callId":"call-provider-finish","operationId":"paperclip_finish","ok":true,"result":result}),
|
||||
),
|
||||
)?;
|
||||
sequence += 2;
|
||||
}
|
||||
write_json(
|
||||
output,
|
||||
&frame(
|
||||
sequence,
|
||||
"runtime.event",
|
||||
json!({"type":"text_delta","text":"Claude finished its own work."}),
|
||||
),
|
||||
)?;
|
||||
sequence += 1;
|
||||
if tokens.iter().any(|token| token == "provider-turn-open") {
|
||||
return Ok(sequence);
|
||||
}
|
||||
write_json(
|
||||
output,
|
||||
&frame(
|
||||
sequence,
|
||||
"runtime.event",
|
||||
json!({"type":"status","tag":"usage_update","text":"terminal prompt usage",
|
||||
"breakdown":{"inputTokens":6,"outputTokens":4,"cachedReadTokens":0,"cachedWriteTokens":0,"thoughtTokens":0}}),
|
||||
),
|
||||
)?;
|
||||
sequence += 1;
|
||||
let live = if tokens.iter().any(|token| token == "provider-terminal-live") {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
write_json(
|
||||
output,
|
||||
&frame(
|
||||
sequence,
|
||||
"runtime.turn_terminal",
|
||||
json!({"status":"completed","stopReason":"end_turn","backgroundTasks":{"live":live}}),
|
||||
),
|
||||
)?;
|
||||
Ok(sequence + 1)
|
||||
}
|
||||
|
||||
fn write_turn_event(
|
||||
output: &mut impl Write,
|
||||
sequence: u64,
|
||||
|
||||
@@ -404,6 +404,20 @@ pub fn project_acpx_state_event(
|
||||
}),
|
||||
)
|
||||
}
|
||||
AcpxProviderStateEvent::ProviderTurnStarted { turn_id } => {
|
||||
require_projected_turn(context, turn_id)?;
|
||||
one(
|
||||
"turn.started",
|
||||
EventPriority::P0,
|
||||
json!({
|
||||
"provider": "acpx",
|
||||
"providerTurnId": turn_id,
|
||||
"status": "inProgress",
|
||||
"turn": {"id": turn_id, "status": "inProgress"},
|
||||
"origin": crate::generated_acpx_sidecar_contract::GENERATED_ACPX_PROVIDER_TURN_ORIGIN,
|
||||
}),
|
||||
)
|
||||
}
|
||||
AcpxProviderStateEvent::Process(details) => one(
|
||||
"harness.diagnostic",
|
||||
EventPriority::P1,
|
||||
|
||||
+292
@@ -227,6 +227,196 @@ fn a_full_settled_turn_ledger_still_rotates_through_a_restart() {
|
||||
session.shutdown("test complete").unwrap();
|
||||
}
|
||||
|
||||
fn provider_session(scratch: &Scratch, background: &str, agent: &str) -> AcpxProviderSession {
|
||||
provider_session_in(scratch, background, agent, "turns-tool-result-terminal")
|
||||
}
|
||||
|
||||
fn provider_session_in(
|
||||
scratch: &Scratch,
|
||||
background: &str,
|
||||
agent: &str,
|
||||
mode: &str,
|
||||
) -> AcpxProviderSession {
|
||||
scratch.background(background);
|
||||
AcpxProviderSession::start(&config(&scratch.0, mode, agent)).unwrap()
|
||||
}
|
||||
|
||||
fn arm(session: &mut AcpxProviderSession) -> bool {
|
||||
session
|
||||
.take_provider_turn_offer()
|
||||
.is_some_and(|turn_id| session.arm_provider_turn(&turn_id))
|
||||
}
|
||||
|
||||
fn poll_until_terminal(
|
||||
session: &mut AcpxProviderSession,
|
||||
turn_id: &str,
|
||||
) -> Vec<AcpxProviderStateEvent> {
|
||||
let mut seen = Vec::new();
|
||||
loop {
|
||||
let events = session.poll_event(Duration::from_secs(1)).unwrap().unwrap();
|
||||
let done = events.iter().any(|event| {
|
||||
matches!(event, AcpxProviderStateEvent::TurnTerminal { turn_id: settled, .. } if settled == turn_id)
|
||||
});
|
||||
seen.extend(events);
|
||||
if done {
|
||||
return seen;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_shell_tasks_at_a_completed_claude_terminal_await_and_admit_the_provider_started_turn() {
|
||||
let scratch = Scratch::new();
|
||||
let mut session = provider_session(&scratch, "live provider-turn", "claude");
|
||||
let pid = session.process_id();
|
||||
session
|
||||
.start_turn("turn-1", "Start background work", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
assert!(arm(&mut session));
|
||||
assert!(!session.continue_after_quiet_turn());
|
||||
|
||||
let started = session
|
||||
.poll_awaited_event(Duration::from_secs(1))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
started.as_slice(),
|
||||
[AcpxProviderStateEvent::ProviderTurnStarted { turn_id }] if turn_id == "provider-1"
|
||||
));
|
||||
assert_eq!(session.state().active_turn_id(), Some("provider-1"));
|
||||
let provider = poll_until_terminal(&mut session, "provider-1");
|
||||
assert!(provider.iter().any(|event| matches!(
|
||||
event,
|
||||
AcpxProviderStateEvent::Activity(activity) if activity.event_type == "usage.reported"
|
||||
)));
|
||||
assert_eq!(session.state().active_turn_id(), None);
|
||||
assert!(session.take_provider_turn_offer().is_none());
|
||||
assert_eq!(session.process_id(), pid);
|
||||
session.shutdown("test complete").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_provider_started_turn_that_leaves_shells_running_awaits_again() {
|
||||
let scratch = Scratch::new();
|
||||
let mut session = provider_session(
|
||||
&scratch,
|
||||
"live provider-turn provider-terminal-live",
|
||||
"claude",
|
||||
);
|
||||
session
|
||||
.start_turn("turn-1", "Start background work", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
assert!(arm(&mut session));
|
||||
session
|
||||
.poll_awaited_event(Duration::from_secs(1))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "provider-1");
|
||||
assert_eq!(
|
||||
session.take_provider_turn_offer().as_deref(),
|
||||
Some("provider-1")
|
||||
);
|
||||
session.shutdown("test complete").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unexpected_provider_started_turn_fails_closed_like_any_idle_event() {
|
||||
for (background, agent, expected) in [
|
||||
(
|
||||
"quiet provider-turn provider-turn-unarmed",
|
||||
"claude",
|
||||
"outside an awaited continuation",
|
||||
),
|
||||
(
|
||||
"provider-turn provider-turn-unarmed",
|
||||
"claude",
|
||||
"outside an awaited continuation",
|
||||
),
|
||||
(
|
||||
"live provider-turn provider-turn-unarmed",
|
||||
"codex",
|
||||
"outside an awaited continuation",
|
||||
),
|
||||
(
|
||||
"live provider-turn provider-turn-id=turn-1",
|
||||
"claude",
|
||||
"reused a settled turn identity",
|
||||
),
|
||||
(
|
||||
"live provider-turn provider-run=run-2",
|
||||
"claude",
|
||||
"bounded contract",
|
||||
),
|
||||
] {
|
||||
let scratch = Scratch::new();
|
||||
let mut session = provider_session(&scratch, background, agent);
|
||||
session
|
||||
.start_turn("turn-1", "Start background work", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
let awaited = arm(&mut session);
|
||||
let error = if awaited {
|
||||
session.poll_awaited_event(Duration::from_secs(1))
|
||||
} else {
|
||||
session.poll_event(Duration::from_secs(1))
|
||||
}
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
error.to_string().contains(expected),
|
||||
"{background} {agent}: {error}"
|
||||
);
|
||||
let _ = session.shutdown("test complete");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_provider_started_turn_cannot_replay_a_settled_tool_call() {
|
||||
let scratch = Scratch::new();
|
||||
let mut session = provider_session(
|
||||
&scratch,
|
||||
"live provider-turn provider-reuse-call=call-1",
|
||||
"claude",
|
||||
);
|
||||
session
|
||||
.start_turn("turn-1", "Start background work", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
assert!(arm(&mut session));
|
||||
session
|
||||
.poll_awaited_event(Duration::from_secs(1))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let error = session.poll_event(Duration::from_secs(1)).unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("reused a completed tool call id"),
|
||||
"{error}"
|
||||
);
|
||||
let _ = session.shutdown("test complete");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_controller_turn_while_awaiting_restarts_the_provider_first() {
|
||||
let scratch = Scratch::new();
|
||||
let mut session = provider_session(&scratch, "live provider-turn", "claude");
|
||||
let pid = session.process_id();
|
||||
session
|
||||
.start_turn("turn-1", "Start background work", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
assert!(arm(&mut session));
|
||||
session
|
||||
.start_turn("turn-2", "Paperclip's next turn", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
assert_ne!(session.process_id(), pid);
|
||||
assert!(session.turn_start_replaced_provider());
|
||||
assert!(session.take_provider_turn_offer().is_none());
|
||||
session.shutdown("test complete").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_generated_reader_matches_the_shared_background_task_vectors() {
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
@@ -248,3 +438,105 @@ fn the_generated_reader_matches_the_shared_background_task_vectors() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_turn_that_reported_its_result_never_awaits_even_with_live_shells() {
|
||||
let scratch = Scratch::new();
|
||||
let mut session = provider_session_in(
|
||||
&scratch,
|
||||
"live provider-turn",
|
||||
"claude",
|
||||
"turns-reserved-result-terminal",
|
||||
);
|
||||
session
|
||||
.start_turn(
|
||||
"turn-1",
|
||||
"Finish with a shell still running",
|
||||
&std::env::temp_dir(),
|
||||
)
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
assert!(session.run_result_reported());
|
||||
assert!(session.take_provider_turn_offer().is_none());
|
||||
assert!(session
|
||||
.poll_event(Duration::from_millis(200))
|
||||
.unwrap()
|
||||
.is_none());
|
||||
session.shutdown("test complete").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_provider_started_turn_that_reports_a_result_settles_and_does_not_await_again() {
|
||||
let scratch = Scratch::new();
|
||||
let mut session = provider_session(
|
||||
&scratch,
|
||||
"live provider-turn provider-result provider-terminal-live",
|
||||
"claude",
|
||||
);
|
||||
session
|
||||
.start_turn("turn-1", "Start background work", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
assert!(arm(&mut session));
|
||||
session
|
||||
.poll_awaited_event(Duration::from_secs(1))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let events = poll_until_terminal(&mut session, "provider-1");
|
||||
assert!(events.iter().any(|event| matches!(
|
||||
event,
|
||||
AcpxProviderStateEvent::TurnTerminal { status, .. }
|
||||
if *status == paperclip_runner_core::acpx_event_payload::AcpxTurnStatus::Completed
|
||||
)));
|
||||
assert!(session.run_result_reported());
|
||||
assert!(session.take_provider_turn_offer().is_none());
|
||||
session.shutdown("test complete").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sidecar_that_cannot_arm_leaves_runnerd_not_awaiting() {
|
||||
let scratch = Scratch::new();
|
||||
let mut session =
|
||||
provider_session(&scratch, "live provider-turn provider-arm-reject", "claude");
|
||||
session
|
||||
.start_turn("turn-1", "Start background work", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
let offer = session
|
||||
.take_provider_turn_offer()
|
||||
.expect("eligible terminal");
|
||||
assert!(!session.arm_provider_turn(&offer));
|
||||
session.shutdown("test complete").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelling_a_provider_started_turn_settles_it_cancelled_and_retires_the_provider() {
|
||||
let scratch = Scratch::new();
|
||||
let mut session = provider_session(&scratch, "live provider-turn provider-turn-open", "claude");
|
||||
session
|
||||
.start_turn("turn-1", "Start background work", &std::env::temp_dir())
|
||||
.unwrap();
|
||||
poll_until_terminal(&mut session, "turn-1");
|
||||
assert!(arm(&mut session));
|
||||
session
|
||||
.poll_awaited_event(Duration::from_secs(1))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(session.state().active_turn_id(), Some("provider-1"));
|
||||
let response = session
|
||||
.interrupt_turn("provider-1", "Operator stop")
|
||||
.unwrap();
|
||||
assert_eq!(response["sessionClosed"], json!(true));
|
||||
assert!(session.runtime_retired());
|
||||
let events = poll_until_terminal(&mut session, "provider-1");
|
||||
assert!(events.iter().any(|event| matches!(
|
||||
event,
|
||||
AcpxProviderStateEvent::TurnTerminal { status, .. }
|
||||
if *status == paperclip_runner_core::acpx_event_payload::AcpxTurnStatus::Cancelled
|
||||
)));
|
||||
assert!(session.take_provider_turn_offer().is_none());
|
||||
assert!(session
|
||||
.start_turn("turn-2", "Paperclip's next turn", &std::env::temp_dir())
|
||||
.is_err());
|
||||
let _ = session.shutdown("test complete");
|
||||
}
|
||||
+642
-2
@@ -75,8 +75,8 @@ fn a_quiet_completed_claude_turn_keeps_its_provider_for_the_next_turns() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_or_unknown_background_work_restarts_at_the_turn_end_like_today() {
|
||||
for mode in ["live", "", "malformed"] {
|
||||
fn unknown_background_work_restarts_at_the_turn_end_like_today() {
|
||||
for mode in ["", "malformed"] {
|
||||
let mut harness = Harness::new();
|
||||
harness.fixture.background(mode);
|
||||
let pid = harness.opened["processId"].clone();
|
||||
@@ -227,3 +227,643 @@ fn a_runner_restart_after_a_kept_quiet_provider_resumes_in_a_fresh_process() {
|
||||
);
|
||||
harness.executor.shutdown().unwrap();
|
||||
}
|
||||
|
||||
fn complete_turn_accepting_provider_turns(harness: &mut Harness, turn: &str) -> Vec<Value> {
|
||||
let started = harness.execute(
|
||||
"turn.start",
|
||||
json!({"turnId": turn, "text": "Complete the work.", "providerStartedTurns": true}),
|
||||
);
|
||||
let seen = started
|
||||
.events
|
||||
.iter()
|
||||
.map(|(kind, _, payload)| json!({"type":kind,"payload":payload}))
|
||||
.collect::<Vec<_>>();
|
||||
drain_until(harness, seen, "turn.completed")
|
||||
}
|
||||
|
||||
fn drain_until(harness: &mut Harness, mut seen: Vec<Value>, kind: &str) -> Vec<Value> {
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
while !seen.iter().any(|event| event["type"] == kind) {
|
||||
assert!(Instant::now() < deadline, "expected {kind}: {seen:?}");
|
||||
seen.extend(harness.drain());
|
||||
}
|
||||
seen
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_shells_at_a_completed_claude_turn_keep_the_provider_and_refuse_park() {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold");
|
||||
let pid = harness.opened["processId"].clone();
|
||||
let events = complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
let completed = events
|
||||
.iter()
|
||||
.find(|event| event["type"] == "turn.completed")
|
||||
.unwrap();
|
||||
assert_eq!(completed["payload"]["awaitingProviderTurn"], json!(true));
|
||||
assert!(!events.iter().any(|event| event["type"] == "run.terminal"));
|
||||
let before = harness.fixture.journal().len();
|
||||
harness.executor.prepare_next_turn().unwrap();
|
||||
assert_eq!(
|
||||
commands_after(&harness.fixture.journal(), before),
|
||||
Vec::<String>::new()
|
||||
);
|
||||
assert_eq!(live_providers(&harness), [pid.clone()]);
|
||||
assert!(!harness.executor.host_park_ready().unwrap());
|
||||
|
||||
let (next, _) = run_turn(&mut harness, "second");
|
||||
assert!(
|
||||
replaced(&next),
|
||||
"a controller turn restarts the awaited provider first"
|
||||
);
|
||||
assert!(!alive(&pid));
|
||||
harness.executor.shutdown().unwrap();
|
||||
assert_eq!(live_providers(&harness), Vec::<Value>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_provider_started_turn_reaches_the_controller_as_ordinary_turn_events() {
|
||||
let mut harness = Harness::new();
|
||||
harness.fixture.background("live provider-turn");
|
||||
let pid = harness.opened["processId"].clone();
|
||||
let first = complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
let events = drain_until(&mut harness, first, "run.terminal");
|
||||
let kinds = events
|
||||
.iter()
|
||||
.map(|event| event["type"].as_str().unwrap().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let first_completed = kinds
|
||||
.iter()
|
||||
.position(|kind| kind == "turn.completed")
|
||||
.unwrap();
|
||||
let provider_started = events
|
||||
.iter()
|
||||
.position(|event| {
|
||||
event["type"] == "turn.started" && event["payload"]["origin"] == "provider"
|
||||
})
|
||||
.expect("provider-started turn");
|
||||
assert!(first_completed < provider_started);
|
||||
assert_eq!(
|
||||
events[provider_started]["payload"]["providerTurnId"],
|
||||
json!("provider-1")
|
||||
);
|
||||
let provider_completed = events
|
||||
.iter()
|
||||
.rposition(|event| event["type"] == "turn.completed")
|
||||
.unwrap();
|
||||
assert!(provider_started < provider_completed);
|
||||
assert_eq!(
|
||||
events[provider_completed]["payload"]["providerTurnId"],
|
||||
json!("provider-1")
|
||||
);
|
||||
assert!(events[provider_completed]["payload"]
|
||||
.get("awaitingProviderTurn")
|
||||
.is_none());
|
||||
assert!(events[provider_started..provider_completed]
|
||||
.iter()
|
||||
.any(|event| event["type"] == "usage.reported"));
|
||||
let run_terminal = kinds
|
||||
.iter()
|
||||
.position(|kind| kind == "run.terminal")
|
||||
.unwrap();
|
||||
assert!(run_terminal > provider_completed);
|
||||
assert_eq!(
|
||||
kinds.iter().filter(|kind| *kind == "run.terminal").count(),
|
||||
1
|
||||
);
|
||||
harness.executor.prepare_next_turn().unwrap();
|
||||
assert!(harness.executor.host_park_ready().unwrap());
|
||||
assert_eq!(live_providers(&harness).len(), 1);
|
||||
assert!(!alive(&pid) || live_providers(&harness) == [pid.clone()]);
|
||||
harness.executor.shutdown().unwrap();
|
||||
assert_eq!(live_providers(&harness), Vec::<Value>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_runner_restart_while_awaiting_resumes_like_today() {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold");
|
||||
let pid = harness.opened["processId"].clone();
|
||||
complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
harness.executor.prepare_next_turn().unwrap();
|
||||
assert_eq!(live_providers(&harness), [pid.clone()]);
|
||||
harness.executor.shutdown().unwrap();
|
||||
assert!(!alive(&pid));
|
||||
harness.executor =
|
||||
NativeProviderCommandExecutor::with_runner_config(&harness.fixture.0, &harness.config);
|
||||
harness.sequence = 90;
|
||||
harness.fixture.background("quiet");
|
||||
let resumed = harness.drain();
|
||||
assert_eq!(
|
||||
resumed
|
||||
.iter()
|
||||
.filter(|event| event["type"] == "run.terminal")
|
||||
.count(),
|
||||
1,
|
||||
"{resumed:?}"
|
||||
);
|
||||
let (events, _) = run_turn(&mut harness, "second");
|
||||
assert!(events.iter().any(|event| event["type"] == "turn.completed"));
|
||||
assert_eq!(harness.fixture.process_starts().len(), 2);
|
||||
harness.executor.shutdown().unwrap();
|
||||
assert_eq!(live_providers(&harness), Vec::<Value>::new());
|
||||
}
|
||||
|
||||
fn controller_view(events: &[Value]) -> Vec<Value> {
|
||||
events
|
||||
.iter()
|
||||
.map(|event| {
|
||||
let mut event = event.clone();
|
||||
for volatile in ["processId", "previousProcessId", "providerTurnId"] {
|
||||
if let Some(payload) = event["payload"].as_object_mut() {
|
||||
payload.remove(volatile);
|
||||
}
|
||||
}
|
||||
event
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_the_controller_capability_live_shells_behave_exactly_as_before() {
|
||||
let outcome = |background: &str| {
|
||||
let mut harness = Harness::new();
|
||||
harness.fixture.background(background);
|
||||
let pid = harness.opened["processId"].clone();
|
||||
let events = harness.complete_turn("first");
|
||||
let park_ready_before_restart = harness.executor.host_park_ready().unwrap();
|
||||
let before = harness.fixture.journal().len();
|
||||
harness.executor.prepare_next_turn().unwrap();
|
||||
let commands = commands_after(&harness.fixture.journal(), before);
|
||||
let restarted = !alive(&pid);
|
||||
let park_ready = harness.executor.host_park_ready().unwrap();
|
||||
harness.executor.shutdown().unwrap();
|
||||
(
|
||||
controller_view(&events),
|
||||
commands,
|
||||
restarted,
|
||||
park_ready,
|
||||
park_ready_before_restart,
|
||||
)
|
||||
};
|
||||
let live = outcome("live provider-turn");
|
||||
let unknown = outcome("");
|
||||
assert_eq!(live.0, unknown.0);
|
||||
assert_eq!(live.1, unknown.1);
|
||||
assert!(live.2 && unknown.2);
|
||||
assert_eq!(live.3, unknown.3);
|
||||
assert_eq!(live.4, unknown.4);
|
||||
assert!(live.4);
|
||||
assert!(live.0.iter().any(|event| event["type"] == "run.terminal"));
|
||||
assert!(!live
|
||||
.0
|
||||
.iter()
|
||||
.any(|event| event["payload"].get("awaitingProviderTurn").is_some()));
|
||||
assert!(!live
|
||||
.1
|
||||
.iter()
|
||||
.any(|command| command == "session.provider_turn"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_provider_lost_while_awaiting_ends_the_wait_with_the_held_run_terminal() {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold provider-lost-after-arm");
|
||||
let pid = harness.opened["processId"].clone();
|
||||
let first = complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
let events = drain_until(&mut harness, first, "run.terminal");
|
||||
let completed = events
|
||||
.iter()
|
||||
.position(|event| event["type"] == "turn.completed")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
events[completed]["payload"]["awaitingProviderTurn"],
|
||||
json!(true)
|
||||
);
|
||||
let terminals = events
|
||||
.iter()
|
||||
.filter(|event| event["type"] == "run.terminal")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(terminals.len(), 1);
|
||||
assert_eq!(
|
||||
terminals[0]["payload"]["turnTerminalState"],
|
||||
json!("completed")
|
||||
);
|
||||
assert!(events
|
||||
.iter()
|
||||
.any(|event| event["payload"]["code"] == "ACPX_PROVIDER_PROCESS_LOST"));
|
||||
harness.executor.prepare_next_turn().unwrap();
|
||||
assert!(
|
||||
!alive(&pid),
|
||||
"the lost provider is replaced at the turn end"
|
||||
);
|
||||
harness.executor.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sidecar_that_dies_while_awaiting_ends_the_wait_with_the_held_run_terminal() {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm");
|
||||
let first = complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
let events = drain_until(&mut harness, first, "run.terminal");
|
||||
let terminals = events
|
||||
.iter()
|
||||
.filter(|event| event["type"] == "run.terminal")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(terminals.len(), 1);
|
||||
assert_eq!(
|
||||
terminals[0]["payload"]["turnTerminalState"],
|
||||
json!("completed")
|
||||
);
|
||||
assert!(!events.iter().any(|event| event["type"] == "turn.failed"));
|
||||
let _ = harness.executor.shutdown();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelling_or_stopping_the_run_while_awaiting_disarms_and_publishes_the_held_run_terminal() {
|
||||
for command in ["run.cancel", "turn.interrupt", "turn.stop"] {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold provider-turn-after-disarm");
|
||||
let first = complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
assert!(
|
||||
!first.iter().any(|event| event["type"] == "run.terminal"),
|
||||
"{command}"
|
||||
);
|
||||
let before = harness.fixture.journal().len();
|
||||
let cancelled = harness.execute(
|
||||
command,
|
||||
json!({"turnId": "first", "reason": "Operator stop"}),
|
||||
);
|
||||
assert_eq!(
|
||||
cancelled.result["status"],
|
||||
json!("already_settled"),
|
||||
"{command}"
|
||||
);
|
||||
let mut later = cancelled
|
||||
.events
|
||||
.iter()
|
||||
.map(|(kind, _, payload)| json!({"type":kind,"payload":payload}))
|
||||
.collect::<Vec<_>>();
|
||||
let journal = harness.fixture.journal();
|
||||
assert!(
|
||||
journal[before..].iter().any(|line| {
|
||||
line["command"] == "session.provider_turn"
|
||||
&& line["params"]["armed"] == json!(false)
|
||||
}),
|
||||
"{command}"
|
||||
);
|
||||
let deadline = Instant::now() + Duration::from_millis(500);
|
||||
while Instant::now() < deadline {
|
||||
match harness.executor.poll_events() {
|
||||
Ok(events) => {
|
||||
later.extend(
|
||||
events
|
||||
.iter()
|
||||
.map(|event| json!({"type":event.event_type,"payload":event.payload})),
|
||||
);
|
||||
let _ = harness.executor.acknowledge_events(events.len());
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
!later
|
||||
.iter()
|
||||
.any(|event| event["payload"]["origin"] == "provider"),
|
||||
"{command}: {later:?}"
|
||||
);
|
||||
let terminals = later
|
||||
.iter()
|
||||
.filter(|event| event["type"] == "run.terminal")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(terminals.len(), 1, "{command}: {later:?}");
|
||||
assert_eq!(
|
||||
terminals[0]["payload"]["turnTerminalState"],
|
||||
json!("completed"),
|
||||
"{command}"
|
||||
);
|
||||
let _ = harness.executor.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_active_goal_at_a_live_shell_turn_end_neither_marks_nor_arms_a_wait() {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold");
|
||||
harness.fixture.fault("goal");
|
||||
let before = harness.fixture.journal().len();
|
||||
let events = complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
let events = [events, harness.drain()].concat();
|
||||
let completed = events
|
||||
.iter()
|
||||
.find(|event| event["type"] == "turn.completed")
|
||||
.unwrap();
|
||||
assert!(completed["payload"].get("awaitingProviderTurn").is_none());
|
||||
assert!(!events.iter().any(|event| event["type"] == "run.terminal"));
|
||||
assert!(!commands_after(&harness.fixture.journal(), before)
|
||||
.iter()
|
||||
.any(|command| command == "session.provider_turn"));
|
||||
harness.executor.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_crash_after_cancelling_or_stopping_a_wait_still_delivers_the_held_run_terminal_once() {
|
||||
for command in ["run.cancel", "turn.interrupt", "turn.stop"] {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold");
|
||||
complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
let lost_with_the_crash = harness.execute(
|
||||
command,
|
||||
json!({"turnId": "first", "reason": "Operator stop"}),
|
||||
);
|
||||
assert_eq!(
|
||||
lost_with_the_crash.result["status"],
|
||||
json!("already_settled"),
|
||||
"{command}"
|
||||
);
|
||||
harness.executor =
|
||||
NativeProviderCommandExecutor::with_runner_config(&harness.fixture.0, &harness.config);
|
||||
harness.fixture.background("quiet");
|
||||
let recovered = harness.drain();
|
||||
assert_eq!(
|
||||
recovered
|
||||
.iter()
|
||||
.filter(|event| event["type"] == "run.terminal")
|
||||
.count(),
|
||||
1,
|
||||
"{command}: {recovered:?}"
|
||||
);
|
||||
let _ = harness.executor.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
fn awaiting_and_quiesced(harness: &mut Harness) {
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold");
|
||||
let first = complete_turn_accepting_provider_turns(harness, "first");
|
||||
let events = [first, harness.drain()].concat();
|
||||
assert!(!events.iter().any(|event| event["type"] == "run.terminal"));
|
||||
assert!(harness.fixture.state().get("providerTurnWait").is_some());
|
||||
harness.quiesce();
|
||||
}
|
||||
|
||||
fn drain_briefly(harness: &mut Harness) -> Vec<Value> {
|
||||
let mut seen = Vec::new();
|
||||
let deadline = Instant::now() + Duration::from_millis(500);
|
||||
while Instant::now() < deadline {
|
||||
seen.extend(harness.drain());
|
||||
}
|
||||
seen
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_different_run_attaching_while_awaiting_ends_the_wait_without_the_old_run_terminal() {
|
||||
let mut harness = Harness::new();
|
||||
awaiting_and_quiesced(&mut harness);
|
||||
let pid = harness.opened["processId"].clone();
|
||||
let before = harness.fixture.journal().len();
|
||||
let (attached, _) = harness.attach("run-2", |_| {});
|
||||
let state = harness.fixture.state();
|
||||
assert_eq!(state["descriptor"]["runId"], "run-2");
|
||||
assert!(state.get("providerTurnWait").is_none(), "{state}");
|
||||
assert!(
|
||||
!state["pendingEvents"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|event| event.to_string().contains("run.terminal")),
|
||||
"{state}"
|
||||
);
|
||||
let mut received = attached
|
||||
.events
|
||||
.iter()
|
||||
.map(|(kind, _, payload)| json!({"type":kind,"payload":payload}))
|
||||
.collect::<Vec<_>>();
|
||||
received.extend(drain_briefly(&mut harness));
|
||||
assert!(
|
||||
!received.iter().any(|event| event["type"] == "run.terminal"),
|
||||
"{received:?}"
|
||||
);
|
||||
let commands = commands_after(&harness.fixture.journal(), before);
|
||||
assert!(
|
||||
commands.iter().any(|command| command == "session.close"),
|
||||
"{commands:?}"
|
||||
);
|
||||
assert!(
|
||||
!commands
|
||||
.iter()
|
||||
.any(|command| command == "session.provider_turn"),
|
||||
"{commands:?}"
|
||||
);
|
||||
assert!(!alive(&pid));
|
||||
let _ = harness.executor.shutdown();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_different_run_attach_while_awaiting_keeps_the_wait_and_its_run_terminal() {
|
||||
for ending in [
|
||||
None,
|
||||
Some("run.cancel"),
|
||||
Some("turn.interrupt"),
|
||||
Some("turn.stop"),
|
||||
] {
|
||||
let mut harness = Harness::new();
|
||||
awaiting_and_quiesced(&mut harness);
|
||||
let pid = harness.opened["processId"].clone();
|
||||
std::process::Command::new("kill")
|
||||
.args(["-9", &pid.to_string()])
|
||||
.status()
|
||||
.unwrap();
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
let payload = harness.attach_payload("run-2", |_| {});
|
||||
harness.sequence += 1;
|
||||
let error = harness
|
||||
.executor
|
||||
.execute(&command(harness.sequence, "run.attach", payload))
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
format!("{error:?}").contains("failed to checkpoint ACPX before attaching a new run"),
|
||||
"{ending:?}: {error:?}"
|
||||
);
|
||||
let state = harness.fixture.state();
|
||||
assert_eq!(state["descriptor"]["runId"], "run-1", "{ending:?}");
|
||||
assert!(
|
||||
state.get("providerTurnWait").is_some(),
|
||||
"{ending:?}: {state}"
|
||||
);
|
||||
let mut recovered = Vec::new();
|
||||
if let Some(ending) = ending {
|
||||
let ended = harness.execute(
|
||||
ending,
|
||||
json!({"turnId": "first", "reason": "Operator stop"}),
|
||||
);
|
||||
assert_eq!(ended.result["status"], json!("already_settled"), "{ending}");
|
||||
recovered.extend(
|
||||
ended
|
||||
.events
|
||||
.iter()
|
||||
.map(|(kind, _, payload)| json!({"type":kind,"payload":payload})),
|
||||
);
|
||||
}
|
||||
recovered.extend(drain_briefly(&mut harness));
|
||||
let terminals = recovered
|
||||
.iter()
|
||||
.filter(|event| event["type"] == "run.terminal")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(terminals.len(), 1, "{ending:?}: {recovered:?}");
|
||||
assert_eq!(
|
||||
terminals[0]["payload"]["turnTerminalState"],
|
||||
json!("completed"),
|
||||
"{ending:?}"
|
||||
);
|
||||
let state = harness.fixture.state();
|
||||
assert_eq!(state["descriptor"]["runId"], "run-1", "{ending:?}");
|
||||
assert!(state.get("providerTurnWait").is_none(), "{ending:?}");
|
||||
let _ = harness.executor.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_same_run_attach_while_awaiting_publishes_the_held_run_terminal_once() {
|
||||
let mut harness = Harness::new();
|
||||
awaiting_and_quiesced(&mut harness);
|
||||
let attached = harness.execute(
|
||||
"run.attach",
|
||||
json!({"provider": harness.descriptor.clone()}),
|
||||
);
|
||||
let mut received = attached
|
||||
.events
|
||||
.iter()
|
||||
.map(|(kind, _, payload)| json!({"type":kind,"payload":payload}))
|
||||
.collect::<Vec<_>>();
|
||||
received.extend(drain_briefly(&mut harness));
|
||||
let terminals = received
|
||||
.iter()
|
||||
.filter(|event| event["type"] == "run.terminal")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(terminals.len(), 1, "{received:?}");
|
||||
assert_eq!(
|
||||
terminals[0]["payload"]["turnTerminalState"],
|
||||
json!("completed")
|
||||
);
|
||||
let _ = harness.executor.shutdown();
|
||||
}
|
||||
|
||||
fn execution_events(execution: &CommandExecution) -> Vec<Value> {
|
||||
execution
|
||||
.events
|
||||
.iter()
|
||||
.map(|(kind, _, payload)| json!({"type":kind,"payload":payload}))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn run_terminals(events: &[Value]) -> usize {
|
||||
events
|
||||
.iter()
|
||||
.filter(|event| event["type"] == "run.terminal")
|
||||
.count()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lost_disarm_reply_is_bounded_and_still_publishes_the_held_run_terminal_once() {
|
||||
for command in ["run.cancel", "turn.interrupt"] {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-turn-hold provider-disarm-reply-lost");
|
||||
let pid = harness.opened["processId"].clone();
|
||||
let first = complete_turn_accepting_provider_turns(&mut harness, "first");
|
||||
let first = [first, harness.drain()].concat();
|
||||
assert_eq!(run_terminals(&first), 0, "{command}");
|
||||
let started = Instant::now();
|
||||
let ended = harness.execute(
|
||||
command,
|
||||
json!({"turnId": "first", "reason": "Operator stop"}),
|
||||
);
|
||||
let elapsed = started.elapsed();
|
||||
assert_eq!(
|
||||
ended.result["status"],
|
||||
json!("already_settled"),
|
||||
"{command}"
|
||||
);
|
||||
assert!(
|
||||
elapsed >= Duration::from_secs(4) && elapsed < Duration::from_secs(10),
|
||||
"{command}: {elapsed:?}"
|
||||
);
|
||||
let mut later = execution_events(&ended);
|
||||
later.extend(drain_briefly(&mut harness));
|
||||
assert_eq!(run_terminals(&later), 1, "{command}: {later:?}");
|
||||
assert_eq!(
|
||||
later
|
||||
.iter()
|
||||
.find(|event| event["type"] == "run.terminal")
|
||||
.unwrap()["payload"]["turnTerminalState"],
|
||||
json!("completed"),
|
||||
"{command}"
|
||||
);
|
||||
assert!(
|
||||
harness.fixture.state().get("providerTurnWait").is_none(),
|
||||
"{command}"
|
||||
);
|
||||
let started = Instant::now();
|
||||
let closed = harness.execute("session.close", json!({}));
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(2),
|
||||
"{command}: {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
assert_eq!(closed.result["status"], json!("closed"), "{command}");
|
||||
later.extend(execution_events(&closed));
|
||||
later.extend(drain_briefly(&mut harness));
|
||||
assert_eq!(run_terminals(&later), 1, "{command}: {later:?}");
|
||||
assert!(!alive(&pid), "{command}");
|
||||
let _ = harness.executor.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lost_arm_reply_is_bounded_and_the_turn_ends_with_its_run_terminal() {
|
||||
let mut harness = Harness::new();
|
||||
harness
|
||||
.fixture
|
||||
.background("live provider-turn provider-arm-reply-lost");
|
||||
let started = Instant::now();
|
||||
let mut seen = execution_events(&harness.execute(
|
||||
"turn.start",
|
||||
json!({"turnId": "first", "text": "Complete the work.", "providerStartedTurns": true}),
|
||||
));
|
||||
let deadline = started + Duration::from_secs(15);
|
||||
while run_terminals(&seen) == 0 {
|
||||
assert!(Instant::now() < deadline, "{seen:?}");
|
||||
seen.extend(harness.drain());
|
||||
}
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(10),
|
||||
"{:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
seen.extend(drain_briefly(&mut harness));
|
||||
let completed = seen
|
||||
.iter()
|
||||
.find(|event| event["type"] == "turn.completed")
|
||||
.unwrap();
|
||||
assert!(completed["payload"].get("awaitingProviderTurn").is_none());
|
||||
assert_eq!(run_terminals(&seen), 1, "{seen:?}");
|
||||
assert!(harness.fixture.state().get("providerTurnWait").is_none());
|
||||
let _ = harness.executor.shutdown();
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { execFileSync, spawn } from "node:child_process";
|
||||
import { closeSync, openSync } from "node:fs";
|
||||
import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
|
||||
import { createServer } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
|
||||
import { QUALIFIED_ACPX_PROFILES } from "../drivers/acpx/qualified-profiles.js";
|
||||
import { createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type RunnerProviderLaunchRequirements } from "./runner-provider-launch.js";
|
||||
import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding, type CapabilityRunnerdCodexTransportOptions } from "./runnerd-codex-transport.js";
|
||||
|
||||
const sha = (value: string | Buffer) => `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`;
|
||||
const kill = (pid: number) => { try { process.kill(-pid, "SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } };
|
||||
|
||||
type CoreEvent = { eventType: string; envelope: { payload?: Record<string, unknown> } };
|
||||
|
||||
async function awaitedTurn(tokens: string, providerStartedTurns: boolean) {
|
||||
const root = await realpath(await mkdtemp(join(tmpdir(), "runner-acpx-provider-wait-")));
|
||||
const journal = join(root, "calls"), fault = join(root, "fault"), wrapper = join(root, "sidecar"), script = join(root, "sidecar-entry");
|
||||
const background = join(root, "background");
|
||||
const binary = resolve(import.meta.dirname, "../../runner/target/debug/paperclip-runnerd"), sidecar = resolve(import.meta.dirname, "../../runner/target/debug/fake-acpx-sidecar");
|
||||
await mkdir(join(root, "acpx"), { mode: 0o700 }); await mkdir(join(root, "diagnostics"), { mode: 0o700 });
|
||||
await writeFile(script, "qualified fixture entry");
|
||||
await writeFile(background, tokens);
|
||||
await writeFile(wrapper, `#!/bin/sh\nexec ${quote(sidecar)} --mode source-reconcile --profile-digest ${quote(QUALIFIED_ACPX_PROFILES.claude.commandDigest)} --journal ${quote(journal)} --source-fault ${quote(fault)} --background-tasks ${quote(background)} --spawn-owned-child\n`, { mode: 0o700 });
|
||||
let authority: DurablePrpControlPlane | null = null;
|
||||
const server = createServer();
|
||||
server.on("upgrade", (request, socket, head) => authority ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy());
|
||||
await new Promise<void>(done => server.listen(0, "127.0.0.1", done));
|
||||
const address = server.address(); if (!address || typeof address === "string") throw new Error("missing fixture listener");
|
||||
const identity = { runnerInstanceId: "runner-wait", environmentLeaseId: "lease-wait", runId: "run-wait", normalizedSessionId: "session-wait", turnId: "turn-wait", itemId: "item-wait" };
|
||||
const runnerPids = new Set<number>();
|
||||
let requirements: RunnerProviderLaunchRequirements | undefined;
|
||||
const options: CapabilityRunnerdCodexTransportOptions = {
|
||||
provider: "acpx", acpxAgent: "claude", runnerBinary: binary, stateDirectory: root, prpIdentity: identity,
|
||||
runnerArtifact: readRunnerdArtifactBinding(binary), codexSourceRecoveryVersion: 1, acpxSourceRecoveryVersion: 1,
|
||||
lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, runnerProcessReconnectGraceMs: null, runnerReconnectGraceMs: 3_000,
|
||||
controlPlaneRegistration: async next => { authority = next; return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: () => { authority = null; } }; },
|
||||
prepareLaunch: async launch => {
|
||||
requirements ??= structuredClone(launch);
|
||||
return { configurationId: `initial-${launch.identity.runId}`, requirementsDigest: runnerLaunchRequirementsDigest(launch),
|
||||
provider: createRunnerProviderLaunch(launch, { options: {}, root, cwd: root, codexHome: root, codexCommand: "", providerNodeCommand: wrapper, opencodeProxyPath: "", acpxSidecarPath: script }) };
|
||||
},
|
||||
runnerProcessLauncher: spec => {
|
||||
const stderr = openSync(join(root, "native.stderr"), "a", 0o600);
|
||||
const child = spawn(binary, [...spec.args, "--acpx-launch-authority-digest", sha("fixture authority"),
|
||||
"--acpx-sidecar-command", wrapper, "--acpx-sidecar-command-sha256", sha(execFileSync("cat", [wrapper])),
|
||||
"--acpx-sidecar-script", script, "--acpx-sidecar-script-sha256", sha("qualified fixture entry")],
|
||||
{ env: { PATH: process.env.PATH, HOME: root, ...spec.environment }, stdio: ["ignore", "ignore", stderr], detached: true });
|
||||
closeSync(stderr);
|
||||
if (child.pid) runnerPids.add(child.pid);
|
||||
return { child, completion: new Promise((done, reject) => { child.once("error", reject); child.once("exit", (code, signal) => done({ code, signal, stdout: "", stderr: "" })); }) };
|
||||
},
|
||||
};
|
||||
const bundles = [createCapabilityRunnerdCodexTransport(options)];
|
||||
const bundle = bundles[0]!;
|
||||
const calls = async () => (await readFile(journal, "utf8").catch(() => "")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line) as Record<string, unknown>);
|
||||
const coreEvents = async () => {
|
||||
const core = JSON.parse(await readFile(join(root, "control-plane", "control-plane-state.json"), "utf8")) as { committedEvents?: CoreEvent[] };
|
||||
return (core.committedEvents ?? []).map(event => ({ eventType: event.eventType,
|
||||
payload: ((event.envelope.payload ?? {}) as { payload?: Record<string, unknown> }).payload ?? {} }));
|
||||
};
|
||||
try {
|
||||
const opened = await bundle.transport.request("thread/start", { cwd: root, model: "explicit-source-model", dynamicTools: [] });
|
||||
const thread = opened.thread as { id: string; sessionId: string };
|
||||
authority!.queueCommand("turn.start", { text: "Install and test.", turnId: "turn-own-1", ...(providerStartedTurns ? { providerStartedTurns: true } : {}) }, undefined, true);
|
||||
await vi.waitFor(async () => expect((await coreEvents()).some(event => event.eventType === "turn.completed")).toBe(true), { timeout: 10_000, interval: 50 });
|
||||
const restartRunner = async () => {
|
||||
const providerIdentity = JSON.parse(await readFile(join(root, "runner", "acpx-provider-state.json"), "utf8")).identity;
|
||||
const pid = bundle.evidence().runnerPid!;
|
||||
await bundle.detachControllerForRestart(); kill(pid);
|
||||
for (const entry of await calls()) if (typeof entry.processStart === "number") kill(entry.processStart);
|
||||
const recovered = createCapabilityRunnerdCodexTransport({ ...options, recoveryLaunch: { requirements: requirements! },
|
||||
resumeProviderSession: { driverSessionId: thread.id, providerSessionId: thread.sessionId, providerIdentity },
|
||||
onRecoveryAccountingEvent: async () => {}, authorityRunId: identity.runId });
|
||||
bundles.push(recovered);
|
||||
await recovered.transport.request("thread/read", {});
|
||||
return recovered;
|
||||
};
|
||||
return { coreEvents, calls, root, authority: authority!, restartRunner };
|
||||
} catch (error) {
|
||||
throw new Error(`${String(error)}\n${await readFile(join(root, "native.stderr"), "utf8").catch(() => "")}`, { cause: error });
|
||||
} finally {
|
||||
cleanups.push(async () => {
|
||||
for (const opened of bundles.reverse()) await opened.transport.close().catch(() => undefined);
|
||||
for (const pid of runnerPids) kill(pid);
|
||||
for (const entry of await calls()) if (typeof entry.processStart === "number") kill(entry.processStart);
|
||||
server.closeAllConnections(); await new Promise<void>(done => server.close(() => done()));
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const cleanups: Array<() => Promise<void>> = [];
|
||||
|
||||
const alive = (pid: number) => {
|
||||
try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; throw error; }
|
||||
try { return !execFileSync("ps", ["-p", String(pid), "-o", "stat="], { encoding: "utf8" }).trim().startsWith("Z"); }
|
||||
catch (error) { if ((error as { status?: number }).status === 1) return false; throw error; }
|
||||
};
|
||||
|
||||
const runTerminals = (events: Array<{ eventType: string; payload: Record<string, unknown> }>) => events.filter(event => event.eventType === "run.terminal");
|
||||
const commands = async (calls: () => Promise<Array<Record<string, unknown>>>) => (await calls()).map(entry => entry.command).filter(Boolean);
|
||||
|
||||
it.skipIf(process.platform === "win32")("ends a wait through the real runnerd when the sidecar exits while a descendant keeps its output open", async () => {
|
||||
try {
|
||||
const run = await awaitedTurn("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm", true);
|
||||
const awaited = (await run.coreEvents()).find(event => event.eventType === "turn.completed");
|
||||
expect(awaited?.payload.awaitingProviderTurn).toBe(true);
|
||||
await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 });
|
||||
const events = await run.coreEvents();
|
||||
expect(runTerminals(events)[0]!.payload).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" });
|
||||
expect(events.findIndex(event => event.eventType === "run.terminal")).toBeGreaterThan(events.findIndex(event => event.eventType === "turn.completed"));
|
||||
expect(await commands(run.calls)).toContain("session.provider_turn");
|
||||
} finally { for (const cleanup of cleanups.splice(0)) await cleanup(); }
|
||||
}, 60_000);
|
||||
|
||||
it.skipIf(process.platform === "win32")("ends a wait through the real runnerd when the sidecar reports the provider lost", async () => {
|
||||
try {
|
||||
const run = await awaitedTurn("live provider-turn provider-turn-hold provider-lost-after-arm", true);
|
||||
await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 });
|
||||
const events = await run.coreEvents();
|
||||
expect(events.find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBe(true);
|
||||
expect(events.some(event => event.payload.code === "ACPX_PROVIDER_PROCESS_LOST")).toBe(true);
|
||||
} finally { for (const cleanup of cleanups.splice(0)) await cleanup(); }
|
||||
}, 60_000);
|
||||
|
||||
it.skipIf(process.platform === "win32")("emits run.terminal with the completed turn and never arms without the controller opt-in", async () => {
|
||||
try {
|
||||
const run = await awaitedTurn("live provider-turn provider-turn-hold provider-sidecar-exit-after-arm", false);
|
||||
await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 5_000, interval: 50 });
|
||||
const events = await run.coreEvents();
|
||||
expect(events.find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBeUndefined();
|
||||
expect(await commands(run.calls)).not.toContain("session.provider_turn");
|
||||
} finally { for (const cleanup of cleanups.splice(0)) await cleanup(); }
|
||||
}, 60_000);
|
||||
|
||||
it.skipIf(process.platform === "win32")("cancelling the run during a wait disarms the provider, publishes the held terminal and admits no later provider turn", async () => {
|
||||
try {
|
||||
const run = await awaitedTurn("live provider-turn provider-turn-hold provider-turn-after-disarm", true);
|
||||
expect(runTerminals(await run.coreEvents())).toHaveLength(0);
|
||||
run.authority.queueCommand("run.cancel", { reason: "Operator stop" }, undefined, true);
|
||||
await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 });
|
||||
expect(runTerminals(await run.coreEvents())[0]!.payload).toMatchObject({ turnTerminalState: "completed" });
|
||||
expect((await run.calls()).some(entry => entry.command === "session.provider_turn" && (entry.params as { armed?: unknown }).armed === false)).toBe(true);
|
||||
await new Promise(done => setTimeout(done, 1_000));
|
||||
const events = await run.coreEvents();
|
||||
expect(events.some(event => event.payload.origin === "provider")).toBe(false);
|
||||
expect(runTerminals(events)).toHaveLength(1);
|
||||
run.authority.queueCommand("session.close", { reason: "Run cancelled" }, undefined, true);
|
||||
const processes = (await run.calls()).flatMap(entry => [entry.processStart, entry.childPid]).filter((pid): pid is number => typeof pid === "number");
|
||||
expect(processes.length).toBeGreaterThan(0);
|
||||
await vi.waitFor(() => { for (const pid of processes) expect(alive(pid)).toBe(false); }, { timeout: 10_000, interval: 100 });
|
||||
} finally { for (const cleanup of cleanups.splice(0)) await cleanup(); }
|
||||
}, 60_000);
|
||||
|
||||
it.skipIf(process.platform === "win32")("publishes the held run.terminal exactly once when runnerd restarts during the wait", async () => {
|
||||
try {
|
||||
const run = await awaitedTurn("live provider-turn provider-turn-hold", true);
|
||||
expect((await run.coreEvents()).find(event => event.eventType === "turn.completed")?.payload.awaitingProviderTurn).toBe(true);
|
||||
expect(runTerminals(await run.coreEvents())).toHaveLength(0);
|
||||
await run.restartRunner();
|
||||
await vi.waitFor(async () => expect(runTerminals(await run.coreEvents())).toHaveLength(1), { timeout: 10_000, interval: 100 });
|
||||
expect(runTerminals(await run.coreEvents())[0]!.payload).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" });
|
||||
await new Promise(done => setTimeout(done, 1_000));
|
||||
expect(runTerminals(await run.coreEvents())).toHaveLength(1);
|
||||
} finally { for (const cleanup of cleanups.splice(0)) await cleanup(); }
|
||||
}, 60_000);
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { NativeExecutionInputV1 } from "./contracts/native-execution.js";
|
||||
import type { NativeSession } from "./contracts/native-session-backend.js";
|
||||
import { executeNativeSession } from "./native-session-runtime.js";
|
||||
import type { PrpEvent } from "./protocol/replay-contract.js";
|
||||
|
||||
const identity = {
|
||||
runId: "run-provider-turn",
|
||||
sessionId: "session-provider-turn",
|
||||
companyId: "company-provider-turn",
|
||||
issueId: "issue-provider-turn",
|
||||
agentId: "agent-provider-turn",
|
||||
};
|
||||
|
||||
const input: NativeExecutionInputV1 = {
|
||||
schema: "paperclip.native-execution-input.v1",
|
||||
binding: { companyId: identity.companyId, runId: identity.runId, issueId: identity.issueId, agentId: identity.agentId,
|
||||
executionWorkspaceId: "workspace-provider-turn" },
|
||||
task: { identifier: "PAP-PROVIDER", title: "Wait for a background install", description: null,
|
||||
prompt: "# PAP-PROVIDER: Wait for a background install", workMode: "standard" },
|
||||
workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1,
|
||||
lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
|
||||
provider: { kind: "codex", model: null },
|
||||
completionContract: { id: "contract-provider-turn", sha256: "contract-provider-turn-sha", schemaVersion: "paperclip.completion-contract.v1",
|
||||
contract: { revision: "1", objective: "Install and test", criteria: [{ id: "objective", requirement: "Install and test" }] } },
|
||||
interactionResponses: [],
|
||||
credentialBindings: [],
|
||||
};
|
||||
|
||||
const event = (sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record<string, unknown>): PrpEvent => ({
|
||||
schema: "paperclip.prp.event.v1",
|
||||
sourceEventId: `runner-provider-turn:${identity.runId}:${sourceSeq}`,
|
||||
sourceSeq,
|
||||
sourceInstanceId: "runner-provider-turn",
|
||||
sourceKind: "runner",
|
||||
runId: identity.runId,
|
||||
normalizedSessionId: identity.sessionId,
|
||||
turnId: "turn-1",
|
||||
eventType,
|
||||
schemaVersion: 1,
|
||||
priority: 0,
|
||||
emittedAt: "2026-11-12T00:00:00.000Z",
|
||||
payload,
|
||||
});
|
||||
|
||||
const usage = { provider: "acpx", model: null, providerSessionId: null, providerRequestId: null,
|
||||
cumulative: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 1, providerCostUsd: 0 },
|
||||
runDeltaAvailable: true,
|
||||
runDelta: { inputTokens: 6, outputTokens: 4, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 1, providerCostUsd: 0 } };
|
||||
|
||||
const turnOne = (awaiting: boolean): PrpEvent[] => [
|
||||
event(1, "turn.started", { provider: "acpx", providerTurnId: "turn-1", status: "inProgress", turn: { id: "turn-1", status: "inProgress" } }),
|
||||
event(2, "usage.reported", usage),
|
||||
event(3, "turn.completed", { provider: "acpx", providerTurnId: "turn-1", status: "completed", error: null,
|
||||
...(awaiting ? { awaitingProviderTurn: true } : {}) }),
|
||||
...(awaiting ? [] : [event(4, "run.terminal", { schema: "paperclip.prp.terminal.v1", status: "succeeded", turnTerminalState: "completed",
|
||||
runTerminalState: "succeeded", reportedWorkDisposition: "done", provider: "acpx" })]),
|
||||
];
|
||||
|
||||
const providerTurn: PrpEvent[] = [
|
||||
event(4, "turn.started", { provider: "acpx", providerTurnId: "provider-1", status: "inProgress",
|
||||
turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }),
|
||||
event(5, "turn.completed", { provider: "acpx", providerTurnId: "provider-1", status: "completed", error: null }),
|
||||
];
|
||||
|
||||
async function finalize(stream: PrpEvent[]) {
|
||||
const appended: PrpEvent[] = [];
|
||||
const completeRun = vi.fn(async () => undefined);
|
||||
const capabilities = { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true };
|
||||
const session: NativeSession = {
|
||||
identity: () => identity,
|
||||
async capabilities() { return capabilities; },
|
||||
async *events() { for (const value of stream) yield structuredClone(value); },
|
||||
async startTurn() { return { turnId: "turn-1" }; },
|
||||
async result() { return null; },
|
||||
async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity,
|
||||
providerSessionId: "provider-session", cursor: String(stream.length), activeTurnId: "turn-1", pendingRuntimeRequests: [], lineage: [] }; },
|
||||
async close() {},
|
||||
};
|
||||
const outcome = await executeNativeSession({
|
||||
input,
|
||||
backend: { async descriptor() { return { kind: "mock", name: "provider-turn", version: "1", capabilities }; }, async openSession() { return session; } },
|
||||
controlPlane: {
|
||||
async openRun() {}, async checkpointSession() {},
|
||||
async appendEvent(value: PrpEvent) {
|
||||
appended.push(structuredClone(value));
|
||||
return { cursor: value.sourceSeq, highestContiguousSourceSeq: value.sourceSeq, disposition: "committed" };
|
||||
},
|
||||
async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
|
||||
completeRun,
|
||||
},
|
||||
runnerInstanceId: "runner-provider-turn", controlPlaneInstanceId: "control-provider-turn", timeoutMs: 1000,
|
||||
resolveMissingResult: async () => null,
|
||||
}).then(() => "finalized", (error: unknown) => String(error instanceof Error ? error.message : error));
|
||||
return { outcome, appended, completeRun };
|
||||
}
|
||||
|
||||
describe("today's controller consumer with provider-started turn sequences", () => {
|
||||
it("finalizes a no-result turn with live shells exactly as before when the controller did not opt in", async () => {
|
||||
const withoutOptIn = await finalize(turnOne(false));
|
||||
expect(withoutOptIn.outcome).toContain("native_finalization_missing");
|
||||
expect(withoutOptIn.completeRun).not.toHaveBeenCalled();
|
||||
expect(withoutOptIn.appended.map(value => value.eventType)).toEqual(["turn.started", "usage.reported", "turn.completed"]);
|
||||
});
|
||||
|
||||
it("would end the run at the awaited terminal and never consume the provider-started turn, so the opt-in must stay off", async () => {
|
||||
const optedIn = await finalize([...turnOne(true), ...providerTurn]);
|
||||
expect(optedIn.outcome).toContain("native_finalization_missing");
|
||||
expect(optedIn.appended.some(value => value.payload.origin === "provider")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -973,6 +973,7 @@ export const LIVE_EVENT_TYPES = [
|
||||
"heartbeat.run.event",
|
||||
"heartbeat.run.log",
|
||||
"heartbeat.run.activity",
|
||||
"heartbeat.run.tool_progress",
|
||||
"agent.session.goal.changed",
|
||||
"agent.status",
|
||||
"agent.runtime_state.updated",
|
||||
|
||||
@@ -1458,6 +1458,9 @@ export type {
|
||||
RejectSecretProposalInput,
|
||||
ConnectToolAppAuthChallenge,
|
||||
ConnectToolAppResult,
|
||||
MachineSignInPublicKey,
|
||||
MachineSignInStatus,
|
||||
MachineSignInSetupResult,
|
||||
ToolAppMetadataPreflightAttempt,
|
||||
ToolAppMetadataPreflightResult,
|
||||
ToolOAuthClientRegistrationSource,
|
||||
@@ -2302,6 +2305,15 @@ export {
|
||||
genericMcpOAuthClientSchema,
|
||||
type GenericMcpAuthMode,
|
||||
type GenericMcpOAuthClient,
|
||||
MACHINE_SIGN_IN_METHODS,
|
||||
machineSignInMethodSchema,
|
||||
createMachineSignInSchema,
|
||||
testMachineSignInSchema,
|
||||
rotateMachineSignInSchema,
|
||||
type MachineSignInMethod,
|
||||
type CreateMachineSignIn,
|
||||
type TestMachineSignIn,
|
||||
type RotateMachineSignIn,
|
||||
reconnectToolAppSchema,
|
||||
createToolApplicationSchema,
|
||||
finishToolAppSchema,
|
||||
@@ -2966,5 +2978,6 @@ export {
|
||||
} from "./issue-activity-identity.js";
|
||||
export { recognizeLinkPreview, type LinkPreviewTarget, type LinkPreviewState, type LinkPreview, type LinkPreviewResult } from "./link-preview.js";
|
||||
export * from "./mcp-apps.js";
|
||||
export * from "./tool-call-progress.js";
|
||||
export * from "./mcp-skills.js";
|
||||
export type { TailscaleConnection, TailscaleDeviceLogin } from "./types/tailscale.js";
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Live progress of one connected-app tool call, from the provider's MCP
|
||||
* `notifications/progress`. Everything here is untrusted provider content:
|
||||
* numbers are checked and the message is plain text, bounded in length.
|
||||
*/
|
||||
export const TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH = 200;
|
||||
|
||||
export interface ToolCallProgress {
|
||||
/** Increases with each update. Never shown as a raw number without `total`. */
|
||||
progress: number;
|
||||
/** Known end value; absent when the provider does not know the total. */
|
||||
total: number | null;
|
||||
/** The provider's latest status line, plain text. */
|
||||
message: string | null;
|
||||
/** When the provider last reported progress (ISO). */
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/** A running tool call's progress, as the task chat reads it. */
|
||||
export interface ToolCallProgressSnapshot extends ToolCallProgress {
|
||||
invocationId: string;
|
||||
runId: string;
|
||||
/** The tool name the agent called through Paperclip. */
|
||||
toolName: string;
|
||||
/** The name native runners give that tool (shortened, with a hash); their transcripts show it. */
|
||||
nativeToolName?: string | null;
|
||||
/** When the call started (ISO), for elapsed time. */
|
||||
startedAt: string;
|
||||
/** True once the call has finished; the row then drops its progress. */
|
||||
done: boolean;
|
||||
}
|
||||
|
||||
function finite(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
/** Plain, single-line text: control characters removed, whitespace collapsed, bounded. */
|
||||
export function boundToolCallProgressMessage(value: unknown): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const plain = value.replace(/[\u0000-\u001f\u007f-\u009f---]+/g, " ").replace(/\s+/g, " ").trim();
|
||||
if (!plain) return null;
|
||||
return plain.length > TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH
|
||||
? `${plain.slice(0, TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH - 1)}…`
|
||||
: plain;
|
||||
}
|
||||
|
||||
/**
|
||||
* The progress a provider reported, or null when it is not usable. `total` is
|
||||
* kept only when it is positive and at least the progress so far.
|
||||
*/
|
||||
export function normalizeToolCallProgress(input: { progress?: unknown; total?: unknown; message?: unknown }, now = new Date()): ToolCallProgress | null {
|
||||
const progress = finite(input.progress);
|
||||
if (progress === null || progress < 0) return null;
|
||||
const total = finite(input.total);
|
||||
return {
|
||||
progress,
|
||||
total: total !== null && total > 0 && total >= progress ? total : null,
|
||||
message: boundToolCallProgressMessage(input.message),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/** A fraction in [0, 1] when the total is known, otherwise null (indeterminate). */
|
||||
export function toolCallProgressFraction(progress: Pick<ToolCallProgress, "progress" | "total">): number | null {
|
||||
if (progress.total === null || progress.total <= 0) return null;
|
||||
return Math.min(1, Math.max(0, progress.progress / progress.total));
|
||||
}
|
||||
@@ -525,6 +525,9 @@ export type {
|
||||
ToolAccessSelector,
|
||||
ConnectToolAppAuthChallenge,
|
||||
ConnectToolAppResult,
|
||||
MachineSignInPublicKey,
|
||||
MachineSignInStatus,
|
||||
MachineSignInSetupResult,
|
||||
ToolAppMetadataPreflightAttempt,
|
||||
ToolAppMetadataPreflightResult,
|
||||
FinishToolAppResult,
|
||||
|
||||
@@ -1221,6 +1221,39 @@ export interface ConnectToolAppAuthChallenge {
|
||||
manualClientRequired?: boolean;
|
||||
}
|
||||
|
||||
/** Public half of a key Paperclip holds for machine sign-in. Safe to show and share. */
|
||||
export interface MachineSignInPublicKey {
|
||||
keyId: string;
|
||||
algorithm: "RS256" | "ES256";
|
||||
/** PEM (SPKI) for providers that take a certificate-style key. */
|
||||
pem: string;
|
||||
/** JWK for providers that take a JSON key. */
|
||||
jwk: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** Machine sign-in settings for one organization connection. Never includes secret or private key values. */
|
||||
export interface MachineSignInStatus {
|
||||
connectionId: string;
|
||||
method: "paperclip_key" | "client_secret";
|
||||
clientId: string | null;
|
||||
/** The sign-in service this connection's credentials are bound to. */
|
||||
issuer: string | null;
|
||||
tokenUrl: string;
|
||||
scopes: string[];
|
||||
publicKey: MachineSignInPublicKey | null;
|
||||
/** A replacement key waiting to be registered with the provider. */
|
||||
pendingPublicKey: MachineSignInPublicKey | null;
|
||||
hasClientSecret: boolean;
|
||||
lastTokenAt: string | null;
|
||||
}
|
||||
|
||||
export interface MachineSignInSetupResult {
|
||||
connectionId: string;
|
||||
connection: ToolConnection;
|
||||
machineSignIn: MachineSignInStatus;
|
||||
}
|
||||
|
||||
export interface ConnectToolAppResult {
|
||||
connectionId: string;
|
||||
application: ToolApplication;
|
||||
|
||||
@@ -919,6 +919,15 @@ export {
|
||||
genericMcpOAuthClientSchema,
|
||||
type GenericMcpAuthMode,
|
||||
type GenericMcpOAuthClient,
|
||||
MACHINE_SIGN_IN_METHODS,
|
||||
machineSignInMethodSchema,
|
||||
createMachineSignInSchema,
|
||||
testMachineSignInSchema,
|
||||
rotateMachineSignInSchema,
|
||||
type MachineSignInMethod,
|
||||
type CreateMachineSignIn,
|
||||
type TestMachineSignIn,
|
||||
type RotateMachineSignIn,
|
||||
reconnectToolAppSchema,
|
||||
finishToolAppSchema,
|
||||
finalizeOAuthAccessSchema,
|
||||
|
||||
@@ -372,6 +372,45 @@ export const genericMcpOAuthClientSchema = z.object({
|
||||
|
||||
export type GenericMcpOAuthClient = z.infer<typeof genericMcpOAuthClientSchema>;
|
||||
|
||||
/**
|
||||
* Machine sign-in (MCP OAuth client credentials) for organization-owned
|
||||
* connections. "paperclip_key" means Paperclip holds a signing key and the admin
|
||||
* registers its public half with the provider; "client_secret" stores a secret
|
||||
* the provider issued. Secret values are write-only.
|
||||
*/
|
||||
export const MACHINE_SIGN_IN_METHODS = ["paperclip_key", "client_secret"] as const;
|
||||
export const machineSignInMethodSchema = z.enum(MACHINE_SIGN_IN_METHODS);
|
||||
export type MachineSignInMethod = z.infer<typeof machineSignInMethodSchema>;
|
||||
|
||||
const machineSignInScopesSchema = z.string().trim().max(2048)
|
||||
.refine((value) => !/[\u0000-\u001f"\\]/.test(value), "Scopes can only contain letters, numbers and punctuation, separated by spaces.");
|
||||
|
||||
export const createMachineSignInSchema = z.object({
|
||||
link: z.string().trim().url("Enter the server's full address, starting with https://").max(4096),
|
||||
name: z.string().trim().min(1).max(160).optional(),
|
||||
/** The sign-in service's address, or its token URL. Omitted means "find it from the server". */
|
||||
authorizationServer: z.string().trim().url("Enter the sign-in service's full address, starting with https://").max(4096).optional(),
|
||||
method: machineSignInMethodSchema,
|
||||
clientId: z.string().trim().min(1).max(4096).optional(),
|
||||
clientSecret: z.string().min(1).max(16384).optional(),
|
||||
scopes: machineSignInScopesSchema.optional(),
|
||||
applicationId: z.string().uuid().optional(),
|
||||
}).strict();
|
||||
export type CreateMachineSignIn = z.infer<typeof createMachineSignInSchema>;
|
||||
|
||||
export const testMachineSignInSchema = z.object({
|
||||
clientId: z.string().trim().min(1).max(4096).optional(),
|
||||
clientSecret: z.string().min(1).max(16384).optional(),
|
||||
scopes: machineSignInScopesSchema.optional(),
|
||||
}).strict();
|
||||
export type TestMachineSignIn = z.infer<typeof testMachineSignInSchema>;
|
||||
|
||||
export const rotateMachineSignInSchema = z.discriminatedUnion("method", [
|
||||
z.object({ method: z.literal("paperclip_key") }).strict(),
|
||||
z.object({ method: z.literal("client_secret"), clientSecret: z.string().min(1).max(16384) }).strict(),
|
||||
]);
|
||||
export type RotateMachineSignIn = z.infer<typeof rotateMachineSignInSchema>;
|
||||
|
||||
/**
|
||||
* Reject `headers.*` credential paths whose header name Paperclip refuses to
|
||||
* send, and any value that could split the outbound request. This runs at the
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
|
||||
const workflows = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".github", "workflows");
|
||||
const workflow = readFileSync(path.join(workflows, "merge-queue-checks.yml"), "utf8");
|
||||
const jobNames = [...workflow.matchAll(/^ name: (.+)$/gm)].map(match => match[1]);
|
||||
|
||||
test("the merge queue reports the check names that pull requests report", () => {
|
||||
const caller = readFileSync(path.join(workflows, "pr.yml"), "utf8");
|
||||
const trusted = readFileSync(path.join(workflows, "pr-trusted.yml"), "utf8");
|
||||
assert.match(caller, /^ ci:\n(?: {4}.*\n)*? {4}uses: .+\/pr-trusted\.yml@/m, "pr.yml reports its checks under the ci job");
|
||||
for (const job of ["verify", "e2e"]) {
|
||||
assert.match(trusted, new RegExp(`^ ${job}:\\n name: ${job}$`, "m"), `pull requests report ci / ${job}`);
|
||||
}
|
||||
assert.deepEqual(jobNames, ["ci / verify", "ci / e2e"]);
|
||||
});
|
||||
|
||||
test("the merge queue checks cannot read the queued commit or any credential", () => {
|
||||
const triggers = workflow.split("\non:\n")[1].split("\n\n")[0];
|
||||
assert.equal(triggers.trim(), "merge_group:", "only the merge queue triggers these checks");
|
||||
assert.match(workflow, /^permissions: \{\}$/m);
|
||||
assert.doesNotMatch(workflow, /^ +permissions:/m, "no job widens the empty token");
|
||||
assert.doesNotMatch(workflow, /secrets\.|uses:|environment:|id-token|runs-on\//);
|
||||
assert.deepEqual([...workflow.matchAll(/^ runs-on: (.+)$/gm)].map(match => match[1]), ["ubuntu-latest", "ubuntu-latest"]);
|
||||
});
|
||||
@@ -21,6 +21,10 @@ export const artifactName = arch => `merge-group-image-${arch}`;
|
||||
const architectures = ["amd64", "arm64"];
|
||||
const indexTypes = ["application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json"];
|
||||
const manifestTypes = ["application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json"];
|
||||
// How long a master push waits for a merge-queue build that is still running.
|
||||
// A build takes about 12 minutes and the exact-source gate about 14.
|
||||
const findWaitMs = 15 * 60_000;
|
||||
const findPollMs = 20_000;
|
||||
const fullSha = /^[a-f0-9]{40}$/;
|
||||
const digestPattern = /^sha256:[a-f0-9]{64}$/;
|
||||
const sha256 = bytes => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
||||
@@ -43,6 +47,11 @@ export function selectVerifiedRun({ sha, runs }) {
|
||||
typeof run.head_branch === "string" && run.head_branch.startsWith("gh-readonly-queue/master/"));
|
||||
if (eligible.length === 0) return { reason: "no merge-queue image build exists for this commit" };
|
||||
const passed = eligible.filter(run => run.status === "completed" && run.conclusion === "success").sort((a, b) => b.id - a.id);
|
||||
// The queue can land a commit while its image is still building. `pending`
|
||||
// tells the caller that waiting can still produce a usable run.
|
||||
if (passed.length === 0 && eligible.some(run => run.status !== "completed")) {
|
||||
return { pending: true, reason: "the merge-queue image build for this commit did not finish in time" };
|
||||
}
|
||||
if (passed.length === 0) return { reason: "the merge-queue image build for this commit did not succeed" };
|
||||
return { runId: passed[0].id };
|
||||
}
|
||||
@@ -163,8 +172,18 @@ async function main(command) {
|
||||
assert.ok(response.ok, `GitHub API ${path}: HTTP ${response.status}`);
|
||||
return response.json();
|
||||
};
|
||||
const { workflow_runs: runs } = await api(`/actions/workflows/docker-merge-group.yml/runs?event=merge_group&head_sha=${sha}&per_page=20`);
|
||||
const run = selectVerifiedRun({ sha, runs });
|
||||
// Publication waits for the exact-source gate in any case, so waiting here
|
||||
// for an unfinished build costs nothing while the build is the shorter of
|
||||
// the two. Past the deadline the push builds from source.
|
||||
const deadline = Date.now() + findWaitMs;
|
||||
let run;
|
||||
for (;;) {
|
||||
const { workflow_runs: runs } = await api(`/actions/workflows/docker-merge-group.yml/runs?event=merge_group&head_sha=${sha}&per_page=20`);
|
||||
run = selectVerifiedRun({ sha, runs });
|
||||
if (!run.pending || Date.now() >= deadline) break;
|
||||
console.log("The merge-queue image build for this commit is still running. Waiting.");
|
||||
await new Promise(resolve => setTimeout(resolve, findPollMs));
|
||||
}
|
||||
if (!run.runId) { console.log(`No promotion: ${run.reason}. Building from source.`); return; }
|
||||
const { artifacts } = await api(`/actions/runs/${run.runId}/artifacts?per_page=100`);
|
||||
const artifact = selectArtifact({ artifacts, arch });
|
||||
|
||||
@@ -30,8 +30,17 @@ test("promotes only a successful merge-queue build of this repository for this e
|
||||
["a queue for another branch", run({ head_branch: "gh-readonly-queue/release/pr-1-x" })],
|
||||
["an ordinary branch", run({ head_branch: "master" })],
|
||||
]) assert.match(selectVerifiedRun({ sha, runs: [candidate] }).reason, /no merge-queue image build/, label);
|
||||
for (const candidate of [run({ conclusion: "failure" }), run({ conclusion: "cancelled" }), run({ status: "in_progress", conclusion: null })]) {
|
||||
assert.match(selectVerifiedRun({ sha, runs: [candidate] }).reason, /did not succeed/);
|
||||
for (const candidate of [run({ conclusion: "failure" }), run({ conclusion: "cancelled" })]) {
|
||||
assert.deepEqual(selectVerifiedRun({ sha, runs: [candidate] }), { reason: "the merge-queue image build for this commit did not succeed" });
|
||||
}
|
||||
// An unfinished build is worth waiting for, even beside a failed attempt. A
|
||||
// finished one is used at once, and an unfinished run of another commit is not.
|
||||
for (const status of ["queued", "in_progress"]) {
|
||||
const unfinished = run({ id: 11, status, conclusion: null });
|
||||
assert.equal(selectVerifiedRun({ sha, runs: [unfinished] }).pending, true);
|
||||
assert.equal(selectVerifiedRun({ sha, runs: [run({ conclusion: "failure" }), unfinished] }).pending, true);
|
||||
assert.deepEqual(selectVerifiedRun({ sha, runs: [run(), unfinished] }), { runId: 10 });
|
||||
assert.equal(selectVerifiedRun({ sha, runs: [run({ id: 11, status, conclusion: null, head_sha: "b".repeat(40) })] }).pending, undefined);
|
||||
}
|
||||
assert.throws(() => selectVerifiedRun({ sha: "master", runs: [] }));
|
||||
});
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, companySecretBindings, connectionGrants, createDb, principalPermissionGrants, toolApplications, toolCatalogEntries, toolConnections } from "@paperclipai/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { syncConnectionCredentialBindings } from "../services/connection-credential-bindings.js";
|
||||
import { connectionManagementService } from "../services/connection-management.js";
|
||||
import { secretService } from "../services/secrets.js";
|
||||
import { toolAccessService } from "../services/tool-access.js";
|
||||
import type { PaperclipCloudConnector } from "../services/paperclip-cloud-connector.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
describe("dedicated agent connection discovery", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let provider: Server;
|
||||
let origin: string;
|
||||
let requests: Array<string | undefined>;
|
||||
let rejectCredential: boolean;
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("agent-connection-catalog-");
|
||||
db = createDb(database.connectionString);
|
||||
provider = createServer(async (req, res) => {
|
||||
if (req.method !== "POST") { res.writeHead(404); res.end(); return; }
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) chunks.push(Buffer.from(chunk));
|
||||
const message = JSON.parse(Buffer.concat(chunks).toString());
|
||||
requests.push(req.headers.authorization);
|
||||
if (rejectCredential || req.headers.authorization !== "Bearer fixture-agent-access") {
|
||||
res.writeHead(401, { "www-authenticate": "Bearer", "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ error: "unauthorized" }));
|
||||
return;
|
||||
}
|
||||
if (message.method === "notifications/initialized") {
|
||||
res.writeHead(202); res.end(); return;
|
||||
}
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ jsonrpc: "2.0", id: message.id, result: message.method === "initialize"
|
||||
? { protocolVersion: "2025-03-26", capabilities: { tools: {} }, serverInfo: { name: "fixture", version: "1" } }
|
||||
: { tools: [{ name: "get_fixture", description: "Read a fixture", inputSchema: { type: "object" }, annotations: { readOnlyHint: true } }] } }));
|
||||
});
|
||||
await new Promise<void>(resolve => provider.listen(0, "127.0.0.1", resolve));
|
||||
origin = `http://127.0.0.1:${(provider.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
requests = [];
|
||||
rejectCredential = false;
|
||||
const localFetch = globalThis.fetch;
|
||||
vi.spyOn(globalThis, "fetch").mockImplementation(async (url, init) => {
|
||||
const href = String(url);
|
||||
if (href.startsWith(`${origin}/`)) return localFetch(url, init);
|
||||
if (href === "https://api.github.com/user") return Response.json({ id: 42, login: "fixture-bot" });
|
||||
if (href.startsWith("https://api.github.com/user/installations?")) return Response.json({ installations: [{ id: 101, repository_selection: "selected", account: { login: "example" } }] });
|
||||
if (href.startsWith("https://api.github.com/user/installations/101/repositories?")) return Response.json({ total_count: 1, repositories: [{ id: 1, full_name: "example/fixture" }] });
|
||||
throw new Error("Unexpected provider request");
|
||||
});
|
||||
});
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
afterAll(async () => {
|
||||
if (provider) await new Promise<void>(resolve => provider.close(() => resolve()));
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await database?.cleanup();
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Agent catalog fixture", issuePrefix: `AC${randomUUID().slice(0, 6)}` }).returning();
|
||||
const [agent, otherAgent] = await db.insert(agents).values(["Owner", "Other"].map(name => ({ companyId: company.id, name, role: "engineer", adapterType: "process" }))).returning();
|
||||
const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "GitHub", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({
|
||||
companyId: company.id, applicationId: application.id, name: "GitHub", uid: randomUUID(),
|
||||
transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_agent", status: "active", enabled: true,
|
||||
healthStatus: "failed", healthMessage: "OAuth credentials have expired and need to be reconnected.",
|
||||
config: { sourceTemplateKey: "github", url: `${origin}/mcp`, oauth: {
|
||||
strategy: "paperclip_cloud_connector", provider: "github", connectorProfile: "github.code",
|
||||
tokenUrl: "https://github.com/login/oauth/access_token", expiresAt: null,
|
||||
} },
|
||||
}).returning();
|
||||
const secret = await secretService(db).create(company.id, { name: "Agent access", key: randomUUID(), provider: "local_encrypted", value: "fixture-agent-access" });
|
||||
const [grant] = await db.insert(connectionGrants).values({
|
||||
companyId: company.id, connectionId: connection.id, kind: "agent", subjectAgentId: agent.id, status: "active",
|
||||
credentialSecretRefs: [{ secretId: secret.id, configPath: "oauth.access_token", versionSelector: "latest", required: true }],
|
||||
providerTenant: { oauth: { strategy: "paperclip_cloud_connector", accessTokenExpiresAt: null },
|
||||
github: { userId: "42", login: "fixture-bot", installationCount: 1, repositoryCount: 1, repositorySelection: "selected", installationIds: ["101"], installationOwnerLogins: ["example"] } },
|
||||
}).returning();
|
||||
await syncConnectionCredentialBindings(db, connection);
|
||||
const connector = { setWebhookBinding: vi.fn(async () => undefined) } as unknown as PaperclipCloudConnector;
|
||||
const service = toolAccessService(db, { deploymentMode: "local_trusted", deploymentExposure: "private", paperclipCloudConnector: connector });
|
||||
return { company, agent, otherAgent, connection, grant, secret, service };
|
||||
}
|
||||
|
||||
it.each(["background", "system", "operator", "agent"] as const)("renames and refreshes an agent-owned GitHub connection without reconnect (%s)", async mode => {
|
||||
const f = await fixture();
|
||||
const actor = mode === "background" ? undefined : mode === "system"
|
||||
? { actorType: "system" as const } : mode === "operator"
|
||||
? { actorType: "user" as const, actorId: "fixture-operator" }
|
||||
: { actorType: "agent" as const, actorId: f.agent.id };
|
||||
await f.service.updateConnection(f.connection.id, { name: "Renamed bot" });
|
||||
const refreshed = await f.service.refreshCatalog(f.connection.id, actor);
|
||||
expect(refreshed.connection).toMatchObject({ name: "Renamed bot", healthStatus: "ok", enabled: true });
|
||||
expect((await f.service.getConnection(f.connection.id, f.company.id)).requiresReauthorization).toBe(false);
|
||||
expect(refreshed.catalog.map(tool => tool.toolName)).toEqual(["get_fixture"]);
|
||||
expect(requests.length).toBeGreaterThan(0);
|
||||
expect(requests.every(header => header === "Bearer fixture-agent-access")).toBe(true);
|
||||
await expect(f.service.checkHealth(f.connection.id, actor)).resolves.toMatchObject({ connection: { healthStatus: "ok" } });
|
||||
const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id));
|
||||
expect(grant.status).toBe("active");
|
||||
expect(grant.credentialSecretRefs).toEqual(f.grant.credentialSecretRefs);
|
||||
const bindings = await db.select().from(companySecretBindings).where(and(eq(companySecretBindings.targetId, f.connection.id), eq(companySecretBindings.secretId, f.secret.id)));
|
||||
expect(bindings).toHaveLength(1);
|
||||
});
|
||||
|
||||
describe.each(["refreshCatalog", "checkHealth"] as const)("%s caller isolation", operation => {
|
||||
it.each(["another agent", "another company", "missing agent identity"])("denies %s without changing the owner's connection", async scope => {
|
||||
const f = await fixture();
|
||||
const actorId = scope === "another company" ? (await fixture()).agent.id : scope === "missing agent identity" ? null : f.otherAgent.id;
|
||||
await f.service.refreshCatalog(f.connection.id, { actorType: "agent", actorId: f.agent.id });
|
||||
const before = await db.select().from(toolConnections).where(eq(toolConnections.id, f.connection.id));
|
||||
const catalog = await db.select().from(toolCatalogEntries).where(eq(toolCatalogEntries.connectionId, f.connection.id));
|
||||
requests = [];
|
||||
vi.mocked(fetch).mockClear();
|
||||
await expect(f.service[operation](f.connection.id, { actorType: "agent", actorId }))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } });
|
||||
expect(requests).toEqual([]);
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(toolConnections).where(eq(toolConnections.id, f.connection.id))).toEqual(before);
|
||||
expect(await db.select().from(toolCatalogEntries).where(eq(toolCatalogEntries.connectionId, f.connection.id))).toEqual(catalog);
|
||||
await expect(f.service.checkHealth(f.connection.id, { actorType: "agent", actorId: f.agent.id }))
|
||||
.resolves.toMatchObject({ connection: { healthStatus: "ok" } });
|
||||
});
|
||||
|
||||
it("keeps the owner's tools healthy when a permitted managing agent lacks the credential", async () => {
|
||||
const f = await fixture();
|
||||
const userId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name: "Manager", email: `${userId}@fixture.invalid`, createdAt: new Date(), updatedAt: new Date() });
|
||||
await db.insert(companyMemberships).values([
|
||||
{ companyId: f.company.id, principalType: "user", principalId: userId, membershipRole: "owner", status: "active" },
|
||||
{ companyId: f.company.id, principalType: "agent", principalId: f.otherAgent.id, status: "active" },
|
||||
]);
|
||||
await db.insert(principalPermissionGrants).values([
|
||||
{ companyId: f.company.id, principalType: "agent", principalId: f.otherAgent.id, permissionKey: "tools:manage_connections" },
|
||||
{ companyId: f.company.id, principalType: "user", principalId: userId, permissionKey: "tools:manage_connections" },
|
||||
]);
|
||||
const managementContext = await connectionManagementService(db).authorize({
|
||||
type: "agent", agentId: f.otherAgent.id, companyId: f.company.id, onBehalfOfUserId: userId,
|
||||
}, "connections:configure", { type: "connection_management", companyId: f.company.id, connectionId: f.connection.id, operation });
|
||||
expect(managementContext.decision.allowed).toBe(true);
|
||||
await f.service.refreshCatalog(f.connection.id, { actorType: "agent", actorId: f.agent.id });
|
||||
const before = await db.select().from(toolConnections).where(eq(toolConnections.id, f.connection.id));
|
||||
requests = [];
|
||||
vi.mocked(fetch).mockClear();
|
||||
await expect(f.service[operation](f.connection.id, { actorType: "agent", actorId: f.otherAgent.id, managementContext }))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } });
|
||||
expect(requests).toEqual([]);
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(toolConnections).where(eq(toolConnections.id, f.connection.id))).toEqual(before);
|
||||
expect((await f.service.listCatalog(f.connection.id, f.company.id)).map(tool => tool.toolName)).toEqual(["get_fixture"]);
|
||||
});
|
||||
});
|
||||
|
||||
it("does not fall back to personal or organization grants when the agent grant is revoked", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, f.grant.id));
|
||||
await db.insert(connectionGrants).values([
|
||||
{ companyId: f.company.id, connectionId: f.connection.id, kind: "organization", status: "active", isDefault: true, credentialSecretRefs: f.grant.credentialSecretRefs },
|
||||
{ companyId: f.company.id, connectionId: f.connection.id, kind: "user", subjectUserId: "fixture-operator", status: "active", credentialSecretRefs: f.grant.credentialSecretRefs },
|
||||
]);
|
||||
await expect(f.service.refreshCatalog(f.connection.id))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } });
|
||||
await expect(f.service.refreshCatalog(f.connection.id, { actorType: "user", actorId: "fixture-operator" }))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } });
|
||||
expect((await f.service.getConnection(f.connection.id, f.company.id)).requiresReauthorization).toBe(true);
|
||||
await expect(f.service.checkHealth(f.connection.id))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_required" } });
|
||||
expect(requests).toEqual([]);
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the same agent grant for generic OAuth MCP health and discovery", async () => {
|
||||
const f = await fixture();
|
||||
await f.service.updateConnection(f.connection.id, { config: { url: `${origin}/mcp`, oauth: {
|
||||
provider: origin, tokenUrl: `${origin}/token`, expiresAt: null,
|
||||
} } });
|
||||
await expect(f.service.refreshCatalog(f.connection.id)).resolves.toMatchObject({ connection: { healthStatus: "ok" } });
|
||||
await expect(f.service.checkHealth(f.connection.id)).resolves.toMatchObject({ connection: { healthStatus: "ok" } });
|
||||
expect(requests.length).toBeGreaterThan(0);
|
||||
expect(requests.every(header => header === "Bearer fixture-agent-access")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not choose between multiple active agent authorizations in background work", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(connectionGrants).values({ companyId: f.company.id, connectionId: f.connection.id,
|
||||
kind: "agent", subjectAgentId: f.otherAgent.id, status: "active", credentialSecretRefs: f.grant.credentialSecretRefs });
|
||||
await expect(f.service.refreshCatalog(f.connection.id))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_ambiguous" } });
|
||||
await expect(f.service.checkHealth(f.connection.id))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_authorization_ambiguous" } });
|
||||
expect(requests).toEqual([]);
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still requires reconnect when the provider rejects the agent's credential", async () => {
|
||||
const f = await fixture();
|
||||
rejectCredential = true;
|
||||
await expect(f.service.refreshCatalog(f.connection.id)).rejects.toThrow();
|
||||
expect(requests.length).toBeGreaterThan(0);
|
||||
expect(requests.every(header => header === "Bearer fixture-agent-access")).toBe(true);
|
||||
const [grant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id));
|
||||
expect(grant.status).toBe("needs_reauthorization");
|
||||
expect((await f.service.getConnection(f.connection.id, f.company.id)).requiresReauthorization).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { activityLog, agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentWakeupRequests, automaticRunBrakes, companies, createDb, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
|
||||
import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat/queue.js";
|
||||
@@ -13,6 +13,7 @@ import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY } from "../services/chat-control-re
|
||||
import { ASSIGNED_WORK_CONTINUED_ACTION, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js";
|
||||
import { CONTINUATION_CAP_WINDOW_MS } from "../services/recovery/assigned-work-invariant.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { automaticRunBrake, automaticRunCreationHeldUntil } from "../services/recovery/automatic-run-brake.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const MINUTE = 60_000;
|
||||
@@ -25,7 +26,7 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o
|
||||
db = createDb(database.connectionString);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
afterEach(() => { vi.restoreAllMocks(); });
|
||||
afterEach(() => { vi.restoreAllMocks(); automaticRunBrake.reset(); });
|
||||
|
||||
async function seed() {
|
||||
const prefix = `S${randomUUID().slice(0, 5)}`;
|
||||
@@ -104,7 +105,7 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o
|
||||
const readRun = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
|
||||
const continuations = async () => db.select().from(activityLog).where(and(eq(activityLog.entityId, issue.id),
|
||||
eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION)));
|
||||
return { companyId, agentId, issue, queuedRun, otherStarts, olderRun, queue, sweep, runsOfTask, readRun, continuations };
|
||||
return { companyId, agentId, issue, other, queuedRun, otherStarts, olderRun, queue, sweep, runsOfTask, readRun, continuations };
|
||||
}
|
||||
|
||||
async function replacementOf(f: Awaited<ReturnType<typeof seed>>, stalledId: string) {
|
||||
@@ -112,6 +113,32 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o
|
||||
return runs.filter(run => run.id !== stalledId);
|
||||
}
|
||||
|
||||
it("a replacement spends the paused company's creation slot", async () => {
|
||||
const f = await seed();
|
||||
const stalled = await f.queuedRun();
|
||||
await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2);
|
||||
await db.insert(automaticRunBrakes).values({ companyId: f.companyId, state: "open", openedAt: new Date(), stateChangedAt: new Date() });
|
||||
expect(await automaticRunCreationHeldUntil(db, f.companyId, { issueId: f.issue.id, sourceRunId: stalled.id })).toBeNull();
|
||||
|
||||
const check = await f.sweep().checkIssue({ companyId: f.companyId, issueId: f.issue.id });
|
||||
expect(check).toMatchObject({ outcome: "replaced_stalled_run" });
|
||||
const [brake] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, f.companyId));
|
||||
expect(brake!.nextCreationAt!.getTime()).toBeGreaterThan(Date.now());
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
it("an old finished request from a person does not exempt a replacement from the pause", async () => {
|
||||
const f = await seed();
|
||||
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "on_demand", triggerDetail: "manual",
|
||||
reason: "task_retry", payload: { issueId: f.issue.id }, status: "completed", requestedByActorType: "user", requestedByActorId: "person-1",
|
||||
requestedAt: new Date(Date.now() - 4 * 60 * MINUTE), finishedAt: new Date(Date.now() - 4 * 60 * MINUTE) });
|
||||
const stalled = await f.queuedRun();
|
||||
await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2);
|
||||
await db.insert(automaticRunBrakes).values({ companyId: f.companyId, state: "open", openedAt: new Date(), stateChangedAt: new Date() });
|
||||
expect(await f.sweep().checkIssue({ companyId: f.companyId, issueId: f.issue.id })).toMatchObject({ outcome: "replaced_stalled_run" });
|
||||
const [brake] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, f.companyId));
|
||||
expect(brake!.nextCreationAt!.getTime()).toBeGreaterThan(Date.now());
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
it("cancels an automatic run the claim keeps refusing and queues exactly one claimable replacement", async () => {
|
||||
const f = await seed();
|
||||
const stalled = await f.queuedRun();
|
||||
@@ -191,6 +218,40 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o
|
||||
expect(await f.continuations()).toHaveLength(0);
|
||||
}, 60_000);
|
||||
|
||||
it("leaves an automatic run that waits behind work on tasks people created", async () => {
|
||||
// Work people wait on goes first by design; a system run behind it is waiting its turn, not stalled.
|
||||
const f = await seed();
|
||||
await db.update(issues).set({ createdByUserId: "person-1" }).where(eq(issues.id, f.other.id));
|
||||
const waiting = await f.queuedRun({ refused: false });
|
||||
await f.otherStarts(OLDEST_DISPATCH_SHARE + 3, 10 * MINUTE);
|
||||
const sweep = f.sweep();
|
||||
expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id }))
|
||||
.toMatchObject({ verdict: { kind: "ok", clause: "live_run", runId: waiting.id } });
|
||||
expect((await f.readRun(waiting.id)).status).toBe("queued");
|
||||
expect(await f.continuations()).toHaveLength(0);
|
||||
}, 60_000);
|
||||
|
||||
it("leaves an automatic run that waits behind continuations of a person's Retry", async () => {
|
||||
// The dispatcher ranks a recovery run as person work when its source chain leads to a person's request.
|
||||
const f = await seed();
|
||||
const waiting = await f.queuedRun({ refused: false });
|
||||
const personSourceAt = new Date(Date.now() - 30 * MINUTE);
|
||||
const [personSource] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, invocationSource: "on_demand",
|
||||
triggerDetail: "manual", status: "failed", createdAt: personSourceAt, startedAt: personSourceAt, finishedAt: personSourceAt,
|
||||
contextSnapshot: { issueId: f.other.id, taskId: f.other.id, wakeReason: "task_retry" } }).returning();
|
||||
for (let index = 0; index < OLDEST_DISPATCH_SHARE + 3; index++) {
|
||||
const startedAt = new Date(Date.now() - 10 * MINUTE + index * 1_000);
|
||||
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, invocationSource: "automation", triggerDetail: "system",
|
||||
status: "succeeded", createdAt: new Date(startedAt.getTime() - 1_000), startedAt, finishedAt: new Date(startedAt.getTime() + 500),
|
||||
contextSnapshot: { issueId: f.other.id, taskId: f.other.id, wakeReason: "issue_recovery_action_restored", previousRunId: personSource!.id } });
|
||||
}
|
||||
const sweep = f.sweep();
|
||||
expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id }))
|
||||
.toMatchObject({ verdict: { kind: "ok", clause: "live_run", runId: waiting.id } });
|
||||
expect((await f.readRun(waiting.id)).status).toBe("queued");
|
||||
expect(await f.continuations()).toHaveLength(0);
|
||||
}, 60_000);
|
||||
|
||||
it("leaves a run that waits behind a full agent", async () => {
|
||||
const f = await seed();
|
||||
const waiting = await f.queuedRun();
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
describe("automatic-run brake module graph", () => {
|
||||
it("loads with the server's heartbeat and recovery modules in the order the server imports them", async () => {
|
||||
await expect(import("../services/heartbeat.js")).resolves.toBeDefined();
|
||||
const brake = await import("../services/recovery/automatic-run-brake.js");
|
||||
const rules = await brake.automaticRunOutcomeRules();
|
||||
expect(rules.isTransientSetupCode("environment_provider_unavailable")).toBe(true);
|
||||
expect(rules.isPersonOwnedFailureCode("budget_hard_stop")).toBe(true);
|
||||
}, 600_000);
|
||||
});
|
||||
@@ -0,0 +1,495 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog, agentWakeupRequests, agents, automaticRunBrakes, companies, createDb, heartbeatRuns, issueRecoveryActions, issues,
|
||||
nativeRunFinalizations,
|
||||
} from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { AUTOMATIC_RUN_BRAKE, automaticRunBrake } from "../services/recovery/automatic-run-brake.js";
|
||||
import { recoveryService } from "../services/recovery/service.js";
|
||||
import { ASSIGNED_WORK_CONTINUED_ACTION } from "../services/recovery/assigned-work-sweep.js";
|
||||
import { taskAutomaticAttempts } from "../services/recovery/task-automatic-attempts.js";
|
||||
import { CONTINUATION_CAP } from "../services/recovery/assigned-work-invariant.js";
|
||||
import { deliverReconciledExecutions } from "../services/execution-recovery-resolution.js";
|
||||
import { continueHeldExecutions, resetBacklogContinuationPacingForTests } from "../services/fresh-continuation-fallback.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const suite = support.supported ? describe : describe.skip;
|
||||
const MINUTE = 60_000;
|
||||
const HOUR = 60 * MINUTE;
|
||||
|
||||
suite("automatic-run brake in the recovery passes", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let companyId = "", agentId = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("paperclip-automatic-run-brake-passes-");
|
||||
db = createDb(temporary.connectionString);
|
||||
}, 60_000);
|
||||
|
||||
afterEach(async () => {
|
||||
automaticRunBrake.reset();
|
||||
resetBacklogContinuationPacingForTests(db);
|
||||
await db.execute(sql`truncate table automatic_run_brakes, issue_recovery_actions, native_run_finalizations, activity_log,
|
||||
agent_wakeup_requests, heartbeat_runs, issues, agents, companies cascade`);
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }, 120_000);
|
||||
|
||||
async function seedCompany() {
|
||||
companyId = randomUUID(); agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Passes", issuePrefix: `P${companyId.replaceAll("-", "").slice(0, 6)}`,
|
||||
requireBoardApprovalForNewAgents: false });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Builder", role: "engineer", status: "idle", adapterType: "codex_local",
|
||||
adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} });
|
||||
}
|
||||
|
||||
const ago = (ms: number) => new Date(Date.now() - ms);
|
||||
|
||||
async function openBrake(options: { startingAutomaticRun?: boolean } = { startingAutomaticRun: true }) {
|
||||
const failures = await db.insert(heartbeatRuns).values(Array.from({ length: 20 }, (_, index) => ({ companyId, agentId,
|
||||
invocationSource: "automation", triggerDetail: "system", runtimeMode: "legacy", status: "failed",
|
||||
errorCode: "environment_provider_unavailable", executionStage: "preparing", createdAt: ago(4 * MINUTE + index),
|
||||
startedAt: ago(3 * MINUTE + index), finishedAt: ago(MINUTE - index * 100),
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
|
||||
contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() } }))).returning();
|
||||
if (options.startingAutomaticRun) {
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", triggerDetail: "system",
|
||||
status: "running", runtimeMode: "legacy", executionStage: "preparing", startedAt: ago(20_000),
|
||||
contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: failures[0]!.id, issueId: randomUUID() } });
|
||||
}
|
||||
expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("open");
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function recover(failures: Array<{ id: string }>) {
|
||||
await db.update(heartbeatRuns).set({ createdAt: sql`created_at - interval '1 hour'`, startedAt: sql`started_at - interval '1 hour'`,
|
||||
finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id)));
|
||||
await db.insert(heartbeatRuns).values(Array.from({ length: 2 }, () => ({ companyId, agentId, invocationSource: "automation",
|
||||
triggerDetail: "system", runtimeMode: "legacy", status: "succeeded", executionStage: "dispatching", createdAt: ago(MINUTE),
|
||||
startedAt: ago(50_000), finishedAt: ago(1_000), contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() } })));
|
||||
await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at - ${`${AUTOMATIC_RUN_BRAKE.minOpenMs} milliseconds`}::interval` })
|
||||
.where(eq(automaticRunBrakes.companyId, companyId));
|
||||
expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("half_open");
|
||||
}
|
||||
|
||||
async function strandedTask() {
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Continue the work", status: "in_progress", assigneeAgentId: agentId,
|
||||
updatedAt: ago(3 * HOUR), createdAt: ago(5 * HOUR) });
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "on_demand", status: "interrupted",
|
||||
errorCode: "server_shutdown_interrupted", startedAt: ago(4 * HOUR), finishedAt: ago(3 * HOUR), livenessState: "needs_followup",
|
||||
createdAt: ago(4 * HOUR), contextSnapshot: { issueId, taskId: issueId },
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } });
|
||||
return issueId;
|
||||
}
|
||||
|
||||
function wakeQueue() {
|
||||
const enqueueWakeup = vi.fn(async (wakeAgentId: string, opts?: { payload?: Record<string, unknown> | null; reason?: string | null }) => {
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: wakeAgentId, source: "automation",
|
||||
triggerDetail: "system", reason: opts?.reason ?? null, payload: opts?.payload ?? null, status: "queued" }).returning();
|
||||
return { id: wake!.id } as never;
|
||||
});
|
||||
return enqueueWakeup;
|
||||
}
|
||||
|
||||
const ledger = async (issueId: string) => (await taskAutomaticAttempts(db, companyId, issueId, new Date())).all.length;
|
||||
const continuedActivity = async (issueId: string) => db.select().from(activityLog)
|
||||
.where(and(eq(activityLog.entityId, issueId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION)));
|
||||
|
||||
it("the assigned-work sweep defers continuations while open without spending the task's attempts, and resumes by itself", async () => {
|
||||
await seedCompany();
|
||||
const issueId = await strandedTask();
|
||||
const failures = await openBrake();
|
||||
const enqueueWakeup = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
for (let pass = 0; pass < 3; pass += 1) {
|
||||
const check = await recovery.assignedWork.checkIssue({ companyId, issueId });
|
||||
expect(check).toMatchObject({ outcome: "deferred_rate_limit" });
|
||||
expect(check!.retryAt!.getTime()).toBeGreaterThan(Date.now() + 30_000);
|
||||
}
|
||||
expect(enqueueWakeup).not.toHaveBeenCalled();
|
||||
expect(await continuedActivity(issueId)).toHaveLength(0);
|
||||
expect(await ledger(issueId)).toBe(0);
|
||||
expect(await recovery.assignedWork.report(companyId)).toMatchObject({ automaticWork: { state: "open",
|
||||
summary: expect.stringMatching(/^Automatic work is paused for this company since .* because the last 5 automatic runs could not get a sandbox\.$/) } });
|
||||
|
||||
await recover(failures);
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" });
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(1);
|
||||
expect(await ledger(issueId)).toBe(1);
|
||||
|
||||
await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at
|
||||
- ${`${AUTOMATIC_RUN_BRAKE.releaseSteps * AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval` })
|
||||
.where(eq(automaticRunBrakes.companyId, companyId));
|
||||
expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("closed");
|
||||
expect((await recovery.assignedWork.report(companyId)).automaticWork).toMatchObject({ state: "closed",
|
||||
summary: "Automatic work is running normally." });
|
||||
}, 120_000);
|
||||
|
||||
it("the sweep still continues one task a minute while open when no automatic run is starting", async () => {
|
||||
await seedCompany();
|
||||
const [first, second] = [await strandedTask(), await strandedTask()];
|
||||
await openBrake({ startingAutomaticRun: false });
|
||||
const enqueueWakeup = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId: first })).toMatchObject({ outcome: "repaired" });
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId: second })).toMatchObject({ outcome: "deferred_rate_limit" });
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(1);
|
||||
}, 120_000);
|
||||
|
||||
it("a task that loses the creation slot to another sweep check gains no extra wait", async () => {
|
||||
await seedCompany();
|
||||
const [first, second] = [await strandedTask(), await strandedTask()];
|
||||
await openBrake({ startingAutomaticRun: false });
|
||||
const enqueueWakeup = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
const checks = await Promise.all([first, second].map(issueId => recovery.assignedWork.checkIssue({ companyId, issueId })));
|
||||
expect(checks.map(check => check!.outcome).sort()).toEqual(["deferred_rate_limit", "repaired"]);
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(1);
|
||||
const loser = checks[0]!.outcome === "deferred_rate_limit" ? first : second;
|
||||
expect(await ledger(loser)).toBe(0);
|
||||
}, 120_000);
|
||||
|
||||
it("leaves every pass exactly as before when the company has no pause record", async () => {
|
||||
await seedCompany();
|
||||
const issueId = await strandedTask();
|
||||
const action = await pendingContinuation();
|
||||
const enqueueWakeup = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record<string, unknown> }) => {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation",
|
||||
triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning();
|
||||
return run!;
|
||||
});
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" });
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(1);
|
||||
expect(wake).toHaveBeenCalledTimes(1);
|
||||
expect(wake.mock.calls[0]![1]).toMatchObject({ reason: "issue_recovery_action_restored", contextSnapshot: { recoveryActionId: action.id } });
|
||||
expect(await db.select().from(automaticRunBrakes)).toHaveLength(0);
|
||||
}, 120_000);
|
||||
|
||||
it("tries again on another agent when the waiting automatic run cannot start, and closes once it works", async () => {
|
||||
await seedCompany();
|
||||
const otherAgent = randomUUID();
|
||||
await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local",
|
||||
adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} });
|
||||
const failures = await openBrake({ startingAutomaticRun: false });
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", triggerDetail: "system", status: "queued",
|
||||
createdAt: ago(10 * MINUTE), contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: failures[0]!.id, issueId: randomUUID() } });
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Review the work", status: "in_progress", assigneeAgentId: otherAgent,
|
||||
updatedAt: ago(3 * HOUR), createdAt: ago(5 * HOUR) });
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId: otherAgent, invocationSource: "on_demand", status: "interrupted",
|
||||
errorCode: "server_shutdown_interrupted", startedAt: ago(4 * HOUR), finishedAt: ago(3 * HOUR), livenessState: "needs_followup",
|
||||
createdAt: ago(4 * HOUR), contextSnapshot: { issueId, taskId: issueId },
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } });
|
||||
const enqueueWakeup = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" });
|
||||
expect(enqueueWakeup.mock.calls[0]![0]).toBe(otherAgent);
|
||||
|
||||
await db.update(heartbeatRuns).set({ finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id)));
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId: otherAgent, invocationSource: "automation", triggerDetail: "system",
|
||||
status: "succeeded", runtimeMode: "legacy", executionStage: "dispatching", startedAt: ago(40_000), finishedAt: ago(1_000),
|
||||
contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID(), issueId } });
|
||||
await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at - ${`${AUTOMATIC_RUN_BRAKE.minOpenMs} milliseconds`}::interval` })
|
||||
.where(eq(automaticRunBrakes.companyId, companyId));
|
||||
expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("half_open");
|
||||
await db.update(automaticRunBrakes).set({ stateChangedAt: sql`state_changed_at
|
||||
- ${`${AUTOMATIC_RUN_BRAKE.releaseSteps * AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval` })
|
||||
.where(eq(automaticRunBrakes.companyId, companyId));
|
||||
expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("closed");
|
||||
}, 120_000);
|
||||
|
||||
it("a delivery its agent refuses gives the creation slot back to the next continuation in the same pass", async () => {
|
||||
await seedCompany();
|
||||
const otherAgent = randomUUID();
|
||||
await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local",
|
||||
adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} });
|
||||
const refused = await pendingContinuation();
|
||||
const accepted = await pendingContinuation(otherAgent, 3 * MINUTE);
|
||||
await openBrake({ startingAutomaticRun: false });
|
||||
const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record<string, unknown> }) => {
|
||||
if (wakeAgentId === agentId) return null;
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation",
|
||||
triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning();
|
||||
return run!;
|
||||
});
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake.mock.calls.map(call => call[0])).toEqual([agentId, otherAgent]);
|
||||
const delivery = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!
|
||||
.evidence.continuationDelivery;
|
||||
expect(await delivery(refused.id)).toBe("pending");
|
||||
expect(await delivery(accepted.id)).toBe("delivered");
|
||||
expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull();
|
||||
}, 120_000);
|
||||
|
||||
it("a capped task gives the creation slot back to the next task the sweep checks", async () => {
|
||||
await seedCompany();
|
||||
const [capped, next] = [await strandedTask(), await strandedTask()];
|
||||
await openBrake({ startingAutomaticRun: false });
|
||||
await db.insert(activityLog).values(Array.from({ length: CONTINUATION_CAP }, (_, index) => ({ companyId, actorType: "system",
|
||||
actorId: "assigned-work-invariant", action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: capped,
|
||||
createdAt: ago(10 * MINUTE + index), details: {} })));
|
||||
const enqueueWakeup = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId: capped })).toMatchObject({ outcome: "deferred_cap" });
|
||||
expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull();
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId: next })).toMatchObject({ outcome: "repaired" });
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(1);
|
||||
}, 120_000);
|
||||
|
||||
for (const createdBeforeError of [true, false]) {
|
||||
it(`a delivery whose wake fails ${createdBeforeError ? "after creating its run keeps" : "before creating a run gives back"} the creation slot`, async () => {
|
||||
await seedCompany();
|
||||
const otherAgent = randomUUID();
|
||||
await db.insert(agents).values({ id: otherAgent, companyId, name: "Reviewer", role: "engineer", status: "idle", adapterType: "codex_local",
|
||||
adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} });
|
||||
await pendingContinuation();
|
||||
await pendingContinuation(otherAgent, 3 * MINUTE);
|
||||
await openBrake({ startingAutomaticRun: false });
|
||||
const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record<string, unknown> }) => {
|
||||
if (wakeAgentId === agentId && !createdBeforeError) throw new Error("wake refused before creating a run");
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation",
|
||||
triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning();
|
||||
if (wakeAgentId === agentId) throw new Error("dispatch failed after the run was committed");
|
||||
return run!;
|
||||
});
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake.mock.calls.map(call => call[0])).toEqual(createdBeforeError ? [agentId] : [agentId, otherAgent]);
|
||||
expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull();
|
||||
}, 120_000);
|
||||
|
||||
it(`a sweep whose wake fails ${createdBeforeError ? "after creating its run keeps" : "before creating a run gives back"} the creation slot`, async () => {
|
||||
await seedCompany();
|
||||
const [first, second] = [await strandedTask(), await strandedTask()];
|
||||
await openBrake({ startingAutomaticRun: false });
|
||||
const enqueueWakeup = vi.fn(async (wakeAgentId: string, opts?: { payload?: Record<string, unknown> | null; contextSnapshot?: Record<string, unknown> }) => {
|
||||
const issueId = String(opts?.payload?.issueId);
|
||||
if (issueId === first && !createdBeforeError) throw new Error("wake refused before creating a run");
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation",
|
||||
triggerDetail: "system", status: "queued", contextSnapshot: opts?.contextSnapshot ?? { issueId } }).returning();
|
||||
if (issueId === first) throw new Error("dispatch failed after the run was committed");
|
||||
return { id: run!.id } as never;
|
||||
});
|
||||
const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
await recovery.assignedWork.checkIssue({ companyId, issueId: first });
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId: second }))
|
||||
.toMatchObject({ outcome: createdBeforeError ? "deferred_rate_limit" : "repaired" });
|
||||
}, 120_000);
|
||||
}
|
||||
|
||||
async function pendingContinuation(ownerAgentId = agentId, endedAgoMs = 2 * MINUTE) {
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Continue the work", status: "todo", assigneeAgentId: ownerAgentId });
|
||||
const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: ownerAgentId, issueId, invocationSource: "assignment", status: "failed",
|
||||
errorCode: "server_shutdown_interrupted", startedAt: ago(endedAgoMs + MINUTE), finishedAt: ago(endedAgoMs), createdAt: ago(endedAgoMs + 2 * MINUTE),
|
||||
contextSnapshot: { issueId, wakeReason: "issue_assigned" } }).returning();
|
||||
const recordedAt = new Date().toISOString();
|
||||
const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog",
|
||||
status: "resolved", outcome: "handed_back", resolvedAt: new Date(), ownerType: "board", returnOwnerAgentId: ownerAgentId,
|
||||
cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${source!.id}`, nextAction: "Continue the task.",
|
||||
attemptCount: 1, evidence: { runId: source!.id, originalFailureCode: "server_shutdown_interrupted", continuationDelivery: "pending",
|
||||
freshContinuation: { schema: "paperclip.fresh-continuation.v1", reason: "exact_resumption_unavailable", recordedAt },
|
||||
freshContinuationSettlement: { recordedAt },
|
||||
executionReconciliation: { runId: source!.id, observedOutcome: "unknown", disposition: "continue_fresh",
|
||||
actorId: "execution-recovery", recordedAt } } }).returning();
|
||||
return action!;
|
||||
}
|
||||
|
||||
it("the continuation delivery leaves pending continuations pending while open and delivers them after", async () => {
|
||||
await seedCompany();
|
||||
const action = await pendingContinuation();
|
||||
const failures = await openBrake();
|
||||
const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record<string, unknown> }) => {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation",
|
||||
triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning();
|
||||
return run!;
|
||||
});
|
||||
const delivery = async () => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0]!
|
||||
.evidence.continuationDelivery;
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).not.toHaveBeenCalled();
|
||||
expect(await delivery()).toBe("pending");
|
||||
await recover(failures);
|
||||
await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running")));
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(1);
|
||||
expect(await delivery()).toBe("delivered");
|
||||
}, 120_000);
|
||||
|
||||
async function heldBacklog() {
|
||||
const issueId = randomUUID(), runId = randomUUID();
|
||||
const finishedAt = ago(HOUR);
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Ship the notes", status: "blocked", assigneeAgentId: agentId,
|
||||
statusVersion: 4, blockedTransitionAt: finishedAt });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "assignment", issueId, nativeIssueId: issueId,
|
||||
runtimeMode: "native", status: "failed", errorCode: "native_restart_recovery_blocked", error: "native_restart_recovery_blocked",
|
||||
startedAt: new Date(finishedAt.getTime() - 10 * MINUTE), finishedAt, createdAt: new Date(finishedAt.getTime() - 11 * MINUTE),
|
||||
contextSnapshot: { issueId, wakeReason: "issue_assigned" }, nativePhase: "terminal_failure" });
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "terminal_failure", recoveryState: "blocked",
|
||||
failureCode: "native_continuation_requires_reconciliation", leaseOwner: null });
|
||||
const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog",
|
||||
status: "resolved", outcome: "blocked", resolvedAt: finishedAt, ownerType: "board", returnOwnerAgentId: agentId,
|
||||
cause: "native_continuation_requires_reconciliation", fingerprint: `native-continuation:${runId}`, nextAction: "Continue.",
|
||||
maxAttempts: 3, createdAt: finishedAt, evidence: { runId, originalFailureCode: "native_restart_recovery_blocked",
|
||||
nativeFailureBlock: { runId, statusVersion: 4 }, automaticRecovery: { policy: "preserve_without_replay_v1", runId, replay: "blocked",
|
||||
actionOutcome: "unknown", recordedAt: finishedAt.toISOString() } } }).returning();
|
||||
return { runId, actionId: action!.id };
|
||||
}
|
||||
|
||||
function runWake() {
|
||||
return vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record<string, unknown> }) => {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: wakeAgentId, invocationSource: "automation",
|
||||
triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning();
|
||||
return run!;
|
||||
});
|
||||
}
|
||||
|
||||
const continuationRecords = async (runId: string) => db.select().from(issueRecoveryActions)
|
||||
.where(sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${runId}`);
|
||||
|
||||
it("the held backlog records its continuation for free and the delivery creates the run only after the pause", async () => {
|
||||
await seedCompany();
|
||||
const held = await heldBacklog();
|
||||
const failures = await openBrake();
|
||||
expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 });
|
||||
expect(await continuationRecords(held.runId)).toHaveLength(1);
|
||||
const wake = runWake();
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).not.toHaveBeenCalled();
|
||||
expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull();
|
||||
await recover(failures);
|
||||
await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running")));
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(1);
|
||||
}, 120_000);
|
||||
|
||||
it("one held task spends one creation slot while open", async () => {
|
||||
await seedCompany();
|
||||
await heldBacklog();
|
||||
await openBrake({ startingAutomaticRun: false });
|
||||
expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 });
|
||||
expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull();
|
||||
const wake = runWake();
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).not.toBeNull();
|
||||
}, 120_000);
|
||||
|
||||
const userWake = (issueId: string, values: Partial<typeof agentWakeupRequests.$inferInsert>) => db.insert(agentWakeupRequests).values({
|
||||
companyId, agentId, source: "on_demand", triggerDetail: "manual", reason: "task_retry", status: "deferred_issue_execution",
|
||||
requestedByActorType: "user", requestedByActorId: "board", payload: { issueId }, ...values });
|
||||
|
||||
const deliveryShapes: Array<[string, (issueId: string, sourceRunId: string) => Promise<unknown>]> = [
|
||||
["a Retry waiting behind the task", issueId => userWake(issueId, {})],
|
||||
["a move-forward Retry queued for the task", issueId => userWake(issueId, { status: "queued",
|
||||
payload: { issueId, taskRetry: { schema: "paperclip.task-retry.v1", sourceRunId: randomUUID() } } })],
|
||||
["a Retry folded into the continuation", issueId => userWake(issueId, { status: "coalesced", finishedAt: new Date(),
|
||||
payload: { issueId, coalescedIntoFreshContinuationRun: randomUUID() } })],
|
||||
["a person's message", issueId => userWake(issueId, { reason: "issue_commented", status: "skipped", finishedAt: new Date() })],
|
||||
["a Retry refused at dispatch while the task was held", issueId => db.insert(heartbeatRuns).values({ companyId, agentId, issueId,
|
||||
invocationSource: "on_demand", triggerDetail: "manual", status: "cancelled", errorCode: "execution_reconciliation_required",
|
||||
createdAt: ago(60_000), startedAt: null, finishedAt: ago(59_000),
|
||||
contextSnapshot: { issueId, wakeReason: "task_retry", taskRetryRequestId: randomUUID() } })],
|
||||
["a continuation of a person's Retry", (_issueId, sourceRunId) => db.update(heartbeatRuns)
|
||||
.set({ contextSnapshot: sql`${heartbeatRuns.contextSnapshot} || '{"wakeReason":"task_retry"}'::jsonb` }).where(eq(heartbeatRuns.id, sourceRunId))],
|
||||
];
|
||||
|
||||
for (const [shape, apply] of deliveryShapes) {
|
||||
it(`delivers at once while paused: ${shape}`, async () => {
|
||||
await seedCompany();
|
||||
const action = await pendingContinuation();
|
||||
await openBrake();
|
||||
const sourceRunId = (action.evidence.executionReconciliation as { runId: string }).runId;
|
||||
await apply(action.sourceIssueId, sourceRunId);
|
||||
const wake = runWake();
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(automaticRunBrakes))[0]).toMatchObject({ state: "open", nextCreationAt: null });
|
||||
}, 120_000);
|
||||
}
|
||||
|
||||
it("a held automatic continuation does not hide a person's continuation for the same agent", async () => {
|
||||
await seedCompany();
|
||||
const personAction = await pendingContinuation();
|
||||
await userWake(personAction.sourceIssueId, {});
|
||||
const automaticAction = await pendingContinuation();
|
||||
await openBrake();
|
||||
const wake = runWake();
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(1);
|
||||
expect(wake.mock.calls[0]![1]).toMatchObject({ payload: { issueId: personAction.sourceIssueId } });
|
||||
const delivery = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!
|
||||
.evidence.continuationDelivery;
|
||||
expect(await delivery(personAction.id)).toBe("delivered");
|
||||
expect(await delivery(automaticAction.id)).toBe("pending");
|
||||
}, 120_000);
|
||||
|
||||
it("held continuations take turns across passes so they cannot fill every page", async () => {
|
||||
await seedCompany();
|
||||
const recent = await pendingContinuation(agentId, 2 * HOUR);
|
||||
const older = await pendingContinuation(agentId, 3 * HOUR);
|
||||
await openBrake();
|
||||
const wake = runWake();
|
||||
const attempted = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!
|
||||
.evidence.continuationDeliveryAttemptedAt;
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).not.toHaveBeenCalled();
|
||||
expect(await attempted(recent.id)).toBeTruthy();
|
||||
expect(await attempted(older.id)).toBeTruthy();
|
||||
}, 120_000);
|
||||
|
||||
it("still pauses the delivery of a continuation no person asked for", async () => {
|
||||
await seedCompany();
|
||||
const action = await pendingContinuation();
|
||||
await openBrake();
|
||||
await userWake(action.sourceIssueId, { status: "completed", requestedAt: ago(HOUR), finishedAt: ago(HOUR) });
|
||||
const wake = runWake();
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).not.toHaveBeenCalled();
|
||||
}, 120_000);
|
||||
|
||||
const sweepShapes: Array<[string, (issueId: string) => Promise<unknown>]> = [
|
||||
["the task's last run was a person's Retry", async issueId => {
|
||||
const [run] = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`);
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId, taskId: issueId, wakeReason: "task_retry" } }).where(eq(heartbeatRuns.id, run!.id));
|
||||
}],
|
||||
["a person assigned the task after its last run", issueId => userWake(issueId, { reason: "issue_assigned", status: "failed", finishedAt: new Date() })],
|
||||
["a person commented after its last run", issueId => userWake(issueId, { reason: "issue_commented", status: "skipped", finishedAt: new Date() })],
|
||||
];
|
||||
|
||||
for (const [shape, apply] of sweepShapes) {
|
||||
it(`the sweep continues at once while paused: ${shape}`, async () => {
|
||||
await seedCompany();
|
||||
const issueId = await strandedTask();
|
||||
await openBrake();
|
||||
await apply(issueId);
|
||||
const enqueueWakeup = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" });
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(1);
|
||||
}, 120_000);
|
||||
}
|
||||
|
||||
it("the held backlog continues a task a person asked for while paused, and its delivery is not paused", async () => {
|
||||
await seedCompany();
|
||||
const held = await heldBacklog();
|
||||
await openBrake();
|
||||
const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, held.runId));
|
||||
await userWake(source!.issueId!, {});
|
||||
expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 1 });
|
||||
const wake = runWake();
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
const records = await continuationRecords(held.runId);
|
||||
expect(records.map(record => [record.evidence.freshContinuation ? 'record' : 'hold', record.evidence.continuationDelivery ?? null]))
|
||||
.not.toContainEqual(['record', 'pending']);
|
||||
expect((await db.select().from(automaticRunBrakes))[0]!.nextCreationAt).toBeNull();
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,441 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, inArray, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agentWakeupRequests, agents, automaticRunBrakes, companies, createDb, heartbeatRuns } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import {
|
||||
AUTOMATIC_RUN_BRAKE, automaticRunOutcomeRules, automaticWorkStatusText, classifyAutomaticRunOutcome, createAutomaticRunBrake,
|
||||
isAutomaticRun,
|
||||
} from "../services/recovery/automatic-run-brake.js";
|
||||
import { taskAutomaticAttempts } from "../services/recovery/task-automatic-attempts.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const suite = support.supported ? describe : describe.skip;
|
||||
const MINUTE = 60_000;
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
|
||||
suite("automatic-run brake", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("paperclip-automatic-run-brake-");
|
||||
db = createDb(temporary.connectionString);
|
||||
}, 60_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.execute(sql`truncate table automatic_run_brakes, agent_wakeup_requests, heartbeat_runs, agents, companies cascade`);
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }, 120_000);
|
||||
|
||||
async function seedCompany() {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Brake", issuePrefix: `B${companyId.replaceAll("-", "").slice(0, 6)}`,
|
||||
requireBoardApprovalForNewAgents: false });
|
||||
const agentIds: string[] = [];
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const agentId = randomUUID();
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: `Worker ${index}`, role: "engineer", status: "idle",
|
||||
adapterType: "codex_local", adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} });
|
||||
agentIds.push(agentId);
|
||||
}
|
||||
return { companyId, agentIds };
|
||||
}
|
||||
|
||||
async function insertRuns(companyId: string, agentId: string, count: number, values: (index: number) => Partial<typeof heartbeatRuns.$inferInsert>) {
|
||||
if (count === 0) return [] as Run[];
|
||||
return db.insert(heartbeatRuns).values(Array.from({ length: count }, (_, index) => ({ companyId, agentId,
|
||||
invocationSource: "automation", triggerDetail: "system", runtimeMode: "legacy",
|
||||
contextSnapshot: { wakeReason: "transient_failure_retry", issueId: randomUUID() }, ...values(index) }))).returning();
|
||||
}
|
||||
|
||||
const ago = (ms: number) => new Date(Date.now() - ms);
|
||||
|
||||
function failedBeforeStarting(companyId: string, agentId: string, count: number, errorCode = "environment_provider_unavailable") {
|
||||
return insertRuns(companyId, agentId, count, index => ({ status: "failed", errorCode, executionStage: "preparing",
|
||||
createdAt: ago(4 * MINUTE + index), startedAt: ago(3 * MINUTE + index), finishedAt: ago(MINUTE - index * 100),
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }));
|
||||
}
|
||||
|
||||
function interruptedWhileWorking(companyId: string, agentId: string, count: number) {
|
||||
const codes = ["controller_lease_lost", "server_shutdown_interrupted", "process_lost", "native_session_interrupted"];
|
||||
return insertRuns(companyId, agentId, count, index => ({ status: index % 2 ? "failed" : "interrupted", errorCode: codes[index % codes.length],
|
||||
executionStage: "dispatching", lastOutputAt: ago(40_000), createdAt: ago(20 * MINUTE), startedAt: ago(19 * MINUTE),
|
||||
finishedAt: ago(30_000 - index * 500), contextSnapshot: { wakeReason: "issue_assigned", issueId: randomUUID() } }));
|
||||
}
|
||||
|
||||
function healthy(companyId: string, agentId: string, count: number, status: "succeeded" | "failed" = "succeeded") {
|
||||
return insertRuns(companyId, agentId, count, index => ({ status, errorCode: status === "failed" ? "adapter_failed" : null,
|
||||
executionStage: "dispatching", lastOutputAt: ago(10_000), createdAt: ago(5 * MINUTE), startedAt: ago(4 * MINUTE),
|
||||
finishedAt: ago(5_000 - index * 10) }));
|
||||
}
|
||||
|
||||
function queuedAutomatic(companyId: string, agentId: string, count: number) {
|
||||
return insertRuns(companyId, agentId, count, index => ({ status: "queued", createdAt: ago(30_000 - index),
|
||||
contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID(), issueId: randomUUID() } }));
|
||||
}
|
||||
|
||||
async function forgetRuns(ids: string[]) {
|
||||
await db.update(heartbeatRuns).set({ createdAt: sql`created_at - interval '1 hour'`, startedAt: sql`started_at - interval '1 hour'`,
|
||||
finishedAt: sql`finished_at - interval '1 hour'` }).where(inArray(heartbeatRuns.id, ids));
|
||||
}
|
||||
|
||||
async function ageBrake(companyId: string, ms: number) {
|
||||
const shift = `${ms} milliseconds`;
|
||||
await db.update(automaticRunBrakes).set({
|
||||
stateChangedAt: sql`state_changed_at - ${shift}::interval`, openedAt: sql`opened_at - ${shift}::interval`,
|
||||
nextProbeAt: sql`next_probe_at - ${shift}::interval`, nextCreationAt: sql`next_creation_at - ${shift}::interval`,
|
||||
releaseWindowStartedAt: sql`release_window_started_at - ${shift}::interval`,
|
||||
}).where(eq(automaticRunBrakes.companyId, companyId));
|
||||
}
|
||||
|
||||
const brakeRow = async (companyId: string) => (await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId)))[0];
|
||||
|
||||
async function openBrake(companyId: string, agentId: string, brake = createAutomaticRunBrake()) {
|
||||
const failures = await failedBeforeStarting(companyId, agentId, 20);
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("open");
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function heldCount(brake: ReturnType<typeof createAutomaticRunBrake>, runs: Run[], options: { personOrigin?: boolean } = {}) {
|
||||
let started = 0;
|
||||
for (const run of runs) if (!await brake.holdAutomaticRunStart(db, run, options)) started += 1;
|
||||
return started;
|
||||
}
|
||||
|
||||
function countingDb() {
|
||||
let queries = 0;
|
||||
const counted = new Set(["select", "insert", "update", "delete", "execute"]);
|
||||
const proxy = new Proxy(db, {
|
||||
get(target, property, receiver) {
|
||||
const value = Reflect.get(target, property, receiver);
|
||||
if (typeof value !== "function") return value;
|
||||
return counted.has(String(property)) ? (...args: unknown[]) => { queries += 1; return value.apply(target, args); } : value.bind(target);
|
||||
},
|
||||
});
|
||||
return { db: proxy as typeof db, queries: () => queries };
|
||||
}
|
||||
|
||||
it("classifies automatic runs and outcomes with the shared person and restart rules", async () => {
|
||||
const rules = await automaticRunOutcomeRules();
|
||||
expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "issue_continuation_needed" } })).toBe(true);
|
||||
expect(isAutomaticRun({ id: "a", companyId: "c", scheduledRetryReason: "transient_failure" })).toBe(true);
|
||||
expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "task_retry", previousRunId: "x" } })).toBe(false);
|
||||
expect(isAutomaticRun({ id: "a", companyId: "c", contextSnapshot: { wakeReason: "issue_assigned" } })).toBe(false);
|
||||
const base = { status: "failed", runtimeMode: "legacy", startedAt: new Date(), hasOutput: false, providerWorkStarted: null };
|
||||
expect(classifyAutomaticRunOutcome({ ...base, errorCode: "environment_provider_unavailable", executionStage: "preparing" }, rules)).toBe("failed_before_starting");
|
||||
expect(classifyAutomaticRunOutcome({ ...base, errorCode: "controller_lease_lost", executionStage: "dispatching" }, rules)).toBe("neutral");
|
||||
expect(classifyAutomaticRunOutcome({ ...base, errorCode: "process_lost", executionStage: "preparing" }, rules)).toBe("neutral");
|
||||
expect(classifyAutomaticRunOutcome({ ...base, errorCode: "budget_hard_stop", executionStage: "preparing" }, rules)).toBe("healthy");
|
||||
expect(classifyAutomaticRunOutcome({ ...base, errorCode: "adapter_failed", executionStage: "dispatching" }, rules)).toBe("healthy");
|
||||
expect(classifyAutomaticRunOutcome({ ...base, errorCode: "adapter_failed", executionStage: null, providerWorkStarted: "false" }, rules)).toBe("failed_before_starting");
|
||||
expect(classifyAutomaticRunOutcome({ ...base, status: "running", errorCode: null, executionStage: "preparing" }, rules)).toBe("neutral");
|
||||
expect(classifyAutomaticRunOutcome({ ...base, status: "cancelled", errorCode: "environment_provider_unavailable", executionStage: "preparing" }, rules)).toBe("neutral");
|
||||
const preDispatch = { version: 1, providerWorkStarted: false };
|
||||
for (const runtimeMode of ["legacy", "native"]) {
|
||||
const shape = { ...base, runtimeMode, executionStage: runtimeMode === "legacy" ? "preparing" : null,
|
||||
providerWorkStarted: runtimeMode === "legacy" ? "false" : null, nativePreDispatchFailure: runtimeMode === "native" ? preDispatch : undefined };
|
||||
for (const errorCode of ["environment_provider_unavailable", "setup_failed", "adapter_failed"]) {
|
||||
expect(classifyAutomaticRunOutcome({ ...shape, errorCode }, rules)).toBe("failed_before_starting");
|
||||
}
|
||||
for (const errorCode of ["controller_lease_lost", "process_lost", "native_session_interrupted", "server_shutdown_interrupted"]) {
|
||||
expect(classifyAutomaticRunOutcome({ ...shape, errorCode }, rules)).toBe("neutral");
|
||||
}
|
||||
const working = { ...base, runtimeMode, executionStage: runtimeMode === "legacy" ? "dispatching" : null, hasOutput: true };
|
||||
expect(classifyAutomaticRunOutcome({ ...working, errorCode: "adapter_failed" }, rules)).toBe("healthy");
|
||||
expect(classifyAutomaticRunOutcome({ ...working, errorCode: "environment_provider_unavailable" }, rules)).toBe("failed_before_starting");
|
||||
}
|
||||
});
|
||||
|
||||
it("opens on native runs that failed before dispatch", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
await insertRuns(companyId, agentId!, 20, index => ({ runtimeMode: "native", status: "failed",
|
||||
errorCode: index % 2 ? "adapter_failed" : "setup_failed", createdAt: ago(4 * MINUTE), startedAt: ago(3 * MINUTE),
|
||||
finishedAt: ago(MINUTE - index * 100), nativePhase: "terminal_failure",
|
||||
resultJson: { nativePreDispatchFailure: { version: 1, providerWorkStarted: false }, failureCause: index % 2 ? "40P01" : "adapter_reported_error" } }));
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId))).toMatchObject({ state: "open",
|
||||
reason: { recentAutomaticRuns: 20, failedBeforeStarting: 20, trigger: "failed_before_starting" } });
|
||||
}, 60_000);
|
||||
|
||||
it("sees failures that finished in the window behind more than 500 newer runs", async () => {
|
||||
const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany();
|
||||
const failures = await failedBeforeStarting(companyId, agentId!, 20, "setup_failed");
|
||||
await db.update(heartbeatRuns).set({ createdAt: sql`now() - interval '90 minutes'`, startedAt: sql`now() - interval '3 minutes'` })
|
||||
.where(inArray(heartbeatRuns.id, failures.map(run => run.id)));
|
||||
await queuedAutomatic(companyId, otherAgent!, 600);
|
||||
await insertRuns(companyId, otherAgent!, 50, () => ({ status: "succeeded", createdAt: ago(MINUTE), contextSnapshot: { wakeReason: "issue_assigned" } }));
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open");
|
||||
}, 120_000);
|
||||
|
||||
it("opens when automatic runs fail before starting, holds all but one probe, and never holds a person's work", async () => {
|
||||
const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany();
|
||||
const brake = createAutomaticRunBrake();
|
||||
await failedBeforeStarting(companyId, agentId!, 20);
|
||||
const snapshot = await brake.refresh(db, companyId);
|
||||
expect(snapshot).toMatchObject({ state: "open", reason: { recentAutomaticRuns: 20, failedBeforeStarting: 20 } });
|
||||
expect(automaticWorkStatusText(snapshot)).toMatch(/^Automatic work is paused for this company since .* because the last 5 automatic runs could not get a sandbox/);
|
||||
expect(automaticWorkStatusText(snapshot)).not.toMatch(/brake|probe|_/);
|
||||
|
||||
const queued = [...await queuedAutomatic(companyId, agentId!, 25), ...await queuedAutomatic(companyId, otherAgent!, 25)];
|
||||
const attemptsBefore = await Promise.all(queued.map(run => taskAutomaticAttempts(db, companyId, String(run.contextSnapshot!.issueId), new Date())));
|
||||
expect(await heldCount(brake, queued, { personOrigin: false })).toBe(1);
|
||||
expect(await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false })).toBe(0);
|
||||
const after = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.id, queued.map(run => run.id)));
|
||||
expect(after.every(run => run.status === "queued" && run.errorCode === null)).toBe(true);
|
||||
const attemptsAfter = await Promise.all(queued.map(run => taskAutomaticAttempts(db, companyId, String(run.contextSnapshot!.issueId), new Date())));
|
||||
expect(attemptsAfter.map(attempts => attempts.all.length)).toEqual(attemptsBefore.map(attempts => attempts.all.length));
|
||||
|
||||
const [retry] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", contextSnapshot: { wakeReason: "task_retry", previousRunId: queued[0]!.id } }));
|
||||
const [assignment] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued", contextSnapshot: { wakeReason: "issue_assigned" } }));
|
||||
const [joinedByPerson] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued",
|
||||
contextSnapshot: { wakeReason: "issue_continuation_needed", previousRunId: randomUUID() } }));
|
||||
await db.insert(agentWakeupRequests).values({ companyId, agentId: agentId!, source: "on_demand", triggerDetail: "manual",
|
||||
reason: "issue_commented", status: "queued", requestedByActorType: "user", requestedByActorId: "board", runId: joinedByPerson!.id });
|
||||
const [continuationOfPersonRetry] = await insertRuns(companyId, agentId!, 1, () => ({ status: "queued",
|
||||
contextSnapshot: { wakeReason: "transient_failure_retry", previousRunId: retry!.id } }));
|
||||
for (const run of [retry!, assignment!, joinedByPerson!, continuationOfPersonRetry!]) {
|
||||
expect(await brake.holdAutomaticRunStart(db, run)).toBe(false);
|
||||
}
|
||||
expect(await brake.holdAutomaticRunStart(db, queued[3]!, { personOrigin: true })).toBe(false);
|
||||
}, 60_000);
|
||||
|
||||
it("stays closed for mixed outcomes, person-owned failures and task failures after the agent started", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
const brake = createAutomaticRunBrake();
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("closed");
|
||||
expect(await brakeRow(companyId)).toBeUndefined();
|
||||
await failedBeforeStarting(companyId, agentId!, 8, "setup_failed");
|
||||
await healthy(companyId, agentId!, 4, "failed");
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("closed");
|
||||
await insertRuns(companyId, agentId!, 30, index => ({ status: "failed", errorCode: index % 2 ? "budget_hard_stop" : "claude_auth_required",
|
||||
executionStage: "preparing", createdAt: ago(3 * MINUTE), startedAt: ago(2 * MINUTE), finishedAt: ago(MINUTE),
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }));
|
||||
await healthy(companyId, agentId!, 30, "failed");
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("closed");
|
||||
expect(await brakeRow(companyId)).toBeUndefined();
|
||||
}, 60_000);
|
||||
|
||||
it("ignores failures older than the window", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
const failures = await failedBeforeStarting(companyId, agentId!, 40);
|
||||
await db.update(heartbeatRuns).set({ finishedAt: sql`now() - interval '11 minutes'` }).where(inArray(heartbeatRuns.id, failures.map(run => run.id)));
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("closed");
|
||||
}, 60_000);
|
||||
|
||||
it("stays closed through a release swap that interrupts working runs, and opens when their continuations cannot get a sandbox", async () => {
|
||||
const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany();
|
||||
const brake = createAutomaticRunBrake();
|
||||
await interruptedWhileWorking(companyId, agentId!, 40);
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("closed");
|
||||
const continuations = [...await queuedAutomatic(companyId, agentId!, 20), ...await queuedAutomatic(companyId, otherAgent!, 20)];
|
||||
expect(await heldCount(createAutomaticRunBrake(), continuations)).toBe(40);
|
||||
expect((await brake.reserveCreation(db, companyId, { issueId: randomUUID(), sourceRunId: randomUUID() })).heldUntil).toBeNull();
|
||||
|
||||
await db.update(heartbeatRuns).set({ status: "failed", errorCode: "environment_provider_unavailable", executionStage: "preparing",
|
||||
startedAt: sql`now() - interval '20 seconds'`, finishedAt: sql`now() - interval '5 seconds'`,
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } })
|
||||
.where(inArray(heartbeatRuns.id, continuations.map(run => run.id)));
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("open");
|
||||
}, 60_000);
|
||||
|
||||
it("opens on five sandbox refusals in a row, below the run count", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
const brake = createAutomaticRunBrake();
|
||||
await failedBeforeStarting(companyId, agentId!, 4);
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("closed");
|
||||
await failedBeforeStarting(companyId, agentId!, 1);
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("open");
|
||||
}, 60_000);
|
||||
|
||||
it("waits out a failed probe, then releases at a doubling rate after two probes reach the agent, and closes", async () => {
|
||||
const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany();
|
||||
const failures = await openBrake(companyId, agentId!);
|
||||
await forgetRuns(failures.map(run => run.id));
|
||||
const queued = [...await queuedAutomatic(companyId, agentId!, 30), ...await queuedAutomatic(companyId, otherAgent!, 30)];
|
||||
const probe = async () => {
|
||||
const brake = createAutomaticRunBrake();
|
||||
for (const run of queued) if (!await brake.holdAutomaticRunStart(db, run, { personOrigin: false })) return run;
|
||||
return null;
|
||||
};
|
||||
const first = await probe();
|
||||
expect(first).not.toBeNull();
|
||||
await db.update(heartbeatRuns).set({ status: "failed", errorCode: "environment_provider_unavailable", executionStage: "preparing",
|
||||
startedAt: sql`now()`, finishedAt: sql`now()`, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } })
|
||||
.where(eq(heartbeatRuns.id, first!.id));
|
||||
queued.splice(queued.indexOf(first!), 1);
|
||||
expect(await probe()).toBeNull();
|
||||
expect((await brakeRow(companyId))).toMatchObject({ state: "open", probeSuccesses: 0, probeRunId: null });
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs);
|
||||
|
||||
for (let success = 0; success < AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease; success += 1) {
|
||||
const run = await probe();
|
||||
expect(run).not.toBeNull();
|
||||
await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()`, executionStage: "dispatching" })
|
||||
.where(eq(heartbeatRuns.id, run!.id));
|
||||
queued.splice(queued.indexOf(run!), 1);
|
||||
await createAutomaticRunBrake().refresh(db, companyId);
|
||||
}
|
||||
expect(await brakeRow(companyId)).toMatchObject({ state: "open", probeSuccesses: 2 });
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs);
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("half_open");
|
||||
|
||||
const startedPerWindow: number[] = [];
|
||||
for (let step = 0; step < AUTOMATIC_RUN_BRAKE.releaseSteps; step += 1) {
|
||||
const startedNow = await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false })
|
||||
+ await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false });
|
||||
startedPerWindow.push(startedNow);
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.releaseIntervalMs);
|
||||
}
|
||||
expect(startedPerWindow).toEqual([2, 4, 8, 16, 32, 64]);
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("closed");
|
||||
expect(await heldCount(createAutomaticRunBrake(), queued, { personOrigin: false })).toBe(queued.length);
|
||||
}, 120_000);
|
||||
|
||||
it("stays open when runs that started fine are followed by newer failures before starting", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
const failures = await openBrake(companyId, agentId!);
|
||||
await forgetRuns(failures.map(run => run.id));
|
||||
await insertRuns(companyId, agentId!, 2, () => ({ status: "running", executionStage: "dispatching", lastOutputAt: ago(80_000),
|
||||
startedAt: ago(90_000), createdAt: ago(100_000) }));
|
||||
await failedBeforeStarting(companyId, agentId!, 10);
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs);
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open");
|
||||
}, 60_000);
|
||||
|
||||
it("reopens from a gradual release when runs fail before starting again", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
const failures = await openBrake(companyId, agentId!);
|
||||
await forgetRuns(failures.map(run => run.id));
|
||||
const recovered = await healthy(companyId, agentId!, 2);
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.minOpenMs);
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("half_open");
|
||||
await forgetRuns(recovered.map(run => run.id));
|
||||
const again = await failedBeforeStarting(companyId, agentId!, 5);
|
||||
await db.update(heartbeatRuns).set({ finishedAt: sql`now()` }).where(inArray(heartbeatRuns.id, again.map(run => run.id)));
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open");
|
||||
}, 60_000);
|
||||
|
||||
it("frees the probe slot when the probe is cancelled, deleted or never starts", async () => {
|
||||
const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany();
|
||||
await openBrake(companyId, agentId!);
|
||||
const queued = [...await queuedAutomatic(companyId, agentId!, 3), ...await queuedAutomatic(companyId, otherAgent!, 3)];
|
||||
const brake = createAutomaticRunBrake();
|
||||
expect(await brake.holdAutomaticRunStart(db, queued[0]!, { personOrigin: false })).toBe(false);
|
||||
expect(await heldCount(createAutomaticRunBrake(), queued.slice(1), { personOrigin: false })).toBe(0);
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs);
|
||||
expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, queued[3]!, { personOrigin: false })).toBe(false);
|
||||
expect((await brakeRow(companyId))?.probeRunId).toBe(queued[3]!.id);
|
||||
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: sql`now()` }).where(eq(heartbeatRuns.id, queued[3]!.id));
|
||||
await createAutomaticRunBrake().refresh(db, companyId);
|
||||
expect((await brakeRow(companyId))?.probeRunId).toBeNull();
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeIntervalMs);
|
||||
expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, queued[4]!, { personOrigin: false })).toBe(false);
|
||||
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, queued[4]!.id));
|
||||
await createAutomaticRunBrake().refresh(db, companyId);
|
||||
expect((await brakeRow(companyId))?.probeRunId).toBeNull();
|
||||
}, 60_000);
|
||||
|
||||
it("lets another run take the probe when the reserved probe never starts within the claim bound", async () => {
|
||||
const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany();
|
||||
await openBrake(companyId, agentId!);
|
||||
const [first] = await queuedAutomatic(companyId, agentId!, 1);
|
||||
const [second] = await queuedAutomatic(companyId, otherAgent!, 1);
|
||||
expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, first!, { personOrigin: false })).toBe(false);
|
||||
expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, second!, { personOrigin: false })).toBe(true);
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeClaimBoundMs + 1_000);
|
||||
expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, second!, { personOrigin: false })).toBe(false);
|
||||
expect((await brakeRow(companyId))?.probeRunId).toBe(second!.id);
|
||||
await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()` }).where(eq(heartbeatRuns.id, second!.id));
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.probeClaimBoundMs + 1_000);
|
||||
expect(await createAutomaticRunBrake().holdAutomaticRunStart(db, first!, { personOrigin: false })).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it("closes by itself when no automatic run started or finished for the quiet bound, even with runs waiting", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
const failures = await openBrake(companyId, agentId!);
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.quietCloseMs - MINUTE);
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId)).state).toBe("open");
|
||||
await forgetRuns(failures.map(run => run.id));
|
||||
await queuedAutomatic(companyId, agentId!, 5);
|
||||
await ageBrake(companyId, 2 * MINUTE);
|
||||
expect((await createAutomaticRunBrake().refresh(db, companyId))).toMatchObject({ state: "closed", reason: { trigger: "no_automatic_work" } });
|
||||
}, 60_000);
|
||||
|
||||
it("lets two servers agree on the state and start one probe between them", async () => {
|
||||
const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany();
|
||||
await failedBeforeStarting(companyId, agentId!, 20);
|
||||
const [left, right] = [createAutomaticRunBrake(), createAutomaticRunBrake()];
|
||||
const [leftState, rightState] = await Promise.all([left.refresh(db, companyId), right.refresh(db, companyId)]);
|
||||
expect([leftState.state, rightState.state]).toEqual(["open", "open"]);
|
||||
expect(leftState.stateChangedAt?.getTime()).toBe(rightState.stateChangedAt?.getTime());
|
||||
const queued = [...await queuedAutomatic(companyId, agentId!, 25), ...await queuedAutomatic(companyId, otherAgent!, 25)];
|
||||
const results = await Promise.all(queued.map((run, index) => (index % 2 ? left : right).holdAutomaticRunStart(db, run, { personOrigin: false })));
|
||||
expect(results.filter(held => !held)).toHaveLength(1);
|
||||
}, 60_000);
|
||||
|
||||
it("allows one new automatic run a minute while open and no automatic run is starting, across servers", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
await openBrake(companyId, agentId!);
|
||||
const work = { issueId: randomUUID(), sourceRunId: randomUUID() };
|
||||
const [left, right] = [createAutomaticRunBrake(), createAutomaticRunBrake()];
|
||||
const decisions = await Promise.all([left, right, left, right].map(async brake => (await brake.reserveCreation(db, companyId, work)).heldUntil));
|
||||
expect(decisions.filter(held => held === null)).toHaveLength(1);
|
||||
const later = decisions.find(Boolean)!;
|
||||
expect(later.getTime()).toBeGreaterThan(Date.now() + 50_000);
|
||||
await queuedAutomatic(companyId, agentId!, 3);
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.creationIntervalMs);
|
||||
expect((await createAutomaticRunBrake().reserveCreation(db, companyId, work)).heldUntil).toBeNull();
|
||||
await insertRuns(companyId, agentId!, 1, () => ({ status: "running", executionStage: "preparing", startedAt: ago(20_000) }));
|
||||
await ageBrake(companyId, AUTOMATIC_RUN_BRAKE.creationIntervalMs);
|
||||
expect((await createAutomaticRunBrake().reserveCreation(db, companyId, work)).heldUntil).not.toBeNull();
|
||||
}, 60_000);
|
||||
|
||||
it("keeps a pass over hundreds of held runs at a bounded number of queries", async () => {
|
||||
const { companyId, agentIds: [agentId, otherAgent] } = await seedCompany();
|
||||
await openBrake(companyId, agentId!);
|
||||
const queued = [...await queuedAutomatic(companyId, agentId!, 125), ...await queuedAutomatic(companyId, otherAgent!, 125)];
|
||||
const counting = countingDb();
|
||||
const brake = createAutomaticRunBrake();
|
||||
let started = 0;
|
||||
for (let pass = 0; pass < 100; pass += 1) {
|
||||
for (const run of queued) {
|
||||
if (await brake.holdAutomaticRunStart(counting.db, run, { personOrigin: false })) continue;
|
||||
started += 1;
|
||||
await db.update(heartbeatRuns).set({ status: "running", startedAt: sql`now()` }).where(eq(heartbeatRuns.id, run.id));
|
||||
}
|
||||
}
|
||||
expect(started).toBe(1);
|
||||
expect(counting.queries()).toBeLessThanOrEqual(6);
|
||||
|
||||
const fallback = countingDb();
|
||||
const second = createAutomaticRunBrake();
|
||||
for (let pass = 0; pass < 100; pass += 1) for (const run of queued) await second.holdAutomaticRunStart(fallback.db, run);
|
||||
expect(fallback.queries()).toBeLessThanOrEqual(queued.length + 5);
|
||||
}, 120_000);
|
||||
|
||||
it("reads as closed and logs once when its run rules cannot load", async () => {
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
await failedBeforeStarting(companyId, agentId!, 20);
|
||||
const loadRules = vi.fn(async () => { throw new Error("module failed to load"); });
|
||||
const brake = createAutomaticRunBrake({ loadRules });
|
||||
const queued = await queuedAutomatic(companyId, agentId!, 3);
|
||||
expect((await brake.refresh(db, companyId)).state).toBe("closed");
|
||||
expect((await brake.reserveCreation(db, companyId, { issueId: randomUUID(), sourceRunId: randomUUID() })).heldUntil).toBeNull();
|
||||
expect(await heldCount(brake, queued)).toBe(3);
|
||||
expect(await brakeRow(companyId)).toBeUndefined();
|
||||
}, 60_000);
|
||||
|
||||
it("keeps its in-process memory bounded across many companies and runs", async () => {
|
||||
const brake = createAutomaticRunBrake();
|
||||
const { companyId, agentIds: [agentId] } = await seedCompany();
|
||||
for (let index = 0; index < AUTOMATIC_RUN_BRAKE.maxCompanies + 100; index += 1) await brake.read(db, randomUUID());
|
||||
await openBrake(companyId, agentId!, brake);
|
||||
for (let index = 0; index < AUTOMATIC_RUN_BRAKE.maxRuns + 200; index += 1) {
|
||||
await brake.holdAutomaticRunStart(db, { id: randomUUID(), companyId, contextSnapshot: { wakeReason: "issue_continuation_needed" } });
|
||||
}
|
||||
expect(brake.cacheSizes()).toEqual({ companies: AUTOMATIC_RUN_BRAKE.maxCompanies, runs: AUTOMATIC_RUN_BRAKE.maxRuns });
|
||||
}, 180_000);
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, asc, sql, type SQL } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
environmentLeases,
|
||||
environments,
|
||||
executionWorkspaces,
|
||||
heartbeatRuns,
|
||||
issueRecoveryActions,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import {
|
||||
nextRetainedOrphanDueAt,
|
||||
releasableRetainedOrphanCondition,
|
||||
retainedOrphanSandboxCondition,
|
||||
softlyHeldSql,
|
||||
} from "../services/environment-retained-orphan-settlement.js";
|
||||
import { abandonedWorkspaceFilesHeldOnLeaseRowSql } from "../services/remote-workspace-abandoned-files-hold.js";
|
||||
import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const suite = support.supported ? describe : describe.skip;
|
||||
const KIND = OWNABLE_KINDS.find(kind => kind.id === "environment_retained_orphan_sandbox")!;
|
||||
const TERMINAL = ["succeeded", "failed", "cancelled", "timed_out", "interrupted"];
|
||||
|
||||
function previousAlwaysHeldSql(): SQL {
|
||||
return sql`(${environmentLeases.metadata} ?| array['nativeWorkspaceExportResume', 'sandboxStopAndRetain',
|
||||
'workspaceRestoreRecovery', 'workspaceRecoveryCleanup', 'runnerTaskSourceConsumer', 'reusableSandboxKeptAfterReplacement']
|
||||
or ${abandonedWorkspaceFilesHeldOnLeaseRowSql()}
|
||||
or coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') = 'true'
|
||||
or coalesce(${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,state}' = 'pending', false)
|
||||
or exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId}
|
||||
and run.status not in ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted'))
|
||||
or exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId}
|
||||
and run.runtime_mode = 'native'
|
||||
and run.runner_profile_json#>>'{nativeWorkspaceSync,schema}' in
|
||||
('paperclip.native-workspace-sync/v1', 'paperclip.native-workspace-sync/v2', 'paperclip.native-workspace-sync/v3')
|
||||
and run.runner_profile_json#>>'{nativeWorkspaceSync,state}' = 'prepared'
|
||||
and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,descriptorSha256}' ~ '^[0-9a-f]{64}$', false)
|
||||
and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,baselineSha256}' ~ '^[0-9a-f]{64}$', false)
|
||||
and coalesce(jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,finalHostSha256}'), 'null') = 'null'
|
||||
and jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,workspaceId}') = 'string'
|
||||
and jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,remoteCwd}') = 'string'
|
||||
and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,resourceDisposition}', 'destroy')
|
||||
in ('keep_running', 'stop_and_retain', 'destroy')
|
||||
and run.runner_profile_json#>>'{nativeWorkspaceSync,leaseId}' = ${environmentLeases.id}::text
|
||||
and run.runner_profile_json#>>'{nativeWorkspaceSync,providerLeaseId}' = ${environmentLeases.providerLeaseId}
|
||||
and coalesce(run.runner_profile_json#>>'{nativeExecutionInput,workspace,access}', '') <> 'none')
|
||||
or exists (select 1 from environment_leases other where other.id <> ${environmentLeases.id}
|
||||
and other.provider = ${environmentLeases.provider} and other.provider_lease_id = ${environmentLeases.providerLeaseId}
|
||||
and other.status in ('active', 'retained', 'pending_cleanup')))`;
|
||||
}
|
||||
|
||||
function previousReleasableCondition(now?: Date) {
|
||||
return and(retainedOrphanSandboxCondition(), sql`not ${previousAlwaysHeldSql()}`, sql`not (${softlyHeldSql(now)})`);
|
||||
}
|
||||
|
||||
function random(seed: number) {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state = (state + 0x6d2b79f5) >>> 0;
|
||||
let value = state;
|
||||
value = Math.imul(value ^ (value >>> 15), value | 1);
|
||||
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
|
||||
return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
type PlanNode = { "Relation Name"?: string; "Actual Loops"?: number; "Actual Rows"?: number;
|
||||
"Rows Removed by Filter"?: number; Plans?: PlanNode[] };
|
||||
|
||||
function heartbeatRunRowsRead(node: PlanNode): number {
|
||||
const own = node["Relation Name"] === "heartbeat_runs"
|
||||
? (node["Actual Loops"] ?? 0) * ((node["Actual Rows"] ?? 0) + (node["Rows Removed by Filter"] ?? 0)) : 0;
|
||||
return own + (node.Plans ?? []).reduce((total, child) => total + heartbeatRunRowsRead(child), 0);
|
||||
}
|
||||
|
||||
suite("retained orphan sandbox listing cost", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
const tenants: Array<{ companyId: string; agentId: string; issueIds: string[]; workspaceIds: string[] }> = [];
|
||||
let environmentId = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("paperclip-retained-orphan-listing-");
|
||||
db = createDb(temporary.connectionString);
|
||||
environmentId = randomUUID();
|
||||
await db.insert(environments).values({ id: environmentId, name: "Fake Sandbox", driver: "sandbox",
|
||||
status: "active", config: { provider: "fake", image: "ubuntu:24.04" } });
|
||||
for (const prefix of ["ROA", "ROB"]) {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: prefix, issuePrefix: prefix,
|
||||
requireBoardApprovalForNewAgents: false });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "idle",
|
||||
adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} });
|
||||
const workspaceIds: string[] = [];
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId, mode: "isolated_workspace",
|
||||
strategyType: "git_worktree", name: `${prefix} workspace ${index}` }).returning({ id: executionWorkspaces.id });
|
||||
workspaceIds.push(workspace!.id);
|
||||
}
|
||||
const issueIds: string[] = [];
|
||||
for (let index = 0; index < 6; index += 1) {
|
||||
const issueId = randomUUID();
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: `${prefix} issue ${index}`,
|
||||
executionWorkspaceId: index < 3 ? workspaceIds[index] : null });
|
||||
issueIds.push(issueId);
|
||||
}
|
||||
tenants.push({ companyId, agentId, issueIds, workspaceIds });
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(issueRecoveryActions);
|
||||
await db.delete(environmentLeases);
|
||||
await db.delete(heartbeatRuns);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await temporary?.cleanup();
|
||||
});
|
||||
|
||||
async function seedVariedFixture(seed: number, count: number) {
|
||||
const next = random(seed);
|
||||
const pick = <T,>(values: readonly T[]): T => values[Math.floor(next() * values.length)]!;
|
||||
const chance = (probability: number) => next() < probability;
|
||||
const runs: Array<typeof heartbeatRuns.$inferInsert> = [];
|
||||
const leases: Array<typeof environmentLeases.$inferInsert> = [];
|
||||
const base = Date.now() - 24 * 60 * 60_000;
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const tenant = pick(tenants);
|
||||
const leaseId = randomUUID();
|
||||
const provider = pick(["fake", "fake", "fake", "local", null]);
|
||||
const providerLeaseId = chance(0.9) ? `sbx-${Math.floor(next() * 300)}` : null;
|
||||
let runId: string | null = null;
|
||||
if (chance(0.9)) {
|
||||
runId = randomUUID();
|
||||
const sync = pick(["none", "prepared", "prepared", "access_none", "other_sandbox", "bad_descriptor", "final"]);
|
||||
const runnerProfileJson = sync === "none" ? {} : {
|
||||
...(sync === "access_none" ? { nativeExecutionInput: { workspace: { access: "none" } } } : {}),
|
||||
nativeWorkspaceSync: {
|
||||
schema: pick(["paperclip.native-workspace-sync/v3", "paperclip.native-workspace-sync/v1", "unknown"]),
|
||||
state: "prepared", descriptorSha256: sync === "bad_descriptor" ? "zz" : "a".repeat(64),
|
||||
baselineSha256: "b".repeat(64), finalHostSha256: sync === "final" ? "c".repeat(64) : null,
|
||||
workspaceId: randomUUID(), remoteCwd: "/workspace", resourceDisposition: pick(["destroy", "keep_running", "odd"]),
|
||||
leaseId, providerLeaseId: sync === "other_sandbox" ? "sbx-elsewhere" : providerLeaseId,
|
||||
},
|
||||
};
|
||||
runs.push({ id: runId, companyId: tenant.companyId, agentId: tenant.agentId, issueId: pick(tenant.issueIds),
|
||||
nativeIssueId: chance(0.2) ? pick(tenant.issueIds) : null, invocationSource: "on_demand",
|
||||
status: chance(0.06) ? pick(["running", "queued", "scheduled_retry"]) : pick(TERMINAL), nextEventSeq: 1,
|
||||
runtimeMode: pick(["native", "native", "legacy"]), runnerProfileJson });
|
||||
}
|
||||
const terminationRunId = chance(0.85) ? runId : randomUUID();
|
||||
const metadata: Record<string, unknown> = {};
|
||||
if (chance(0.75)) metadata.orphanedRunLeaseRecovery = { recoveredAt: "x" };
|
||||
if (chance(0.85)) metadata.sandboxStopAndRetainReceipt = { stoppedAt: "x" };
|
||||
if (chance(0.75)) metadata.remoteExecutionTermination = { state: pick(["stopped", "stopped", "running"]),
|
||||
leaseId: chance(0.85) ? leaseId : randomUUID(), runId: terminationRunId };
|
||||
for (const key of ["nativeWorkspaceExportResume", "sandboxStopAndRetain", "workspaceRestoreRecovery",
|
||||
"workspaceRecoveryCleanup", "runnerTaskSourceConsumer", "reusableSandboxKeptAfterReplacement"]) {
|
||||
if (chance(0.04)) metadata[key] = { at: "x" };
|
||||
}
|
||||
if (chance(0.1)) metadata.pendingCleanupInFlight = pick([true, false, "true", "maybe"]);
|
||||
if (chance(0.06)) metadata.legacyShutdownWorkspaceSettlement = { state: pick(["pending", "done"]) };
|
||||
const expiresRoll = next();
|
||||
leases.push({ id: leaseId, companyId: tenant.companyId, environmentId,
|
||||
executionWorkspaceId: chance(0.7) ? pick(tenant.workspaceIds) : null,
|
||||
issueId: chance(0.9) ? pick(tenant.issueIds) : null, heartbeatRunId: runId,
|
||||
status: pick(["released", "released", "released", "failed", "failed", "retained", "retained", "expired", "active", "pending_cleanup"]),
|
||||
leasePolicy: pick(["ephemeral", "ephemeral", "reuse_by_environment", "reuse_by_environment", "retain_on_failure"]),
|
||||
provider, providerLeaseId, cleanupStatus: pick(["success", "success", "success", "success", "failed", null]),
|
||||
metadata: chance(0.03) ? null : metadata,
|
||||
expiresAt: expiresRoll < 0.3 ? null : new Date(Date.now() + (expiresRoll < 0.6 ? -1 : 1) * 60 * 60_000),
|
||||
createdAt: new Date(base + Math.floor(next() * 5) * 1000) });
|
||||
}
|
||||
for (let index = 0; index < runs.length; index += 200) await db.insert(heartbeatRuns).values(runs.slice(index, index + 200));
|
||||
for (let index = 0; index < leases.length; index += 200) {
|
||||
await db.insert(environmentLeases).values(leases.slice(index, index + 200));
|
||||
}
|
||||
for (const tenant of tenants) {
|
||||
for (const issueId of tenant.issueIds) {
|
||||
if (!chance(0.25)) continue;
|
||||
await db.insert(issueRecoveryActions).values({ companyId: tenant.companyId, sourceIssueId: issueId,
|
||||
kind: "active_run_watchdog", status: pick(["active", "resolved", "cancelled", "escalated"]),
|
||||
cause: "fixture", fingerprint: randomUUID(), nextAction: "Recover the workspace." });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function previousPage(limit: number, offset: number, now?: Date) {
|
||||
const rows = await db.select({ id: environmentLeases.id }).from(environmentLeases)
|
||||
.where(previousReleasableCondition(now)).orderBy(asc(environmentLeases.id)).limit(limit + 1).offset(offset);
|
||||
return { truncated: rows.length > limit, ids: rows.slice(0, limit).map(row => row.id) };
|
||||
}
|
||||
|
||||
it("lists exactly the rows the previous listing listed, page by page, over a varied fixture", async () => {
|
||||
for (const seed of [11, 29, 47, 83]) {
|
||||
await seedVariedFixture(seed, 600);
|
||||
const expected = await previousPage(10_000, 0);
|
||||
expect(expected.ids.length).toBeGreaterThan(0);
|
||||
expect(expected.ids.length).toBeLessThan(600);
|
||||
for (const limit of [7, 200]) {
|
||||
for (let offset = 0; ; offset += limit) {
|
||||
const [listed, previous] = await Promise.all([KIND.listInProgress!(db, limit, offset), previousPage(limit, offset)]);
|
||||
expect(listed.records.map(record => record.recordId)).toEqual(previous.ids);
|
||||
expect(listed.truncated).toBe(previous.truncated);
|
||||
if (!previous.truncated) break;
|
||||
}
|
||||
}
|
||||
const now = new Date();
|
||||
const [current] = await db.select({ count: sql<number>`count(*)::int` }).from(environmentLeases)
|
||||
.where(releasableRetainedOrphanCondition(now));
|
||||
expect(current!.count).toBe(expected.ids.length);
|
||||
const [previousDue] = await db.select({ due: sql<string | null>`min(${environmentLeases.expiresAt})` })
|
||||
.from(environmentLeases).where(and(retainedOrphanSandboxCondition(), sql`not ${previousAlwaysHeldSql()}`,
|
||||
sql`${softlyHeldSql(now)}`, sql`${environmentLeases.expiresAt} is not null`));
|
||||
expect((await nextRetainedOrphanDueAt(db, now))?.getTime() ?? null)
|
||||
.toBe(previousDue?.due ? new Date(previousDue.due).getTime() : null);
|
||||
await db.delete(issueRecoveryActions);
|
||||
await db.delete(environmentLeases);
|
||||
await db.delete(heartbeatRuns);
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
async function seedHistory(count: number) {
|
||||
const tenant = tenants[0]!;
|
||||
await db.execute(sql`
|
||||
with history as (
|
||||
select gen_random_uuid() as run_id, gen_random_uuid() as lease_id, g from generate_series(1, ${count}) g
|
||||
), inserted as (
|
||||
insert into heartbeat_runs (id, company_id, agent_id, issue_id, invocation_source, status, next_event_seq,
|
||||
runtime_mode, runner_profile_json)
|
||||
select run_id, ${tenant.companyId}::uuid, ${tenant.agentId}::uuid, ${tenant.issueIds[0]}::uuid, 'on_demand',
|
||||
'succeeded', 1, 'native', jsonb_build_object('nativeWorkspaceSync', jsonb_build_object(
|
||||
'schema', 'paperclip.native-workspace-sync/v3', 'state', 'finalized', 'leaseId', lease_id::text))
|
||||
from history returning id
|
||||
)
|
||||
insert into environment_leases (id, company_id, environment_id, execution_workspace_id, issue_id, heartbeat_run_id,
|
||||
status, lease_policy, provider, provider_lease_id, cleanup_status, metadata, expires_at)
|
||||
select lease_id, ${tenant.companyId}::uuid, ${environmentId}::uuid, ${tenant.workspaceIds[0]}::uuid,
|
||||
${tenant.issueIds[0]}::uuid, run_id,
|
||||
case when g % 100 < 71 then 'expired' when g % 100 < 96 then 'released' else 'failed' end,
|
||||
case when g % 3 = 0 then 'reuse_by_environment' else 'ephemeral' end,
|
||||
'fake', 'sbx-history-' || (g % 50), 'success', jsonb_build_object('sandboxStopAndRetainReceipt', '{}'::jsonb),
|
||||
now() + interval '1 day'
|
||||
from history`);
|
||||
await db.execute(sql`analyze heartbeat_runs; analyze environment_leases`);
|
||||
}
|
||||
|
||||
async function listingPlan() {
|
||||
const query = db.select({ id: environmentLeases.id }).from(environmentLeases)
|
||||
.where(releasableRetainedOrphanCondition()).orderBy(asc(environmentLeases.id)).limit(201).offset(0).toSQL();
|
||||
const rows = await db.$client.unsafe(`explain (analyze, format json) ${query.sql}`, query.params as never[]);
|
||||
return (rows[0] as unknown as { "QUERY PLAN": Array<{ Plan: PlanNode }> })["QUERY PLAN"][0]!.Plan;
|
||||
}
|
||||
|
||||
it("reads heartbeat runs only for retained orphan candidates, not for the whole lease history", async () => {
|
||||
await seedHistory(2_000);
|
||||
await seedVariedFixture(5, 20);
|
||||
const [{ candidates }] = await db.select({ candidates: sql<number>`count(*)::int` }).from(environmentLeases)
|
||||
.where(retainedOrphanSandboxCondition()) as Array<{ candidates: number }>;
|
||||
const plan = await listingPlan();
|
||||
expect(heartbeatRunRowsRead(plan)).toBeLessThanOrEqual(10 * candidates + 50);
|
||||
}, 120_000);
|
||||
|
||||
it("lets a schema change take the heartbeat_runs lock within a 5 second lock timeout while listings run", async () => {
|
||||
await seedHistory(5_000);
|
||||
let stop = false;
|
||||
let listings = 0;
|
||||
const load = (async () => {
|
||||
while (!stop) {
|
||||
await KIND.listInProgress!(db, 200, 0);
|
||||
listings += 1;
|
||||
}
|
||||
})();
|
||||
const ddl = createDb(temporary!.connectionString);
|
||||
try {
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
const started = Date.now();
|
||||
await ddl.transaction(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '5s'`);
|
||||
await tx.execute(sql`alter table heartbeat_runs add column retained_orphan_listing_probe jsonb`);
|
||||
await tx.execute(sql`alter table heartbeat_runs drop column retained_orphan_listing_probe`);
|
||||
});
|
||||
expect(Date.now() - started).toBeLessThan(5_000);
|
||||
} finally {
|
||||
stop = true;
|
||||
await load;
|
||||
await ddl.$client.end({ timeout: 1 });
|
||||
}
|
||||
expect(listings).toBeGreaterThan(0);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -65,6 +65,7 @@ function githubConnectionHealth() {
|
||||
discovery: {} as never,
|
||||
setupPolicy: { assertConnectorConnectionSetupEnabled: async () => undefined } as never,
|
||||
resolveCredentialHeaders: vi.fn() as never,
|
||||
vaultGrantForConnection: vi.fn() as never,
|
||||
refreshManagedGitHubGrantAccess: (async () => {
|
||||
await loadGitHubGrantMetadata("ghu_secret", suspendedGitHub);
|
||||
}) as never,
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { eq, inArray, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints } from "@paperclipai/db";
|
||||
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints, databaseWorkPending,
|
||||
reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { appendHeartbeatRunEvent, appendHeartbeatRunEventBatch } from "../services/heartbeat-run-events.js";
|
||||
import { nativeSha256 } from "../services/native-runtime/canonical.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
@@ -293,6 +295,7 @@ describeEmbeddedPostgres("atomic ordinary run events", () => {
|
||||
locked();
|
||||
await released;
|
||||
});
|
||||
void holder.catch(() => undefined);
|
||||
await lockedPromise;
|
||||
return { release: () => { release(); return holder; } };
|
||||
};
|
||||
@@ -485,17 +488,143 @@ describeEmbeddedPostgres("atomic ordinary run events", () => {
|
||||
const runId = await makeRun();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Progress task" }).returning();
|
||||
await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId));
|
||||
const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed")));
|
||||
// BEGIN, lock, duplicate probe, the wake's transaction id, the append, COMMIT.
|
||||
expect(first.statements).toHaveLength(6);
|
||||
const wake = first.statements.findIndex(text => text.includes("pg_current_xact_id"));
|
||||
expect(wake).toBeGreaterThan(0);
|
||||
expect(wake).toBeLessThan(first.statements.findIndex(text => text.includes("insert into heartbeat_run_events")));
|
||||
expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 });
|
||||
const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed")));
|
||||
expect(replay.statements.some(text => text.includes("pg_current_xact_id"))).toBe(false);
|
||||
expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 });
|
||||
expect(await readEvents(runId)).toEqual([first.result.row]);
|
||||
const wakes: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, event => { wakes.push(event); });
|
||||
try {
|
||||
const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed")));
|
||||
expect(first.statements).toHaveLength(4);
|
||||
expect(first.statements[0]).toMatch(/^begin/i);
|
||||
expect(first.statements[1]).toMatch(/^with "snapshot_run" as .*pg_current_xact_id\(\).* for no key update.*"replay_found"/);
|
||||
expect(first.statements[2]).toMatch(/insert into heartbeat_run_events/);
|
||||
expect(first.statements[3]).toMatch(/^commit/i);
|
||||
expect(wakes).toEqual(["intent", "settled"]);
|
||||
expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 });
|
||||
wakes.length = 0;
|
||||
const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed")));
|
||||
expect(wakes).toEqual([]);
|
||||
expect(replay.statements).toHaveLength(4);
|
||||
expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 });
|
||||
expect(await readEvents(runId)).toEqual([first.result.row]);
|
||||
} finally { unsubscribe(); }
|
||||
});
|
||||
|
||||
it("gives the wake the transaction id of the transaction that took the run lock", async () => {
|
||||
const runId = await makeRun();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Primed wake" }).returning();
|
||||
await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId));
|
||||
const topic = DELIVERY_QUEUES.chatNativeProgress;
|
||||
const options = db.$client.options as { debug?: unknown };
|
||||
const sent: Array<{ text: string; parameters: unknown[] }> = [];
|
||||
let ownXid: string | undefined;
|
||||
options.debug = (_connection: number, text: string, parameters: unknown[]) => {
|
||||
sent.push({ text: text.trim().replace(/\s+/g, " "), parameters });
|
||||
};
|
||||
try {
|
||||
await expect(db.transaction(async tx => {
|
||||
await appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 1, "item.completed"));
|
||||
expect(sent.filter(statement => /^select pg_current_xact_id\(\)/.test(statement.text))).toEqual([]);
|
||||
expect(sent.filter(statement => statement.text.includes("pg_current_xact_id()"))).toHaveLength(1);
|
||||
const [own] = await tx.execute<{ xid: string; pid: number }>(sql`select pg_current_xact_id()::text as xid, pg_backend_pid() as pid`);
|
||||
ownXid = own!.xid;
|
||||
await otherDb.execute(sql`select pg_terminate_backend(${own!.pid})`);
|
||||
await tx.execute(sql`select 1`);
|
||||
})).rejects.toThrow();
|
||||
expect(databaseWorkPending(db, topic)).toBe(true);
|
||||
sent.length = 0;
|
||||
await reconcileDatabaseWork(db, topic);
|
||||
} finally { options.debug = undefined; }
|
||||
const statusRead = sent.find(statement => statement.text.includes("pg_xact_status"));
|
||||
expect(ownXid).toMatch(/^\d+$/);
|
||||
expect(statusRead?.parameters).toEqual([ownXid]);
|
||||
expect(databaseWorkPending(db, topic)).toBe(false);
|
||||
expect(await readEvents(runId)).toEqual([]);
|
||||
});
|
||||
|
||||
it("primes nothing when the lock read fails, so the transaction is not left uncertain", async () => {
|
||||
const runId = await makeRun();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Failed lock read" }).returning();
|
||||
await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId));
|
||||
let release!: () => void, locked!: () => void;
|
||||
const released = new Promise<void>(resolve => { release = resolve; });
|
||||
const lockedRun = new Promise<void>(resolve => { locked = resolve; });
|
||||
const holder = otherDb.transaction(async tx => {
|
||||
await tx.execute(sql`select id from heartbeat_runs where id = ${runId} for update`);
|
||||
locked();
|
||||
await released;
|
||||
});
|
||||
void holder.catch(() => undefined);
|
||||
await lockedRun;
|
||||
const wakes: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, wake => { wakes.push(wake); });
|
||||
const options = db.$client.options as { debug?: unknown };
|
||||
const sent: string[] = [];
|
||||
try {
|
||||
await expect(db.transaction(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '100ms'`);
|
||||
await appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 1, "item.completed"));
|
||||
})).rejects.toThrow();
|
||||
expect(databaseWorkPending(db, DELIVERY_QUEUES.chatNativeProgress)).toBe(false);
|
||||
await expect(db.transaction(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '100ms'`);
|
||||
await expect(tx.transaction(savepoint =>
|
||||
appendHeartbeatRunEvent(savepoint as unknown as typeof db, nativeEvent(runId, 1, "item.completed")))).rejects.toThrow();
|
||||
options.debug = (_connection: number, text: string) => { sent.push(text.trim().replace(/\s+/g, " ")); };
|
||||
await signalDatabaseWork(tx, "lock-read-failed");
|
||||
options.debug = undefined;
|
||||
throw new Error("roll back");
|
||||
})).rejects.toThrow("roll back");
|
||||
} finally {
|
||||
options.debug = undefined;
|
||||
unsubscribe();
|
||||
release();
|
||||
await holder;
|
||||
}
|
||||
expect(sent.filter(text => /^select pg_current_xact_id\(\)/.test(text))).toHaveLength(1);
|
||||
await reconcileDatabaseWork(db, "lock-read-failed");
|
||||
expect(databaseWorkPending(db, "lock-read-failed")).toBe(false);
|
||||
expect(wakes).toEqual([]);
|
||||
expect(databaseWorkPending(db, DELIVERY_QUEUES.chatNativeProgress)).toBe(false);
|
||||
expect(await readEvents(runId)).toEqual([]);
|
||||
});
|
||||
|
||||
it("finds a replay that another writer committed while the lock read waited, and registers no wake for it", async () => {
|
||||
const runId = await makeRun();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Progress race" }).returning();
|
||||
await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId));
|
||||
const event = nativeEvent(runId, 1, "item.completed");
|
||||
let release!: () => void, locked!: () => void;
|
||||
const released = new Promise<void>(resolve => { release = resolve; });
|
||||
const lockedRun = new Promise<void>(resolve => { locked = resolve; });
|
||||
const holder = otherDb.transaction(async tx => {
|
||||
await tx.execute(sql`select id from heartbeat_runs where id = ${runId} for update`);
|
||||
await tx.insert(heartbeatRunEvents).values({ companyId, agentId, runId, seq: 1, eventType: event.eventType, stream: "system",
|
||||
level: "info", payload: event.payload, sourceInstanceId: "native-writer", sourceEventId: event.nativeSource.sourceEventId,
|
||||
sourceSeq: 1, sourcePayloadSha256: nativeSha256(event.nativeSource.canonicalPayload), protocolSchemaVersion: 1 });
|
||||
await tx.update(heartbeatRuns).set({ nextEventSeq: 2 }).where(eq(heartbeatRuns.id, runId));
|
||||
locked();
|
||||
await released;
|
||||
});
|
||||
void holder.catch(() => undefined);
|
||||
await lockedRun;
|
||||
const wakes: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, wake => { wakes.push(wake); });
|
||||
try {
|
||||
const appended = sentStatements(db.$client, () => appendHeartbeatRunEvent(db, event));
|
||||
void appended.catch(() => undefined);
|
||||
await vi.waitFor(async () => {
|
||||
const [waiting] = await otherDb.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity
|
||||
where datname = current_database() and wait_event_type = 'Lock' and query like '%"locked_run"%'`);
|
||||
expect(waiting?.count).toBe(1);
|
||||
}, { timeout: 10_000, interval: 20 });
|
||||
release();
|
||||
await holder;
|
||||
const { result, statements } = await appended;
|
||||
expect(result).toMatchObject({ disposition: "duplicate", row: { seq: 1, sourceSeq: 1 } });
|
||||
expect(statements.some(text => /^select \* from heartbeat_run_events/.test(text))).toBe(true);
|
||||
expect(wakes).toEqual([]);
|
||||
} finally { unsubscribe(); }
|
||||
expect((await readEvents(runId)).map(row => row.seq)).toEqual([1]);
|
||||
expect((await readRun(runId)).nextEventSeq).toBe(2);
|
||||
});
|
||||
|
||||
it("keeps one ordered row per source event when two servers append the same stream", async () => {
|
||||
@@ -601,11 +730,11 @@ describeEmbeddedPostgres("atomic ordinary run events", () => {
|
||||
expect(replay.highestContiguousSourceSeq).toBe(kind === "native" ? 1 : 0);
|
||||
expect((await readRun(runId)).nextEventSeq).toBe(2);
|
||||
expect(await readEvents(runId)).toEqual([committed.row]);
|
||||
const lockReads = statements.filter(query => query.startsWith("select ") &&
|
||||
query.includes('from "heartbeat_runs"') && query.endsWith("for no key update"));
|
||||
const lockReads = statements.filter(query => /^(select|with) /.test(query) &&
|
||||
query.includes('from "heartbeat_runs"') && query.includes("for no key update"));
|
||||
expect(lockReads).toHaveLength(2);
|
||||
for (const query of lockReads) {
|
||||
expect(query).toMatch(/^select "company_id", "agent_id"(, \("native_source_cursors" ->> \$1\)::bigint as "source_cursor")? from "heartbeat_runs" /);
|
||||
expect(query).toMatch(/select "company_id", "agent_id"(, \("native_source_cursors" ->> \$\d+\)::bigint as "source_cursor")?(, pg_current_xact_id\(\)::text as "xid")?(, "next_event_seq")? from "heartbeat_runs" /);
|
||||
expect(query).not.toMatch(/context_snapshot|runner_profile_json|result_json|stdout_excerpt/);
|
||||
}
|
||||
});
|
||||
@@ -631,9 +760,11 @@ describeEmbeddedPostgres("atomic ordinary run events", () => {
|
||||
holderLocked();
|
||||
await held;
|
||||
});
|
||||
void holder.catch(() => undefined);
|
||||
await locked;
|
||||
let secondSettled = false;
|
||||
const second = appendHeartbeatRunEvent(db, native(2)).finally(() => { secondSettled = true; });
|
||||
void second.catch(() => undefined);
|
||||
try {
|
||||
// A controller checkpoint row references the run row by foreign key.
|
||||
// Its insert must not wait for the open append.
|
||||
|
||||
@@ -1755,6 +1755,50 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
}, { timeout: 10_000 });
|
||||
});
|
||||
|
||||
describe("a queued system recovery run re-checks that its task still needs it when its turn comes", () => {
|
||||
async function queueRecovery(task: Partial<typeof issues.$inferInsert>) {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Waited behind people's work",
|
||||
status: "in_progress", assigneeAgentId: agentId, ...task }).returning();
|
||||
const queued = await seedQueuedRun({ companyId, agentId, issueId: issue.id,
|
||||
wakeReason: "issue_recovery_action_restored", invocationSource: "automation" });
|
||||
return { companyId, agentId, issueId: issue.id, ...queued };
|
||||
}
|
||||
async function expectDroppedQuietly(runId: string, wakeupRequestId: string) {
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
expect(run).toMatchObject({ status: "cancelled", errorCode: "recovery_no_longer_needed",
|
||||
resultJson: expect.objectContaining({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }) });
|
||||
expect(countExecuteCallsForRun(runId)).toBe(0);
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)))[0]?.status)
|
||||
.toBe("skipped");
|
||||
}
|
||||
|
||||
it("drops it when the task was reassigned while it waited", async () => {
|
||||
const { issueId, runId, wakeupRequestId } = await queueRecovery({});
|
||||
await db.update(issues).set({ assigneeAgentId: null, assigneeUserId: "responsible-user" }).where(eq(issues.id, issueId));
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await expectDroppedQuietly(runId, wakeupRequestId);
|
||||
});
|
||||
|
||||
it("drops it when a newer run already continued the task", async () => {
|
||||
const { companyId, agentId, issueId, runId, wakeupRequestId } = await queueRecovery({});
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "assignment", triggerDetail: "system",
|
||||
status: "succeeded", createdAt: new Date(Date.now() + 1_000), startedAt: new Date(Date.now() + 1_000),
|
||||
finishedAt: new Date(Date.now() + 2_000), contextSnapshot: { issueId, wakeReason: "issue_commented" } });
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await expectDroppedQuietly(runId, wakeupRequestId);
|
||||
});
|
||||
|
||||
it("starts it when the task still needs it", async () => {
|
||||
const { runId } = await queueRecovery({ status: "todo" });
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(countExecuteCallsForRun(runId)).toBe(1);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
it("cancels queued max-turn continuations when another continuation owns the issue lock", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent();
|
||||
const issueId = randomUUID();
|
||||
|
||||
@@ -12,6 +12,9 @@ import { REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE } from "../services/remote-work
|
||||
import { LEGACY_WORKSPACE_REHOME_OUTCOME, rehomeStrandedLegacyWorkspaceTasks } from "../services/recovery/legacy-workspace-rehome.js";
|
||||
import { recoveryActionSweepCondition } from "../services/recovery/blocker-stall-wakeups.js";
|
||||
import { recoveryService } from "../services/recovery/service.js";
|
||||
import { assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js";
|
||||
import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js";
|
||||
import { QUEUED_RECOVERY_RUNS_PER_AGENT } from "../services/recovery-queue-admission.js";
|
||||
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
|
||||
const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip;
|
||||
@@ -225,6 +228,47 @@ describePostgres("stranded subtasks bound to a parent's held workspace", () => {
|
||||
expect(last.nextWakeRetryAt).toBeNull();
|
||||
}, 60_000);
|
||||
|
||||
it("holds a move's wake while its agent's recovery queue is full, and the assigned-work sweep sees it as pending", async () => {
|
||||
const f = await seedSharedFamily();
|
||||
const hero = await f.strand(f.hero.id);
|
||||
expect((await rehomeStrandedLegacyWorkspaceTasks(db, null, { companyId: f.companyId })).rehomed).toEqual([f.hero.id]);
|
||||
const queuedIds: string[] = [];
|
||||
for (let index = 0; index < QUEUED_RECOVERY_RUNS_PER_AGENT; index += 1) {
|
||||
const [busy] = await db.insert(issues).values({ companyId: f.companyId, title: `Busy ${index}`, status: "in_progress",
|
||||
assigneeAgentId: f.agentId }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, issueId: busy!.id, status: "queued",
|
||||
invocationSource: "automation", triggerDetail: "system",
|
||||
contextSnapshot: { issueId: busy!.id, wakeReason: "issue_recovery_action_restored" } }).returning();
|
||||
queuedIds.push(run!.id);
|
||||
}
|
||||
const woken: string[] = [];
|
||||
const wake = async (_agentId: string, options: { payload: Record<string, unknown> }) => {
|
||||
woken.push(String(options.payload.issueId)); return { id: randomUUID() };
|
||||
};
|
||||
const held = await rehomeStrandedLegacyWorkspaceTasks(db, wake as never, { companyId: f.companyId });
|
||||
expect(woken).toEqual([]);
|
||||
const pending = (await action(hero.actionId)).evidence.legacyWorkspaceRehome as Record<string, unknown>;
|
||||
// Waiting its turn is not a refusal: no failure is counted.
|
||||
expect(pending).toMatchObject({ wake: "pending" });
|
||||
expect(pending.wakeFailures).toBeUndefined();
|
||||
expect(held.nextWakeRetryAt).toBe(Number(pending.wakeRetryAt));
|
||||
|
||||
// The sweep reads the held wake as the task's pending path, so it queues no uncapped continuation.
|
||||
const sweep = assignedWorkSweep(db, { enqueueWakeup: async () => null });
|
||||
const facts = (await sweep.loadFacts(f.companyId, [await issue(f.hero.id)])).get(f.hero.id)!;
|
||||
expect(facts.pendingWakes).toBeGreaterThan(0);
|
||||
expect(evaluateAssignedWork(facts)).toMatchObject({ kind: "ok", clause: "pending_intent" });
|
||||
|
||||
// A queued recovery run leaves the queue; the held wake is delivered on its next turn.
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", startedAt: new Date(), finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, queuedIds[0]!));
|
||||
await db.update(issueRecoveryActions).set({ evidence: { ...(await action(hero.actionId)).evidence,
|
||||
legacyWorkspaceRehome: { ...pending, wakeRetryAt: Date.now() - 1 } } }).where(eq(issueRecoveryActions.id, hero.actionId));
|
||||
await rehomeStrandedLegacyWorkspaceTasks(db, wake as never, { companyId: f.companyId });
|
||||
expect(woken).toEqual([f.hero.id]);
|
||||
expect((await action(hero.actionId)).evidence).toMatchObject({ legacyWorkspaceRehome: { wake: "delivered" } });
|
||||
}, 60_000);
|
||||
|
||||
it("pages past escalations the guards reject", async () => {
|
||||
const f = await seedSharedFamily();
|
||||
// Thirty escalations that match the query but fail a guard (another setup error).
|
||||
|
||||
@@ -119,3 +119,93 @@ describe("parseMcpHttpResponseBody", () => {
|
||||
expect(() => parseMcpHttpResponseBody("event: ping\n\n", "text/event-stream")).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("MCP progress notifications", () => {
|
||||
it("sends the progress token in the request's _meta and keeps other metadata", async () => {
|
||||
const { mcpHttpRequestInit } = await import("../services/mcp-http.js");
|
||||
const init = mcpHttpRequestInit({ "MCP-Protocol-Version": "2026-07-28" }, {
|
||||
id: "call-1", method: "tools/call", params: { name: "export", arguments: {}, _meta: { other: true } },
|
||||
}, { progressToken: "token-1" });
|
||||
const meta = JSON.parse(init.body).params._meta;
|
||||
expect(meta.progressToken).toBe("token-1");
|
||||
expect(meta.other).toBe(true);
|
||||
expect(meta["io.modelcontextprotocol/protocolVersion"]).toBe("2026-07-28");
|
||||
const without = mcpHttpRequestInit(undefined, { id: "call-2", method: "tools/call", params: { name: "export" } });
|
||||
expect(JSON.parse(without.body).params._meta?.progressToken).toBeUndefined();
|
||||
});
|
||||
|
||||
it("reads progress interleaved with the result on one SSE stream, before the result", async () => {
|
||||
const { mcpProgressNotificationOf } = await import("../services/mcp-http.js");
|
||||
const frames = [
|
||||
{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "token-1", progress: 1, total: 3, message: "Exporting rows" } },
|
||||
{ jsonrpc: "2.0", method: "notifications/message", params: { level: "info", data: "unrelated" } },
|
||||
{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "someone-else", progress: 9 } },
|
||||
{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "token-1", progress: 2, total: 3 } },
|
||||
{ jsonrpc: "2.0", id: "call-1", result: { content: [{ type: "text", text: "done" }] } },
|
||||
{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "token-1", progress: 3, total: 3 } },
|
||||
];
|
||||
// Split mid-event so frames cross chunk boundaries.
|
||||
const text = frames.map((frame) => `event: message\r\ndata: ${JSON.stringify(frame)}\r\n\r\n`).join("");
|
||||
const encoded = new TextEncoder().encode(text);
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
for (let i = 0; i < encoded.length; i += 37) controller.enqueue(encoded.slice(i, i + 37));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
const seen: unknown[] = [];
|
||||
let resultSeenBeforeProgress = false;
|
||||
const result = await readMcpHttpResponse(new Response(body, { headers: { "content-type": "text/event-stream" } }), "call-1", {
|
||||
onNotification: (message) => {
|
||||
const update = mcpProgressNotificationOf(message, "token-1");
|
||||
if (update) seen.push(update);
|
||||
},
|
||||
onProgress: (progress) => { if (progress.matchedResponse && seen.length === 0) resultSeenBeforeProgress = true; },
|
||||
});
|
||||
expect(result).toMatchObject({ id: "call-1", result: { content: [{ text: "done" }] } });
|
||||
expect(resultSeenBeforeProgress).toBe(false);
|
||||
expect(seen).toEqual([
|
||||
{ progress: 1, total: 3, message: "Exporting rows" },
|
||||
{ progress: 2, total: 3, message: undefined },
|
||||
]);
|
||||
});
|
||||
|
||||
it("never lets a failing progress observer fail the call", async () => {
|
||||
const text = [
|
||||
{ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "t", progress: 1 } },
|
||||
{ jsonrpc: "2.0", id: 7, result: { ok: true } },
|
||||
].map((frame) => `data: ${JSON.stringify(frame)}\n\n`).join("");
|
||||
const result = await readMcpHttpResponse(new Response(text, { headers: { "content-type": "text/event-stream" } }), 7, {
|
||||
onNotification: () => { throw new Error("observer broke"); },
|
||||
});
|
||||
expect(result).toMatchObject({ result: { ok: true } });
|
||||
});
|
||||
|
||||
it("ignores requests, responses and other tokens", async () => {
|
||||
const { mcpProgressNotificationOf } = await import("../services/mcp-http.js");
|
||||
expect(mcpProgressNotificationOf({ method: "notifications/progress", params: { progressToken: "t", progress: 1 } }, undefined)).toBeNull();
|
||||
expect(mcpProgressNotificationOf({ id: 1, method: "notifications/progress", params: { progressToken: "t", progress: 1 } }, "t")).toBeNull();
|
||||
expect(mcpProgressNotificationOf({ method: "notifications/progress", params: { progressToken: "x", progress: 1 } }, "t")).toBeNull();
|
||||
expect(mcpProgressNotificationOf({ method: "notifications/progress", params: { progressToken: "t", progress: 1 } }, "t")).toEqual({ progress: 1, total: undefined, message: undefined });
|
||||
});
|
||||
|
||||
it("does not count handled progress against the result's size limit", async () => {
|
||||
const note = `data: ${JSON.stringify({ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: "t", progress: 1, message: "x".repeat(200) } })}\n\n`;
|
||||
const text = note.repeat(50) + `data: ${JSON.stringify({ jsonrpc: "2.0", id: 1, result: { ok: true } })}\n\n`;
|
||||
expect(Buffer.byteLength(text)).toBeGreaterThan(5_000);
|
||||
const result = await readMcpHttpResponse(new Response(text, { headers: { "content-type": "text/event-stream" } }), 1, { maxBytes: 1_000, limitPerMessage: true });
|
||||
expect(result).toMatchObject({ result: { ok: true } });
|
||||
// Callers without an overall deadline (catalog discovery) keep the total limit,
|
||||
// so a stream of small notifications cannot keep them waiting forever.
|
||||
await expect(readMcpHttpResponse(new Response(text, { headers: { "content-type": "text/event-stream" } }), 1, { maxBytes: 1_000 }))
|
||||
.rejects.toMatchObject({ reason: "too_large" });
|
||||
});
|
||||
|
||||
it("still rejects one SSE message over the limit, complete or not", async () => {
|
||||
const big = `data: ${JSON.stringify({ jsonrpc: "2.0", id: 1, result: { text: "x".repeat(2_000) } })}`;
|
||||
for (const text of [`${big}\n\n`, big]) {
|
||||
await expect(readMcpHttpResponse(new Response(text, { headers: { "content-type": "text/event-stream" } }), 1, { maxBytes: 1_000, limitPerMessage: true }))
|
||||
.rejects.toMatchObject({ reason: "too_large" });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -15,10 +15,12 @@ import {
|
||||
toolMcpTasks,
|
||||
} from "@paperclipai/db";
|
||||
import { and, eq, lte, ne } from "drizzle-orm";
|
||||
import type { AskUserQuestionsPayload } from "@paperclipai/shared";
|
||||
import { forgetMcpHttpSessions } from "../services/mcp-http.js";
|
||||
import { createToolGatewayService } from "../services/tool-gateway.js";
|
||||
import { toolAccessService } from "../services/tool-access.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { issueThreadInteractionService } from "../services/issue-thread-interactions.js";
|
||||
import { adapterExecutionControls, createAdapterExecutionControl } from "../services/adapter-execution-control.js";
|
||||
import {
|
||||
createTasksDemoHttpServer,
|
||||
@@ -144,11 +146,14 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
expect(first).toMatchObject({ reasonCode: "elicitation_required" });
|
||||
const [card] = await db.select().from(issueThreadInteractions).where(and(eq(issueThreadInteractions.companyId, org.id), eq(issueThreadInteractions.issueId, issue.id)));
|
||||
expect(card).toMatchObject({ kind: "ask_user_questions", status: "pending" });
|
||||
await db.update(issueThreadInteractions).set({ status: "answered", result: { version: 1, answers: [
|
||||
{ questionId: "decision", optionIds: ["choice_1"] },
|
||||
const decision = (card!.payload as unknown as AskUserQuestionsPayload).questions.find((question) => question.id === "decision")!;
|
||||
const approve = decision.options.find((option) => option.label === "approve");
|
||||
expect(approve).toBeDefined();
|
||||
await issueThreadInteractionService(db).answerQuestions({ id: issue.id, companyId: org.id }, card!.id, { answers: [
|
||||
{ questionId: decision.id, optionIds: [approve!.id] },
|
||||
{ questionId: "note", optionIds: [], otherText: "go ahead" },
|
||||
] } }).where(eq(issueThreadInteractions.id, card!.id));
|
||||
const pending = call("needs_approval", { action: "rotate the demo key" });
|
||||
] }, { userId: actor.actorId });
|
||||
const pending = call("needs_approval", { action: "rotate the demo key" }).catch((error) => error);
|
||||
await vi.waitFor(() => expect(demoTask("needs_approval")?.status).toBe("working"), { timeout: 5_000 });
|
||||
clock += 3_000;
|
||||
expect(await pending).toMatchObject({ status: "completed", result: { content: "Approved: rotate the demo key was carried out. Note: go ahead" } });
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_DELIVERY_BOUND_MS, RECOVERY_CONTINUATION_MAX_HOLD_MS,
|
||||
deliverReconciledExecutions,
|
||||
} from "../services/execution-recovery-resolution.js";
|
||||
import { QUEUED_RECOVERY_RUNS_PER_AGENT } from "../services/recovery-queue-admission.js";
|
||||
import { PENDING_CONTINUATION_BOUND_MS, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js";
|
||||
import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js";
|
||||
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
|
||||
@@ -239,9 +240,12 @@ suite("recovery wake herd", () => {
|
||||
|
||||
it("gives the longest-waiting queued run a free slot after three fresher starts passed it over", async () => {
|
||||
const agentId = await seedCompany(1);
|
||||
const old = await queuedRecoveryRun(agentId, 150, 152);
|
||||
const people: Array<typeof heartbeatRuns.$inferSelect> = [];
|
||||
for (let index = 0; index < 3; index += 1) people.push(await queuedPersonRetry(agentId));
|
||||
// A person's run on a todo task, waiting 150 minutes behind fresher person runs on in-progress tasks.
|
||||
const old = await queuedPersonRetry(agentId);
|
||||
await db.update(issues).set({ status: "todo" }).where(eq(issues.id, old.issueId!));
|
||||
await db.update(heartbeatRuns).set({ createdAt: new Date(Date.now() - 150 * 60_000) }).where(eq(heartbeatRuns.id, old.id));
|
||||
const fresher: Array<typeof heartbeatRuns.$inferSelect> = [];
|
||||
for (let index = 0; index < 3; index += 1) fresher.push(await queuedPersonRetry(agentId));
|
||||
const issue = await task(agentId);
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
const startedAt = new Date(Date.now() - (3 - index) * 60_000);
|
||||
@@ -253,7 +257,29 @@ suite("recovery wake herd", () => {
|
||||
try {
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
expect((await readRun(old.id)).startedAt).not.toBeNull();
|
||||
for (const run of people) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null });
|
||||
for (const run of fresher) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null });
|
||||
} finally {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("keeps a long-waiting recovery run behind people's queued work, even when the oldest-run share is due", async () => {
|
||||
const agentId = await seedCompany(1);
|
||||
const old = await queuedRecoveryRun(agentId, 150, 152);
|
||||
const people: Array<typeof heartbeatRuns.$inferSelect> = [];
|
||||
for (let index = 0; index < 3; index += 1) people.push(await queuedPersonRetry(agentId));
|
||||
const issue = await task(agentId);
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
const startedAt = new Date(Date.now() - (3 - index) * 60_000);
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "on_demand",
|
||||
triggerDetail: "manual", status: "succeeded", createdAt: new Date(startedAt.getTime() - 1_000), startedAt,
|
||||
finishedAt: new Date(startedAt.getTime() + 1_000), contextSnapshot: { issueId: issue.id, wakeReason: "issue_commented" } });
|
||||
}
|
||||
const heartbeat = heartbeatService(db);
|
||||
try {
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
expect(await readRun(old.id)).toMatchObject({ status: "queued", startedAt: null });
|
||||
expect((await Promise.all(people.map(run => readRun(run.id)))).filter(run => run.startedAt)).toHaveLength(1);
|
||||
} finally {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
}
|
||||
@@ -285,6 +311,11 @@ suite("recovery wake herd", () => {
|
||||
}
|
||||
const deliveryOf = async (id: string) =>
|
||||
(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!.evidence.continuationDelivery;
|
||||
// The agent starts its queued recovery runs, which frees their turns in the queue.
|
||||
const startQueuedRecovery = async (agentId: string) => db.update(heartbeatRuns)
|
||||
.set({ status: "succeeded", startedAt: new Date(), finishedAt: new Date() })
|
||||
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' in ('issue_recovery_action_restored', 'issue_continuation_needed')`));
|
||||
const finishQueued = async (agentId: string) => db.update(heartbeatRuns).set({ status: "succeeded", startedAt: new Date(), finishedAt: new Date() })
|
||||
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued")));
|
||||
|
||||
@@ -379,6 +410,7 @@ suite("recovery wake herd", () => {
|
||||
await pendingContinuation(agentId, 1);
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
if (await deliveryOf(old.action.id) === "delivered") admittedAt = pass;
|
||||
await startQueuedRecovery(agentId);
|
||||
}
|
||||
expect(admittedAt).toBeGreaterThan(0);
|
||||
expect(admittedAt).toBeLessThanOrEqual(bound);
|
||||
@@ -576,6 +608,10 @@ suite("recovery wake herd", () => {
|
||||
const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString();
|
||||
await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence},
|
||||
'{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, held.action.id));
|
||||
// Past the bound it is still waiting its turn behind the agent's queued recovery run.
|
||||
const waiting = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!;
|
||||
expect(waiting.pendingWakes).toBe(facts.pendingWakes);
|
||||
await startQueuedRecovery(agentId);
|
||||
const stale = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!;
|
||||
expect(stale.pendingWakes).toBe(facts.pendingWakes - 1);
|
||||
}, 60_000);
|
||||
@@ -627,7 +663,10 @@ suite("recovery wake herd", () => {
|
||||
const wake = fakeWake();
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(deliveredIds(wake)).toEqual([fresh.action.id, backlog[0]!.action.id]);
|
||||
for (let pass = 0; pass < backlog.length; pass += 1) await deliverReconciledExecutions(db, wake as never);
|
||||
for (let pass = 0; pass < backlog.length; pass += 1) {
|
||||
await startQueuedRecovery(agentId);
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
}
|
||||
expect(await Promise.all(backlog.map(record => deliveryOf(record.action.id)))).toEqual(backlog.map(() => "delivered"));
|
||||
expect(wake).toHaveBeenCalledTimes(backlog.length + 1);
|
||||
}, 60_000);
|
||||
@@ -641,8 +680,38 @@ suite("recovery wake herd", () => {
|
||||
for (let pass = 0; pass < records.length; pass += 1) {
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(pass + 1);
|
||||
await startQueuedRecovery(agentId);
|
||||
}
|
||||
expect(deliveredIds(wake)).toEqual(records.map(record => record.action.id));
|
||||
}, 60_000);
|
||||
|
||||
it("queues at most a few recovery runs per agent, holds the rest, and admits the next when one leaves the queue", async () => {
|
||||
// paperclip.staging, 2026-11-13: 139 continuations for one agent entered its queue within an hour,
|
||||
// because a continuation held past the hold bound was admitted however deep the queue was.
|
||||
const agentId = await seedCompany(20);
|
||||
await queueOrdinaryWork(agentId, 25);
|
||||
const records: Array<Awaited<ReturnType<typeof pendingContinuation>>> = [];
|
||||
for (let index = 0; index < 12; index += 1) {
|
||||
const record = await pendingContinuation(agentId, index % 2 ? 2 : 200);
|
||||
await holdPastBound(record, 60 * 60_000);
|
||||
records.push(record);
|
||||
}
|
||||
const wake = fakeWake();
|
||||
for (let pass = 0; pass < 12; pass += 1) await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(QUEUED_RECOVERY_RUNS_PER_AGENT);
|
||||
expect((await Promise.all(records.map(record => deliveryOf(record.action.id)))).filter(state => state === "pending"))
|
||||
.toHaveLength(records.length - QUEUED_RECOVERY_RUNS_PER_AGENT);
|
||||
expect(await executionControlSchedule(db).continuation()).not.toBeNull();
|
||||
|
||||
for (let pass = 0; pass < 2 * records.length && wake.mock.calls.length < records.length; pass += 1) {
|
||||
await startQueuedRecovery(agentId);
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
const [queued] = await db.select({ count: sql<number>`count(*)::int` }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_recovery_action_restored'`));
|
||||
expect(Number(queued!.count)).toBeLessThanOrEqual(QUEUED_RECOVERY_RUNS_PER_AGENT);
|
||||
}
|
||||
expect(await Promise.all(records.map(record => deliveryOf(record.action.id)))).toEqual(records.map(() => "delivered"));
|
||||
await deliverReconciledExecutions(db, wake as never);
|
||||
expect(wake).toHaveBeenCalledTimes(records.length);
|
||||
}, 60_000);
|
||||
@@ -882,12 +951,20 @@ suite("recovery wake herd", () => {
|
||||
contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: "issue_recovery_action_restored" } }).returning();
|
||||
await db.update(agentWakeupRequests).set({ runId: run!.id }).where(eq(agentWakeupRequests.id, request!.id));
|
||||
}
|
||||
const people = [await queuedPersonRetry(agentId), await queuedPersonRetry(agentId)];
|
||||
// Recovery's share comes out of agent work; with a person's run waiting it would wait too.
|
||||
const work: Array<typeof heartbeatRuns.$inferSelect> = [];
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const busy = await task(agentId);
|
||||
const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "assignment",
|
||||
triggerDetail: "system", status: "queued", contextSnapshot: { issueId: busy.id, taskId: busy.id, wakeReason: "issue_assigned" } })
|
||||
.returning();
|
||||
work.push(row!);
|
||||
}
|
||||
const heartbeat = heartbeatService(db);
|
||||
try {
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
expect((await readRun(system.id)).startedAt).not.toBeNull();
|
||||
for (const run of people) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null });
|
||||
for (const run of work) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null });
|
||||
} finally {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,682 @@
|
||||
import { createPublicKey, randomUUID, verify, type KeyObject } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
applyPendingMigrations,
|
||||
authUsers,
|
||||
companies,
|
||||
companySecretBindings,
|
||||
companySecrets,
|
||||
connectionGrants,
|
||||
createDb,
|
||||
secretAccessEvents,
|
||||
toolAccessAuditEvents,
|
||||
toolApplications,
|
||||
toolCatalogEntries,
|
||||
toolConnections,
|
||||
} from "@paperclipai/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { toolAccessService } from "../services/tool-access.js";
|
||||
import { secretService } from "../services/secrets.js";
|
||||
import { toolAccessRoutes } from "../routes/tool-access.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { createOAuthClientProtocol } from "../services/tool-access/oauth/client.js";
|
||||
import {
|
||||
CLIENT_ASSERTION_LIFETIME_SECONDS,
|
||||
generateClientKeyPair,
|
||||
signClientAssertion,
|
||||
} from "../services/tool-access/oauth/client-credentials.js";
|
||||
import { MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID, mcpClientCapabilities } from "../services/mcp-http.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const ISSUER = "https://auth.example.test";
|
||||
const TOKEN_URL = `${ISSUER}/oauth/token`;
|
||||
const OTHER_ISSUER = "https://other-auth.example.test";
|
||||
const MCP_URL = "https://mcp.example.test/mcp";
|
||||
const CLIENT_ID = "paperclip-automation";
|
||||
const CLIENT_SECRET = "s3cret-value-never-shown";
|
||||
|
||||
function decodeJwt(token: string) {
|
||||
const [header, claims, signature] = token.split(".");
|
||||
return {
|
||||
header: JSON.parse(Buffer.from(header!, "base64url").toString()) as Record<string, unknown>,
|
||||
claims: JSON.parse(Buffer.from(claims!, "base64url").toString()) as Record<string, unknown>,
|
||||
signingInput: `${header}.${claims}`,
|
||||
signature: Buffer.from(signature!, "base64url"),
|
||||
};
|
||||
}
|
||||
|
||||
function verifyJwt(token: string, key: KeyObject): boolean {
|
||||
const { header, signingInput, signature } = decodeJwt(token);
|
||||
return verify(
|
||||
"sha256",
|
||||
Buffer.from(signingInput),
|
||||
header.alg === "ES256" ? { key, dsaEncoding: "ieee-p1363" } : key,
|
||||
signature,
|
||||
);
|
||||
}
|
||||
|
||||
function json(payload: unknown, status = 200, headers: Record<string, string> = {}) {
|
||||
return new Response(JSON.stringify(payload), { status, headers: { "content-type": "application/json", ...headers } });
|
||||
}
|
||||
|
||||
/**
|
||||
* A fake sign-in service and MCP server behind the outbound HTTP transport.
|
||||
* The sign-in service verifies real signatures against keys an "admin"
|
||||
* registered, exactly as a provider would.
|
||||
*/
|
||||
function fakeProvider() {
|
||||
const state = {
|
||||
advertisedIssuers: [ISSUER],
|
||||
registeredKeys: new Map<string, KeyObject>(),
|
||||
clientSecret: CLIENT_SECRET,
|
||||
tokenRequests: [] as Array<{ url: string; body: URLSearchParams; authorization: string | null }>,
|
||||
otherIssuerRequests: 0,
|
||||
mcpRequests: [] as Array<{ authorization: string | null; body: Record<string, unknown> }>,
|
||||
acceptedTokens: new Set<string>(),
|
||||
issued: 0,
|
||||
tokenDelayMs: 0,
|
||||
rejectAllTokens: false,
|
||||
/** Other clients the sign-in service knows (id → secret). */
|
||||
extraClients: new Map<string, string>(),
|
||||
/** Clients whose tokens the MCP server refuses. */
|
||||
mcpDeniedClients: new Set<string>(),
|
||||
tokenClients: new Map<string, string>(),
|
||||
/** Serve protected-resource metadata only at the address the challenge names. */
|
||||
metadataOnlyAtHint: false,
|
||||
/** Runs once when the server first receives a signed-in request (to stage races). */
|
||||
onAuthorizedMcpRequest: null as null | (() => Promise<void>),
|
||||
};
|
||||
const metadata = (issuer: string) => ({
|
||||
issuer,
|
||||
token_endpoint: `${issuer}/oauth/token`,
|
||||
grant_types_supported: ["client_credentials"],
|
||||
token_endpoint_auth_methods_supported: ["private_key_jwt", "client_secret_basic", "client_secret_post"],
|
||||
token_endpoint_auth_signing_alg_values_supported: ["RS256", "ES256"],
|
||||
});
|
||||
const handle = async (input: string | URL | Request, init?: RequestInit): Promise<Response> => {
|
||||
const url = String(input instanceof Request ? input.url : input);
|
||||
const headers = new Headers(init?.headers);
|
||||
if (url.startsWith(OTHER_ISSUER)) {
|
||||
state.otherIssuerRequests += 1;
|
||||
if (url.includes("/.well-known/")) return json(metadata(OTHER_ISSUER));
|
||||
return json({ access_token: "stolen", expires_in: 3600 });
|
||||
}
|
||||
if (url === `${new URL(MCP_URL).origin}/meta/custom-resource` && state.metadataOnlyAtHint) {
|
||||
return json({ resource: MCP_URL, authorization_servers: state.advertisedIssuers });
|
||||
}
|
||||
if (url === `${new URL(MCP_URL).origin}/.well-known/oauth-protected-resource/mcp`
|
||||
|| url === `${new URL(MCP_URL).origin}/.well-known/oauth-protected-resource`) {
|
||||
if (state.metadataOnlyAtHint) return new Response("not found", { status: 404 });
|
||||
return json({ resource: MCP_URL, authorization_servers: state.advertisedIssuers });
|
||||
}
|
||||
if (url === `${ISSUER}/.well-known/oauth-authorization-server`) return json(metadata(ISSUER));
|
||||
if (url.startsWith(ISSUER) && url.includes("/.well-known/")) return new Response("not found", { status: 404 });
|
||||
if (url === TOKEN_URL) {
|
||||
const body = new URLSearchParams(String(init?.body));
|
||||
state.tokenRequests.push({ url, body, authorization: headers.get("authorization") });
|
||||
if (state.tokenDelayMs) await new Promise((resolve) => setTimeout(resolve, state.tokenDelayMs));
|
||||
if (body.get("grant_type") !== "client_credentials") return json({ error: "unsupported_grant_type" }, 400);
|
||||
const assertion = body.get("client_assertion");
|
||||
let authenticated = false;
|
||||
let clientId = CLIENT_ID;
|
||||
if (assertion) {
|
||||
const { header, claims } = decodeJwt(assertion);
|
||||
const key = state.registeredKeys.get(String(header.kid));
|
||||
authenticated = Boolean(key && verifyJwt(assertion, key)
|
||||
&& claims.iss === CLIENT_ID && claims.sub === CLIENT_ID && claims.aud === TOKEN_URL
|
||||
&& typeof claims.exp === "number" && claims.exp * 1000 > Date.now());
|
||||
} else {
|
||||
const basic = headers.get("authorization");
|
||||
const expected = `Basic ${Buffer.from(`${CLIENT_ID}:${state.clientSecret}`).toString("base64")}`;
|
||||
authenticated = basic === expected;
|
||||
for (const [id, secret] of state.extraClients) {
|
||||
if (basic === `Basic ${Buffer.from(`${id}:${secret}`).toString("base64")}`) {
|
||||
authenticated = true;
|
||||
clientId = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!authenticated) return json({ error: "invalid_client" }, 401);
|
||||
state.issued += 1;
|
||||
const token = `machine-token-${state.issued}`;
|
||||
state.acceptedTokens.add(token);
|
||||
state.tokenClients.set(token, clientId);
|
||||
return json({ access_token: token, token_type: "Bearer", expires_in: 3600, scope: body.get("scope") ?? undefined });
|
||||
}
|
||||
if (url === MCP_URL) {
|
||||
const body = JSON.parse(String(init?.body ?? "{}")) as Record<string, unknown>;
|
||||
const authorization = headers.get("authorization");
|
||||
state.mcpRequests.push({ authorization, body });
|
||||
const bearer = authorization?.replace(/^Bearer /, "") ?? "";
|
||||
if (state.rejectAllTokens || !state.acceptedTokens.has(bearer) || state.mcpDeniedClients.has(state.tokenClients.get(bearer) ?? "")) {
|
||||
return new Response("", {
|
||||
status: 401,
|
||||
headers: { "www-authenticate": `Bearer resource_metadata="${new URL(MCP_URL).origin}${state.metadataOnlyAtHint ? "/meta/custom-resource" : "/.well-known/oauth-protected-resource/mcp"}"` },
|
||||
});
|
||||
}
|
||||
if (state.onAuthorizedMcpRequest) {
|
||||
const hook = state.onAuthorizedMcpRequest;
|
||||
state.onAuthorizedMcpRequest = null;
|
||||
await hook();
|
||||
}
|
||||
if (body.method === "server/discover") {
|
||||
return json({ jsonrpc: "2.0", id: body.id, result: { supportedVersions: ["2026-07-28"], capabilities: {} } });
|
||||
}
|
||||
if (body.method === "tools/list") {
|
||||
return json({ jsonrpc: "2.0", id: body.id, result: { tools: [
|
||||
{ name: "list_reports", description: "List reports", inputSchema: { type: "object", properties: {} }, annotations: { readOnlyHint: true } },
|
||||
] } });
|
||||
}
|
||||
return json({ jsonrpc: "2.0", id: body.id, result: {} });
|
||||
}
|
||||
throw new Error(`unexpected request ${url}`);
|
||||
};
|
||||
return { state, handle };
|
||||
}
|
||||
|
||||
describe("client credentials protocol", () => {
|
||||
it("creates keys without blocking other work on the server", async () => {
|
||||
let otherWorkRan = false;
|
||||
const generating = generateClientKeyPair("RS256");
|
||||
setImmediate(() => { otherWorkRan = true; });
|
||||
await generating;
|
||||
expect(otherWorkRan).toBe(true);
|
||||
});
|
||||
|
||||
it("signs an RS256 assertion with iss = sub = client ID, the token endpoint as audience, a short expiry and a unique jti", async () => {
|
||||
const { privateKeyPem, publicKey } = await generateClientKeyPair("RS256");
|
||||
const now = new Date("2026-11-13T12:00:00Z");
|
||||
const first = signClientAssertion({ privateKeyPem, algorithm: "RS256", keyId: publicKey.keyId, clientId: CLIENT_ID, audience: TOKEN_URL, now });
|
||||
const second = signClientAssertion({ privateKeyPem, algorithm: "RS256", keyId: publicKey.keyId, clientId: CLIENT_ID, audience: TOKEN_URL, now });
|
||||
const decoded = decodeJwt(first);
|
||||
expect(decoded.header).toEqual({ alg: "RS256", typ: "JWT", kid: publicKey.keyId });
|
||||
expect(decoded.claims).toMatchObject({ iss: CLIENT_ID, sub: CLIENT_ID, aud: TOKEN_URL, iat: now.getTime() / 1000 });
|
||||
expect(Number(decoded.claims.exp) - Number(decoded.claims.iat)).toBe(CLIENT_ASSERTION_LIFETIME_SECONDS);
|
||||
expect(CLIENT_ASSERTION_LIFETIME_SECONDS).toBeLessThanOrEqual(60);
|
||||
expect(decoded.claims.jti).not.toEqual(decodeJwt(second).claims.jti);
|
||||
expect(verifyJwt(first, createPublicKey(publicKey.publicKeyPem))).toBe(true);
|
||||
// The JWK the admin registers verifies the same signature.
|
||||
expect(verifyJwt(first, createPublicKey({ key: publicKey.publicJwk, format: "jwk" }))).toBe(true);
|
||||
expect(publicKey.publicJwk).toMatchObject({ kid: publicKey.keyId, alg: "RS256", use: "sig", kty: "RSA" });
|
||||
expect(JSON.stringify(publicKey)).not.toContain("PRIVATE KEY");
|
||||
expect(publicKey.publicJwk).not.toHaveProperty("d");
|
||||
});
|
||||
|
||||
it("signs ES256 assertions in JOSE (raw r||s) form", async () => {
|
||||
const { privateKeyPem, publicKey } = await generateClientKeyPair("ES256");
|
||||
const token = signClientAssertion({ privateKeyPem, algorithm: "ES256", keyId: publicKey.keyId, clientId: CLIENT_ID, audience: TOKEN_URL });
|
||||
expect(decodeJwt(token).signature).toHaveLength(64);
|
||||
expect(verifyJwt(token, createPublicKey(publicKey.publicKeyPem))).toBe(true);
|
||||
});
|
||||
|
||||
it("sends only the signed assertion as client authentication", async () => {
|
||||
const sent: Array<{ body: URLSearchParams; headers: Headers }> = [];
|
||||
const { exchangeOAuthToken } = createOAuthClientProtocol({
|
||||
assertOAuthEndpointUrl: (_kind, value) => String(value),
|
||||
fetchRemoteHttpUrl: async (_url, init) => {
|
||||
sent.push({ body: new URLSearchParams(String(init?.body)), headers: new Headers(init?.headers) });
|
||||
return json({ access_token: "t", expires_in: 60 });
|
||||
},
|
||||
});
|
||||
await exchangeOAuthToken({
|
||||
tokenUrl: TOKEN_URL, clientId: CLIENT_ID, grantType: "client_credentials", scopes: ["reports.read"],
|
||||
clientAssertion: "a.b.c", resource: MCP_URL,
|
||||
});
|
||||
const { body, headers } = sent[0]!;
|
||||
expect(body.get("grant_type")).toBe("client_credentials");
|
||||
expect(body.get("client_assertion_type")).toBe("urn:ietf:params:oauth:client-assertion-type:jwt-bearer");
|
||||
expect(body.get("client_assertion")).toBe("a.b.c");
|
||||
expect(body.get("scope")).toBe("reports.read");
|
||||
expect(body.get("resource")).toBe(MCP_URL);
|
||||
expect(body.has("client_id")).toBe(false);
|
||||
expect(body.has("client_secret")).toBe(false);
|
||||
expect(headers.get("authorization")).toBeNull();
|
||||
});
|
||||
|
||||
it("declares the extension only when a connection uses machine sign-in", () => {
|
||||
expect(JSON.stringify(mcpClientCapabilities("2026-07-28"))).not.toContain(MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID);
|
||||
expect(mcpClientCapabilities("2026-07-28", { declareClientCredentials: true })).toMatchObject({
|
||||
extensions: { [MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID]: {} },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TOOL_ACCESS_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
describeEmbeddedPostgres("machine sign-in connections", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
const transport = {
|
||||
remoteHttpEndpointLookup: async () => [{ address: "8.8.8.8", family: 4 as const }],
|
||||
remoteHttpRequest: async (url: string, init: RequestInit) => fetch(url, init),
|
||||
};
|
||||
const service = () => toolAccessService(db, transport);
|
||||
const actor = { actorType: "user" as const, actorId: "board-user" };
|
||||
|
||||
beforeAll(async () => {
|
||||
if (externalDatabaseUrl) {
|
||||
await applyPendingMigrations(externalDatabaseUrl);
|
||||
db = createDb(externalDatabaseUrl, { maxConnections: 6 });
|
||||
return;
|
||||
}
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-client-credentials-");
|
||||
db = createDb(tempDb.connectionString, { maxConnections: 6 });
|
||||
}, 120_000);
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await db.delete(secretAccessEvents);
|
||||
await db.delete(companySecretBindings);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(toolAccessAuditEvents);
|
||||
await db.delete(toolCatalogEntries);
|
||||
await db.delete(connectionGrants);
|
||||
await db.delete(toolConnections);
|
||||
await db.delete(companySecrets);
|
||||
await db.delete(toolApplications);
|
||||
await db.delete(companies);
|
||||
await db.delete(authUsers);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function createCompany() {
|
||||
const [company] = await db.insert(companies).values({
|
||||
name: `Machine sign-in ${randomUUID()}`,
|
||||
issuePrefix: `MS${randomUUID().slice(0, 6).toUpperCase()}`,
|
||||
}).returning();
|
||||
return company!;
|
||||
}
|
||||
|
||||
function useProvider() {
|
||||
const provider = fakeProvider();
|
||||
vi.spyOn(globalThis, "fetch").mockImplementation(provider.handle);
|
||||
return provider;
|
||||
}
|
||||
|
||||
function registerKey(provider: ReturnType<typeof fakeProvider>, key: { keyId: string; pem: string }) {
|
||||
provider.state.registeredKeys.set(key.keyId, createPublicKey(key.pem));
|
||||
}
|
||||
|
||||
async function organizationGrant(connectionId: string) {
|
||||
const [grant] = await db.select().from(connectionGrants)
|
||||
.where(and(eq(connectionGrants.connectionId, connectionId), eq(connectionGrants.kind, "organization")));
|
||||
return grant!;
|
||||
}
|
||||
|
||||
async function setupKeyConnection(companyId: string, provider: ReturnType<typeof fakeProvider>) {
|
||||
const svc = service();
|
||||
const created = await svc.createMachineSignIn(companyId, { link: MCP_URL, method: "paperclip_key", scopes: "reports.read" }, actor);
|
||||
registerKey(provider, created.machineSignIn.publicKey!);
|
||||
const result = await svc.testMachineSignIn(companyId, created.connectionId, { clientId: CLIENT_ID }, actor);
|
||||
return { svc, created, result };
|
||||
}
|
||||
|
||||
it("sets up a Paperclip key connection: finds the sign-in service, shows the public key, signs in and lists tools", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const svc = service();
|
||||
const created = await svc.createMachineSignIn(company.id, { link: MCP_URL, method: "paperclip_key", scopes: "reports.read" }, actor);
|
||||
expect(created.machineSignIn).toMatchObject({ method: "paperclip_key", issuer: ISSUER, tokenUrl: TOKEN_URL, clientId: null, scopes: ["reports.read"] });
|
||||
expect(created.machineSignIn.publicKey?.pem).toContain("BEGIN PUBLIC KEY");
|
||||
expect(created.connection.status).toBe("draft");
|
||||
// The private key is in the vault, nowhere else.
|
||||
const serialized = JSON.stringify(created);
|
||||
expect(serialized).not.toContain("PRIVATE KEY");
|
||||
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId));
|
||||
expect(JSON.stringify(row!.config)).not.toContain("PRIVATE KEY");
|
||||
expect(row!.credentialSecretRefs.map((ref) => ref.configPath)).toEqual(["oauth.client_private_key"]);
|
||||
|
||||
// Testing before the key is registered fails clearly and keeps the draft and its key.
|
||||
await expect(svc.testMachineSignIn(company.id, created.connectionId, { clientId: CLIENT_ID }, actor))
|
||||
.rejects.toMatchObject({ status: 422, details: expect.objectContaining({ code: "client_credentials_rejected" }) });
|
||||
expect((await svc.getMachineSignIn(company.id, created.connectionId)).publicKey?.keyId).toBe(created.machineSignIn.publicKey!.keyId);
|
||||
|
||||
registerKey(provider, created.machineSignIn.publicKey!);
|
||||
const result = await svc.testMachineSignIn(company.id, created.connectionId, { clientId: CLIENT_ID }, actor);
|
||||
expect(result.connection.status).toBe("active");
|
||||
expect(result.catalog.map((entry) => entry.name)).toContain("list_reports");
|
||||
|
||||
const accepted = provider.state.tokenRequests.at(-1)!;
|
||||
expect(accepted.body.get("scope")).toBe("reports.read");
|
||||
expect(accepted.body.get("resource")).toBe(MCP_URL);
|
||||
const { header, claims } = decodeJwt(accepted.body.get("client_assertion")!);
|
||||
expect(header.kid).toBe(created.machineSignIn.publicKey!.keyId);
|
||||
expect(claims).toMatchObject({ iss: CLIENT_ID, sub: CLIENT_ID, aud: TOKEN_URL });
|
||||
const authorized = provider.state.mcpRequests.filter((call) => call.authorization === "Bearer machine-token-1");
|
||||
expect(authorized.length).toBeGreaterThan(0);
|
||||
const listCall = provider.state.mcpRequests.find((call) => call.body.method === "tools/list" && call.authorization);
|
||||
expect(JSON.stringify(listCall?.body)).toContain(MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID);
|
||||
const grant = await organizationGrant(created.connectionId);
|
||||
expect(grant.status).toBe("active");
|
||||
expect(grant.credentialSecretRefs.map((ref) => ref.configPath)).toContain("oauth.access_token");
|
||||
});
|
||||
|
||||
it("uses a client secret with client_secret_basic and never returns the secret", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const svc = service();
|
||||
const created = await svc.createMachineSignIn(company.id, {
|
||||
link: MCP_URL, method: "client_secret", authorizationServer: ISSUER, clientId: CLIENT_ID, clientSecret: CLIENT_SECRET,
|
||||
}, actor);
|
||||
expect(created.machineSignIn).toMatchObject({ method: "client_secret", hasClientSecret: true, publicKey: null });
|
||||
const result = await svc.testMachineSignIn(company.id, created.connectionId, {}, actor);
|
||||
expect(result.connection.status).toBe("active");
|
||||
const tokenRequest = provider.state.tokenRequests.at(-1)!;
|
||||
expect(tokenRequest.authorization).toBe(`Basic ${Buffer.from(`${CLIENT_ID}:${CLIENT_SECRET}`).toString("base64")}`);
|
||||
expect(tokenRequest.body.has("client_secret")).toBe(false);
|
||||
for (const value of [created, result, await svc.getMachineSignIn(company.id, created.connectionId)]) {
|
||||
expect(JSON.stringify(value)).not.toContain(CLIENT_SECRET);
|
||||
}
|
||||
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId));
|
||||
expect(JSON.stringify(row)).not.toContain(CLIENT_SECRET);
|
||||
});
|
||||
|
||||
it("follows the server's challenge to metadata published outside the well-known addresses", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
provider.state.metadataOnlyAtHint = true;
|
||||
const created = await service().createMachineSignIn(company.id, { link: MCP_URL, method: "paperclip_key" }, actor);
|
||||
expect(created.machineSignIn).toMatchObject({ issuer: ISSUER, tokenUrl: TOKEN_URL });
|
||||
});
|
||||
|
||||
it("accepts a token URL directly when the provider publishes no sign-in metadata", async () => {
|
||||
const company = await createCompany();
|
||||
useProvider();
|
||||
const created = await service().createMachineSignIn(company.id, {
|
||||
link: MCP_URL, method: "client_secret", authorizationServer: TOKEN_URL, clientId: CLIENT_ID, clientSecret: CLIENT_SECRET,
|
||||
}, actor);
|
||||
expect(created.machineSignIn).toMatchObject({ issuer: null, tokenUrl: TOKEN_URL });
|
||||
});
|
||||
|
||||
it("refuses to follow the server to a different sign-in service and never sends credentials there", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const svc = service();
|
||||
const created = await svc.createMachineSignIn(company.id, { link: MCP_URL, method: "paperclip_key" }, actor);
|
||||
registerKey(provider, created.machineSignIn.publicKey!);
|
||||
provider.state.advertisedIssuers = [OTHER_ISSUER];
|
||||
const tokenRequestsBefore = provider.state.tokenRequests.length;
|
||||
await expect(svc.testMachineSignIn(company.id, created.connectionId, { clientId: CLIENT_ID }, actor))
|
||||
.rejects.toMatchObject({ status: 422, details: expect.objectContaining({ code: "client_credentials_issuer_changed" }) });
|
||||
expect(provider.state.tokenRequests.length).toBe(tokenRequestsBefore);
|
||||
expect(provider.state.otherIssuerRequests).toBe(0);
|
||||
expect((await svc.getMachineSignIn(company.id, created.connectionId)).tokenUrl).toBe(TOKEN_URL);
|
||||
});
|
||||
|
||||
it("keeps the pinned token endpoint when an active server's challenge points elsewhere", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const { svc, created } = await setupKeyConnection(company.id, provider);
|
||||
provider.state.advertisedIssuers = [OTHER_ISSUER];
|
||||
provider.state.rejectAllTokens = true;
|
||||
await expect(svc.checkHealth(created.connectionId, actor)).rejects.toMatchObject({
|
||||
message: expect.stringContaining("didn't accept this connection's machine sign-in"),
|
||||
});
|
||||
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId));
|
||||
expect((row!.config.oauth as Record<string, unknown>).tokenUrl).toBe(TOKEN_URL);
|
||||
expect((row!.config.oauth as Record<string, unknown>).issuer).toBe(ISSUER);
|
||||
expect(provider.state.otherIssuerRequests).toBe(0);
|
||||
});
|
||||
|
||||
it("reuses a cached token and shares one token request across concurrent callers", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const { created } = await setupKeyConnection(company.id, provider);
|
||||
const grant = await organizationGrant(created.connectionId);
|
||||
const issuedAfterSetup = provider.state.issued;
|
||||
|
||||
// A live token is reused: no token request at all.
|
||||
const first = service();
|
||||
await first.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, actor });
|
||||
expect(provider.state.issued).toBe(issuedAfterSetup);
|
||||
|
||||
// Shortly before expiry, many agents on two server processes ask at once.
|
||||
await db.update(connectionGrants).set({
|
||||
providerTenant: { ...(grant.providerTenant ?? {}), oauth: { ...((grant.providerTenant ?? {}).oauth as object ?? {}), accessTokenExpiresAt: new Date(Date.now() + 10_000).toISOString() } },
|
||||
}).where(eq(connectionGrants.id, grant.id));
|
||||
provider.state.tokenDelayMs = 150;
|
||||
const second = service();
|
||||
const input = { companyId: company.id, connectionId: created.connectionId, grantId: grant.id, actor };
|
||||
await Promise.all([
|
||||
first.refreshOAuthGrantCredentials(input), first.refreshOAuthGrantCredentials(input), first.refreshOAuthGrantCredentials(input),
|
||||
second.refreshOAuthGrantCredentials(input), second.refreshOAuthGrantCredentials(input),
|
||||
]);
|
||||
expect(provider.state.issued).toBe(issuedAfterSetup + 1);
|
||||
const refreshed = await organizationGrant(created.connectionId);
|
||||
const expiresAt = Date.parse(String((refreshed.providerTenant?.oauth as Record<string, unknown>).accessTokenExpiresAt));
|
||||
expect(expiresAt).toBeGreaterThan(Date.now() + 3_000_000);
|
||||
});
|
||||
|
||||
it("requests a new token when the server rejects the current one", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const { svc, created } = await setupKeyConnection(company.id, provider);
|
||||
const issuedAfterSetup = provider.state.issued;
|
||||
provider.state.acceptedTokens.clear();
|
||||
const health = await svc.checkHealth(created.connectionId, actor);
|
||||
expect(health.connection.healthStatus).toBe("ok");
|
||||
expect(provider.state.issued).toBe(issuedAfterSetup + 1);
|
||||
expect(provider.state.mcpRequests.at(-1)?.authorization).toBe(`Bearer machine-token-${issuedAfterSetup + 1}`);
|
||||
});
|
||||
|
||||
it("rotates the key without downtime: the old key works until the new one is registered and confirmed", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const { svc, created } = await setupKeyConnection(company.id, provider);
|
||||
const oldKeyId = created.machineSignIn.publicKey!.keyId;
|
||||
const started = await svc.rotateMachineSignIn(company.id, created.connectionId, { method: "paperclip_key" }, actor);
|
||||
expect(started.completed).toBe(false);
|
||||
const pending = started.machineSignIn.pendingPublicKey!;
|
||||
expect(pending.keyId).not.toBe(oldKeyId);
|
||||
expect(started.machineSignIn.publicKey?.keyId).toBe(oldKeyId);
|
||||
|
||||
// Tokens keep coming from the current key while the new one is pending.
|
||||
const grant = await organizationGrant(created.connectionId);
|
||||
await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor });
|
||||
expect(decodeJwt(provider.state.tokenRequests.at(-1)!.body.get("client_assertion")!).header.kid).toBe(oldKeyId);
|
||||
|
||||
// Completing before the provider knows the new key fails and changes nothing.
|
||||
await expect(svc.completeMachineSignInRotation(company.id, created.connectionId, actor))
|
||||
.rejects.toMatchObject({ details: expect.objectContaining({ code: "client_credentials_rejected" }) });
|
||||
expect((await svc.getMachineSignIn(company.id, created.connectionId)).publicKey?.keyId).toBe(oldKeyId);
|
||||
|
||||
registerKey(provider, pending);
|
||||
const completed = await svc.completeMachineSignInRotation(company.id, created.connectionId, actor);
|
||||
expect(completed.machineSignIn.publicKey?.keyId).toBe(pending.keyId);
|
||||
expect(completed.machineSignIn.pendingPublicKey).toBeNull();
|
||||
provider.state.registeredKeys.delete(oldKeyId);
|
||||
await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor });
|
||||
expect(decodeJwt(provider.state.tokenRequests.at(-1)!.body.get("client_assertion")!).header.kid).toBe(pending.keyId);
|
||||
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId));
|
||||
expect(row!.credentialSecretRefs.map((ref) => ref.configPath)).not.toContain("oauth.client_private_key_next");
|
||||
});
|
||||
|
||||
it("signs with a key ID that matches the key a refresh actually reads", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const { svc, created } = await setupKeyConnection(company.id, provider);
|
||||
// A rotation lands between a refresh loading the settings and reading the
|
||||
// vault: the vault holds the new key while config still names the old one.
|
||||
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId));
|
||||
const keyRef = row!.credentialSecretRefs.find((ref) => ref.configPath === "oauth.client_private_key")!;
|
||||
const next = await generateClientKeyPair("RS256");
|
||||
registerKey(provider, { keyId: next.publicKey.keyId, pem: next.publicKey.publicKeyPem });
|
||||
await secretService(db).rotate(keyRef.secretId, { value: next.privateKeyPem });
|
||||
const grant = await organizationGrant(created.connectionId);
|
||||
const before = provider.state.issued;
|
||||
await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor });
|
||||
expect(provider.state.issued).toBe(before + 1);
|
||||
expect(decodeJwt(provider.state.tokenRequests.at(-1)!.body.get("client_assertion")!).header.kid).toBe(next.publicKey.keyId);
|
||||
});
|
||||
|
||||
it("never promotes a staged key that changed after it was registered", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const { svc, created } = await setupKeyConnection(company.id, provider);
|
||||
const started = await svc.rotateMachineSignIn(company.id, created.connectionId, { method: "paperclip_key" }, actor);
|
||||
registerKey(provider, started.machineSignIn.pendingPublicKey!);
|
||||
// Another rotation overwrites the staged private key in the vault.
|
||||
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId));
|
||||
const staged = row!.credentialSecretRefs.find((ref) => ref.configPath === "oauth.client_private_key_next")!;
|
||||
const other = await generateClientKeyPair("RS256");
|
||||
registerKey(provider, { keyId: other.publicKey.keyId, pem: other.publicKey.publicKeyPem });
|
||||
await secretService(db).rotate(staged.secretId, { value: other.privateKeyPem });
|
||||
await expect(svc.completeMachineSignInRotation(company.id, created.connectionId, actor))
|
||||
.rejects.toMatchObject({ status: 409, details: expect.objectContaining({ code: "client_credentials_rotation_changed" }) });
|
||||
const status = await svc.getMachineSignIn(company.id, created.connectionId);
|
||||
expect(status.publicKey?.keyId).toBe(created.machineSignIn.publicKey!.keyId);
|
||||
// The current key still signs in.
|
||||
const grant = await organizationGrant(created.connectionId);
|
||||
await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor });
|
||||
expect(decodeJwt(provider.state.tokenRequests.at(-1)!.body.get("client_assertion")!).header.kid).toBe(created.machineSignIn.publicKey!.keyId);
|
||||
});
|
||||
|
||||
it("keeps working credentials when a retest's client can't use the server", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const svc = service();
|
||||
const created = await svc.createMachineSignIn(company.id, {
|
||||
link: MCP_URL, method: "client_secret", clientId: CLIENT_ID, clientSecret: CLIENT_SECRET,
|
||||
}, actor);
|
||||
await svc.testMachineSignIn(company.id, created.connectionId, {}, actor);
|
||||
provider.state.extraClients.set("other-client", "other-secret");
|
||||
provider.state.mcpDeniedClients.add("other-client");
|
||||
await expect(svc.testMachineSignIn(company.id, created.connectionId, { clientId: "other-client", clientSecret: "other-secret" }, actor))
|
||||
.rejects.toMatchObject({ details: expect.objectContaining({ code: "client_credentials_rejected" }) });
|
||||
const status = await svc.getMachineSignIn(company.id, created.connectionId);
|
||||
expect(status.clientId).toBe(CLIENT_ID);
|
||||
// The working connection still serves tools with the original client.
|
||||
const health = await svc.checkHealth(created.connectionId, actor);
|
||||
expect(health.connection.healthStatus).toBe("ok");
|
||||
expect(provider.state.tokenClients.get(provider.state.mcpRequests.at(-1)!.authorization!.replace("Bearer ", ""))).toBe(CLIENT_ID);
|
||||
|
||||
// A retest that passes gives the organization grant the new token's expiry.
|
||||
const grant = await organizationGrant(created.connectionId);
|
||||
await db.update(connectionGrants).set({
|
||||
providerTenant: { ...(grant.providerTenant ?? {}), oauth: { ...((grant.providerTenant ?? {}).oauth as object ?? {}), accessTokenExpiresAt: "2000-01-01T00:00:00.000Z" } },
|
||||
}).where(eq(connectionGrants.id, grant.id));
|
||||
await svc.testMachineSignIn(company.id, created.connectionId, {}, actor);
|
||||
const retested = await organizationGrant(created.connectionId);
|
||||
expect(Date.parse(String((retested.providerTenant?.oauth as Record<string, unknown>).accessTokenExpiresAt))).toBeGreaterThan(Date.now() + 3_000_000);
|
||||
});
|
||||
|
||||
it("never reactivates a removed connection", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const { svc, created } = await setupKeyConnection(company.id, provider);
|
||||
await svc.archiveConnection(created.connectionId, company.id, actor);
|
||||
await expect(svc.testMachineSignIn(company.id, created.connectionId, { clientSecret: CLIENT_SECRET }, actor))
|
||||
.rejects.toMatchObject({ details: expect.objectContaining({ code: "client_credentials_connection_removed" }) });
|
||||
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId));
|
||||
expect(row!.status).toBe("archived");
|
||||
});
|
||||
|
||||
it("does not undo a rotation that lands while a test is running", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const { svc, created } = await setupKeyConnection(company.id, provider);
|
||||
provider.state.clientSecret = "switched-secret";
|
||||
// While the test talks to the server, an admin switches to a client secret.
|
||||
provider.state.onAuthorizedMcpRequest = async () => {
|
||||
await svc.rotateMachineSignIn(company.id, created.connectionId, { method: "client_secret", clientSecret: "switched-secret" }, actor);
|
||||
};
|
||||
await expect(svc.testMachineSignIn(company.id, created.connectionId, {}, actor))
|
||||
.rejects.toMatchObject({ status: 409, details: expect.objectContaining({ code: "client_credentials_changed" }) });
|
||||
const status = await svc.getMachineSignIn(company.id, created.connectionId);
|
||||
expect(status.method).toBe("client_secret");
|
||||
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, created.connectionId));
|
||||
expect(row!.credentialSecretRefs.map((ref) => ref.configPath)).not.toContain("oauth.client_private_key");
|
||||
// The next token request uses the secret, not the deleted key.
|
||||
const grant = await organizationGrant(created.connectionId);
|
||||
await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor });
|
||||
expect(provider.state.tokenRequests.at(-1)!.authorization).toBe(`Basic ${Buffer.from(`${CLIENT_ID}:switched-secret`).toString("base64")}`);
|
||||
});
|
||||
|
||||
it("tries a new client secret before replacing the old one", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const svc = service();
|
||||
const created = await svc.createMachineSignIn(company.id, {
|
||||
link: MCP_URL, method: "client_secret", clientId: CLIENT_ID, clientSecret: CLIENT_SECRET,
|
||||
}, actor);
|
||||
await svc.testMachineSignIn(company.id, created.connectionId, {}, actor);
|
||||
await expect(svc.rotateMachineSignIn(company.id, created.connectionId, { method: "client_secret", clientSecret: "wrong-new-secret" }, actor))
|
||||
.rejects.toMatchObject({ details: expect.objectContaining({ code: "client_credentials_rejected" }) });
|
||||
const grant = await organizationGrant(created.connectionId);
|
||||
// The old secret still works.
|
||||
await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor });
|
||||
provider.state.clientSecret = "rotated-secret";
|
||||
const rotated = await svc.rotateMachineSignIn(company.id, created.connectionId, { method: "client_secret", clientSecret: "rotated-secret" }, actor);
|
||||
expect(rotated.completed).toBe(true);
|
||||
await svc.refreshOAuthGrantCredentials({ companyId: company.id, connectionId: created.connectionId, grantId: grant.id, forceRefresh: true, actor });
|
||||
expect(provider.state.tokenRequests.at(-1)!.authorization).toBe(`Basic ${Buffer.from(`${CLIENT_ID}:rotated-secret`).toString("base64")}`);
|
||||
});
|
||||
|
||||
it("refuses machine sign-in on a personal connection", async () => {
|
||||
const company = await createCompany();
|
||||
useProvider();
|
||||
const svc = service();
|
||||
const created = await svc.createMachineSignIn(company.id, { link: MCP_URL, method: "client_secret", clientId: CLIENT_ID, clientSecret: CLIENT_SECRET }, actor);
|
||||
await db.update(toolConnections).set({ credentialPolicy: "per_user" }).where(eq(toolConnections.id, created.connectionId));
|
||||
await expect(svc.testMachineSignIn(company.id, created.connectionId, {}, actor))
|
||||
.rejects.toMatchObject({ status: 422, details: expect.objectContaining({ code: "client_credentials_personal_refused" }) });
|
||||
});
|
||||
|
||||
it("logs setup, verification, rotation and removal without any secret material", async () => {
|
||||
const company = await createCompany();
|
||||
const provider = useProvider();
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", userId: "board-user", userName: "Board User", userEmail: null, isInstanceAdmin: true, source: "local_implicit" };
|
||||
next();
|
||||
});
|
||||
app.use("/api", toolAccessRoutes(db, transport));
|
||||
app.use(errorHandler);
|
||||
|
||||
const created = await request(app).post(`/api/companies/${company.id}/tools/machine-sign-in`)
|
||||
.send({ link: MCP_URL, method: "paperclip_key", clientId: CLIENT_ID });
|
||||
expect(created.status).toBe(201);
|
||||
registerKey(provider, created.body.machineSignIn.publicKey);
|
||||
const connectionId = created.body.connectionId as string;
|
||||
expect((await request(app).post(`/api/tool-connections/${connectionId}/machine-sign-in/test`).send({})).status).toBe(200);
|
||||
expect((await request(app).post(`/api/tool-connections/${connectionId}/machine-sign-in/rotate`).send({ method: "client_secret", clientSecret: CLIENT_SECRET })).status).toBe(200);
|
||||
expect((await request(app).get(`/api/tool-connections/${connectionId}/machine-sign-in`)).body).toMatchObject({ method: "client_secret", hasClientSecret: true });
|
||||
expect((await request(app).delete(`/api/tool-connections/${connectionId}`)).status).toBe(200);
|
||||
|
||||
const rows = await db.select().from(activityLog).where(eq(activityLog.entityId, connectionId));
|
||||
expect(rows.map((row) => row.action)).toEqual(expect.arrayContaining([
|
||||
"tool_connection.machine_sign_in_created",
|
||||
"tool_connection.machine_sign_in_verified",
|
||||
"tool_connection.machine_sign_in_rotated",
|
||||
"tool_connection.archived",
|
||||
]));
|
||||
expect(rows.find((row) => row.action === "tool_connection.archived")?.details).toMatchObject({ machineSignIn: true });
|
||||
const logged = JSON.stringify(rows);
|
||||
expect(logged).not.toContain(CLIENT_SECRET);
|
||||
expect(logged).not.toContain("PRIVATE KEY");
|
||||
expect(logged).not.toContain("machine-token-");
|
||||
// Removal destroys the vault entries the connection owned.
|
||||
const remaining = await db.select({ id: companySecrets.id, status: companySecrets.status }).from(companySecrets)
|
||||
.where(eq(companySecrets.companyId, company.id));
|
||||
expect(remaining.filter((secret) => secret.status === "active")).toEqual([]);
|
||||
});
|
||||
|
||||
it("refuses a validation error in plain words", async () => {
|
||||
const company = await createCompany();
|
||||
useProvider();
|
||||
await expect(service().createMachineSignIn(company.id, {
|
||||
link: "https://mcp.example.test/mcp?api_key=abc", method: "client_secret",
|
||||
}, actor)).rejects.toMatchObject({ status: 400, message: expect.stringContaining("Remove the key from the server address") });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, companies, createDb, heartbeatRuns, issues, toolInvocations, type Db } from "@paperclipai/db";
|
||||
import { listToolCallProgress, saveToolCallProgress } from "../services/tool-call-progress.js";
|
||||
import { assignedToolName } from "../services/native-runtime/assigned-mcp-tools.js";
|
||||
import { startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
|
||||
describe("saved tool call progress", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>>;
|
||||
let db: Db;
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startRecoveryPostgres("tool-call-progress-");
|
||||
db = createDb(temporary.connectionString);
|
||||
}, 120_000);
|
||||
afterAll(async () => { await temporary?.cleanup(); });
|
||||
|
||||
async function task(prefix: string) {
|
||||
const companyId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: prefix, issuePrefix: prefix });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Agent", adapterType: "process" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent!.id, status: "running" }).returning();
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Export" });
|
||||
const call = async (values: Partial<typeof toolInvocations.$inferInsert> = {}) => (await db.insert(toolInvocations).values({
|
||||
companyId, issueId, runId: run!.id, agentId: agent!.id, toolName: "kv_demo__slow_with_progress",
|
||||
status: "executing", startedAt: new Date("2026-11-13T10:00:00Z"), ...values,
|
||||
}).returning())[0]!;
|
||||
return { companyId, issueId, runId: run!.id, call };
|
||||
}
|
||||
const progress = (value: number) => ({ progress: value, total: 10, message: "Exporting", updatedAt: "2026-11-13T10:00:05.000Z" });
|
||||
|
||||
it("returns a task's running calls with their latest progress, and nothing from another company or task", async () => {
|
||||
const mine = await task("PRGA");
|
||||
const theirs = await task("PRGB");
|
||||
const running = await mine.call();
|
||||
await saveToolCallProgress(db, mine.companyId, running.id)(progress(3));
|
||||
await saveToolCallProgress(db, mine.companyId, running.id)(progress(4));
|
||||
const other = await theirs.call();
|
||||
await saveToolCallProgress(db, theirs.companyId, other.id)(progress(9));
|
||||
// A write naming the wrong company never lands.
|
||||
await saveToolCallProgress(db, mine.companyId, other.id)(progress(1));
|
||||
const now = new Date("2026-11-13T10:01:00Z");
|
||||
|
||||
expect(await listToolCallProgress(db, mine.companyId, mine.issueId, now)).toEqual([{
|
||||
invocationId: running.id, runId: mine.runId, toolName: "kv_demo__slow_with_progress",
|
||||
nativeToolName: assignedToolName("kv_demo__slow_with_progress"),
|
||||
startedAt: "2026-11-13T10:00:00.000Z", progress: 4, total: 10, message: "Exporting",
|
||||
updatedAt: "2026-11-13T10:00:05.000Z", done: false,
|
||||
}]);
|
||||
expect(await listToolCallProgress(db, mine.companyId, theirs.issueId, now)).toEqual([]);
|
||||
expect(await listToolCallProgress(db, theirs.companyId, mine.issueId, now)).toEqual([]);
|
||||
expect((await db.select().from(toolInvocations).where(eq(toolInvocations.id, other.id)))[0]!.progress).toMatchObject({ progress: 9 });
|
||||
});
|
||||
|
||||
it("never writes progress onto a finished call, and reports it as done", async () => {
|
||||
const mine = await task("PRGC");
|
||||
const running = await mine.call({ createdAt: new Date() });
|
||||
await saveToolCallProgress(db, mine.companyId, running.id)(progress(2));
|
||||
expect(await listToolCallProgress(db, mine.companyId, mine.issueId)).toHaveLength(1);
|
||||
await db.update(toolInvocations).set({ status: "succeeded", completedAt: new Date() }).where(eq(toolInvocations.id, running.id));
|
||||
// A late, throttled save after the result must not overwrite it.
|
||||
await saveToolCallProgress(db, mine.companyId, running.id)(progress(8));
|
||||
expect((await db.select().from(toolInvocations).where(eq(toolInvocations.id, running.id)))[0]!.progress).toMatchObject({ progress: 2 });
|
||||
// A chat that missed the end learns it; a call that never saved progress is reported too.
|
||||
const notSaved = await mine.call({ createdAt: new Date() });
|
||||
await db.update(toolInvocations).set({ status: "succeeded", completedAt: new Date() }).where(eq(toolInvocations.id, notSaved.id));
|
||||
const listed = await listToolCallProgress(db, mine.companyId, mine.issueId);
|
||||
expect(listed.map((entry) => [entry.invocationId, entry.done, entry.progress]).sort()).toEqual(
|
||||
[[running.id, true, 2], [notSaved.id, true, 0]].sort(),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH } from "@paperclipai/shared";
|
||||
import {
|
||||
createToolCallProgressReporter,
|
||||
TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS,
|
||||
TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS,
|
||||
} from "../services/tool-call-progress.js";
|
||||
import { assignedToolName } from "../services/native-runtime/assigned-mcp-tools.js";
|
||||
|
||||
function reporter() {
|
||||
const published: Array<{ companyId: string; type: string; payload: Record<string, unknown> }> = [];
|
||||
const saved: unknown[] = [];
|
||||
const instance = createToolCallProgressReporter({
|
||||
companyId: "company-a",
|
||||
runId: "run-a",
|
||||
issueId: "issue-a",
|
||||
agentId: "agent-a",
|
||||
invocationId: "invocation-a",
|
||||
toolName: "kv_demo__slow_with_progress",
|
||||
startedAt: new Date("2026-11-13T10:00:00Z"),
|
||||
publish: (event) => { published.push(event); },
|
||||
save: async (progress) => { saved.push(progress); },
|
||||
});
|
||||
return { instance, published, saved };
|
||||
}
|
||||
|
||||
describe("tool call progress reporter", () => {
|
||||
beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-11-13T10:00:01Z")); });
|
||||
afterEach(() => { vi.useRealTimers(); });
|
||||
|
||||
it("publishes at most one update per interval and always the latest", () => {
|
||||
const { instance, published } = reporter();
|
||||
for (let step = 1; step <= 50; step++) instance.report({ progress: step, total: 50, message: `Step ${step}` });
|
||||
// The first update goes out at once; the rest wait for the interval.
|
||||
expect(published.map((event) => event.payload.progress)).toEqual([1]);
|
||||
vi.advanceTimersByTime(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS);
|
||||
expect(published.map((event) => event.payload.progress)).toEqual([1, 50]);
|
||||
vi.advanceTimersByTime(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS * 4);
|
||||
expect(published).toHaveLength(2);
|
||||
// A steady stream for one second publishes about four times, never per update.
|
||||
for (let tick = 0; tick < 100; tick++) {
|
||||
instance.report({ progress: 51 + tick, total: 200 });
|
||||
vi.advanceTimersByTime(10);
|
||||
}
|
||||
expect(published.length - 2).toBeLessThanOrEqual(1_000 / TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS + 1);
|
||||
});
|
||||
|
||||
it("saves on the invocation less often than it publishes", async () => {
|
||||
const { instance, saved } = reporter();
|
||||
for (let tick = 0; tick < 200; tick++) {
|
||||
instance.report({ progress: tick, total: 200 });
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
}
|
||||
expect(saved.length).toBeLessThanOrEqual(2_000 / TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS + 1);
|
||||
expect(saved.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("ties every event to its company, run and tool call", () => {
|
||||
const { instance, published } = reporter();
|
||||
instance.report({ progress: 1, total: 4, message: "Reading" });
|
||||
expect(published[0]).toEqual({
|
||||
companyId: "company-a",
|
||||
type: "heartbeat.run.tool_progress",
|
||||
payload: {
|
||||
invocationId: "invocation-a",
|
||||
runId: "run-a",
|
||||
issueId: "issue-a",
|
||||
agentId: "agent-a",
|
||||
toolName: "kv_demo__slow_with_progress",
|
||||
nativeToolName: assignedToolName("kv_demo__slow_with_progress"),
|
||||
startedAt: "2026-11-13T10:00:00.000Z",
|
||||
updatedAt: "2026-11-13T10:00:01.000Z",
|
||||
progress: 1,
|
||||
total: 4,
|
||||
message: "Reading",
|
||||
done: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("treats provider text as untrusted: plain, single-line and bounded", () => {
|
||||
const { instance, published } = reporter();
|
||||
instance.report({ progress: 1, message: `<b>bold</b>\n\u0007line${"x".repeat(1_000)}` });
|
||||
const message = published[0]!.payload.message as string;
|
||||
expect(message.startsWith("<b>bold</b> line ")).toBe(true);
|
||||
expect(message).not.toMatch(/[\n\u0007]/);
|
||||
expect(message.length).toBe(TOOL_CALL_PROGRESS_MESSAGE_MAX_LENGTH);
|
||||
});
|
||||
|
||||
it("drops unusable and out-of-order updates", () => {
|
||||
const { instance, published } = reporter();
|
||||
instance.report({ progress: "1", total: 2 });
|
||||
instance.report({ progress: Number.NaN });
|
||||
instance.report({ progress: -1 });
|
||||
expect(published).toEqual([]);
|
||||
instance.report({ progress: 5, total: 2 });
|
||||
expect(published[0]!.payload).toMatchObject({ progress: 5, total: null });
|
||||
vi.advanceTimersByTime(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS);
|
||||
instance.report({ progress: 3, total: 10 });
|
||||
vi.advanceTimersByTime(TOOL_CALL_PROGRESS_PUBLISH_INTERVAL_MS);
|
||||
expect(published).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("on finish stops pending work and tells viewers to drop the progress", () => {
|
||||
const { instance, published, saved } = reporter();
|
||||
instance.report({ progress: 1, total: 3 });
|
||||
instance.report({ progress: 2, total: 3 });
|
||||
instance.finish();
|
||||
vi.advanceTimersByTime(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS * 3);
|
||||
expect(published.map((event) => [event.payload.progress, event.payload.done])).toEqual([[1, false], [2, true]]);
|
||||
expect(saved).toHaveLength(1);
|
||||
instance.report({ progress: 3, total: 3 });
|
||||
expect(published).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("a call that never reported progress finishes silently", () => {
|
||||
const { instance, published } = reporter();
|
||||
instance.finish();
|
||||
expect(published).toEqual([]);
|
||||
});
|
||||
|
||||
it("never throws into the call when publishing or saving fails", async () => {
|
||||
const instance = createToolCallProgressReporter({
|
||||
companyId: "c", runId: "r", issueId: null, agentId: null, invocationId: "i", toolName: "t",
|
||||
publish: () => { throw new Error("socket gone"); },
|
||||
save: async () => { throw new Error("db gone"); },
|
||||
});
|
||||
expect(() => instance.report({ progress: 1 })).not.toThrow();
|
||||
expect(() => instance.finish()).not.toThrow();
|
||||
await vi.runAllTimersAsync();
|
||||
});
|
||||
|
||||
it("keeps one save in flight, only the newest update waits, and finish drops it", async () => {
|
||||
let release!: () => void;
|
||||
const saves: number[] = [];
|
||||
const instance = createToolCallProgressReporter({
|
||||
companyId: "c", runId: "r", issueId: null, agentId: null, invocationId: "i", toolName: "t",
|
||||
publish: () => undefined,
|
||||
save: (progress) => { saves.push(progress.progress); return new Promise<void>((resolve) => { release = resolve; }); },
|
||||
});
|
||||
instance.report({ progress: 1 });
|
||||
// The database is slow: three more save intervals pass while the first write runs.
|
||||
for (let step = 2; step <= 4; step++) {
|
||||
vi.advanceTimersByTime(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS);
|
||||
instance.report({ progress: step });
|
||||
}
|
||||
vi.advanceTimersByTime(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS);
|
||||
expect(saves).toEqual([1]);
|
||||
release();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(saves).toEqual([1, 4]);
|
||||
instance.report({ progress: 5 });
|
||||
vi.advanceTimersByTime(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS);
|
||||
instance.finish();
|
||||
release();
|
||||
await vi.advanceTimersByTimeAsync(TOOL_CALL_PROGRESS_SAVE_INTERVAL_MS * 3);
|
||||
expect(saves).toEqual([1, 4]);
|
||||
});
|
||||
|
||||
it("names the tool the way native runners name it, so their rows can find the call", () => {
|
||||
const instance = createToolCallProgressReporter({
|
||||
companyId: "c", runId: "r", issueId: null, agentId: null, invocationId: "i",
|
||||
toolName: "mcp.kv-demo-1a2b3c4d:slow-with-progress",
|
||||
publish: (event) => { names.push(event.payload.nativeToolName); },
|
||||
});
|
||||
const names: unknown[] = [];
|
||||
instance.report({ progress: 1 });
|
||||
expect(names).toEqual(["app_mcp_kv_demo_1a2b3c4d_slow_with_progress_889fb9a8d98e"]);
|
||||
});
|
||||
|
||||
it("gives each call its own progress token", () => {
|
||||
expect(reporter().instance.token).not.toBe(reporter().instance.token);
|
||||
});
|
||||
});
|
||||
@@ -3297,7 +3297,7 @@ describeEmbeddedPostgres("tool gateway service", () => {
|
||||
const calls = provider.mock.calls.map(([, init]) => JSON.parse(String(init.body)))
|
||||
.filter((payload) => payload.method === "tools/call");
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0].params).toEqual({ name: "search_messages", arguments: parameters });
|
||||
expect(calls[0].params).toEqual({ name: "search_messages", arguments: parameters, _meta: { progressToken: expect.stringMatching(/^paperclip-progress-/) } });
|
||||
});
|
||||
|
||||
it("explains Google Workspace preview enrollment when a tool call is denied", async () => {
|
||||
|
||||
@@ -6,6 +6,9 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import express from "express";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import type { LiveEvent } from "@paperclipai/shared";
|
||||
import { subscribeCompanyLiveEvents } from "../services/live-events.js";
|
||||
import { assignedToolName as nativeAssignedToolName } from "../services/native-runtime/assigned-mcp-tools.js";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
@@ -3584,6 +3587,7 @@ rl.on("line", (line) => {
|
||||
"kv_get",
|
||||
"kv_list",
|
||||
"kv_set",
|
||||
"slow_with_progress",
|
||||
]);
|
||||
|
||||
await allowAllToolsForAgent(db, company.id, agent.id);
|
||||
@@ -3618,6 +3622,61 @@ rl.on("line", (line) => {
|
||||
}
|
||||
});
|
||||
|
||||
it("shows a run's viewers the provider's progress for a long call, saves it on the call, and returns the same result", async () => {
|
||||
const company = await createCompany(db);
|
||||
const other = await createCompany(db);
|
||||
const agent = await createAgent(db, company.id);
|
||||
const { issue, run } = await createIssueAndRun(db, company.id, agent.id);
|
||||
const kvDemo: KvDemoHttpServer = createKvDemoHttpServer();
|
||||
const port = await kvDemo.listen(0, "127.0.0.1");
|
||||
const events: LiveEvent[] = [];
|
||||
const otherEvents: LiveEvent[] = [];
|
||||
const stop = subscribeCompanyLiveEvents(company.id, (event) => { events.push(event); });
|
||||
const stopOther = subscribeCompanyLiveEvents(other.id, (event) => { otherEvents.push(event); });
|
||||
try {
|
||||
const access = toolAccessService(db);
|
||||
const connection = await access.createConnection(company.id, {
|
||||
name: "KV demo progress fixture", connectionPurpose: "tool", transport: "mcp_remote", authKind: "none",
|
||||
ownership: "customer", connectionKind: "managed",
|
||||
config: { url: `http://127.0.0.1:${port}/mcp` }, transportConfig: { url: `http://127.0.0.1:${port}/mcp` },
|
||||
credentialSecretRefs: [], enabled: true, status: "active",
|
||||
});
|
||||
await access.refreshCatalog(connection.id, { actorType: "user", actorId: "board" });
|
||||
await allowAllToolsForAgent(db, company.id, agent.id);
|
||||
const gateway = createTestToolGatewayService(db);
|
||||
const session = await gateway.createSession({ companyId: company.id, agentId: agent.id, runId: run.id, issueId: issue.id });
|
||||
const slowTool = (await gateway.listToolsForSession(session.token))
|
||||
.find((tool) => tool.upstreamToolName === "slow_with_progress");
|
||||
expect(slowTool).toBeTruthy();
|
||||
|
||||
const result = await gateway.executeTool({
|
||||
sessionToken: session.token,
|
||||
tool: slowTool!.name,
|
||||
parameters: { steps: 6, stepMs: 200 },
|
||||
});
|
||||
expect(result).toMatchObject({ status: "completed", result: { data: { isError: false, transport: "mcp_http" } } });
|
||||
|
||||
const progress = events.filter((event) => event.type === "heartbeat.run.tool_progress");
|
||||
// Throttled: six provider updates over ~1.2s, a handful of events, the last one marks the end.
|
||||
expect(progress.length).toBeGreaterThanOrEqual(3);
|
||||
expect(progress.length).toBeLessThanOrEqual(7);
|
||||
expect(progress.every((event) => event.companyId === company.id && event.payload.runId === run.id
|
||||
&& event.payload.issueId === issue.id && event.payload.toolName === slowTool!.name
|
||||
&& event.payload.nativeToolName === nativeAssignedToolName(slowTool!.name))).toBe(true);
|
||||
expect(progress[0]!.payload).toMatchObject({ progress: 1, total: 6, message: "Finished step 1 of 6", done: false });
|
||||
expect(progress.at(-1)!.payload).toMatchObject({ done: true });
|
||||
expect(otherEvents.filter((event) => event.type === "heartbeat.run.tool_progress")).toEqual([]);
|
||||
const [invocation] = await db.select().from(toolInvocations)
|
||||
.where(eq(toolInvocations.id, String(progress[0]!.payload.invocationId)));
|
||||
expect(invocation).toMatchObject({ companyId: company.id, runId: run.id, status: "succeeded" });
|
||||
expect(invocation!.progress).toMatchObject({ total: 6 });
|
||||
} finally {
|
||||
stop();
|
||||
stopOther();
|
||||
await kvDemo.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("expires legacy managed-connector approvals before provider dispatch", async () => {
|
||||
const company = await createCompany(db);
|
||||
const agent = await createAgent(db, company.id);
|
||||
|
||||
@@ -303,3 +303,15 @@ it("keeps a read set at 249 ms and replaces it at 250 ms", async () => {
|
||||
f.delivery.dispose();
|
||||
} finally { clock.mockRestore(); }
|
||||
});
|
||||
it("sends tool call progress only to clients watching that run", async () => {
|
||||
const f = fixture(); f.subscribe(["run-1"]);
|
||||
const progress = (id: number, runId: string): LiveEvent => ({ id, companyId: "company", type: "heartbeat.run.tool_progress",
|
||||
createdAt: "2026-11-13T00:00:00Z", payload: { runId, agentId: "agent", invocationId: `call-${id}`, progress: id, total: 10, message: "Exporting" } });
|
||||
f.delivery.enqueue(progress(1, "run-1")); f.delivery.enqueue(progress(2, "run-2")); await settle();
|
||||
const delivered = f.frames.filter(frame => frame.type === "heartbeat.run.tool_progress");
|
||||
expect(delivered.map(frame => frame.payload.runId)).toEqual(["run-1"]);
|
||||
// Another run's progress carries no content to this client.
|
||||
expect(JSON.stringify(f.frames)).not.toContain("call-2");
|
||||
expect(f.decorate).not.toHaveBeenCalled();
|
||||
f.delivery.dispose();
|
||||
});
|
||||
@@ -16,7 +16,10 @@ export function sharedDeliveryRead<T>(reads: LiveDeliveryReads | undefined, key:
|
||||
pending.catch(() => { if (reads.get(key) === pending) reads.delete(key); });
|
||||
return pending;
|
||||
}
|
||||
const transcript = (event: LiveEvent) => event.type === "heartbeat.run.event" || event.type === "heartbeat.run.log";
|
||||
// Per-run streams: sent only to clients that asked for the run. Tool progress
|
||||
// follows its run's transcript, since only an open transcript shows it.
|
||||
const transcript = (event: LiveEvent) => event.type === "heartbeat.run.event" || event.type === "heartbeat.run.log"
|
||||
|| event.type === "heartbeat.run.tool_progress";
|
||||
const runId = (event: LiveEvent) => typeof event.payload.runId === "string" ? event.payload.runId : null;
|
||||
interface Socket {
|
||||
readyState: number;
|
||||
|
||||
@@ -315,7 +315,7 @@ export function setupLiveEventsWebSocketServer(
|
||||
: payload.entityType === "issue" && typeof payload.entityId === "string" ? payload.entityId : null;
|
||||
const runId = typeof payload.runId === "string" ? payload.runId
|
||||
: payload.entityType === "heartbeat_run" && typeof payload.entityId === "string" ? payload.entityId : null;
|
||||
if (event.type === "heartbeat.run.event" || event.type === "heartbeat.run.log") {
|
||||
if (event.type === "heartbeat.run.event" || event.type === "heartbeat.run.log" || event.type === "heartbeat.run.tool_progress") {
|
||||
const allowed = await sharedDeliveryRead(reads, "transcript-scope", async () => (await access.decide({
|
||||
actor: context!.actor, action: "company_scope:read",
|
||||
resource: { type: "company", companyId: event.companyId } })).allowed);
|
||||
|
||||
@@ -15,6 +15,7 @@ import { assertBoard, getAccessibleResource } from "./authz.js";
|
||||
import { connectionRequestUserId } from "./connection-management-authz.js";
|
||||
import { authorizationService } from "../services/authorization.js";
|
||||
import { mcpAppViewsService, toMcpAppViewSummary } from "../services/mcp-app-views.js";
|
||||
import { listToolCallProgress } from "../services/tool-call-progress.js";
|
||||
import { ToolGatewayHttpError, type ToolGatewayService } from "../services/tool-gateway.js";
|
||||
|
||||
const uuidSchema = z.string().uuid();
|
||||
@@ -34,7 +35,7 @@ export function mcpAppsRoutes(db: Db, toolGateway: McpAppsGateway) {
|
||||
const router = Router();
|
||||
const views = mcpAppViewsService(db);
|
||||
|
||||
router.use("/issues/:issueId/mcp-app-views", async (req, res, next) => {
|
||||
const readableTask = async (req: Request, res: Response, next: () => void) => {
|
||||
assertBoard(req);
|
||||
if (!uuidSchema.safeParse(req.params.issueId).success) {
|
||||
res.status(404).json({ error: "Task not found" });
|
||||
@@ -57,7 +58,9 @@ export function mcpAppsRoutes(db: Db, toolGateway: McpAppsGateway) {
|
||||
res.locals.mcpAppIssue = issue;
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
next();
|
||||
});
|
||||
};
|
||||
router.use("/issues/:issueId/mcp-app-views", readableTask);
|
||||
router.use("/issues/:issueId/tool-call-progress", readableTask);
|
||||
|
||||
async function viewFor(req: Request, res: Response) {
|
||||
const issue = res.locals.mcpAppIssue as { id: string; companyId: string };
|
||||
@@ -162,6 +165,12 @@ export function mcpAppsRoutes(db: Db, toolGateway: McpAppsGateway) {
|
||||
res.status(201).json({ commentId: comment.id });
|
||||
});
|
||||
|
||||
// The latest progress of the task's running app tool calls (and recently finished ones, as done), for a reload mid-call.
|
||||
router.get("/issues/:issueId/tool-call-progress", async (_req, res) => {
|
||||
const issue = res.locals.mcpAppIssue as { id: string; companyId: string };
|
||||
res.json(await listToolCallProgress(db, issue.companyId, issue.id));
|
||||
});
|
||||
|
||||
router.get("/issues/:issueId/mcp-app-views", async (req, res) => {
|
||||
const issue = res.locals.mcpAppIssue as { id: string; companyId: string };
|
||||
res.json(await views.listForIssue(issue.companyId, issue.id));
|
||||
|
||||
@@ -291,6 +291,9 @@ import {
|
||||
workspaceFileResourceQuerySchema,
|
||||
// Tool access
|
||||
connectToolAppSchema,
|
||||
createMachineSignInSchema,
|
||||
testMachineSignInSchema,
|
||||
rotateMachineSignInSchema,
|
||||
configureRailwaySshSchema,
|
||||
createToolApplicationSchema,
|
||||
updateToolApplicationSchema,
|
||||
@@ -1447,6 +1450,7 @@ const browserUseOperations = [
|
||||
// MCP Apps views in the task chat. Board only; reads follow task read access.
|
||||
const mcpAppsOperations = [
|
||||
["get", "/api/issues/{issueId}/mcp-app-views", "List a task's MCP Apps views", undefined],
|
||||
["get", "/api/issues/{issueId}/tool-call-progress", "Read the latest progress of a task's running app tool calls", undefined],
|
||||
["get", "/api/issues/{issueId}/mcp-app-views/{viewId}", "Read an MCP Apps view with its document", undefined],
|
||||
["post", "/api/issues/{issueId}/mcp-app-views/{viewId}/tool-calls", "Call a tool from an MCP Apps view", mcpAppToolCallSchema],
|
||||
["put", "/api/issues/{issueId}/mcp-app-views/{viewId}/model-context", "Store an MCP Apps view's context for the task's next run", mcpAppModelContextUpdateSchema],
|
||||
@@ -12544,6 +12548,68 @@ registerCurrentRoute({
|
||||
summary: "Archive a tool connection",
|
||||
});
|
||||
|
||||
const MACHINE_SIGN_IN_ERRORS = {
|
||||
400: r.badRequest,
|
||||
401: r.unauthorized,
|
||||
403: r.forbidden,
|
||||
404: r.notFound,
|
||||
422: r.unprocessable,
|
||||
};
|
||||
|
||||
registerCurrentRoute({
|
||||
"x-paperclip-transport": "rest",
|
||||
"x-paperclip-connection-management": "connections:configure",
|
||||
method: "post",
|
||||
path: "/api/companies/{companyId}/tools/machine-sign-in",
|
||||
tags: ["tool-access"],
|
||||
summary: "Start machine sign-in (OAuth client credentials) for an organization connection",
|
||||
description: "Creates a draft organization connection to a remote MCP server that signs in as a registered client, with no person involved. With method paperclip_key, Paperclip creates a signing key, keeps the private half in the company vault, and returns the public key to register with the provider. Secrets are write-only.",
|
||||
body: createMachineSignInSchema,
|
||||
responses: { 201: r.ok(), ...MACHINE_SIGN_IN_ERRORS },
|
||||
});
|
||||
|
||||
registerCurrentRoute({
|
||||
"x-paperclip-transport": "rest",
|
||||
"x-paperclip-connection-management": "connections:configure",
|
||||
method: "get",
|
||||
path: "/api/tool-connections/{connectionId}/machine-sign-in",
|
||||
tags: ["tool-access"],
|
||||
summary: "Read a connection's machine sign-in settings (no secret values)",
|
||||
responses: { 200: r.ok(), ...MACHINE_SIGN_IN_ERRORS },
|
||||
});
|
||||
|
||||
registerCurrentRoute({
|
||||
"x-paperclip-transport": "rest",
|
||||
"x-paperclip-connection-management": "connections:configure",
|
||||
method: "post",
|
||||
path: "/api/tool-connections/{connectionId}/machine-sign-in/test",
|
||||
tags: ["tool-access"],
|
||||
summary: "Test machine sign-in against the provider and server, then turn the connection on",
|
||||
body: testMachineSignInSchema,
|
||||
responses: { 200: r.ok(), ...MACHINE_SIGN_IN_ERRORS },
|
||||
});
|
||||
|
||||
registerCurrentRoute({
|
||||
"x-paperclip-transport": "rest",
|
||||
"x-paperclip-connection-management": "connections:configure",
|
||||
method: "post",
|
||||
path: "/api/tool-connections/{connectionId}/machine-sign-in/rotate",
|
||||
tags: ["tool-access"],
|
||||
summary: "Replace the machine sign-in key or client secret without downtime",
|
||||
body: rotateMachineSignInSchema,
|
||||
responses: { 200: r.ok(), ...MACHINE_SIGN_IN_ERRORS },
|
||||
});
|
||||
|
||||
registerCurrentRoute({
|
||||
"x-paperclip-transport": "rest",
|
||||
"x-paperclip-connection-management": "connections:configure",
|
||||
method: "post",
|
||||
path: "/api/tool-connections/{connectionId}/machine-sign-in/rotate/complete",
|
||||
tags: ["tool-access"],
|
||||
summary: "Switch to the new machine sign-in key once the provider accepts it",
|
||||
responses: { 200: r.ok(), ...MACHINE_SIGN_IN_ERRORS },
|
||||
});
|
||||
|
||||
registerCurrentRoute({
|
||||
"x-paperclip-transport": "rest",
|
||||
"x-paperclip-connection-management": "connections:configure",
|
||||
|
||||
@@ -23,6 +23,10 @@ import {
|
||||
type ToolConnection,
|
||||
type ToolConnectionCreateCapabilities,
|
||||
connectToolAppSchema,
|
||||
createMachineSignInSchema,
|
||||
testMachineSignInSchema,
|
||||
rotateMachineSignInSchema,
|
||||
type MachineSignInStatus,
|
||||
configureRailwaySshSchema,
|
||||
createConnectionGrantDelegationSchema,
|
||||
createToolStdioCommandTemplateSchema,
|
||||
@@ -105,6 +109,21 @@ const ORGANIZATION_GRANT_DENIAL_REASON =
|
||||
type Heartbeat = ReturnType<typeof heartbeatService>;
|
||||
|
||||
/** Allowlist (e.g. Google Sheets allowed spreadsheet ids) lives in connection config. */
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
/** Activity for machine sign-in carries identifiers only: never a secret, key or token. */
|
||||
function machineSignInActivityDetails(status: MachineSignInStatus) {
|
||||
return {
|
||||
method: status.method,
|
||||
issuer: status.issuer,
|
||||
keyId: status.publicKey?.keyId ?? null,
|
||||
pendingKeyId: status.pendingPublicKey?.keyId ?? null,
|
||||
scopeCount: status.scopes.length,
|
||||
};
|
||||
}
|
||||
|
||||
function allowlistIds(config: Record<string, unknown> | null | undefined): string[] {
|
||||
const raw = config?.allowedSpreadsheetIds;
|
||||
if (!Array.isArray(raw)) return [];
|
||||
@@ -1005,6 +1024,87 @@ function connectorEnrollmentPrincipal(req: Request): string {
|
||||
}
|
||||
});
|
||||
|
||||
// Machine sign-in (MCP OAuth client credentials): organization connections
|
||||
// that authenticate as a registered client, with no person signing in.
|
||||
router.post("/companies/:companyId/tools/machine-sign-in", managementGuard("connections:configure", "POST /companies/:companyId/tools/machine-sign-in"), validate(createMachineSignInSchema), async (req, res) => {
|
||||
const companyId = req.params.companyId as string;
|
||||
assertToolAppMutationAccess(req, companyId);
|
||||
if (!await isToolConnectionManagerQuiet(req, companyId)) throw forbidden(ORGANIZATION_GRANT_DENIAL_REASON);
|
||||
try {
|
||||
const result = await svc.createMachineSignIn(companyId, req.body, connectionRequestActor(req));
|
||||
await logActivity(db, {
|
||||
companyId,
|
||||
...connectionRequestActor(req),
|
||||
action: "tool_connection.machine_sign_in_created",
|
||||
entityType: "tool_connection",
|
||||
entityId: result.connectionId,
|
||||
details: machineSignInActivityDetails(result.machineSignIn),
|
||||
});
|
||||
res.status(201).json(result);
|
||||
} catch (error) {
|
||||
svc.ensureNoDuplicateNameError(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/tool-connections/:connectionId/machine-sign-in", managementGuard("connections:configure", "GET /tool-connections/:connectionId/machine-sign-in"), async (req, res) => {
|
||||
const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found");
|
||||
if (!existing) return;
|
||||
await assertToolConnectionConfigureAccess(req, existing);
|
||||
res.json(await svc.getMachineSignIn(existing.companyId, existing.id));
|
||||
});
|
||||
|
||||
router.post("/tool-connections/:connectionId/machine-sign-in/test", managementGuard("connections:configure", "POST /tool-connections/:connectionId/machine-sign-in/test"), validate(testMachineSignInSchema), async (req, res) => {
|
||||
const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found");
|
||||
if (!existing) return;
|
||||
await assertToolConnectionConfigureAccess(req, existing);
|
||||
const result = await svc.testMachineSignIn(existing.companyId, existing.id, req.body, connectionRequestActor(req));
|
||||
await logActivity(db, {
|
||||
companyId: existing.companyId,
|
||||
...connectionRequestActor(req),
|
||||
action: "tool_connection.machine_sign_in_verified",
|
||||
entityType: "tool_connection",
|
||||
entityId: existing.id,
|
||||
details: {
|
||||
...machineSignInActivityDetails(await svc.getMachineSignIn(existing.companyId, existing.id)),
|
||||
clientSecretReplaced: Boolean(req.body.clientSecret),
|
||||
catalogEntryCount: result.catalog.length,
|
||||
},
|
||||
});
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
router.post("/tool-connections/:connectionId/machine-sign-in/rotate", managementGuard("connections:configure", "POST /tool-connections/:connectionId/machine-sign-in/rotate"), validate(rotateMachineSignInSchema), async (req, res) => {
|
||||
const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found");
|
||||
if (!existing) return;
|
||||
await assertToolConnectionConfigureAccess(req, existing);
|
||||
const result = await svc.rotateMachineSignIn(existing.companyId, existing.id, req.body, connectionRequestActor(req));
|
||||
await logActivity(db, {
|
||||
companyId: existing.companyId,
|
||||
...connectionRequestActor(req),
|
||||
action: result.completed ? "tool_connection.machine_sign_in_rotated" : "tool_connection.machine_sign_in_rotation_started",
|
||||
entityType: "tool_connection",
|
||||
entityId: existing.id,
|
||||
details: machineSignInActivityDetails(result.machineSignIn),
|
||||
});
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
router.post("/tool-connections/:connectionId/machine-sign-in/rotate/complete", managementGuard("connections:configure", "POST /tool-connections/:connectionId/machine-sign-in/rotate/complete"), async (req, res) => {
|
||||
const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found");
|
||||
if (!existing) return;
|
||||
await assertToolConnectionConfigureAccess(req, existing);
|
||||
const result = await svc.completeMachineSignInRotation(existing.companyId, existing.id, connectionRequestActor(req));
|
||||
await logActivity(db, {
|
||||
companyId: existing.companyId,
|
||||
...connectionRequestActor(req),
|
||||
action: "tool_connection.machine_sign_in_rotated",
|
||||
entityType: "tool_connection",
|
||||
entityId: existing.id,
|
||||
details: machineSignInActivityDetails(result.machineSignIn),
|
||||
});
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
router.post(
|
||||
"/companies/:companyId/tools/connections/:connectionId/start-authorization", managementGuard("connections:configure", "POST /companies/:companyId/tools/connections/:connectionId/start-authorization", { personal: true }),
|
||||
validate(startConnectionAuthorizationSchema),
|
||||
@@ -2286,7 +2386,11 @@ function connectorEnrollmentPrincipal(req: Request): string {
|
||||
action: "tool_connection.archived",
|
||||
entityType: "tool_connection",
|
||||
entityId: connection.id,
|
||||
details: { transport: connection.transport, ...removal },
|
||||
details: {
|
||||
transport: connection.transport,
|
||||
...(asRecord(asRecord(existing.config).oauth).grantType === "client_credentials" ? { machineSignIn: true } : {}),
|
||||
...removal,
|
||||
},
|
||||
});
|
||||
if (applicationBefore.status !== "archived" && applicationAfter.status === "archived") {
|
||||
await logActivity(db, {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, eq, inArray, ne, notInArray, or, sql } from "drizzle-orm";
|
||||
import { and, eq, inArray, ne, notInArray, or, sql, type SQL } from "drizzle-orm";
|
||||
import { environmentLeases, heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db";
|
||||
import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
import { nativeWorkspaceExportPendingForLease } from "./native-runtime/native-workspace-finalization-state.js";
|
||||
@@ -97,7 +97,7 @@ function orphanedRunSandboxCondition() {
|
||||
* `heldBy`, so a held orphan is not listed (and not polled) until its last
|
||||
* reference clears or its retention window ends. The settle re-checks all of
|
||||
* them, plus the pending native export, under the row lock. */
|
||||
function softlyHeldSql(now?: Date) {
|
||||
export function softlyHeldSql(now?: Date) {
|
||||
const at = now ? sql`${now.toISOString()}::timestamptz` : sql`now()`;
|
||||
const issue = sql`coalesce(run.native_issue_id, run.issue_id, ${environmentLeases.issueId})`;
|
||||
return sql`(${environmentLeases.expiresAt} is null or ${environmentLeases.expiresAt} > ${at}) and (
|
||||
@@ -116,8 +116,7 @@ function alwaysHeldSql() {
|
||||
or ${abandonedWorkspaceFilesHeldOnLeaseRowSql()}
|
||||
or coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') = 'true'
|
||||
or coalesce(${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,state}' = 'pending', false)
|
||||
or exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId}
|
||||
and run.status not in ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted'))
|
||||
or coalesce((select run.status not in ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')
|
||||
-- A prepared native workspace bound to this exact sandbox, for a run with
|
||||
-- workspace access. This is deliberately no wider than the settle's own
|
||||
-- export check: an orphan the settle would release must never be hidden
|
||||
@@ -125,21 +124,23 @@ function alwaysHeldSql() {
|
||||
-- are listed and held by the settle until their export settles.
|
||||
-- The descriptor must also be one the settle's reader accepts
|
||||
-- (readNativeWorkspaceSyncReference); an invalid one holds nothing.
|
||||
or exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId}
|
||||
and run.runtime_mode = 'native'
|
||||
and run.runner_profile_json#>>'{nativeWorkspaceSync,schema}' in
|
||||
('paperclip.native-workspace-sync/v1', 'paperclip.native-workspace-sync/v2', 'paperclip.native-workspace-sync/v3')
|
||||
and run.runner_profile_json#>>'{nativeWorkspaceSync,state}' = 'prepared'
|
||||
and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,descriptorSha256}' ~ '^[0-9a-f]{64}$', false)
|
||||
and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,baselineSha256}' ~ '^[0-9a-f]{64}$', false)
|
||||
and coalesce(jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,finalHostSha256}'), 'null') = 'null'
|
||||
and jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,workspaceId}') = 'string'
|
||||
and jsonb_typeof(run.runner_profile_json#>'{nativeWorkspaceSync,remoteCwd}') = 'string'
|
||||
and coalesce(run.runner_profile_json#>>'{nativeWorkspaceSync,resourceDisposition}', 'destroy')
|
||||
in ('keep_running', 'stop_and_retain', 'destroy')
|
||||
and run.runner_profile_json#>>'{nativeWorkspaceSync,leaseId}' = ${environmentLeases.id}::text
|
||||
and run.runner_profile_json#>>'{nativeWorkspaceSync,providerLeaseId}' = ${environmentLeases.providerLeaseId}
|
||||
and coalesce(run.runner_profile_json#>>'{nativeExecutionInput,workspace,access}', '') <> 'none')
|
||||
or (run.runtime_mode = 'native'
|
||||
and profile.sync->>'schema' in
|
||||
('paperclip.native-workspace-sync/v1', 'paperclip.native-workspace-sync/v2', 'paperclip.native-workspace-sync/v3')
|
||||
and profile.sync->>'state' = 'prepared'
|
||||
and coalesce(profile.sync->>'descriptorSha256' ~ '^[0-9a-f]{64}$', false)
|
||||
and coalesce(profile.sync->>'baselineSha256' ~ '^[0-9a-f]{64}$', false)
|
||||
and coalesce(jsonb_typeof(profile.sync->'finalHostSha256'), 'null') = 'null'
|
||||
and jsonb_typeof(profile.sync->'workspaceId') = 'string'
|
||||
and jsonb_typeof(profile.sync->'remoteCwd') = 'string'
|
||||
and coalesce(profile.sync->>'resourceDisposition', 'destroy') in ('keep_running', 'stop_and_retain', 'destroy')
|
||||
and profile.sync->>'leaseId' = ${environmentLeases.id}::text
|
||||
and profile.sync->>'providerLeaseId' = ${environmentLeases.providerLeaseId}
|
||||
and coalesce(profile.access, '') <> 'none')
|
||||
from heartbeat_runs run
|
||||
cross join lateral (select run.runner_profile_json->'nativeWorkspaceSync' as sync,
|
||||
run.runner_profile_json#>>'{nativeExecutionInput,workspace,access}' as access offset 0) profile
|
||||
where run.id = ${environmentLeases.heartbeatRunId}), false)
|
||||
or exists (select 1 from environment_leases other where other.id <> ${environmentLeases.id}
|
||||
and other.provider = ${environmentLeases.provider} and other.provider_lease_id = ${environmentLeases.providerLeaseId}
|
||||
and other.status in ('active', 'retained', 'pending_cleanup')))`;
|
||||
@@ -148,15 +149,31 @@ function alwaysHeldSql() {
|
||||
/** Retained orphans that nothing appears to reference now. */
|
||||
// The listing is built once, so it reads the database clock.
|
||||
export function releasableRetainedOrphanCondition(now?: Date) {
|
||||
return and(retainedOrphanSandboxCondition(), sql`not ${alwaysHeldSql()}`, sql`not (${softlyHeldSql(now)})`);
|
||||
return onlyForRetainedOrphans(sql`not ${alwaysHeldSql()} and not (${softlyHeldSql(now)})`);
|
||||
}
|
||||
|
||||
function retainedOrphanCandidateCondition() {
|
||||
return and(eq(environmentLeases.cleanupStatus, "success"), sql`${environmentLeases.heartbeatRunId} is not null`,
|
||||
sql`${environmentLeases.provider} is not null`, sql`${environmentLeases.providerLeaseId} is not null`,
|
||||
or(and(eq(environmentLeases.status, "released"), eq(environmentLeases.leasePolicy, "ephemeral"),
|
||||
sql`${environmentLeases.metadata} ? ${ORPHANED_SANDBOX_RECOVERY_KEY}`,
|
||||
sql`${environmentLeases.metadata} ? 'sandboxStopAndRetainReceipt'`),
|
||||
and(eq(environmentLeases.leasePolicy, "reuse_by_environment"), inArray(environmentLeases.status, ["released", "failed"]),
|
||||
sql`coalesce(${environmentLeases.metadata}#>>'{remoteExecutionTermination,state}' = 'stopped', false)`),
|
||||
and(eq(environmentLeases.status, "retained"), eq(environmentLeases.leasePolicy, "reuse_by_environment"),
|
||||
sql`${environmentLeases.executionWorkspaceId} is not null`)));
|
||||
}
|
||||
|
||||
function onlyForRetainedOrphans(condition: SQL, candidate: SQL | undefined = retainedOrphanSandboxCondition()) {
|
||||
return and(retainedOrphanCandidateCondition(), sql`(case when ${candidate} then ${condition} else false end)`);
|
||||
}
|
||||
|
||||
/** When the next retained orphan's retention window ends while only soft
|
||||
* references hold it; the reconciler wakes then to release it. */
|
||||
export async function nextRetainedOrphanDueAt(db: Db, now: Date): Promise<Date | null> {
|
||||
const [row] = await db.select({ due: sql<string | null>`min(${environmentLeases.expiresAt})` }).from(environmentLeases)
|
||||
.where(and(retainedOrphanSandboxCondition(), sql`not ${alwaysHeldSql()}`, sql`${softlyHeldSql(now)}`,
|
||||
sql`${environmentLeases.expiresAt} is not null`));
|
||||
.where(onlyForRetainedOrphans(sql`not ${alwaysHeldSql()} and ${softlyHeldSql(now)}`,
|
||||
and(retainedOrphanSandboxCondition(), sql`${environmentLeases.expiresAt} is not null`)));
|
||||
return row?.due ? new Date(row.due) : null;
|
||||
}
|
||||
|
||||
|
||||
@@ -80,6 +80,44 @@ const minutesAgo = (minutes: number) => new Date(now.getTime() - minutes * 60_00
|
||||
});
|
||||
});
|
||||
|
||||
it("names the routine's open execution that holds a queued routine task, before the agent's limit", async () => {
|
||||
const f = await fixture({ maxConcurrentRuns: 20, running: 0 });
|
||||
const routineId = randomUUID();
|
||||
const holderIssueId = randomUUID(), holderRunId = randomUUID();
|
||||
const waitingIssueId = randomUUID(), waitingRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: holderRunId, companyId: f.companyId, agentId: f.agentId, status: "running",
|
||||
startedAt: minutesAgo(18), createdAt: minutesAgo(30), issueId: holderIssueId, contextSnapshot: { issueId: holderIssueId } });
|
||||
await db.insert(issues).values([
|
||||
{ id: holderIssueId, companyId: f.companyId, identifier: "Q-3093", title: "Nightly test gardening", status: "in_progress",
|
||||
assigneeAgentId: f.agentId, originKind: "routine_execution", originId: routineId, executionRunId: holderRunId },
|
||||
{ id: waitingIssueId, companyId: f.companyId, identifier: "Q-3636", title: "Nightly test gardening", status: "in_progress",
|
||||
assigneeAgentId: f.agentId, originKind: "routine_execution", originId: routineId },
|
||||
]);
|
||||
await db.insert(heartbeatRuns).values({ id: waitingRunId, companyId: f.companyId, agentId: f.agentId, status: "queued",
|
||||
createdAt: minutesAgo(8), issueId: waitingIssueId, contextSnapshot: { issueId: waitingIssueId } });
|
||||
|
||||
expect((await executionProjectionsForRuns(db, f.companyId, [waitingRunId], now)).get(waitingRunId)).toMatchObject({
|
||||
phase: "queued",
|
||||
label: "Waiting for Q-3093 to finish",
|
||||
cause: "routine_execution_in_progress",
|
||||
});
|
||||
// The holder's own run is not waiting on itself.
|
||||
expect((await executionProjectionsForRuns(db, f.companyId, [holderRunId], now)).get(holderRunId)?.cause).not.toBe("routine_execution_in_progress");
|
||||
|
||||
// A private holder is not named to readers of the waiting task.
|
||||
await db.update(issues).set({ visibility: "private" }).where(eq(issues.id, holderIssueId));
|
||||
const privateHold = (await executionProjectionsForRuns(db, f.companyId, [waitingRunId], now)).get(waitingRunId);
|
||||
expect(privateHold).toMatchObject({ label: "Waiting for another task in this routine to finish", cause: "routine_execution_in_progress" });
|
||||
expect(JSON.stringify(privateHold)).not.toContain("Q-3093");
|
||||
await db.update(issues).set({ visibility: "open" }).where(eq(issues.id, holderIssueId));
|
||||
|
||||
// Once the holder closes, the routine slot is free and the run is plainly queued.
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, holderIssueId));
|
||||
expect((await executionProjectionsForRuns(db, f.companyId, [waitingRunId], now)).get(waitingRunId)).toMatchObject({
|
||||
label: "Queued", cause: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves out queued runs that a preview worktree's cutoff keeps from starting", async () => {
|
||||
const f = await fixture({ maxConcurrentRuns: 1, running: 1 });
|
||||
const copiedOld = await f.queue({ priority: "critical", createdMinutesAgo: 120 });
|
||||
|
||||
@@ -24,6 +24,7 @@ import { isAgentAwaitingSetup } from "../modules/agent-lifecycle/index.js";
|
||||
import { NATIVE_RESPONSE_WAKE_WAITING_REASON } from "./native-runtime/status-arbiter.js";
|
||||
import { agentQueueLinesForRuns } from "./heartbeat/queued-run-line.js";
|
||||
const MAX_PREDECESSOR_CLEANUP_CHECKS = 16;
|
||||
const ANOTHER_ROUTINE_TASK = "another task in this routine";
|
||||
const text = (v: unknown) => (typeof v === "string" ? v : null);
|
||||
const executionRunColumns = {
|
||||
id: heartbeatRuns.id,
|
||||
@@ -211,6 +212,32 @@ export async function executionProjectionsForRuns(
|
||||
: new Map<string, NonNullable<ExecutionProjection["agentQueue"]>>();
|
||||
const issueIdOf = (run: Run) => run.nativeIssueId ?? text(run.contextSnapshot?.issueId);
|
||||
const queuedIssueIds = [...new Set(queuedRuns.map(issueIdOf).filter((id): id is string => !!id))];
|
||||
// A routine runs one execution at a time (issues_open_routine_execution_uq):
|
||||
// a never-started run of a routine task waits while another open task of
|
||||
// the same routine holds an execution. One read for the batch.
|
||||
const routineHolderByIssue = new Map<string, string>();
|
||||
const unstartedIssueIds = [...new Set(queuedRuns.filter(run => !run.startedAt).map(issueIdOf).filter((id): id is string => !!id))];
|
||||
if (unstartedIssueIds.length) {
|
||||
const holders = await read(async () => db.execute<{ issueId: string; holderIdentifier: string | null }>(sql`
|
||||
-- A private holder is never named: this projection has no reader to check access for.
|
||||
select waiting.id as "issueId", case when holder.visibility = 'open' then holder.identifier end as "holderIdentifier"
|
||||
from ${issues} waiting
|
||||
join ${issues} holder on holder.company_id = waiting.company_id
|
||||
and holder.origin_kind = 'routine_execution'
|
||||
and holder.origin_id = waiting.origin_id
|
||||
and holder.origin_fingerprint is not distinct from waiting.origin_fingerprint
|
||||
and holder.id <> waiting.id
|
||||
and holder.hidden_at is null
|
||||
and holder.execution_run_id is not null
|
||||
and holder.status in ('backlog', 'todo', 'in_progress', 'in_review', 'blocked')
|
||||
where waiting.company_id = ${companyId}
|
||||
and waiting.origin_kind = 'routine_execution'
|
||||
and waiting.origin_id is not null
|
||||
and waiting.id in (${sql.join(unstartedIssueIds.map(id => sql`${id}::uuid`), sql`, `)})`));
|
||||
for (const row of [...holders]) {
|
||||
if (!routineHolderByIssue.has(row.issueId)) routineHolderByIssue.set(row.issueId, row.holderIdentifier ?? ANOTHER_ROUTINE_TASK);
|
||||
}
|
||||
}
|
||||
if (queuedIssueIds.length) {
|
||||
const issueKey = sql<string>`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.contextSnapshot}->>'issueId')`;
|
||||
// Two set-based reads for the whole batch, then one bounded, parallel
|
||||
@@ -285,6 +312,7 @@ export async function executionProjectionsForRuns(
|
||||
setupByAgent.get(run.agentId),
|
||||
currentResponseWait(run),
|
||||
agentQueueByRun.get(run.id),
|
||||
run.status === "queued" && !run.startedAt ? routineHolderByIssue.get(issueIdOf(run) ?? "") : undefined,
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -313,6 +341,8 @@ export function projectExecution(
|
||||
setupAgent?: { lifecycleState: string; lifecycleHolds: readonly string[]; retryRequired?: boolean },
|
||||
responseWaitCurrent = false,
|
||||
agentQueue?: ExecutionProjection["agentQueue"],
|
||||
/** The open task of the same routine that holds the routine's one execution. */
|
||||
routineHolder?: string,
|
||||
): ExecutionProjection {
|
||||
const detail = coordinator?.failureDetail ?? {};
|
||||
const successorRunId = text(detail.successorRunId);
|
||||
@@ -561,6 +591,12 @@ export function projectExecution(
|
||||
projection.nextAction = "The previous run on this task is still releasing its runner. This turn starts automatically when it settles.";
|
||||
return set("queued", "Waiting for previous run to stop");
|
||||
}
|
||||
// Checked before the agent's limit: a free slot would not start this run.
|
||||
if (run.status === "queued" && !run.startedAt && routineHolder) {
|
||||
projection.cause = "routine_execution_in_progress";
|
||||
projection.nextAction = `This routine runs one task at a time and ${routineHolder === ANOTHER_ROUTINE_TASK ? "another of its tasks" : routineHolder} is still open with its run. This task starts automatically when that one finishes.`;
|
||||
return set("queued", `Waiting for ${routineHolder} to finish`);
|
||||
}
|
||||
if (run.status === "queued" && !run.startedAt && agentQueue) {
|
||||
projection.cause = "agent_at_capacity";
|
||||
projection.agentQueue = agentQueue;
|
||||
|
||||
@@ -39,10 +39,13 @@ import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { readSystemRecovery } from "./system-recovery-state.js";
|
||||
import { isReattachPhaseActive } from "./heartbeat/run-admission.js";
|
||||
import { provenGoneLegacyControllerBootId } from "./legacy-controller-lease.js";
|
||||
import { reserveAutomaticRunCreation } from "./recovery/automatic-run-brake.js";
|
||||
import { recordContinuationDeliveryAttempt } from "./recovery/continuation-delivery-attempt.js";
|
||||
import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck,
|
||||
settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js";
|
||||
import { alias } from "drizzle-orm/pg-core";
|
||||
import { RECOVERY_BACKLOG_AGE_MS, RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js";
|
||||
import { agentRecoveryQueueIsFull } from "./recovery-queue-admission.js";
|
||||
import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js";
|
||||
|
||||
import { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_BACKLOG_SHARE_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS,
|
||||
@@ -394,6 +397,7 @@ export async function deliverReconciledExecutions(
|
||||
] : [
|
||||
sql`${pastBound} desc`,
|
||||
sql`${overdueTurn} asc nulls last`,
|
||||
sql`${attemptedAt} asc nulls first`,
|
||||
sql`${sourceRun.finishedAt} desc nulls last`,
|
||||
sql`${issueRecoveryActions.updatedAt} asc`,
|
||||
];
|
||||
@@ -444,23 +448,19 @@ export async function deliverReconciledExecutions(
|
||||
] : [
|
||||
sql`${candidates.pastBound} desc`,
|
||||
sql`${candidates.heldAt} asc nulls last`,
|
||||
sql`${candidates.attemptedAt} asc nulls first`,
|
||||
sql`${candidates.sourceFinishedAt} desc nulls last`,
|
||||
sql`${candidates.actionUpdatedAt} asc`,
|
||||
]))
|
||||
.limit(25);
|
||||
return rows.map(row => row.action);
|
||||
};
|
||||
const deferRefusedContinuation = (action: typeof issueRecoveryActions.$inferSelect) => db.update(issueRecoveryActions)
|
||||
.set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify(
|
||||
{ continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` })
|
||||
.where(and(
|
||||
eq(issueRecoveryActions.companyId, action.companyId),
|
||||
eq(issueRecoveryActions.id, action.id),
|
||||
eq(issueRecoveryActions.status, "resolved"),
|
||||
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`,
|
||||
));
|
||||
const deferRefusedContinuation = (action: typeof issueRecoveryActions.$inferSelect) => recordContinuationDeliveryAttempt(db, action);
|
||||
const agentHasRoom = async (action: typeof issueRecoveryActions.$inferSelect, agentId: string, kind: ContinuationClass) => {
|
||||
if ((admittedByCompany.get(action.companyId) ?? 0) >= companyLimit) return false;
|
||||
// A few at a time per agent, fresh or not: the rest waits here, durably,
|
||||
// until a queued recovery run starts or is dropped.
|
||||
if (await agentRecoveryQueueIsFull(db, action.companyId, agentId)) return false;
|
||||
if (kind === "fresh") return true;
|
||||
const [agent] = await db.select().from(agents).where(and(eq(agents.companyId, action.companyId), eq(agents.id, agentId)));
|
||||
if (!agent) return true;
|
||||
@@ -549,6 +549,8 @@ export async function deliverReconciledExecutions(
|
||||
seenAgents.add(action.returnOwnerAgentId);
|
||||
continue;
|
||||
}
|
||||
const slot = await reserveAutomaticRunCreation(db, action.companyId, { issueId: task.id, sourceRunId: decision.runId });
|
||||
if (slot.heldUntil) { await deferRefusedContinuation(action); continue; }
|
||||
const run = await wake(action.returnOwnerAgentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
@@ -569,9 +571,10 @@ export async function deliverReconciledExecutions(
|
||||
...((action.evidence.freshContinuation as { isolation?: unknown } | undefined)?.isolation === "fresh_isolated_workspace"
|
||||
? { freshContinuationIsolation: action.id } : {}),
|
||||
},
|
||||
});
|
||||
}).catch(async error => { await slot.release(); throw error; });
|
||||
seenAgents.add(action.returnOwnerAgentId);
|
||||
if (!run) {
|
||||
await slot.release();
|
||||
await deferRefusedContinuation(action);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import { boundRunIssueIdSql } from "./run-task-binding.js";
|
||||
import { and, asc, eq, getTableColumns, is, sql } from "drizzle-orm";
|
||||
import { PgTransaction } from "drizzle-orm/pg-core";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db";
|
||||
import { heartbeatRunEvents, heartbeatRuns, primeDatabaseWorkTransactionId } from "@paperclipai/db";
|
||||
import { nativeSha256 } from "./native-runtime/canonical.js";
|
||||
import { isUniqueViolation } from "../db-errors.js";
|
||||
|
||||
@@ -345,9 +345,9 @@ function decodeRunEventRow(raw: Record<string, unknown>): typeof heartbeatRunEve
|
||||
/**
|
||||
* A native source event: one lock read, then one statement that finds a
|
||||
* replayed event or reserves the sequence and inserts, and reads how much of
|
||||
* the source is retained. The lock must stay a statement of its own. A
|
||||
* statement that waits for the run lock keeps the snapshot it took before
|
||||
* waiting, so it would not see an event the lock holder had just committed.
|
||||
* the source is retained. A statement that waits for the run lock keeps the
|
||||
* snapshot it took before waiting, so it would not see an event the lock
|
||||
* holder had just committed.
|
||||
*/
|
||||
async function appendNativeSourceEvent(
|
||||
tx: Db,
|
||||
@@ -361,15 +361,26 @@ async function appendNativeSourceEvent(
|
||||
// checkpoints, that are written for the same run while events stream.
|
||||
// Only progress needs the task binding. Other native/recovery events
|
||||
// must not read a potentially large context snapshot to append a log.
|
||||
const replayed = sql`run_id = ${input.runId} and (source_event_id = ${source.sourceEventId}
|
||||
or (source_instance_id = ${source.sourceInstanceId} and source_seq = ${source.sourceSeq}))`;
|
||||
const probes = wakes.reassessment || wakes.progress || wakes.response || wakes.retention;
|
||||
const columns = sql`"company_id", "agent_id", ("native_source_cursors" ->> ${source.sourceInstanceId})::bigint as "source_cursor"${
|
||||
wakes.progress || wakes.response || wakes.reassessment ? sql`, "runtime_mode"` : sql.empty()}${
|
||||
wakes.progress ? sql`, ${boundRunIssueIdSql()} as "issue_id"` : sql.empty()}${
|
||||
Object.values(wakes).some(Boolean) ? sql`, pg_current_xact_id()::text as "xid"` : sql.empty()}`;
|
||||
const [run] = await tx.execute<{ company_id: string; agent_id: string; source_cursor: string | number | null;
|
||||
runtime_mode?: string | null; issue_id?: string | null }>(
|
||||
sql`select "company_id", "agent_id", ("native_source_cursors" ->> ${source.sourceInstanceId})::bigint as "source_cursor"${
|
||||
wakes.progress || wakes.response || wakes.reassessment ? sql`, "runtime_mode"` : sql.empty()}${
|
||||
wakes.progress ? sql`, ${boundRunIssueIdSql()} as "issue_id"` : sql.empty()
|
||||
} from "heartbeat_runs" where "id" = ${input.runId} for no key update`);
|
||||
runtime_mode?: string | null; issue_id?: string | null; xid?: string; probe_current?: boolean; replay_found?: boolean }>(probes
|
||||
? sql`with "snapshot_run" as (select "next_event_seq" from "heartbeat_runs" where "id" = ${input.runId}),
|
||||
"locked_run" as (select ${columns}, "next_event_seq" from "heartbeat_runs" where "id" = ${input.runId} for no key update)
|
||||
select "locked_run".*, "locked_run"."next_event_seq" = (select "next_event_seq" from "snapshot_run") as "probe_current",
|
||||
exists (select 1 from heartbeat_run_events where ${replayed}) as "replay_found"
|
||||
from "locked_run"`
|
||||
: sql`select ${columns} from "heartbeat_runs" where "id" = ${input.runId} for no key update`);
|
||||
if (!run || run.company_id !== input.companyId || run.agent_id !== input.agentId) {
|
||||
throw new Error("heartbeat_run_event_binding_mismatch");
|
||||
}
|
||||
if (run.xid) primeDatabaseWorkTransactionId(tx, run.xid);
|
||||
const knownNew = run.probe_current === true && run.replay_found === false;
|
||||
// The stored cursor says the run holds every event of this source up to it.
|
||||
// An event at or just past it moves the cursor without counting the run's
|
||||
// events; any other event, or a run without a cursor, recounts them.
|
||||
@@ -380,15 +391,14 @@ async function appendNativeSourceEvent(
|
||||
if (wakes.recovery) await notifyLiveRunRecovery(tx, input);
|
||||
|
||||
const sourceHash = nativeSha256(source.canonicalPayload);
|
||||
const replayed = sql`run_id = ${input.runId} and (source_event_id = ${source.sourceEventId}
|
||||
or (source_instance_id = ${source.sourceInstanceId} and source_seq = ${source.sourceSeq}))`;
|
||||
const native = run.runtime_mode === "native";
|
||||
const reassessment = wakes.reassessment && native, progress = wakes.progress && native && run.issue_id;
|
||||
const response = wakes.response && native;
|
||||
if (reassessment || progress || response || wakes.retention) {
|
||||
// A wake is registered before the write it announces, and a replayed
|
||||
// event registers none. So these events look for their replay first.
|
||||
const [existing] = await tx.execute<Record<string, unknown>>(sql`select * from heartbeat_run_events where ${replayed} limit 1`);
|
||||
const [existing] = knownNew ? []
|
||||
: await tx.execute<Record<string, unknown>>(sql`select * from heartbeat_run_events where ${replayed} limit 1`);
|
||||
if (existing) {
|
||||
const row = decodeRunEventRow(existing);
|
||||
if (row.sourcePayloadSha256 !== sourceHash) throw new HeartbeatRunEventConflictError();
|
||||
|
||||
@@ -12,7 +12,7 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche
|
||||
import { deferredWorkIntentWait } from "../work-intent-outbox.js";
|
||||
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
|
||||
import { createHeartbeatTimerPolicy } from "./timer-policy.js";
|
||||
import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js";
|
||||
import { QUEUED_RUN_AGING_MS, RECOVERY_WAKE_REASONS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js";
|
||||
import { queuedRunDispatchFacts } from "./queued-run-line.js";
|
||||
import { notifyExecutionRunWork } from "../execution-control-notifications.js";
|
||||
import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
@@ -908,6 +908,47 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (claimedRun && isRunAdmissionClosed()) throw new RunAdmissionClosedDuringClaim();
|
||||
}
|
||||
|
||||
const CLOSED_TASK_CONTINUATION = { error: "The task was closed before this continuation started.",
|
||||
errorCode: "issue_closed_before_start" };
|
||||
const RECOVERY_NO_LONGER_NEEDED = { error: "The task no longer needed this recovery run when its turn came.",
|
||||
errorCode: "recovery_no_longer_needed" };
|
||||
|
||||
/**
|
||||
* A system recovery run waits behind people's work, so by its turn the task
|
||||
* may have moved on. It is still needed only while its agent still owns the
|
||||
* task and no other run continues it: none started after it was queued,
|
||||
* none running, and no queued non-recovery run (a person's or an
|
||||
* assignment's). Of two queued recovery runs for one task, the newer one stays.
|
||||
*/
|
||||
async function recoveryRunNoLongerNeeded(run: typeof heartbeatRuns.$inferSelect, issueId: string,
|
||||
task: { assigneeAgentId: string | null }) {
|
||||
if (run.wakeupRequestId) {
|
||||
const [request] = await db.select({ actor: agentWakeupRequests.requestedByActorType }).from(agentWakeupRequests)
|
||||
.where(and(eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.id, run.wakeupRequestId)));
|
||||
// A person asked for this one (a board restore): it is theirs to keep.
|
||||
if (request?.actor === "user") return false;
|
||||
}
|
||||
if (task.assigneeAgentId !== run.agentId) return true;
|
||||
const recoveryReasons = sql.join([...RECOVERY_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `);
|
||||
// The run it recovers is never newer work, whatever the two rows' timestamps say.
|
||||
const notOwnRuns = [run.id, run.retryOfRunId, readNonEmptyString(parseObject(run.contextSnapshot).previousRunId)]
|
||||
.filter((id): id is string => Boolean(id && isUuidLike(id)));
|
||||
const [other] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, run.companyId),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
|
||||
notInArray(heartbeatRuns.id, notOwnRuns),
|
||||
or(
|
||||
// Ran after this one was queued: the task already continued.
|
||||
gt(heartbeatRuns.startedAt, run.createdAt),
|
||||
eq(heartbeatRuns.status, "running"),
|
||||
and(eq(heartbeatRuns.status, "queued"), or(
|
||||
sql`coalesce(${heartbeatRuns.contextSnapshot}->>'wakeReason', '') not in (${recoveryReasons})`,
|
||||
gt(heartbeatRuns.createdAt, run.createdAt))),
|
||||
),
|
||||
)).limit(1);
|
||||
return Boolean(other);
|
||||
}
|
||||
|
||||
/** System continuations that only make sense for an open task. */
|
||||
const SYSTEM_CONTINUATION_WAKE_REASONS = new Set(["issue_recovery_action_restored", "issue_continuation_needed",
|
||||
"issue_assignment_recovery", ISSUE_BLOCKER_STALLED_WAKE_REASON, "issue_children_completed"]);
|
||||
@@ -925,13 +966,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
* attached after the check changes the run, the cancel matches nothing, and
|
||||
* the check runs again.
|
||||
*/
|
||||
async function cancelClosedTaskContinuation(run: typeof heartbeatRuns.$inferSelect, issueId: string) {
|
||||
async function cancelClosedTaskContinuation(run: typeof heartbeatRuns.$inferSelect, issueId: string,
|
||||
outcome: { error: string; errorCode: string } = CLOSED_TASK_CONTINUATION) {
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
const decision = await checkClosedTaskContinuation(run, issueId);
|
||||
if (decision.kind !== "cancel") return decision;
|
||||
const write = await setRunStatusFromLive(run.id, "cancelled", ["queued"], {
|
||||
finishedAt: new Date(), error: "The task was closed before this continuation started.",
|
||||
errorCode: "issue_closed_before_start",
|
||||
finishedAt: new Date(), ...outcome,
|
||||
}, undefined, sql`${heartbeatRuns}.xmin::text = ${decision.rowVersion}`);
|
||||
if (write.updated && write.run) return { kind: "cancelled" as const, run: write.run };
|
||||
if (write.run?.status !== "queued") return { kind: "not_queued" as const };
|
||||
@@ -992,13 +1033,19 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
const continuedIssueId = readNonEmptyString(runContext.issueId);
|
||||
if (systemContinuationReason && SYSTEM_CONTINUATION_WAKE_REASONS.has(systemContinuationReason) && continuedIssueId
|
||||
&& runContext.onboardingCompletion !== true) {
|
||||
const [target] = await db.select({ status: issues.status }).from(issues)
|
||||
const [target] = await db.select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId }).from(issues)
|
||||
.where(and(eq(issues.companyId, run.companyId), eq(issues.id, continuedIssueId)));
|
||||
if (target && ["done", "cancelled"].includes(target.status)) {
|
||||
const closed = await cancelClosedTaskContinuation(run, continuedIssueId);
|
||||
const closedTask = Boolean(target && ["done", "cancelled"].includes(target.status));
|
||||
// Dropped quietly: the run is cancelled before it started and its wake is
|
||||
// skipped, the same as a continuation whose task closed. Nothing failed.
|
||||
const outcome = closedTask ? CLOSED_TASK_CONTINUATION
|
||||
: target && RECOVERY_WAKE_REASONS.has(systemContinuationReason)
|
||||
&& await recoveryRunNoLongerNeeded(run, continuedIssueId, target) ? RECOVERY_NO_LONGER_NEEDED : null;
|
||||
if (outcome) {
|
||||
const closed = await cancelClosedTaskContinuation(run, continuedIssueId, outcome);
|
||||
if (closed.kind === "cancelled") {
|
||||
const cancelled = closed.run;
|
||||
await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: cancelled.finishedAt ?? new Date(), error: "issue_closed_before_start" });
|
||||
await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: cancelled.finishedAt ?? new Date(), error: outcome.errorCode });
|
||||
await releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: true, deferredPostCommitEffects });
|
||||
}
|
||||
// A person's undelivered message rides on this run: it runs as their
|
||||
@@ -2425,7 +2472,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
agent: typeof agents.$inferSelect,
|
||||
queuedRuns: QueuedRunCandidate[],
|
||||
dependencyReadiness: Map<string, { isDependencyReady: boolean }>,
|
||||
issueById: Map<string, { status: string; priority: string | null; updatedAt: Date; executionRunId: string | null }>,
|
||||
issueById: Map<string, { status: string; priority: string | null; updatedAt: Date; executionRunId: string | null;
|
||||
createdByUserId: string | null }>,
|
||||
) {
|
||||
const now = new Date();
|
||||
pruneDeclinedAgedClaims(now);
|
||||
@@ -2531,7 +2579,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}))];
|
||||
const issueRows = queuedIssueIds.length > 0
|
||||
? await db.select({ id: issues.id, status: issues.status, priority: issues.priority, updatedAt: issues.updatedAt,
|
||||
executionRunId: issues.executionRunId }).from(issues)
|
||||
executionRunId: issues.executionRunId, createdByUserId: issues.createdByUserId }).from(issues)
|
||||
.where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds)))
|
||||
: [];
|
||||
const issueById = new Map(issueRows.map((row) => [row.id, row]));
|
||||
|
||||
@@ -4,9 +4,12 @@ import {
|
||||
QUEUED_RUN_AGING_MS,
|
||||
RECOVERY_BACKLOG_AGE_MS,
|
||||
createQueuedRunDispatcher,
|
||||
dispatchCountersFromStarts,
|
||||
queuedRunStartOf,
|
||||
RECOVERY_DISPATCH_SHARE,
|
||||
isRecoveryWakeReason,
|
||||
orderQueuedRunsForDispatch,
|
||||
queuedRunDispatchLine,
|
||||
queuedRunDispatchClass,
|
||||
type QueuedRunDispatchFacts,
|
||||
} from "./queued-run-dispatch-order.js";
|
||||
@@ -58,10 +61,17 @@ describe("queued run dispatch order", () => {
|
||||
expect(ordered.slice(0, 2).map(entry => entry.id).sort()).toEqual([chat.id, assignment.id].sort());
|
||||
});
|
||||
|
||||
it("keeps draining recovery wakes while people keep sending work", () => {
|
||||
it("holds recovery wakes while people keep sending work", () => {
|
||||
const burst = Array.from({ length: 10 }, () => backlogRecovery());
|
||||
const { started, pending } = dispatch(burst, { steps: 40, arrivals: () => [personRetry()] });
|
||||
expect(started.every(entry => entry.wakeReason === "task_retry")).toBe(true);
|
||||
expect(pending.filter(entry => entry.wakeReason === "issue_recovery_action_restored")).toHaveLength(burst.length);
|
||||
});
|
||||
|
||||
it("keeps draining recovery wakes at their share while agents keep sending work", () => {
|
||||
const burst = Array.from({ length: 10 }, () => backlogRecovery());
|
||||
const steps = burst.length * RECOVERY_DISPATCH_SHARE;
|
||||
const { started } = dispatch(burst, { steps, arrivals: () => [personRetry()] });
|
||||
const { started } = dispatch(burst, { steps, arrivals: () => [run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) })] });
|
||||
const recoveryStarts = started.filter(entry => entry.wakeReason === "issue_recovery_action_restored").length;
|
||||
expect(recoveryStarts).toBe(burst.length);
|
||||
for (let window = 0; window + RECOVERY_DISPATCH_SHARE <= started.length; window += 1) {
|
||||
@@ -69,18 +79,22 @@ describe("queued run dispatch order", () => {
|
||||
if (window + RECOVERY_DISPATCH_SHARE <= steps)
|
||||
expect(slice.some(entry => entry.wakeReason === "issue_recovery_action_restored")).toBe(true);
|
||||
}
|
||||
expect(started.filter(entry => entry.wakeReason === "task_retry").length).toBe(steps - burst.length);
|
||||
expect(started.filter(entry => entry.wakeReason === "issue_assigned").length).toBe(steps - burst.length);
|
||||
});
|
||||
|
||||
it("keeps recovery's share when one dispatch fills many slots", () => {
|
||||
const people = Array.from({ length: 12 }, () => personRetry());
|
||||
it("keeps recovery's share of agent work when one dispatch fills many slots, and none of a person's", () => {
|
||||
const work = Array.from({ length: 12 }, () => run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }));
|
||||
const recovery = Array.from({ length: 6 }, () => backlogRecovery());
|
||||
const batch = orderQueuedRunsForDispatch([...recovery, ...people], { now, startsSinceRecovery: 0 }).slice(0, 8);
|
||||
expect(batch.map(entry => entry.wakeReason === "task_retry" ? "P" : "R").join("")).toBe("PPPRPPPR");
|
||||
const batch = orderQueuedRunsForDispatch([...recovery, ...work], { now, startsSinceRecovery: 0 }).slice(0, 8);
|
||||
expect(batch.map(entry => entry.wakeReason === "issue_assigned" ? "A" : "R").join("")).toBe("AAARAAAR");
|
||||
const people = Array.from({ length: 4 }, () => personRetry());
|
||||
const mixed = orderQueuedRunsForDispatch([...recovery, ...people, ...work], { now, startsSinceRecovery: 0 }).slice(0, 8);
|
||||
expect(mixed.map(entry => entry.wakeReason === "task_retry" ? "P" : entry.wakeReason === "issue_assigned" ? "A" : "R").join(""))
|
||||
.toBe("PPPPRAAA");
|
||||
});
|
||||
|
||||
it("offers the next recovery run when the reserved recovery run cannot be claimed", () => {
|
||||
const people = Array.from({ length: 4 }, () => personRetry());
|
||||
const people = Array.from({ length: 4 }, () => run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }));
|
||||
const stuck = backlogRecovery(59);
|
||||
const claimable = backlogRecovery(30);
|
||||
const dispatcher = createQueuedRunDispatcher([...people, stuck, claimable], { now, startsSinceRecovery: RECOVERY_DISPATCH_SHARE - 1 });
|
||||
@@ -89,7 +103,7 @@ describe("queued run dispatch order", () => {
|
||||
const second = dispatcher.next();
|
||||
expect(second!.id).toBe(claimable.id);
|
||||
dispatcher.claimed(second!);
|
||||
expect(dispatcher.next()!.wakeReason).toBe("task_retry");
|
||||
expect(dispatcher.next()!.wakeReason).toBe("issue_assigned");
|
||||
});
|
||||
|
||||
it("does not put a fresh interruption's continuation behind old-backlog recovery wakes", () => {
|
||||
@@ -125,29 +139,22 @@ describe("queued run dispatch order", () => {
|
||||
expect(ordered.map(entry => entry.id)).toEqual([urgent.id, earlier.id, later.id, inProgress.id, todo.id]);
|
||||
});
|
||||
|
||||
it("starts an old queued continuation within a bounded number of starts while fresh person and system work keeps arriving", () => {
|
||||
it("starts an old queued continuation within a bounded number of starts while fresh agent and recovery work keeps arriving", () => {
|
||||
const continuation = run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(150), sourceEndedAt: minutesAgo(152) });
|
||||
const started: QueuedRunDispatchFacts[] = [];
|
||||
let pending: QueuedRunDispatchFacts[] = [continuation];
|
||||
const steps = 40;
|
||||
for (let step = 0; step < steps; step += 1) {
|
||||
pending.push(personRetry(), run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }),
|
||||
pending.push(run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }),
|
||||
run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(1), sourceEndedAt: minutesAgo(2) }));
|
||||
const recent = started.slice(-RECOVERY_DISPATCH_SHARE).reverse();
|
||||
const lastRecovery = recent.findIndex(entry => isRecoveryWakeReason(entry.wakeReason));
|
||||
const oldest = pending.reduce((left, right) => left.createdAt <= right.createdAt ? left : right);
|
||||
const recentForOldest = started.slice(-OLDEST_DISPATCH_SHARE).reverse();
|
||||
const lastOldest = recentForOldest.findIndex(entry => entry.createdAt <= oldest.createdAt);
|
||||
const [next] = orderQueuedRunsForDispatch(pending, { now,
|
||||
startsSinceRecovery: lastRecovery < 0 ? recent.length : lastRecovery,
|
||||
startsSinceOldest: lastOldest < 0 ? recentForOldest.length : lastOldest });
|
||||
const [next] = orderQueuedRunsForDispatch(pending, { now, ...dispatchCountersFromStarts(pending, [...started].reverse().map(run => queuedRunStartOf(run, now)), now) });
|
||||
started.push(next!);
|
||||
pending = pending.filter(entry => entry.id !== next!.id);
|
||||
}
|
||||
const position = started.findIndex(entry => entry.id === continuation.id);
|
||||
expect(position).toBeGreaterThanOrEqual(0);
|
||||
expect(position).toBeLessThan(OLDEST_DISPATCH_SHARE);
|
||||
expect(started.slice(0, position).every(entry => entry.wakeReason === "task_retry" || entry.wakeReason === "issue_continuation_needed")).toBe(true);
|
||||
expect(position).toBeLessThan(RECOVERY_DISPATCH_SHARE);
|
||||
expect(started.slice(0, position).every(entry => entry.wakeReason === "issue_assigned")).toBe(true);
|
||||
});
|
||||
|
||||
it("serves the longest-waiting run once every share even when fresher work outranks it", () => {
|
||||
@@ -166,12 +173,11 @@ describe("queued run dispatch order", () => {
|
||||
expect(ordered[0]!.wakeReason).toBe("task_retry");
|
||||
});
|
||||
|
||||
it("lets an old-backlog recovery run age into the work tier", () => {
|
||||
const aged = run({ wakeReason: "issue_continuation_needed", createdAt: new Date(now.getTime() - QUEUED_RUN_AGING_MS),
|
||||
sourceEndedAt: minutesAgo(200) });
|
||||
it("never lets a long-waiting recovery run age ahead of agent work or into the oldest-run share", () => {
|
||||
const aged = run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(400), sourceEndedAt: minutesAgo(402) });
|
||||
const fresh = run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) });
|
||||
const ordered = orderQueuedRunsForDispatch([fresh, aged], { now, startsSinceRecovery: 0 });
|
||||
expect(ordered.map(entry => entry.id)).toEqual([aged.id, fresh.id]);
|
||||
const ordered = orderQueuedRunsForDispatch([fresh, aged], { now, startsSinceRecovery: 0, startsSinceOldest: OLDEST_DISPATCH_SHARE - 1 });
|
||||
expect(ordered.map(entry => entry.id)).toEqual([fresh.id, aged.id]);
|
||||
});
|
||||
|
||||
it("passes the oldest-run share to the next oldest run while the oldest one keeps failing to claim", () => {
|
||||
@@ -194,17 +200,105 @@ describe("queued run dispatch order", () => {
|
||||
for (let step = 0; step < 3 * OLDEST_DISPATCH_SHARE; step += 1) {
|
||||
pending.push(run({ wakeReason: "issue_recovery_action_restored", createdAt: minutesAgo(1), sourceEndedAt: minutesAgo(2) }),
|
||||
run({ wakeReason: "issue_assigned", readinessRank: 1, createdAt: minutesAgo(1) }));
|
||||
const recent = started.slice(-RECOVERY_DISPATCH_SHARE).reverse();
|
||||
const lastRecovery = recent.findIndex(entry => isRecoveryWakeReason(entry.wakeReason));
|
||||
const oldest = pending.reduce((left, right) => left.createdAt <= right.createdAt ? left : right);
|
||||
const recentForOldest = started.slice(-OLDEST_DISPATCH_SHARE).reverse();
|
||||
const lastOldest = recentForOldest.findIndex(entry => entry.createdAt <= oldest.createdAt);
|
||||
const [next] = orderQueuedRunsForDispatch(pending, { now,
|
||||
startsSinceRecovery: lastRecovery < 0 ? recent.length : lastRecovery,
|
||||
startsSinceOldest: lastOldest < 0 ? recentForOldest.length : lastOldest });
|
||||
const [next] = orderQueuedRunsForDispatch(pending, { now, ...dispatchCountersFromStarts(pending, [...started].reverse().map(run => queuedRunStartOf(run, now)), now) });
|
||||
started.push(next!);
|
||||
pending = pending.filter(entry => entry.id !== next!.id);
|
||||
}
|
||||
for (const continuation of continuations) expect(started.map(entry => entry.id)).toContain(continuation.id);
|
||||
});
|
||||
});
|
||||
|
||||
describe("deep per-agent queue: a recovery burst behind 20 busy slots (paperclip.staging, 2026-11-13)", () => {
|
||||
// Codie, maxConcurrentRuns 20, every slot busy: each freed slot is one dispatch.
|
||||
const at = new Date("2026-11-13T14:30:00.000Z");
|
||||
const clock = (iso: string) => new Date(`2026-11-13T${iso}:00.000Z`);
|
||||
const addMinutes = (date: Date, minutes: number) => new Date(date.getTime() + minutes * 60_000);
|
||||
let id = 0;
|
||||
const make = (input: Partial<QueuedRunDispatchFacts> & { wakeReason: string; createdAt: Date }): QueuedRunDispatchFacts =>
|
||||
({ id: `codie-${++id}`, requestedByUser: false, sourceEndedAt: null, readinessRank: 1, priorityRank: 2, ...input });
|
||||
const isPerson = (run: QueuedRunDispatchFacts) => run.requestedByUser;
|
||||
|
||||
function todaysQueue() {
|
||||
// Agent-made assignments queued before the people's tasks.
|
||||
const assignments = Array.from({ length: 60 }, (_, index) => make({ wakeReason: "issue_assigned",
|
||||
createdAt: addMinutes(clock("05:00"), index * 2) }));
|
||||
// Two tasks people created, stalled by a restart at 07:25 and waiting since 07:29 on system wakes:
|
||||
// a monitor check and the assigned-work sweep's continuation. Neither wake was a person's.
|
||||
const waitingPeople = [
|
||||
make({ wakeReason: "issue_monitor_due", personTask: true, readinessRank: 0, createdAt: clock("07:29") }),
|
||||
make({ wakeReason: "issue_continuation_needed", personTask: true, readinessRank: 0, sourceEndedAt: clock("07:27"),
|
||||
createdAt: addMinutes(clock("07:29"), 1) }),
|
||||
];
|
||||
// Later person-requested work on tasks already in progress, at higher priority.
|
||||
const laterPeople = Array.from({ length: 30 }, (_, index) => make({ wakeReason: "issue_commented", requestedByUser: true,
|
||||
readinessRank: 0, priorityRank: 1, createdAt: addMinutes(clock("08:00"), index * 12) }));
|
||||
// The burst: 139 system recovery wakes between 10:46 and 11:47 for in-progress tasks.
|
||||
const burst = Array.from({ length: 139 }, (_, index) => make({ wakeReason: "issue_recovery_action_restored",
|
||||
readinessRank: 0, sourceEndedAt: clock("09:00"), createdAt: addMinutes(clock("10:46"), Math.floor(index * 61 / 139)) }));
|
||||
return { assignments, waitingPeople, laterPeople, burst };
|
||||
}
|
||||
|
||||
function dispatchOneAtATime(queue: QueuedRunDispatchFacts[], steps: number) {
|
||||
const started: Array<{ run: QueuedRunDispatchFacts; personWaiting: boolean }> = [];
|
||||
let pending = [...queue];
|
||||
for (let step = 0; step < steps; step += 1) {
|
||||
// People and agents keep adding work while the queue drains.
|
||||
if (step % 2 === 0) pending.push(make({ wakeReason: "issue_commented", requestedByUser: true, readinessRank: 0,
|
||||
priorityRank: 1, createdAt: at }));
|
||||
pending.push(make({ wakeReason: "issue_assigned", createdAt: at }));
|
||||
const history = started.map(entry => entry.run);
|
||||
const [next] = orderQueuedRunsForDispatch(pending, { now: at, ...dispatchCountersFromStarts(pending, [...history].reverse().map(run => queuedRunStartOf(run, at)), at) });
|
||||
started.push({ run: next!, personWaiting: pending.some(entry => entry !== next && isPerson(entry) && entry.readinessRank < 3) });
|
||||
pending = pending.filter(entry => entry.id !== next!.id);
|
||||
}
|
||||
return started;
|
||||
}
|
||||
|
||||
it("starts the oldest run on a task a person is waiting on within the oldest-run share, and never starts recovery while a person waits", () => {
|
||||
const { assignments, waitingPeople, laterPeople, burst } = todaysQueue();
|
||||
const started = dispatchOneAtATime([...burst, ...assignments, ...laterPeople, ...waitingPeople], 40);
|
||||
const position = started.findIndex(entry => entry.run.id === waitingPeople[0]!.id);
|
||||
expect(position).toBeGreaterThanOrEqual(0);
|
||||
expect(position).toBeLessThan(OLDEST_DISPATCH_SHARE);
|
||||
// The next one starts within the following share.
|
||||
const second = started.findIndex(entry => entry.run.id === waitingPeople[1]!.id);
|
||||
expect(second).toBeGreaterThanOrEqual(0);
|
||||
expect(second).toBeLessThan(2 * OLDEST_DISPATCH_SHARE);
|
||||
const recoveryAheadOfPeople = started.filter(entry => entry.run.wakeReason === "issue_recovery_action_restored" && entry.personWaiting);
|
||||
expect(recoveryAheadOfPeople).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not let older agent work use up the turn of a run on a task a person created", () => {
|
||||
// No person-requested arrivals: normal order serves the older, higher-priority assignments.
|
||||
const assignments = Array.from({ length: 60 }, (_, index) => make({ wakeReason: "issue_assigned", priorityRank: 1,
|
||||
createdAt: addMinutes(clock("05:00"), index * 2) }));
|
||||
const monitor = make({ wakeReason: "issue_monitor_due", personTask: true, priorityRank: 3, createdAt: clock("07:29") });
|
||||
const started: QueuedRunDispatchFacts[] = [];
|
||||
let pending = [...assignments, monitor];
|
||||
for (let step = 0; step < 2 * OLDEST_DISPATCH_SHARE && pending.length; step += 1) {
|
||||
const [next] = orderQueuedRunsForDispatch(pending, { now: at,
|
||||
...dispatchCountersFromStarts(pending, [...started].reverse().map(run => queuedRunStartOf(run, at)), at) });
|
||||
started.push(next!);
|
||||
pending = pending.filter(entry => entry.id !== next!.id);
|
||||
}
|
||||
const position = started.findIndex(entry => entry.id === monitor.id);
|
||||
expect(position).toBeGreaterThanOrEqual(0);
|
||||
expect(position).toBeLessThan(OLDEST_DISPATCH_SHARE);
|
||||
// The place-in-line order agrees.
|
||||
const line = queuedRunDispatchLine([...assignments, monitor], { now: at, startsSinceRecovery: 0, startsSinceOldest: 0 });
|
||||
expect(line.findIndex(entry => entry.id === monitor.id)).toBeLessThan(OLDEST_DISPATCH_SHARE);
|
||||
});
|
||||
|
||||
it("keeps recovery behind agent assignments once people are served, and still drains it at its share", () => {
|
||||
const { assignments, burst } = todaysQueue();
|
||||
const started: QueuedRunDispatchFacts[] = [];
|
||||
let pending = [...burst.slice(0, 3), ...assignments];
|
||||
for (let step = 0; step < 4 * RECOVERY_DISPATCH_SHARE && pending.length; step += 1) {
|
||||
const [next] = orderQueuedRunsForDispatch(pending, { now: at, ...dispatchCountersFromStarts(pending, [...started].reverse().map(run => queuedRunStartOf(run, at)), at) });
|
||||
started.push(next!);
|
||||
pending = pending.filter(entry => entry.id !== next!.id);
|
||||
}
|
||||
expect(started.filter(entry => entry.wakeReason === "issue_recovery_action_restored")).toHaveLength(3);
|
||||
expect(started[0]!.wakeReason).toBe("issue_assigned");
|
||||
});
|
||||
});
|
||||
@@ -27,6 +27,8 @@ export type QueuedRunDispatchFacts = {
|
||||
readinessRank: number;
|
||||
priorityRank: number;
|
||||
shareDeferred?: boolean;
|
||||
/** The task was created by a person, who waits on its outcome whichever system wake moves it. */
|
||||
personTask?: boolean;
|
||||
};
|
||||
|
||||
export type QueuedRunDispatchClass = "person" | "work" | "recovery" | "recovery_backlog";
|
||||
@@ -48,8 +50,18 @@ export function queuedRunDispatchClass(run: QueuedRunDispatchFacts, now: Date):
|
||||
return "work";
|
||||
}
|
||||
|
||||
const CLASS_TIER: Record<QueuedRunDispatchClass, number> = { person: 0, work: 1, recovery: 1, recovery_backlog: 2 };
|
||||
// System recovery sits below all other work and never ages out of it: a
|
||||
// person's run always goes first, and recovery takes only its share of the
|
||||
// starts that would otherwise go to agent work.
|
||||
const CLASS_TIER: Record<QueuedRunDispatchClass, number> = { person: 0, work: 1, recovery: 2, recovery_backlog: 3 };
|
||||
const NOT_READY_RANK = 3;
|
||||
const NOT_READY_TIER = 4;
|
||||
|
||||
const isReady = (run: QueuedRunDispatchFacts) => run.readinessRank < NOT_READY_RANK;
|
||||
const isPersonRun = (run: QueuedRunDispatchFacts, now: Date) => queuedRunDispatchClass(run, now) === "person";
|
||||
/** A run a person asked for, or any run on a task a person created. */
|
||||
const isPersonWaitingRun = (run: QueuedRunDispatchFacts, now: Date) => Boolean(run.personTask) || isPersonRun(run, now);
|
||||
const isOldestShareEligible = (run: QueuedRunDispatchFacts) => isReady(run) && !run.shareDeferred;
|
||||
|
||||
export type QueuedRunDispatchInput = { now: Date; startsSinceRecovery: number; startsSinceOldest?: number };
|
||||
|
||||
@@ -57,20 +69,102 @@ export function hasQueuedRunAged(run: Pick<QueuedRunDispatchFacts, "createdAt">,
|
||||
return now.getTime() - run.createdAt.getTime() >= QUEUED_RUN_AGING_MS;
|
||||
}
|
||||
|
||||
export function oldestReadyQueuedRun<T extends QueuedRunDispatchFacts>(runs: readonly T[]): T | null {
|
||||
export function oldestReadyQueuedRun<T extends QueuedRunDispatchFacts>(runs: readonly T[],
|
||||
include: (run: T) => boolean = () => true): T | null {
|
||||
let oldest: T | null = null;
|
||||
for (const run of runs) {
|
||||
if (run.readinessRank >= NOT_READY_RANK || run.shareDeferred) continue;
|
||||
if (!isOldestShareEligible(run) || !include(run)) continue;
|
||||
if (!oldest || run.createdAt.getTime() < oldest.createdAt.getTime()) oldest = run;
|
||||
}
|
||||
return oldest;
|
||||
}
|
||||
|
||||
/**
|
||||
* The run the oldest-run share serves next: the longest-waiting run past the
|
||||
* aging bound that a person asked for or whose task a person created (a
|
||||
* system continuation of a person's stalled task included), else the
|
||||
* longest-waiting agent-work run past it. Other system recovery is never
|
||||
* served by this share; it has its own.
|
||||
*
|
||||
* So such a run that has waited past the bound starts within
|
||||
* OLDEST_DISPATCH_SHARE x (aged person-task runs queued before it + 1) starts,
|
||||
* however much newer or higher-priority work arrives. Waiting never fails or
|
||||
* cancels it; it only moves it forward.
|
||||
*/
|
||||
export function oldestShareCandidate<T extends QueuedRunDispatchFacts>(runs: readonly T[], now: Date): T | null {
|
||||
const aged = (run: T) => hasQueuedRunAged(run, now);
|
||||
return oldestReadyQueuedRun(runs, run => aged(run) && isPersonWaitingRun(run, now))
|
||||
?? oldestReadyQueuedRun(runs, run => aged(run) && !isSystemRecoveryRun(run));
|
||||
}
|
||||
|
||||
/**
|
||||
* One of the agent's recent starts. `personOrigin`: a person asked for it (its
|
||||
* wake or source chain). `personWaiting`: that, or its task was created by a
|
||||
* person.
|
||||
*/
|
||||
export type QueuedRunStart = { wakeReason: string | null; createdAt: Date; startedAt?: Date | null; personOrigin?: boolean;
|
||||
personWaiting?: boolean };
|
||||
|
||||
/** A dispatched run as the start history records it. */
|
||||
export function queuedRunStartOf(run: QueuedRunDispatchFacts, now: Date): QueuedRunStart {
|
||||
return { wakeReason: run.wakeReason, createdAt: run.createdAt, personOrigin: isPersonRun(run, now),
|
||||
personWaiting: isPersonWaitingRun(run, now) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a start used up the oldest-run share's turn: it was at least as old
|
||||
* as the candidate, and, while the candidate is a run people wait on, it was
|
||||
* one too. Older agent work never uses up a person's turn.
|
||||
*/
|
||||
function servesOldestShare(started: { createdAt: Date; personWaiting: boolean }, candidate: QueuedRunDispatchFacts, now: Date) {
|
||||
return started.createdAt.getTime() <= candidate.createdAt.getTime()
|
||||
&& (started.personWaiting || !isPersonWaitingRun(candidate, now));
|
||||
}
|
||||
|
||||
/** Where the start history must reach back to for `dispatchCountersFromStarts`, or null when no counter needs it. */
|
||||
export function dispatchHistoryStart<T extends QueuedRunDispatchFacts>(runs: readonly T[], now: Date): Date | null {
|
||||
const recovery = runs.filter(run => isSystemRecoveryRun(run));
|
||||
const countsRecovery = recovery.length > 0 && recovery.length < runs.length;
|
||||
const candidate = runs.length > 1 ? oldestShareCandidate(runs, now) : null;
|
||||
const at = Math.min(countsRecovery ? Math.min(...recovery.map(run => run.createdAt.getTime())) : Infinity,
|
||||
candidate?.createdAt.getTime() ?? Infinity);
|
||||
return Number.isFinite(at) ? new Date(at) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives the share counters from this agent's recent starts, newest first.
|
||||
* The dispatcher is rebuilt for every pass, so its shares carry over through
|
||||
* the agent's own start history rather than process memory. A person's
|
||||
* recovery restart (`personOrigin`) is person work, so it does not use up
|
||||
* system recovery's share.
|
||||
*/
|
||||
export function dispatchCountersFromStarts<T extends QueuedRunDispatchFacts>(runs: readonly T[],
|
||||
recentStarts: readonly QueuedRunStart[], now: Date): Pick<QueuedRunDispatchInput, "startsSinceRecovery" | "startsSinceOldest"> {
|
||||
const startsSince = (at: number) => recentStarts.filter(start => !start.startedAt || start.startedAt.getTime() >= at);
|
||||
const recovery = runs.filter(run => isSystemRecoveryRun(run));
|
||||
let startsSinceRecovery = 0;
|
||||
if (recovery.length > 0 && recovery.length < runs.length) {
|
||||
const starts = startsSince(Math.min(...recovery.map(run => run.createdAt.getTime()))).slice(0, RECOVERY_DISPATCH_SHARE);
|
||||
const last = starts.findIndex(start => isRecoveryWakeReason(start.wakeReason) && !start.personOrigin);
|
||||
startsSinceRecovery = last < 0 ? starts.length : last;
|
||||
}
|
||||
let startsSinceOldest = 0;
|
||||
const candidate = runs.length > 1 ? oldestShareCandidate(runs, now) : null;
|
||||
if (candidate) {
|
||||
const starts = startsSince(candidate.createdAt.getTime()).slice(0, OLDEST_DISPATCH_SHARE);
|
||||
const last = starts.findIndex(start => servesOldestShare({ createdAt: start.createdAt,
|
||||
personWaiting: Boolean(start.personWaiting) }, candidate, now));
|
||||
startsSinceOldest = last < 0 ? starts.length : last;
|
||||
}
|
||||
return { startsSinceRecovery, startsSinceOldest };
|
||||
}
|
||||
|
||||
function sortForDispatch<T extends QueuedRunDispatchFacts>(runs: readonly T[], now: Date): T[] {
|
||||
const tier = (run: T) => {
|
||||
if (run.readinessRank >= NOT_READY_RANK) return 3;
|
||||
const classTier = CLASS_TIER[queuedRunDispatchClass(run, now)];
|
||||
return hasQueuedRunAged(run, now) ? Math.min(classTier, CLASS_TIER.work) : classTier;
|
||||
if (!isReady(run)) return NOT_READY_TIER;
|
||||
const runClass = queuedRunDispatchClass(run, now);
|
||||
// Old backlog that has waited past the aging bound rejoins the other recovery, never more.
|
||||
return runClass === "recovery_backlog" && hasQueuedRunAged(run, now) ? CLASS_TIER.recovery : CLASS_TIER[runClass];
|
||||
};
|
||||
return [...runs].sort((left, right) =>
|
||||
tier(left) - tier(right)
|
||||
@@ -89,17 +183,24 @@ export function createQueuedRunDispatcher<T extends QueuedRunDispatchFacts>(
|
||||
return {
|
||||
next(): T | null {
|
||||
if (!remaining.length) return null;
|
||||
const oldest = sinceOldest >= OLDEST_DISPATCH_SHARE - 1 ? oldestReadyQueuedRun(remaining) : null;
|
||||
if (oldest && hasQueuedRunAged(oldest, input.now)) return remaining.splice(remaining.indexOf(oldest), 1)[0]!;
|
||||
const recoveryIndex = sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1
|
||||
? remaining.findIndex(run => run.readinessRank < NOT_READY_RANK && isSystemRecoveryRun(run))
|
||||
: -1;
|
||||
const oldest = sinceOldest >= OLDEST_DISPATCH_SHARE - 1 ? oldestShareCandidate(remaining, input.now) : null;
|
||||
if (oldest) return remaining.splice(remaining.indexOf(oldest), 1)[0]!;
|
||||
// Recovery's share comes out of agent work, never out of a person's turn.
|
||||
const personWaiting = remaining.some(run => isReady(run) && isPersonRun(run, input.now));
|
||||
// The share goes to the longest-waiting recovery run once one has waited
|
||||
// past the aging bound, so fresh recovery cannot keep an old one waiting.
|
||||
const agedRecovery = oldestReadyQueuedRun(remaining, run => isSystemRecoveryRun(run) && hasQueuedRunAged(run, input.now));
|
||||
const recoveryIndex = personWaiting || sinceRecovery < RECOVERY_DISPATCH_SHARE - 1 ? -1
|
||||
: agedRecovery ? remaining.indexOf(agedRecovery)
|
||||
: remaining.findIndex(run => isReady(run) && isSystemRecoveryRun(run));
|
||||
return remaining.splice(recoveryIndex < 0 ? 0 : recoveryIndex, 1)[0]!;
|
||||
},
|
||||
claimed(run: T) {
|
||||
sinceRecovery = isSystemRecoveryRun(run) ? 0 : sinceRecovery + 1;
|
||||
const oldest = oldestReadyQueuedRun(remaining);
|
||||
sinceOldest = !oldest || oldest.createdAt.getTime() >= run.createdAt.getTime() ? 0 : sinceOldest + 1;
|
||||
// The same rule dispatchCountersFromStarts reads from the start history.
|
||||
const oldest = oldestShareCandidate([...remaining, run], input.now);
|
||||
sinceOldest = !oldest || servesOldestShare({ createdAt: run.createdAt, personWaiting: isPersonWaitingRun(run, input.now) },
|
||||
oldest, input.now) ? 0 : sinceOldest + 1;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -128,50 +229,50 @@ export function queuedRunDispatchLine<T extends QueuedRunDispatchFacts>(
|
||||
runs: readonly T[],
|
||||
input: QueuedRunDispatchInput,
|
||||
): T[] {
|
||||
const order = sortForDispatch(runs, input.now);
|
||||
const now = input.now;
|
||||
const order = sortForDispatch(runs, now);
|
||||
const removed = new Uint8Array(order.length);
|
||||
// Eligible for the oldest share, oldest first; ties keep dispatch order,
|
||||
// as oldestReadyQueuedRun keeps the first of equal ages.
|
||||
const byAge = order.map((_, index) => index)
|
||||
.filter(index => order[index]!.readinessRank < NOT_READY_RANK && !order[index]!.shareDeferred)
|
||||
.sort((left, right) => order[left]!.createdAt.getTime() - order[right]!.createdAt.getTime() || left - right);
|
||||
const recovery = order.map((_, index) => index)
|
||||
.filter(index => order[index]!.readinessRank < NOT_READY_RANK && isSystemRecoveryRun(order[index]!));
|
||||
let head = 0;
|
||||
let agePointer = 0;
|
||||
let recoveryPointer = 0;
|
||||
const firstOf = (indexes: number[] | null, pointer: number) => {
|
||||
if (!indexes) {
|
||||
while (head < order.length && removed[head]) head += 1;
|
||||
return head < order.length ? head : -1;
|
||||
}
|
||||
while (pointer < indexes.length && removed[indexes[pointer]!]) pointer += 1;
|
||||
return pointer;
|
||||
const indexes = (include: (run: T) => boolean) => order.map((_, index) => index).filter(index => include(order[index]!));
|
||||
// Oldest first; ties keep dispatch order, as oldestReadyQueuedRun keeps the first of equal ages.
|
||||
const byAge = (list: number[]) =>
|
||||
list.sort((left, right) => order[left]!.createdAt.getTime() - order[right]!.createdAt.getTime() || left - right);
|
||||
// A list's first entry not yet started; lists stay in their order, so each pointer only moves forward.
|
||||
const cursor = (list: number[]) => {
|
||||
let pointer = 0;
|
||||
return () => {
|
||||
while (pointer < list.length && removed[list[pointer]!]) pointer += 1;
|
||||
return pointer < list.length ? list[pointer]! : -1;
|
||||
};
|
||||
};
|
||||
const oldestIndex = () => {
|
||||
agePointer = firstOf(byAge, agePointer);
|
||||
return agePointer < byAge.length ? byAge[agePointer]! : -1;
|
||||
const agedFirst = (next: () => number) => () => {
|
||||
const index = next();
|
||||
return index >= 0 && hasQueuedRunAged(order[index]!, now) ? index : -1;
|
||||
};
|
||||
const oldestPersonWaiting = agedFirst(cursor(byAge(indexes(run => isOldestShareEligible(run) && isPersonWaitingRun(run, now)))));
|
||||
const oldestWork = agedFirst(cursor(byAge(indexes(run => isOldestShareEligible(run) && !isSystemRecoveryRun(run)))));
|
||||
const oldestShare = () => { const person = oldestPersonWaiting(); return person >= 0 ? person : oldestWork(); };
|
||||
const oldestRecovery = agedFirst(cursor(byAge(indexes(run => isOldestShareEligible(run) && isSystemRecoveryRun(run)))));
|
||||
const firstRecovery = cursor(indexes(run => isReady(run) && isSystemRecoveryRun(run)));
|
||||
const head = cursor(order.map((_, index) => index));
|
||||
let readyPeople = order.filter(run => isReady(run) && isPersonRun(run, now)).length;
|
||||
let sinceRecovery = input.startsSinceRecovery;
|
||||
let sinceOldest = input.startsSinceOldest ?? 0;
|
||||
const result: T[] = [];
|
||||
while (result.length < order.length) {
|
||||
let pick = -1;
|
||||
if (sinceOldest >= OLDEST_DISPATCH_SHARE - 1) {
|
||||
const oldest = oldestIndex();
|
||||
if (oldest >= 0 && hasQueuedRunAged(order[oldest]!, input.now)) pick = oldest;
|
||||
const candidate = oldestShare();
|
||||
let pick = sinceOldest >= OLDEST_DISPATCH_SHARE - 1 ? candidate : -1;
|
||||
if (pick < 0 && readyPeople === 0 && sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1) {
|
||||
pick = oldestRecovery();
|
||||
if (pick < 0) pick = firstRecovery();
|
||||
}
|
||||
if (pick < 0 && sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1) {
|
||||
recoveryPointer = firstOf(recovery, recoveryPointer);
|
||||
if (recoveryPointer < recovery.length) pick = recovery[recoveryPointer]!;
|
||||
}
|
||||
if (pick < 0) pick = firstOf(null, 0);
|
||||
if (pick < 0) pick = head();
|
||||
removed[pick] = 1;
|
||||
const run = order[pick]!;
|
||||
if (isReady(run) && isPersonRun(run, now)) readyPeople -= 1;
|
||||
result.push(run);
|
||||
sinceRecovery = isSystemRecoveryRun(run) ? 0 : sinceRecovery + 1;
|
||||
const oldest = oldestIndex();
|
||||
sinceOldest = oldest < 0 || order[oldest]!.createdAt.getTime() >= run.createdAt.getTime() ? 0 : sinceOldest + 1;
|
||||
sinceOldest = candidate < 0 || servesOldestShare({ createdAt: run.createdAt, personWaiting: isPersonWaitingRun(run, now) },
|
||||
order[candidate]!, now) ? 0 : sinceOldest + 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -35,6 +35,7 @@ function queue(seed: number, size: number): QueuedRunDispatchFacts[] {
|
||||
readinessRank: pick([0, 0, 1, 2, 3]),
|
||||
priorityRank: pick([0, 1, 2, 3, 4]),
|
||||
shareDeferred: next() < 0.1,
|
||||
personTask: next() < 0.3,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, asc, desc, eq, gte, inArray, isNotNull, sql } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, gte, inArray, isNotNull, sql, type SQL } from "drizzle-orm";
|
||||
import { agentWakeupRequests, agents, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
import type { ExecutionAgentQueue } from "@paperclipai/shared";
|
||||
import { parseObject } from "../../adapters/utils.js";
|
||||
@@ -6,10 +6,8 @@ import {
|
||||
OLDEST_DISPATCH_SHARE,
|
||||
PERSON_WAKE_REASONS,
|
||||
RECOVERY_DISPATCH_SHARE,
|
||||
hasQueuedRunAged,
|
||||
isRecoveryWakeReason,
|
||||
isSystemRecoveryRun,
|
||||
oldestReadyQueuedRun,
|
||||
dispatchCountersFromStarts,
|
||||
dispatchHistoryStart,
|
||||
queuedRunDispatchLine,
|
||||
type QueuedRunDispatchFacts,
|
||||
type QueuedRunDispatchInput,
|
||||
@@ -40,6 +38,33 @@ export function issueRunPriorityRank(priority: string | null | undefined) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* SQL: whether the run `runId` was asked for by a person, itself or through
|
||||
* its source chain (`previousRunId`, else `retryOfRunId`), the same rule as
|
||||
* {@link listPersonOriginSourceRunIds}. For use inside another query.
|
||||
*/
|
||||
export function personOriginRunSql(companyId: SQL, runId: SQL) {
|
||||
const uuidPattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$";
|
||||
return sql`exists (
|
||||
with recursive origin_chain(run_id, depth) as (
|
||||
select ${runId}, 0
|
||||
union all
|
||||
select parent.id, origin_chain.depth + 1
|
||||
from origin_chain
|
||||
join heartbeat_runs origin_child on origin_child.id = origin_chain.run_id
|
||||
join heartbeat_runs parent on parent.company_id = ${companyId} and parent.id = coalesce(
|
||||
case when origin_child.context_snapshot->>'previousRunId' ~* ${uuidPattern}
|
||||
then (origin_child.context_snapshot->>'previousRunId')::uuid end,
|
||||
origin_child.retry_of_run_id)
|
||||
where origin_chain.depth < ${PERSON_ORIGIN_CHAIN_DEPTH}
|
||||
)
|
||||
select 1 from origin_chain join heartbeat_runs origin_run on origin_run.id = origin_chain.run_id
|
||||
where origin_run.context_snapshot->>'wakeReason' in (${sql.join([...PERSON_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `)})
|
||||
or origin_run.context_snapshot->>'taskRetryRequestId' is not null
|
||||
or exists (select 1 from agent_wakeup_requests origin_wake where origin_wake.company_id = ${companyId}
|
||||
and origin_wake.requested_by_actor_type = 'user' and (origin_wake.run_id = origin_run.id or origin_wake.id = origin_run.wakeup_request_id)))`;
|
||||
}
|
||||
|
||||
export async function listPersonOriginSourceRunIds(db: Db, companyId: string, sourceRunIds: string[]) {
|
||||
const uuidPattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$";
|
||||
const result = await db.execute<{ root_id: string }>(sql`
|
||||
@@ -77,7 +102,7 @@ export async function queuedRunDispatchFacts<R extends QueuedRunDispatchRun>(
|
||||
agent: Pick<Agent, "id" | "companyId">;
|
||||
queuedRuns: R[];
|
||||
dependencyReadiness: Map<string, { isDependencyReady: boolean }>;
|
||||
issueById: Map<string, { status: string; priority: string | null }>;
|
||||
issueById: Map<string, { status: string; priority: string | null; createdByUserId?: string | null }>;
|
||||
shareDeferred?: (runId: string) => boolean;
|
||||
now: Date;
|
||||
},
|
||||
@@ -135,39 +160,30 @@ export async function queuedRunDispatchFacts<R extends QueuedRunDispatchRun>(
|
||||
readinessRank: issueId ? (ready ? (issue?.status === "in_progress" ? 0 : 1) : 3) : 2,
|
||||
priorityRank: issueRunPriorityRank(issue?.priority),
|
||||
shareDeferred: input.shareDeferred?.(run.id) ?? false,
|
||||
personTask: Boolean(issue?.createdByUserId),
|
||||
};
|
||||
});
|
||||
const recoveryRuns = facts.filter(entry => isSystemRecoveryRun(entry));
|
||||
const oldestReady = oldestReadyQueuedRun(facts);
|
||||
const agedOldest = oldestReady && hasQueuedRunAged(oldestReady, now) ? oldestReady : null;
|
||||
const countsRecovery = recoveryRuns.length > 0 && recoveryRuns.length < facts.length;
|
||||
let startsSinceRecovery = 0;
|
||||
let startsSinceOldest = 0;
|
||||
if (countsRecovery || (agedOldest && facts.length > 1)) {
|
||||
const oldestRecoveryAt = countsRecovery ? Math.min(...recoveryRuns.map(entry => entry.createdAt.getTime())) : Infinity;
|
||||
const since = new Date(Math.min(oldestRecoveryAt, agedOldest?.createdAt.getTime() ?? Infinity));
|
||||
const recentStarts = await db.select({ id: heartbeatRuns.id, wakeReason: sql<string | null>`${heartbeatRuns.contextSnapshot}->>'wakeReason'`,
|
||||
startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt })
|
||||
.from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id),
|
||||
isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, since)))
|
||||
.orderBy(desc(heartbeatRuns.startedAt))
|
||||
.limit(Math.max(RECOVERY_DISPATCH_SHARE, OLDEST_DISPATCH_SHARE));
|
||||
const startsSince = (at: number) => recentStarts.filter(row => row.startedAt && row.startedAt.getTime() >= at);
|
||||
if (countsRecovery) {
|
||||
const starts = startsSince(oldestRecoveryAt).slice(0, RECOVERY_DISPATCH_SHARE);
|
||||
const recoveryStartIds = starts.flatMap(row => isRecoveryWakeReason(row.wakeReason) ? [row.id] : []);
|
||||
const personStarts = new Set(recoveryStartIds.length
|
||||
? await listPersonOriginSourceRunIds(db, agent.companyId, recoveryStartIds) : []);
|
||||
const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason) && !personStarts.has(row.id));
|
||||
startsSinceRecovery = lastRecovery < 0 ? starts.length : lastRecovery;
|
||||
}
|
||||
if (agedOldest && facts.length > 1) {
|
||||
const starts = startsSince(agedOldest.createdAt.getTime()).slice(0, OLDEST_DISPATCH_SHARE);
|
||||
const lastOldest = starts.findIndex(row => row.createdAt.getTime() <= agedOldest.createdAt.getTime());
|
||||
startsSinceOldest = lastOldest < 0 ? starts.length : lastOldest;
|
||||
}
|
||||
}
|
||||
const since = dispatchHistoryStart(facts, now);
|
||||
const startRows = since ? await db.select({ id: heartbeatRuns.id, wakeReason: sql<string | null>`${heartbeatRuns.contextSnapshot}->>'wakeReason'`,
|
||||
startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt,
|
||||
issueId: sql<string | null>`${heartbeatRuns.contextSnapshot}->>'issueId'` })
|
||||
.from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id),
|
||||
isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, since)))
|
||||
.orderBy(desc(heartbeatRuns.startedAt))
|
||||
.limit(Math.max(RECOVERY_DISPATCH_SHARE, OLDEST_DISPATCH_SHARE)) : [];
|
||||
// A person's recovery restart is person work, so it does not use up system recovery's share;
|
||||
// only a start people wait on uses up their oldest-run turn.
|
||||
const personStarts = new Set(startRows.length
|
||||
? await listPersonOriginSourceRunIds(db, agent.companyId, startRows.map(row => row.id)) : []);
|
||||
const startedTaskIds = [...new Set(startRows.flatMap(row => row.issueId && RUN_ID_PATTERN.test(row.issueId) ? [row.issueId] : []))];
|
||||
const personTaskIds = new Set(startedTaskIds.length
|
||||
? (await db.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, agent.companyId),
|
||||
inArray(issues.id, startedTaskIds), isNotNull(issues.createdByUserId)))).map(row => row.id)
|
||||
: []);
|
||||
const recentStarts = startRows.map(row => ({ ...row, personOrigin: personStarts.has(row.id),
|
||||
personWaiting: personStarts.has(row.id) || Boolean(row.issueId && personTaskIds.has(row.issueId)) }));
|
||||
const { startsSinceRecovery, startsSinceOldest } = dispatchCountersFromStarts(facts, recentStarts, now);
|
||||
return { facts, dispatchInput: { now, startsSinceRecovery, startsSinceOldest } };
|
||||
}
|
||||
|
||||
@@ -255,7 +271,7 @@ export async function agentQueuedRunLine(
|
||||
const [dependencyReadiness, issueRows] = issueIds.length
|
||||
? await Promise.all([
|
||||
issuesSvc.listDependencyReadiness(agent.companyId, issueIds),
|
||||
db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues)
|
||||
db.select({ id: issues.id, status: issues.status, priority: issues.priority, createdByUserId: issues.createdByUserId }).from(issues)
|
||||
.where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, issueIds))),
|
||||
])
|
||||
: [new Map<string, { isDependencyReady: boolean }>(), []];
|
||||
|
||||
@@ -36,6 +36,9 @@ const MODERN_ERROR_CODES = new Set([MCP_ERROR_HEADER_MISMATCH, MCP_ERROR_MISSING
|
||||
export const MCP_TASKS_EXTENSION_ID = "io.modelcontextprotocol/tasks";
|
||||
const MCP_TASK_METHODS = new Set(["tasks/get", "tasks/update", "tasks/cancel"]);
|
||||
|
||||
/** MCP OAuth Client Credentials extension; declared only by connections that use machine sign-in. */
|
||||
export const MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID = "io.modelcontextprotocol/oauth-client-credentials";
|
||||
|
||||
const META_PROTOCOL_VERSION = "io.modelcontextprotocol/protocolVersion";
|
||||
const META_CLIENT_CAPABILITIES = "io.modelcontextprotocol/clientCapabilities";
|
||||
const META_CLIENT_INFO = "io.modelcontextprotocol/clientInfo";
|
||||
@@ -54,7 +57,7 @@ export function isModernMcpProtocolVersion(version: unknown): version is string
|
||||
* they may then return URL input requests, which Paperclip shows as a link.
|
||||
* Legacy sessions keep the handshake capabilities they have always received.
|
||||
*/
|
||||
export function mcpClientCapabilities(protocolVersion: string, options: { declareTasks?: boolean; elicitationForm?: boolean } = {}): Record<string, unknown> {
|
||||
export function mcpClientCapabilities(protocolVersion: string, options: { declareTasks?: boolean; elicitationForm?: boolean; declareClientCredentials?: boolean } = {}): Record<string, unknown> {
|
||||
const version: string = protocolVersion;
|
||||
// The skills and tasks extensions are defined against stateless MCP only.
|
||||
// Tasks are declared only where Paperclip can wait on one: a provider tool
|
||||
@@ -66,6 +69,7 @@ export function mcpClientCapabilities(protocolVersion: string, options: { declar
|
||||
return { elicitation: options.elicitationForm ? { form: {}, url: {} } : { url: {} }, extensions: {
|
||||
...MCP_APPS_CLIENT_CAPABILITY, ...MCP_SKILLS_CLIENT_CAPABILITY,
|
||||
...(options.declareTasks ? { [MCP_TASKS_EXTENSION_ID]: {} } : {}),
|
||||
...(options.declareClientCredentials ? { [MCP_OAUTH_CLIENT_CREDENTIALS_EXTENSION_ID]: {} } : {}),
|
||||
} };
|
||||
}
|
||||
// 2025-06-18 knows only form questions (`{}`); 2025-11-25 added URL mode.
|
||||
@@ -209,18 +213,20 @@ export type McpJsonRpcRequest = { id?: string | number; method: string; params?:
|
||||
export function mcpHttpRequestInit(
|
||||
headers: Record<string, string> | undefined,
|
||||
message: McpJsonRpcRequest,
|
||||
options: { toolInputSchema?: unknown; declareTasks?: boolean; elicitationForm?: boolean } = {},
|
||||
options: { toolInputSchema?: unknown; declareTasks?: boolean; elicitationForm?: boolean; declareClientCredentials?: boolean; progressToken?: string } = {},
|
||||
): { method: "POST"; headers: Record<string, string>; body: string } {
|
||||
const requestHeaders = mcpHttpRequestHeaders(headers);
|
||||
const protocolVersion = mcpProtocolVersionOf(headers);
|
||||
let params = message.params ?? {};
|
||||
// Asks the server to stream `notifications/progress` for this request.
|
||||
if (options.progressToken) params = { ...params, _meta: { ...record(params._meta), progressToken: options.progressToken } };
|
||||
if (isModernMcpProtocolVersion(protocolVersion)) {
|
||||
params = {
|
||||
...params,
|
||||
_meta: {
|
||||
...record(params._meta),
|
||||
[META_PROTOCOL_VERSION]: protocolVersion,
|
||||
[META_CLIENT_CAPABILITIES]: mcpClientCapabilities(protocolVersion, { declareTasks: options.declareTasks, elicitationForm: options.elicitationForm }),
|
||||
[META_CLIENT_CAPABILITIES]: mcpClientCapabilities(protocolVersion, { declareTasks: options.declareTasks, elicitationForm: options.elicitationForm, declareClientCredentials: options.declareClientCredentials }),
|
||||
[META_CLIENT_INFO]: MCP_CLIENT_INFO,
|
||||
},
|
||||
};
|
||||
@@ -666,6 +672,15 @@ export function parseMcpHttpResponseBody(bodyText: string, contentType: string |
|
||||
throw new SyntaxError("MCP SSE response contained no data events");
|
||||
}
|
||||
|
||||
/** A `notifications/progress` for this request's token, or null for any other message. */
|
||||
export function mcpProgressNotificationOf(message: unknown, progressToken: string | undefined): { progress: unknown; total: unknown; message: unknown } | null {
|
||||
const payload = record(message);
|
||||
if (!progressToken || !payload || payload.method !== "notifications/progress" || "id" in payload) return null;
|
||||
const params = record(payload.params);
|
||||
if (!params || params.progressToken !== progressToken) return null;
|
||||
return { progress: params.progress, total: params.total, message: params.message };
|
||||
}
|
||||
|
||||
export type McpHttpResponseProgress = {
|
||||
bytesReceived: number;
|
||||
chunksReceived: number;
|
||||
@@ -683,6 +698,15 @@ export async function readMcpHttpResponse(
|
||||
maxBytes?: number;
|
||||
onRequest?: (message: Record<string, unknown>) => Promise<void>;
|
||||
onProgress?: (progress: McpHttpResponseProgress) => void;
|
||||
/**
|
||||
* Bound an SSE stream per message instead of in total, so handled
|
||||
* notifications (such as progress) do not use up the result's limit. Only
|
||||
* for callers with an overall deadline: a stream of small notifications
|
||||
* could otherwise never end.
|
||||
*/
|
||||
limitPerMessage?: boolean;
|
||||
/** Notifications sent on this request's stream before its result. Must not block. */
|
||||
onNotification?: (message: Record<string, unknown>) => void;
|
||||
} = {},
|
||||
): Promise<unknown> {
|
||||
const progress: McpHttpResponseProgress = { bytesReceived: 0, chunksReceived: 0, eventsReceived: 0, matchedResponse: false, ended: false };
|
||||
@@ -714,6 +738,10 @@ export async function readMcpHttpResponse(
|
||||
return record;
|
||||
}
|
||||
if ("method" in record && "id" in record) await options.onRequest?.(record);
|
||||
else if ("method" in record) {
|
||||
// A notification observer never fails or delays the call.
|
||||
try { options.onNotification?.(record); } catch { /* ignored */ }
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const event = async (value: string) => {
|
||||
@@ -729,17 +757,22 @@ export async function readMcpHttpResponse(
|
||||
progress.bytesReceived = bytes;
|
||||
if (value) progress.chunksReceived += 1;
|
||||
progress.ended = done;
|
||||
if (bytes > maxBytes) throw new McpHttpResponseError("too_large", "MCP response exceeded the size limit");
|
||||
// A JSON body is one message. An SSE stream is bounded in total unless the
|
||||
// caller asks for a per-message bound (see limitPerMessage).
|
||||
const perMessage = isStream && options.limitPerMessage === true;
|
||||
if (!perMessage && bytes > maxBytes) throw new McpHttpResponseError("too_large", "MCP response exceeded the size limit");
|
||||
buffer += decoder.decode(value, { stream: !done });
|
||||
if (isStream) {
|
||||
// Normalize CRLF after concatenating chunks, including split CR/LF pairs.
|
||||
buffer = buffer.replace(/\r\n/g, "\n");
|
||||
let boundary: number;
|
||||
while ((boundary = buffer.indexOf("\n\n")) >= 0) {
|
||||
if (perMessage && Buffer.byteLength(buffer.slice(0, boundary), "utf8") > maxBytes) throw new McpHttpResponseError("too_large", "MCP response exceeded the size limit");
|
||||
const result = await event(buffer.slice(0, boundary));
|
||||
buffer = buffer.slice(boundary + 2);
|
||||
if (result !== undefined) return result;
|
||||
}
|
||||
if (perMessage && Buffer.byteLength(buffer, "utf8") > maxBytes) throw new McpHttpResponseError("too_large", "MCP response exceeded the size limit");
|
||||
}
|
||||
if (done) break;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { agents, checkpointDb, closeCheckpointDb, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints,
|
||||
subscribeDatabaseWork } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
(support.supported ? describe : describe.skip)("native runner checkpoint save", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => {
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY ??= randomBytes(32).toString("base64");
|
||||
temporary = await startEmbeddedPostgresTestDatabase("paperclip-checkpoint-save-");
|
||||
db = createDb(temporary.connectionString);
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
if (db) await closeCheckpointDb(db, 1);
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await temporary?.cleanup();
|
||||
});
|
||||
|
||||
const fixture = async (leaseExpiresAt: Date) => {
|
||||
const [company] = await db.insert(companies).values({ name: "Checkpoint save", issuePrefix: `CS${randomUUID().slice(0, 4)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Checkpoint agent", adapterType: "process" }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId: company!.id, title: "Checkpoint task" }).returning();
|
||||
const runId = randomUUID(), leaseOwner = randomUUID(), bootId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId: company!.id, agentId: agent!.id, status: "running",
|
||||
runtimeMode: "native", nativeIssueId: issue!.id, issueId: issue!.id });
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId: company!.id, issueId: issue!.id, phase: "observed", attempt: 0,
|
||||
controllerBootId: bootId, controllerPid: process.pid, controllerProcessStartedAt: new Date(), controllerGeneration: 1,
|
||||
leaseOwner, leaseExpiresAt });
|
||||
const write = nativeRunnerCheckpointWriter(db, { companyId: company!.id, issueId: issue!.id, runId, authorityRunId: runId,
|
||||
bootId, generation: 1, leaseOwner });
|
||||
const checkpoint = (changed: Map<string, string>, reset: boolean) => ({
|
||||
runId, records: new Map(changed), changedRecords: changed, removedRecords: [], reset, workCommandIds: [], predecessor: null,
|
||||
runnerVersion: "test", runnerDigest: `sha256:${"a".repeat(64)}`,
|
||||
recoveryEvidence: { runnerInstanceId: "test", normalizedSessionId: "test", authenticatedConnectionCount: 1, highestRecordedSourceSeq: 0 },
|
||||
restoreSessionDirectory() {},
|
||||
}) as unknown as Parameters<typeof write>[0];
|
||||
return { runId, write, checkpoint };
|
||||
};
|
||||
|
||||
const sentStatements = async (action: () => Promise<unknown>) => {
|
||||
const statements: string[] = [];
|
||||
const options = checkpointDb(db).$client.options as { debug?: unknown };
|
||||
options.debug = (_connection: number, text: string) => { statements.push(text.trim().replace(/\s+/g, " ")); };
|
||||
try { await action(); return statements; } finally { options.debug = undefined; }
|
||||
};
|
||||
|
||||
it("saves a checkpoint with the owner lock, the write and the commit, and registers its wake before the write", async () => {
|
||||
const f = await fixture(new Date(Date.now() + 3_600_000));
|
||||
await f.write(f.checkpoint(new Map([["header", "{}"], ["cursor", "{\"ackedSourceSeq\":0}"]]), true));
|
||||
const events: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.nativeProcessSessions, event => { events.push(event); });
|
||||
let statements: string[];
|
||||
try {
|
||||
statements = await sentStatements(() => f.write(f.checkpoint(new Map([["event:1", "{}"], ["cursor", "{\"ackedSourceSeq\":1}"]]), false)));
|
||||
} finally { unsubscribe(); }
|
||||
expect(statements).toHaveLength(4);
|
||||
expect(statements[0]).toMatch(/^begin/i);
|
||||
expect(statements[1]).toMatch(/pg_current_xact_id\(\)/);
|
||||
expect(statements[2]).toMatch(/^insert into "native_runner_checkpoints"/);
|
||||
expect(statements[3]).toMatch(/^commit/i);
|
||||
expect(events).toEqual(["intent", "settled"]);
|
||||
expect((await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints)
|
||||
.where(eq(nativeRunnerCheckpoints.runId, f.runId))).map(row => row.part).sort()).toEqual(["cursor", "event:1", "header"]);
|
||||
});
|
||||
|
||||
it("refuses a save after the owner's lease expired and writes nothing", async () => {
|
||||
const f = await fixture(new Date(Date.now() + 3_600_000));
|
||||
await f.write(f.checkpoint(new Map([["header", "{}"]]), true));
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
await expect(f.write(f.checkpoint(new Map([["event:1", "{}"]]), false))).rejects.toThrow("native_runner_checkpoint_owner_lost");
|
||||
expect((await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints)
|
||||
.where(eq(nativeRunnerCheckpoints.runId, f.runId))).map(row => row.part)).toEqual(["header"]);
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import { NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js";
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql, type SQL } from "drizzle-orm";
|
||||
import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
|
||||
import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, primeDatabaseWorkTransactionId, type Db } from "@paperclipai/db";
|
||||
import { inspectDurablePrpCheckpointHeader, inspectUnauthenticatedBootstrapCheckpoint, inspectUnclaimedBootstrapCheckpoint, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret } from "../../secrets/local-encrypted-provider.js";
|
||||
import { legacyControllerBootId } from "../legacy-controller-lease.js";
|
||||
@@ -76,9 +76,12 @@ export async function lockNativeRunnerCheckpointOwner(
|
||||
// wall time. Sampling in the locking SELECT can precede its lock wait.
|
||||
// Keep both steps in one statement without another database round trip.
|
||||
const locked = nativeRunnerCheckpointOwnerLock(tx, owner);
|
||||
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
|
||||
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, phase: locked.phase, now: sql<Date>`clock_timestamp()` }).from(locked);
|
||||
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
|
||||
const [row] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
|
||||
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, phase: locked.phase, now: sql<Date>`clock_timestamp()`,
|
||||
xid: sql<string>`pg_current_xact_id()::text` }).from(locked);
|
||||
if (!row) throw new Error("native_runner_checkpoint_owner_lost");
|
||||
const { xid, ...owned } = row;
|
||||
primeDatabaseWorkTransactionId(tx, xid);
|
||||
return owned;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
export function runnerApiSensitiveResponse(method: string, path: string): boolean {
|
||||
if (method === "POST" && /^\/api\/chat-endpoints\/\{[^}]+\}\/setup$/.test(path)) return true;
|
||||
if (/(\/apps\/connect$|\/email\/connections$|setup-secret|\/github\/(setup|identity\/start)|\/slack\/resume|start-authorization|finalize-oauth-access|\/oauth\/|\/registration|\/managed\/authorize|\/install$|\/login|\/setup-token-login-sessions|\/local\/(attempts|check)(\/|$)|\/test-calls(\/|$))/.test(path)) return true;
|
||||
if (/(\/apps\/connect$|\/email\/connections$|setup-secret|\/github\/(setup|identity\/start)|\/slack\/resume|start-authorization|finalize-oauth-access|\/machine-sign-in|\/oauth\/|\/registration|\/managed\/authorize|\/install$|\/login|\/setup-token-login-sessions|\/local\/(attempts|check)(\/|$)|\/test-calls(\/|$))/.test(path)) return true;
|
||||
if (/\/secrets\/\{[^}]+\}\/value$/.test(path)) return true;
|
||||
return /^\/api\/tool-gateway\/sessions(?:\/|$)/.test(path)
|
||||
|| /^\/api\/companies\/\{[^}]+\}\/exports?(?:\/|$)/.test(path)
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { and, eq, sql, type SQL } from "drizzle-orm";
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js";
|
||||
|
||||
/**
|
||||
* System recovery joins an agent's queue a few runs at a time. The rest stays
|
||||
* pending with its producer (a resolved recovery action), which delivers the
|
||||
* next one when a queued recovery run starts or is dropped. Recovery waits
|
||||
* behind people's work in the dispatcher, so a deep recovery queue would only
|
||||
* hold stale runs; pending recovery is re-read when it is admitted instead.
|
||||
*/
|
||||
export const QUEUED_RECOVERY_RUNS_PER_AGENT = 3;
|
||||
|
||||
/**
|
||||
* Evidence keys under which a restored task records the wake it is owed
|
||||
* (legacy-workspace rehome, released workspace writer). While its assignee's
|
||||
* recovery queue is full, that wake waits pending here.
|
||||
*/
|
||||
export const HELD_RECOVERY_WAKE_KEYS = ["legacyWorkspaceRehome", "releasedWorkspaceWriter"] as const;
|
||||
|
||||
/** Queued recovery runs of `agentId`, as SQL on a `heartbeat_runs` alias. */
|
||||
export function queuedRecoveryRunsOf(companyId: SQL | string, agentId: SQL | string) {
|
||||
const reasons = sql.join([...RECOVERY_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `);
|
||||
return sql`select 1 from ${heartbeatRuns} queued_recovery
|
||||
where queued_recovery.company_id = ${companyId} and queued_recovery.agent_id = ${agentId}
|
||||
and queued_recovery.status = 'queued' and queued_recovery.context_snapshot->>'wakeReason' in (${reasons})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* A soft cap, read before the producer's wake: producers that run at the same
|
||||
* moment can each see room, so the queue can briefly hold one more run per
|
||||
* concurrent producer (two today: continuation delivery and restored-task
|
||||
* wakes). That stays "a few at a time"; the dispatcher keeps every recovery
|
||||
* run behind people's work and re-checks it before it starts.
|
||||
*/
|
||||
export async function agentRecoveryQueueIsFull(db: Db, companyId: string, agentId: string) {
|
||||
const reasons = [...RECOVERY_WAKE_REASONS];
|
||||
const [row] = await db.select({ count: sql<number>`count(*)::int` }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' in (${sql.join(reasons.map(reason => sql`${reason}`), sql`, `)})`));
|
||||
return Number(row?.count ?? 0) >= QUEUED_RECOVERY_RUNS_PER_AGENT;
|
||||
}
|
||||
@@ -31,10 +31,12 @@ import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE
|
||||
import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js";
|
||||
import { workspaceRecoverySourceEligible } from "../workspace-recovery-source.js";
|
||||
import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js";
|
||||
import { OLDEST_DISPATCH_SHARE, QUEUED_RUN_AGING_MS } from "../heartbeat/queued-run-dispatch-order.js";
|
||||
import { HELD_RECOVERY_WAKE_KEYS, queuedRecoveryRunsOf } from "../recovery-queue-admission.js";
|
||||
import { listPersonOriginSourceRunIds } from "../heartbeat/queued-run-line.js";
|
||||
import { aliasedRun, queuedRunStalledSql } from "./stalled-queued-run-sql.js";
|
||||
import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js";
|
||||
import { ASSIGNED_WORK_CONTINUED_ACTION, taskAutomaticAttempts } from "./task-automatic-attempts.js";
|
||||
import { ASSIGNED_WORK_CONTINUED_ACTION, PAUSED_FOR_AUTOMATIC_WORK, taskAutomaticAttempts } from "./task-automatic-attempts.js";
|
||||
import { automaticRunCreationHeldUntil, readAutomaticWorkStatus, reserveAutomaticRunCreation } from "./automatic-run-brake.js";
|
||||
import { isReattachPhaseActive, reattachPhaseRecheckAt } from "../heartbeat/run-admission.js";
|
||||
import { ensureFreshContinuation, type FreshContinuationResult } from "../fresh-continuation-fallback.js";
|
||||
import { canContinueRecoveryRunAutomatically } from "../continuation-cause.js";
|
||||
@@ -343,11 +345,23 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids),
|
||||
eq(issueRecoveryActions.status, "resolved"),
|
||||
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`,
|
||||
sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text,
|
||||
// Recent, or waiting its turn behind the agent's queued recovery runs.
|
||||
sql`(coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text,
|
||||
${issueRecoveryActions.updatedAt}::text)::timestamptz
|
||||
> ${at.toISOString()}::timestamptz - ${`${PENDING_CONTINUATION_BOUND_MS} milliseconds`}::interval`));
|
||||
> ${at.toISOString()}::timestamptz - ${`${PENDING_CONTINUATION_BOUND_MS} milliseconds`}::interval
|
||||
or exists (${queuedRecoveryRunsOf(sql`${issueRecoveryActions.companyId}`, sql`${issueRecoveryActions.returnOwnerAgentId}`)}))`));
|
||||
// A restored task's wake that is waiting its turn behind its assignee's
|
||||
// queued recovery runs (see HELD_RECOVERY_WAKE_KEYS) is its pending path too.
|
||||
const heldWakes = await database.select({ issueId: issueRecoveryActions.sourceIssueId, agentId: issues.assigneeAgentId })
|
||||
.from(issueRecoveryActions)
|
||||
.innerJoin(issues, and(eq(issues.companyId, issueRecoveryActions.companyId), eq(issues.id, issueRecoveryActions.sourceIssueId)))
|
||||
.where(and(
|
||||
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids),
|
||||
eq(issueRecoveryActions.status, "resolved"),
|
||||
or(...HELD_RECOVERY_WAKE_KEYS.map(key => sql`${issueRecoveryActions.evidence} #>> array[${key}::text, 'wake'] = 'pending'`)),
|
||||
sql`exists (${queuedRecoveryRunsOf(sql`${issues.companyId}`, sql`${issues.assigneeAgentId}`)})`));
|
||||
const wakeCounts = counts(wakes);
|
||||
for (const continuation of pendingContinuations) {
|
||||
for (const continuation of [...pendingContinuations, ...heldWakes]) {
|
||||
const issue = issueRows.find(row => row.id === continuation.issueId);
|
||||
if (!issue || !continuation.agentId || issue.assigneeAgentId !== continuation.agentId) continue;
|
||||
wakeCounts.set(continuation.issueId, (wakeCounts.get(continuation.issueId) ?? 0) + 1);
|
||||
@@ -614,6 +628,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
if (repairTimes.length >= ASSIGNED_WORK_REPAIRS_PER_MINUTE) {
|
||||
return { verdict, outcome: "deferred_rate_limit", retryAt: new Date(repairTimes[0] + 60_000 + Math.floor(Math.random() * 5_000)) };
|
||||
}
|
||||
const heldUntil = facts.latestRun
|
||||
? await automaticRunCreationHeldUntil(db, issue.companyId, { issueId: issue.id, sourceRunId: facts.latestRun.id }) : null;
|
||||
if (heldUntil) return { verdict, outcome: "deferred_rate_limit", retryAt: heldUntil };
|
||||
const claimed = await claimContinuation(issue, at);
|
||||
if (claimed.kind !== "claimed") return { verdict, outcome: claimed.kind === "capped" ? "deferred_cap" : claimed.kind === "changed" ? "changed" : "deferred_rate_limit",
|
||||
retryAt: claimed.kind === "changed" ? null : claimed.retryAt };
|
||||
@@ -635,6 +652,15 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
await settleContinuationActivity(claimed, claimed.reason, "not_started", { waitingUntil: fresh.retryAt.toISOString() });
|
||||
return { verdict: claimed.verdict, outcome: fresh.kind === "backoff" ? "deferred_cap" : "waiting_on_recovery", retryAt: fresh.retryAt };
|
||||
}
|
||||
const slot = facts.latestRun
|
||||
? await reserveAutomaticRunCreation(db, issue.companyId, { issueId: issue.id, sourceRunId: facts.latestRun.id }) : null;
|
||||
if (slot?.heldUntil) {
|
||||
if (claimed.restoredTo) await revertRestoredBlock(issue, claimed);
|
||||
await settleContinuationActivity(claimed, claimed.reason, "not_started",
|
||||
{ waitingUntil: slot.heldUntil.toISOString(), [PAUSED_FOR_AUTOMATIC_WORK]: "true" });
|
||||
return { verdict: claimed.verdict, outcome: "deferred_rate_limit", retryAt: slot.heldUntil };
|
||||
}
|
||||
const release = async () => { if (slot && !slot.heldUntil) await slot.release(); };
|
||||
const run = await deps.enqueueWakeup(issue.assigneeAgentId!, {
|
||||
source: facts.latestRun ? "automation" : "assignment", triggerDetail: "system",
|
||||
reason: facts.latestRun ? "issue_continuation_needed" : "issue_assigned",
|
||||
@@ -650,6 +676,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
// A refused continuation stays a violation; the claim spaces the next try.
|
||||
// A block lifted for it is put back, so the task never sits unblocked
|
||||
// with nothing pending.
|
||||
if (!run) await release();
|
||||
if (!run && claimed.restoredTo) await revertRestoredBlock(issue, claimed);
|
||||
await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started",
|
||||
fresh?.kind === "person_owned" ? { waitingFor: fresh.reason } : {});
|
||||
@@ -667,12 +694,19 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
}
|
||||
|
||||
async function replaceStalledRun(issue: IssueRow, claimed: Claimed, runId: string): Promise<AssignedWorkCheck> {
|
||||
const slot = await reserveAutomaticRunCreation(db, issue.companyId, { issueId: issue.id, sourceRunId: runId });
|
||||
if (slot.heldUntil) {
|
||||
await settleContinuationActivity(claimed, claimed.reason, "not_started",
|
||||
{ waitingUntil: slot.heldUntil.toISOString(), [PAUSED_FOR_AUTOMATIC_WORK]: "true" });
|
||||
return { verdict: claimed.verdict, outcome: "deferred_rate_limit", retryAt: slot.heldUntil };
|
||||
}
|
||||
const cancelled = !claimed.stalledFence ? false : await deps.cancelStalledQueuedRun!({ companyId: issue.companyId, runId,
|
||||
...claimed.stalledFence }).catch(error => {
|
||||
logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id, runId }, "assigned-work sweep could not cancel a stalled queued run");
|
||||
return false;
|
||||
});
|
||||
if (!cancelled) {
|
||||
await slot.release();
|
||||
await db.delete(activityLog).where(eq(activityLog.id, claimed.activityId));
|
||||
return { verdict: claimed.verdict, outcome: "changed", retryAt: null };
|
||||
}
|
||||
@@ -686,6 +720,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work replacement for a stalled run was not admitted");
|
||||
return null;
|
||||
});
|
||||
if (!run) await slot.release();
|
||||
await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started");
|
||||
return { verdict: claimed.verdict, outcome: run ? "replaced_stalled_run" : "enqueue_refused", retryAt: claimed.nextCheckAt };
|
||||
}
|
||||
@@ -929,7 +964,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
const queuedByAgent = new Map(queuedRows.map(row => [row.agentId, Number(row.queuedRuns)]));
|
||||
const slotRows = stalledRows.map(row => ({ agentName: row.agentName, stalledRuns: Number(row.stalledRuns),
|
||||
queuedRuns: queuedByAgent.get(row.agentId) ?? 0 }));
|
||||
return { ...summary, agentsWithStalledSlots: slotRows };
|
||||
return { ...summary, agentsWithStalledSlots: slotRows, automaticWork: await readAutomaticWorkStatus(db, companyId) };
|
||||
}
|
||||
|
||||
return { checkIssue, listSuspects, report, loadFacts, flushPassLog: passLog.flush };
|
||||
@@ -944,24 +979,6 @@ function interactionNotSupersededSql(companyId: SQLWrapper, issueIdText: SQLWrap
|
||||
and retry.created_at > ${createdAt})`;
|
||||
}
|
||||
|
||||
type StalledRunColumns = { id: SQLWrapper; companyId: SQLWrapper; agentId: SQLWrapper; status: SQLWrapper; createdAt: SQLWrapper };
|
||||
|
||||
function queuedRunStalledSql(run: StalledRunColumns, at: Date) {
|
||||
const agedBefore = new Date(at.getTime() - QUEUED_RUN_AGING_MS).toISOString();
|
||||
return sql<boolean>`(${run.status} = 'queued' and ${run.createdAt} <= ${agedBefore}::timestamptz
|
||||
and not exists (select 1 from heartbeat_runs older where older.company_id = ${run.companyId}
|
||||
and older.agent_id = ${run.agentId} and older.status = 'queued' and older.created_at < ${run.createdAt})
|
||||
and (select count(*) from (select 1 from heartbeat_runs started where started.company_id = ${run.companyId}
|
||||
and started.agent_id = ${run.agentId} and started.id <> ${run.id}
|
||||
and started.started_at > greatest(${run.createdAt} + ${`${QUEUED_RUN_AGING_MS} milliseconds`}::interval,
|
||||
(select max(coalesce(prior.started_at, prior.finished_at)) from heartbeat_runs prior
|
||||
where prior.company_id = ${run.companyId} and prior.agent_id = ${run.agentId} and prior.created_at < ${run.createdAt}
|
||||
and (prior.started_at >= ${run.createdAt} or (prior.started_at is null and prior.finished_at >= ${run.createdAt}))))
|
||||
limit ${OLDEST_DISPATCH_SHARE + 1}) passed_over) > ${OLDEST_DISPATCH_SHARE})`;
|
||||
}
|
||||
|
||||
const aliasedRun = (alias: string): StalledRunColumns => ({ id: sql.raw(`${alias}.id`), companyId: sql.raw(`${alias}.company_id`),
|
||||
agentId: sql.raw(`${alias}.agent_id`), status: sql.raw(`${alias}.status`), createdAt: sql.raw(`${alias}.created_at`) });
|
||||
|
||||
/** The status a block replaced, from the activity entry that blocked the task:
|
||||
* the normal update route records it as `_previous.status`, recovery as
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { automaticRunBrakes, type Db } from "@paperclipai/db";
|
||||
import { logger } from "../../middleware/logger.js";
|
||||
import { PERSON_WAKE_REASONS, RECOVERY_WAKE_REASONS } from "../heartbeat/queued-run-dispatch-order.js";
|
||||
import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js";
|
||||
|
||||
export const AUTOMATIC_RUN_BRAKE = {
|
||||
windowMs: 10 * 60_000,
|
||||
startProgressMs: 2 * 60_000,
|
||||
minCounted: 12,
|
||||
failureShare: 0.8,
|
||||
consecutiveProviderRefusals: 5,
|
||||
minOpenMs: 2 * 60_000,
|
||||
quietCloseMs: 15 * 60_000,
|
||||
probeIntervalMs: 60_000,
|
||||
probeClaimBoundMs: 10_000,
|
||||
probeRunningBoundMs: 10 * 60_000,
|
||||
probeSuccessesToRelease: 2,
|
||||
releaseIntervalMs: 60_000,
|
||||
releaseInitialRate: 2,
|
||||
releaseSteps: 6,
|
||||
creationIntervalMs: 60_000,
|
||||
cacheTtlMs: 5_000,
|
||||
maxCompanies: 256,
|
||||
maxRuns: 1_024,
|
||||
rowLimit: 500,
|
||||
runningLimit: 200,
|
||||
} as const;
|
||||
|
||||
export type AutomaticRunBrakeState = "closed" | "open" | "half_open";
|
||||
|
||||
export type AutomaticRunFacts = {
|
||||
id: string;
|
||||
companyId: string;
|
||||
retryOfRunId?: string | null;
|
||||
scheduledRetryReason?: string | null;
|
||||
contextSnapshot?: Record<string, unknown> | null;
|
||||
};
|
||||
|
||||
const EXTRA_AUTOMATIC_WAKE_REASONS: ReadonlySet<string> = new Set(["issue_assignment_recovery", "transient_failure_retry"]);
|
||||
|
||||
export function isAutomaticRun(run: AutomaticRunFacts): boolean {
|
||||
const context = run.contextSnapshot ?? {};
|
||||
const reason = typeof context.wakeReason === "string" ? context.wakeReason : null;
|
||||
if (context.explicitUserContinuation || context.taskRetryRequestId || (reason && PERSON_WAKE_REASONS.has(reason))) return false;
|
||||
return Boolean(run.retryOfRunId || run.scheduledRetryReason || context.previousRunId || context.retryReason
|
||||
|| context.freshContinuationIsolation
|
||||
|| (reason && (RECOVERY_WAKE_REASONS.has(reason) || EXTRA_AUTOMATIC_WAKE_REASONS.has(reason))));
|
||||
}
|
||||
|
||||
export type AutomaticRunOutcome = "failed_before_starting" | "healthy" | "neutral";
|
||||
|
||||
export type AutomaticRunOutcomeFacts = {
|
||||
status: string;
|
||||
errorCode: string | null;
|
||||
runtimeMode: string | null;
|
||||
executionStage: string | null;
|
||||
startedAt: Date | null;
|
||||
hasOutput: boolean;
|
||||
providerWorkStarted: string | null;
|
||||
nativePreDispatchFailure?: unknown;
|
||||
};
|
||||
|
||||
export type AutomaticRunOutcomeRules = {
|
||||
isPersonOwnedFailureCode: (code: string | null) => boolean;
|
||||
restartCodes: readonly string[];
|
||||
isTransientSetupCode: (code: string | null) => boolean;
|
||||
hasNativePreDispatchFailureRecord: (resultJson: unknown) => boolean;
|
||||
listPersonOriginSourceRunIds: (db: Db, companyId: string, runIds: string[]) => Promise<string[]>;
|
||||
};
|
||||
|
||||
let outcomeRules: Promise<AutomaticRunOutcomeRules> | null = null;
|
||||
|
||||
export function automaticRunOutcomeRules(): Promise<AutomaticRunOutcomeRules> {
|
||||
outcomeRules ??= Promise.all([import("../fresh-continuation-fallback.js"), import("../heartbeat/retries.js"),
|
||||
import("../heartbeat/run-failure-cause.js"), import("../heartbeat/queued-run-line.js")])
|
||||
.then(([fallback, retries, failureCause, queuedRunLine]) => ({
|
||||
isPersonOwnedFailureCode: fallback.isPersonOwnedFailureCode,
|
||||
restartCodes: fallback.BACKLOG_CONTINUATION_SOURCE_CODES,
|
||||
isTransientSetupCode: retries.isTransientWorkspacePreparationCode,
|
||||
hasNativePreDispatchFailureRecord: failureCause.hasNativePreDispatchFailureRecord,
|
||||
listPersonOriginSourceRunIds: queuedRunLine.listPersonOriginSourceRunIds,
|
||||
}))
|
||||
.catch(error => { outcomeRules = null; throw error; });
|
||||
return outcomeRules;
|
||||
}
|
||||
|
||||
export function classifyAutomaticRunOutcome(run: AutomaticRunOutcomeFacts, rules: AutomaticRunOutcomeRules): AutomaticRunOutcome {
|
||||
if (run.status === "running") return run.executionStage === "dispatching" || run.hasOutput ? "healthy" : "neutral";
|
||||
if (!run.startedAt || run.status === "cancelled" || !["succeeded", "failed", "timed_out", "interrupted"].includes(run.status)) return "neutral";
|
||||
if (run.status === "succeeded" || rules.isPersonOwnedFailureCode(run.errorCode)) return "healthy";
|
||||
if (run.errorCode && rules.restartCodes.includes(run.errorCode)) return "neutral";
|
||||
const startPhase = run.providerWorkStarted === "false" || rules.isTransientSetupCode(run.errorCode)
|
||||
|| rules.hasNativePreDispatchFailureRecord({ nativePreDispatchFailure: run.nativePreDispatchFailure })
|
||||
|| (run.runtimeMode === "legacy" && run.executionStage === "preparing");
|
||||
return startPhase ? "failed_before_starting" : "healthy";
|
||||
}
|
||||
|
||||
type BrakeRow = typeof automaticRunBrakes.$inferSelect;
|
||||
|
||||
type SignalRow = {
|
||||
db_now: string | Date;
|
||||
id: string;
|
||||
status: string;
|
||||
error_code: string | null;
|
||||
runtime_mode: string | null;
|
||||
execution_stage: string | null;
|
||||
started_at: string | Date | null;
|
||||
finished_at: string | Date | null;
|
||||
created_at: string | Date;
|
||||
has_output: boolean;
|
||||
provider_work_started: string | null;
|
||||
native_pre_dispatch: unknown;
|
||||
retry_of_run_id: string | null;
|
||||
scheduled_retry_reason: string | null;
|
||||
context: Record<string, unknown> | null;
|
||||
};
|
||||
|
||||
type Evidence = {
|
||||
counted: number;
|
||||
failedBeforeStarting: number;
|
||||
healthy: number;
|
||||
providerRefusalStreak: boolean;
|
||||
recoveredSince: boolean;
|
||||
lastStartAt: number | null;
|
||||
lastActivityAt: number | null;
|
||||
};
|
||||
|
||||
export type AutomaticRunBrakeSnapshot = {
|
||||
companyId: string;
|
||||
state: AutomaticRunBrakeState;
|
||||
pausedSince: Date | null;
|
||||
stateChangedAt: Date | null;
|
||||
reason: Record<string, unknown>;
|
||||
nextProbeAt: Date | null;
|
||||
nextCreationAt: Date | null;
|
||||
probeRunId: string | null;
|
||||
probeSuccesses: number;
|
||||
dbNow: Date;
|
||||
evidence: Evidence;
|
||||
};
|
||||
|
||||
const toDate = (value: string | Date | null | undefined) => value == null ? null : value instanceof Date ? value : new Date(value);
|
||||
|
||||
function boundedMap<K, V>(max: number) {
|
||||
const map = new Map<K, V>();
|
||||
return {
|
||||
get(key: K) {
|
||||
const value = map.get(key);
|
||||
if (value !== undefined) { map.delete(key); map.set(key, value); }
|
||||
return value;
|
||||
},
|
||||
set(key: K, value: V) {
|
||||
map.delete(key);
|
||||
map.set(key, value);
|
||||
while (map.size > max) map.delete(map.keys().next().value as K);
|
||||
},
|
||||
delete(key: K) { map.delete(key); },
|
||||
clear() { map.clear(); },
|
||||
get size() { return map.size; },
|
||||
};
|
||||
}
|
||||
|
||||
function rampStep(stateChangedAt: Date, now: Date) {
|
||||
return Math.max(0, Math.floor((now.getTime() - stateChangedAt.getTime()) / AUTOMATIC_RUN_BRAKE.releaseIntervalMs));
|
||||
}
|
||||
|
||||
const outcomeFacts = (row: SignalRow): AutomaticRunOutcomeFacts => ({ status: row.status, errorCode: row.error_code,
|
||||
runtimeMode: row.runtime_mode, executionStage: row.execution_stage, startedAt: toDate(row.started_at), hasOutput: row.has_output,
|
||||
providerWorkStarted: row.provider_work_started, nativePreDispatchFailure: row.native_pre_dispatch });
|
||||
|
||||
const latest = (values: Array<number | null | undefined>) => {
|
||||
const known = values.filter((value): value is number => typeof value === "number");
|
||||
return known.length ? Math.max(...known) : null;
|
||||
};
|
||||
|
||||
function evaluate(rows: SignalRow[], state: AutomaticRunBrakeState, lower: number, now: Date, rules: AutomaticRunOutcomeRules): Evidence {
|
||||
const automatic = rows.filter(row => isAutomaticRun({ id: row.id, companyId: "", retryOfRunId: row.retry_of_run_id,
|
||||
scheduledRetryReason: row.scheduled_retry_reason, contextSnapshot: row.context }));
|
||||
const outcomes = automatic.flatMap(row => {
|
||||
const finishedAt = toDate(row.finished_at);
|
||||
if (row.status !== "running" && (!finishedAt || finishedAt.getTime() < lower)) return [];
|
||||
const startedAt = toDate(row.started_at);
|
||||
if (row.status === "running" && (!startedAt || startedAt.getTime() < lower)) return [];
|
||||
const outcome = classifyAutomaticRunOutcome(outcomeFacts(row), rules);
|
||||
return outcome === "neutral" ? [] : [{ outcome, row, at: (finishedAt ?? startedAt!).getTime() }];
|
||||
}).sort((left, right) => right.at - left.at);
|
||||
const failedBeforeStarting = outcomes.filter(entry => entry.outcome === "failed_before_starting").length;
|
||||
const streak = outcomes.slice(0, AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals);
|
||||
const lastStartAt = latest(automatic.map(row => toDate(row.started_at)?.getTime()));
|
||||
const lastActivityAt = latest([lastStartAt, ...automatic.map(row => toDate(row.finished_at)?.getTime())]);
|
||||
return {
|
||||
counted: outcomes.length,
|
||||
failedBeforeStarting,
|
||||
healthy: outcomes.length - failedBeforeStarting,
|
||||
providerRefusalStreak: streak.length === AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals
|
||||
&& streak.every(entry => entry.row.error_code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE),
|
||||
recoveredSince: state === "open" && outcomes.length > 0 && (outcomes[0]!.row.status === "succeeded"
|
||||
|| (outcomes.length >= AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease
|
||||
&& outcomes.slice(0, AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease).every(entry => entry.outcome === "healthy"))),
|
||||
lastStartAt,
|
||||
lastActivityAt,
|
||||
};
|
||||
}
|
||||
|
||||
function trips(evidence: Evidence) {
|
||||
return evidence.providerRefusalStreak || (evidence.counted >= AUTOMATIC_RUN_BRAKE.minCounted
|
||||
&& evidence.failedBeforeStarting / evidence.counted >= AUTOMATIC_RUN_BRAKE.failureShare);
|
||||
}
|
||||
|
||||
export type AutomaticWork = { issueId: string; sourceRunId: string | null };
|
||||
|
||||
export type AutomaticRunCreationSlot = { heldUntil: Date } | { heldUntil: null; release: () => Promise<void> };
|
||||
|
||||
const NO_RESERVATION = async () => undefined;
|
||||
|
||||
type CacheEntry = { snapshot: AutomaticRunBrakeSnapshot; readAt: number; skewMs: number; heldUntil: number };
|
||||
|
||||
export function createAutomaticRunBrake(options: { loadRules?: () => Promise<AutomaticRunOutcomeRules> } = {}) {
|
||||
const loadRules = options.loadRules ?? automaticRunOutcomeRules;
|
||||
const companies = boundedMap<string, CacheEntry>(AUTOMATIC_RUN_BRAKE.maxCompanies);
|
||||
const personOrigin = boundedMap<string, boolean>(AUTOMATIC_RUN_BRAKE.maxRuns);
|
||||
let rulesFailureLogged = false;
|
||||
|
||||
async function rulesOrNull() {
|
||||
try {
|
||||
return await loadRules();
|
||||
} catch (error) {
|
||||
if (!rulesFailureLogged) {
|
||||
rulesFailureLogged = true;
|
||||
logger.error({ err: error, event: "automatic_run_brake.rules_unavailable" },
|
||||
"Automatic work pause could not load its run rules; automatic work is not paused");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function readRow(db: Db, companyId: string): Promise<BrakeRow | null> {
|
||||
const [row] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId));
|
||||
return row ?? null;
|
||||
}
|
||||
|
||||
async function readSignal(db: Db, companyId: string, row: BrakeRow | null) {
|
||||
const state = row?.state ?? "closed";
|
||||
const changedAt = row?.stateChangedAt ?? null;
|
||||
const lower = state === "open"
|
||||
? sql`coalesce(${changedAt?.toISOString() ?? null}::timestamptz, now() - ${`${AUTOMATIC_RUN_BRAKE.windowMs} milliseconds`}::interval)`
|
||||
: sql`greatest(now() - ${`${AUTOMATIC_RUN_BRAKE.windowMs} milliseconds`}::interval, ${changedAt?.toISOString() ?? null}::timestamptz)`;
|
||||
const columns = sql`r.id, r.status, r.error_code, r.runtime_mode, r.execution_stage, r.started_at, r.finished_at, r.created_at,
|
||||
r.last_output_at is not null as has_output, r.result_json #>> '{executionRecovery,providerWorkStarted}' as provider_work_started,
|
||||
r.result_json -> 'nativePreDispatchFailure' as native_pre_dispatch, r.retry_of_run_id, r.scheduled_retry_reason,
|
||||
jsonb_build_object('wakeReason', r.context_snapshot->'wakeReason', 'previousRunId', r.context_snapshot->'previousRunId',
|
||||
'retryReason', r.context_snapshot->'retryReason', 'freshContinuationIsolation', r.context_snapshot->'freshContinuationIsolation',
|
||||
'explicitUserContinuation', r.context_snapshot->'explicitUserContinuation',
|
||||
'taskRetryRequestId', r.context_snapshot->'taskRetryRequestId') as context`;
|
||||
const rows = [...await db.execute<SignalRow & { lower_at: string | Date }>(sql`
|
||||
with clock as (select now() as db_now, ${lower} as lower_at)
|
||||
select clock.db_now, clock.lower_at, runs.* from clock left join lateral (
|
||||
(select ${columns} from heartbeat_runs r
|
||||
where r.company_id = ${companyId} and r.finished_at is not null and r.finished_at >= clock.lower_at
|
||||
order by r.finished_at desc limit ${AUTOMATIC_RUN_BRAKE.rowLimit})
|
||||
union all
|
||||
(select ${columns} from heartbeat_runs r
|
||||
where r.company_id = ${companyId} and r.status = 'running'
|
||||
order by r.started_at desc nulls last limit ${AUTOMATIC_RUN_BRAKE.runningLimit})
|
||||
) runs on true`)];
|
||||
return { rows: rows.filter(row => row.id), now: toDate(rows[0]!.db_now)!, lower: toDate(rows[0]!.lower_at)!.getTime() };
|
||||
}
|
||||
|
||||
async function transition(db: Db, companyId: string, row: BrakeRow | null, to: AutomaticRunBrakeState,
|
||||
reason: Record<string, unknown>): Promise<BrakeRow | null> {
|
||||
const from = row?.state ?? "closed";
|
||||
const values = {
|
||||
state: to,
|
||||
reason,
|
||||
stateChangedAt: sql`date_trunc('milliseconds', now())`,
|
||||
openedAt: to === "open" ? (from === "half_open" && row?.openedAt ? row.openedAt : sql`date_trunc('milliseconds', now())`) : to === "closed" ? null : row?.openedAt ?? null,
|
||||
nextProbeAt: null,
|
||||
probeRunId: null,
|
||||
probeSuccesses: 0,
|
||||
releaseWindowStartedAt: null,
|
||||
releaseWindowStarts: 0,
|
||||
nextCreationAt: null,
|
||||
updatedAt: sql`now()`,
|
||||
};
|
||||
const expectedChangedAt = row?.stateChangedAt?.toISOString() ?? null;
|
||||
const [updated] = await db.insert(automaticRunBrakes).values({ companyId, ...values })
|
||||
.onConflictDoUpdate({
|
||||
target: automaticRunBrakes.companyId,
|
||||
set: values,
|
||||
setWhere: sql`${automaticRunBrakes.state} = ${from}
|
||||
and date_trunc('milliseconds', ${automaticRunBrakes.stateChangedAt}) is not distinct from ${expectedChangedAt}::timestamptz`,
|
||||
}).returning();
|
||||
if (!updated) return readRow(db, companyId);
|
||||
const event = to === "open" ? (from === "half_open" ? "reopened" : "opened") : to;
|
||||
logger[to === "open" ? "warn" : "info"]({ event: `automatic_run_brake.${event}`, companyId, from, to, ...reason },
|
||||
to === "open" ? "Automatic work paused for a company while its runs fail before starting"
|
||||
: to === "half_open" ? "Automatic work for a company is resuming gradually" : "Automatic work for a company is running normally again");
|
||||
return updated;
|
||||
}
|
||||
|
||||
async function settleProbe(db: Db, row: BrakeRow, rows: SignalRow[], rules: AutomaticRunOutcomeRules): Promise<BrakeRow> {
|
||||
if (row.state !== "open" || !row.probeRunId) return row;
|
||||
const known = rows.find(candidate => candidate.id === row.probeRunId);
|
||||
const [probe] = known ? [known] : [...await db.execute<SignalRow>(sql`
|
||||
select r.id, r.status, r.error_code, r.runtime_mode, r.execution_stage, r.started_at, r.finished_at, r.created_at,
|
||||
r.last_output_at is not null as has_output, r.result_json #>> '{executionRecovery,providerWorkStarted}' as provider_work_started,
|
||||
r.result_json -> 'nativePreDispatchFailure' as native_pre_dispatch
|
||||
from heartbeat_runs r where r.company_id = ${row.companyId} and r.id = ${row.probeRunId}`)];
|
||||
if (probe && (probe.status === "queued" || probe.status === "scheduled_retry")) return row;
|
||||
const outcome = probe ? classifyAutomaticRunOutcome(outcomeFacts(probe), rules) : "neutral";
|
||||
if (probe?.status === "running" && outcome === "neutral") return row;
|
||||
const completed = probe?.status === "succeeded";
|
||||
const successes = outcome === "healthy" ? row.probeSuccesses + (completed ? AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease : 1)
|
||||
: outcome === "failed_before_starting" ? 0 : row.probeSuccesses;
|
||||
const [updated] = await db.update(automaticRunBrakes).set({
|
||||
probeRunId: null, probeSuccesses: successes, updatedAt: sql`now()`,
|
||||
...(outcome === "healthy" ? { nextProbeAt: sql`now()` } : {}),
|
||||
}).where(sql`${automaticRunBrakes.companyId} = ${row.companyId} and ${automaticRunBrakes.state} = 'open'
|
||||
and ${automaticRunBrakes.probeRunId} = ${row.probeRunId}`).returning();
|
||||
return updated ?? (await readRow(db, row.companyId)) ?? row;
|
||||
}
|
||||
|
||||
async function refresh(db: Db, companyId: string): Promise<CacheEntry> {
|
||||
const rules = await rulesOrNull();
|
||||
if (!rules) {
|
||||
const now = new Date();
|
||||
const entry: CacheEntry = { readAt: now.getTime(), skewMs: 0, heldUntil: 0, snapshot: { companyId, state: "closed", pausedSince: null,
|
||||
stateChangedAt: null, reason: {}, nextProbeAt: null, nextCreationAt: null, probeRunId: null, probeSuccesses: 0, dbNow: now,
|
||||
evidence: { counted: 0, failedBeforeStarting: 0, healthy: 0, providerRefusalStreak: false, recoveredSince: false,
|
||||
lastStartAt: null, lastActivityAt: null } } };
|
||||
companies.set(companyId, entry);
|
||||
return entry;
|
||||
}
|
||||
let row = await readRow(db, companyId);
|
||||
let signal = await readSignal(db, companyId, row);
|
||||
if (row) row = await settleProbe(db, row, signal.rows, rules);
|
||||
let state: AutomaticRunBrakeState = row?.state ?? "closed";
|
||||
let evidence = evaluate(signal.rows, state, signal.lower, signal.now, rules);
|
||||
const numbers = { windowMinutes: AUTOMATIC_RUN_BRAKE.windowMs / 60_000, recentAutomaticRuns: evidence.counted,
|
||||
failedBeforeStarting: evidence.failedBeforeStarting, healthy: evidence.healthy };
|
||||
const changedAt = row?.stateChangedAt?.getTime() ?? 0;
|
||||
const elapsed = signal.now.getTime() - changedAt;
|
||||
let target: { to: AutomaticRunBrakeState; reason: Record<string, unknown> } | null = null;
|
||||
if (state === "closed" && trips(evidence)) {
|
||||
target = { to: "open", reason: { ...numbers, trigger: evidence.providerRefusalStreak ? "sandbox_refusals" : "failed_before_starting" } };
|
||||
} else if (state === "half_open" && trips(evidence)) {
|
||||
target = { to: "open", reason: { ...numbers, trigger: evidence.providerRefusalStreak ? "sandbox_refusals" : "failed_before_starting" } };
|
||||
} else if (state === "half_open" && rampStep(row!.stateChangedAt, signal.now) >= AUTOMATIC_RUN_BRAKE.releaseSteps) {
|
||||
target = { to: "closed", reason: { ...numbers, trigger: "released" } };
|
||||
} else if (state === "open" && elapsed >= AUTOMATIC_RUN_BRAKE.minOpenMs
|
||||
&& (evidence.recoveredSince || (row?.probeSuccesses ?? 0) >= AUTOMATIC_RUN_BRAKE.probeSuccessesToRelease)) {
|
||||
target = { to: "half_open", reason: { ...numbers, trigger: "runs_starting_again" } };
|
||||
} else if (state === "open" && elapsed >= AUTOMATIC_RUN_BRAKE.quietCloseMs
|
||||
&& (evidence.lastActivityAt ?? 0) < signal.now.getTime() - AUTOMATIC_RUN_BRAKE.quietCloseMs) {
|
||||
target = { to: "closed", reason: { ...numbers, trigger: "no_automatic_work" } };
|
||||
}
|
||||
if (target) {
|
||||
row = await transition(db, companyId, row, target.to, target.reason);
|
||||
state = row?.state ?? "closed";
|
||||
if (state !== "closed") {
|
||||
signal = await readSignal(db, companyId, row);
|
||||
evidence = evaluate(signal.rows, state, signal.lower, signal.now, rules);
|
||||
}
|
||||
}
|
||||
const snapshot: AutomaticRunBrakeSnapshot = {
|
||||
companyId, state, pausedSince: row?.openedAt ?? null, stateChangedAt: row?.stateChangedAt ?? null,
|
||||
reason: row?.reason ?? {}, nextProbeAt: row?.nextProbeAt ?? null, nextCreationAt: row?.nextCreationAt ?? null,
|
||||
probeRunId: row?.probeRunId ?? null, probeSuccesses: row?.probeSuccesses ?? 0, dbNow: signal.now, evidence,
|
||||
};
|
||||
const readAt = Date.now();
|
||||
const entry = { snapshot, readAt, skewMs: signal.now.getTime() - readAt, heldUntil: companies.get(companyId)?.heldUntil ?? 0 };
|
||||
companies.set(companyId, entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
async function current(db: Db, companyId: string): Promise<CacheEntry> {
|
||||
const entry = companies.get(companyId);
|
||||
if (entry && Date.now() - entry.readAt < AUTOMATIC_RUN_BRAKE.cacheTtlMs) return entry;
|
||||
return refresh(db, companyId);
|
||||
}
|
||||
|
||||
async function read(db: Db, companyId: string): Promise<AutomaticRunBrakeSnapshot> {
|
||||
return (await current(db, companyId)).snapshot;
|
||||
}
|
||||
|
||||
async function isPersonWork(db: Db, companyId: string, work: AutomaticWork) {
|
||||
if (!work.sourceRunId) return true;
|
||||
const personReasons = sql.join([...PERSON_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `);
|
||||
const [asked] = [...await db.execute<{ asked: boolean }>(sql`
|
||||
with source as (select max(coalesce(r.finished_at, r.created_at)) as finished_at from heartbeat_runs r
|
||||
where r.company_id = ${companyId} and r.id = ${work.sourceRunId}::uuid)
|
||||
select exists (select 1 from agent_wakeup_requests w, source
|
||||
where w.company_id = ${companyId} and w.requested_by_actor_type = 'user' and w.payload->>'issueId' = ${work.issueId}
|
||||
and ((w.run_id is null and w.status in ('queued', 'deferred_issue_execution', 'claimed'))
|
||||
or w.requested_at > coalesce(source.finished_at, '-infinity'::timestamptz)))
|
||||
or exists (select 1 from heartbeat_runs h, source
|
||||
where h.company_id = ${companyId} and h.context_snapshot->>'issueId' = ${work.issueId}
|
||||
and h.created_at > coalesce(source.finished_at, '-infinity'::timestamptz)
|
||||
and (h.context_snapshot->>'wakeReason' in (${personReasons}) or h.context_snapshot ? 'taskRetryRequestId'
|
||||
or jsonb_typeof(h.context_snapshot->'explicitUserContinuation') = 'object')) as asked`)];
|
||||
if (asked?.asked) return true;
|
||||
const rules = await rulesOrNull();
|
||||
return !rules || (await rules.listPersonOriginSourceRunIds(db, companyId, [work.sourceRunId])).includes(work.sourceRunId);
|
||||
}
|
||||
|
||||
async function creationHeldUntil(db: Db, companyId: string, work: AutomaticWork): Promise<Date | null> {
|
||||
const entry = await current(db, companyId);
|
||||
if (entry.snapshot.state !== "open") return null;
|
||||
const now = Date.now();
|
||||
const lastStartAt = entry.snapshot.evidence.lastStartAt;
|
||||
const startHold = lastStartAt !== null && lastStartAt - entry.skewMs > now - AUTOMATIC_RUN_BRAKE.startProgressMs
|
||||
? now + AUTOMATIC_RUN_BRAKE.creationIntervalMs : 0;
|
||||
const slotHold = entry.snapshot.nextCreationAt ? entry.snapshot.nextCreationAt.getTime() - entry.skewMs : 0;
|
||||
const until = Math.max(startHold, slotHold);
|
||||
if (until <= now) return null;
|
||||
return await isPersonWork(db, companyId, work) ? null : new Date(until);
|
||||
}
|
||||
|
||||
async function reserveCreation(db: Db, companyId: string, work: AutomaticWork): Promise<AutomaticRunCreationSlot> {
|
||||
const entry = await current(db, companyId);
|
||||
if (entry.snapshot.state !== "open") return { heldUntil: null, release: NO_RESERVATION };
|
||||
if (await isPersonWork(db, companyId, work)) return { heldUntil: null, release: NO_RESERVATION };
|
||||
const recheck = new Date(Date.now() + AUTOMATIC_RUN_BRAKE.creationIntervalMs);
|
||||
const lastStartAt = entry.snapshot.evidence.lastStartAt;
|
||||
if (lastStartAt !== null && lastStartAt - entry.skewMs > Date.now() - AUTOMATIC_RUN_BRAKE.startProgressMs) return { heldUntil: recheck };
|
||||
const [seeded] = await db.update(automaticRunBrakes).set({
|
||||
nextCreationAt: sql`date_trunc('milliseconds', now()) + ${`${AUTOMATIC_RUN_BRAKE.creationIntervalMs} milliseconds`}::interval`,
|
||||
updatedAt: sql`now()`,
|
||||
}).where(sql`${automaticRunBrakes.companyId} = ${companyId} and ${automaticRunBrakes.state} = 'open'
|
||||
and (${automaticRunBrakes.nextCreationAt} is null or ${automaticRunBrakes.nextCreationAt} <= now())`)
|
||||
.returning({ nextCreationAt: automaticRunBrakes.nextCreationAt });
|
||||
if (seeded?.nextCreationAt) {
|
||||
const token = seeded.nextCreationAt.toISOString();
|
||||
entry.snapshot = { ...entry.snapshot, nextCreationAt: seeded.nextCreationAt };
|
||||
return { heldUntil: null, release: async () => {
|
||||
await db.update(automaticRunBrakes).set({ nextCreationAt: null, updatedAt: sql`now()` })
|
||||
.where(sql`${automaticRunBrakes.companyId} = ${companyId} and ${automaticRunBrakes.nextCreationAt} = ${token}::timestamptz
|
||||
and not exists (select 1 from heartbeat_runs r where r.company_id = ${companyId}
|
||||
and r.context_snapshot->>'issueId' = ${work.issueId}
|
||||
and r.created_at >= ${token}::timestamptz - ${`${AUTOMATIC_RUN_BRAKE.creationIntervalMs} milliseconds`}::interval)`);
|
||||
companies.delete(companyId);
|
||||
} };
|
||||
}
|
||||
const row = await readRow(db, companyId);
|
||||
if (row?.state !== "open") { companies.delete(companyId); return { heldUntil: null, release: NO_RESERVATION }; }
|
||||
const at = row.nextCreationAt ? row.nextCreationAt.getTime() - entry.skewMs : recheck.getTime();
|
||||
return { heldUntil: new Date(Math.max(Date.now(), at)) };
|
||||
}
|
||||
|
||||
async function isPersonOrigin(db: Db, run: AutomaticRunFacts) {
|
||||
const known = personOrigin.get(run.id);
|
||||
if (known !== undefined) return known;
|
||||
const rules = await rulesOrNull();
|
||||
if (!rules) return true;
|
||||
const origin = (await rules.listPersonOriginSourceRunIds(db, run.companyId, [run.id])).includes(run.id);
|
||||
personOrigin.set(run.id, origin);
|
||||
return origin;
|
||||
}
|
||||
|
||||
async function grantProbe(db: Db, entry: CacheEntry, run: AutomaticRunFacts) {
|
||||
const [granted] = await db.update(automaticRunBrakes).set({
|
||||
probeRunId: run.id, updatedAt: sql`now()`,
|
||||
nextProbeAt: sql`now() + ${`${AUTOMATIC_RUN_BRAKE.probeIntervalMs} milliseconds`}::interval`,
|
||||
}).where(sql`${automaticRunBrakes.companyId} = ${run.companyId} and ${automaticRunBrakes.state} = 'open'
|
||||
and (((${automaticRunBrakes.nextProbeAt} is null or ${automaticRunBrakes.nextProbeAt} <= now())
|
||||
and (${automaticRunBrakes.probeRunId} is null or not exists (select 1 from heartbeat_runs p
|
||||
where p.company_id = ${automaticRunBrakes.companyId} and p.id = ${automaticRunBrakes.probeRunId} and p.status = 'running'
|
||||
and p.started_at > now() - ${`${AUTOMATIC_RUN_BRAKE.probeRunningBoundMs} milliseconds`}::interval)))
|
||||
or (${automaticRunBrakes.probeRunId} is not null
|
||||
and ${automaticRunBrakes.nextProbeAt} <= now() + ${`${AUTOMATIC_RUN_BRAKE.probeIntervalMs - AUTOMATIC_RUN_BRAKE.probeClaimBoundMs} milliseconds`}::interval
|
||||
and not exists (select 1 from heartbeat_runs p where p.company_id = ${automaticRunBrakes.companyId}
|
||||
and p.id = ${automaticRunBrakes.probeRunId} and p.started_at is not null)))`)
|
||||
.returning({ nextProbeAt: automaticRunBrakes.nextProbeAt });
|
||||
const now = Date.now();
|
||||
if (granted) {
|
||||
entry.heldUntil = now + AUTOMATIC_RUN_BRAKE.probeClaimBoundMs;
|
||||
entry.snapshot = { ...entry.snapshot, probeRunId: run.id, nextProbeAt: granted.nextProbeAt };
|
||||
return true;
|
||||
}
|
||||
const nextProbeAt = entry.snapshot.nextProbeAt ? entry.snapshot.nextProbeAt.getTime() - entry.skewMs : 0;
|
||||
entry.heldUntil = Math.max(now + AUTOMATIC_RUN_BRAKE.cacheTtlMs, nextProbeAt);
|
||||
return false;
|
||||
}
|
||||
|
||||
async function grantRelease(db: Db, entry: CacheEntry, companyId: string) {
|
||||
const [granted] = [...await db.execute<{ starts: number; window_ends_at: string | Date }>(sql`
|
||||
with ramp as (
|
||||
select b.company_id, b.state_changed_at + floor(extract(epoch from now() - b.state_changed_at) * 1000
|
||||
/ ${AUTOMATIC_RUN_BRAKE.releaseIntervalMs}) * ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval as window_start,
|
||||
${AUTOMATIC_RUN_BRAKE.releaseInitialRate} * power(2, floor(extract(epoch from now() - b.state_changed_at) * 1000
|
||||
/ ${AUTOMATIC_RUN_BRAKE.releaseIntervalMs}))::int as rate
|
||||
from automatic_run_brakes b where b.company_id = ${companyId} and b.state = 'half_open'
|
||||
)
|
||||
update automatic_run_brakes b set
|
||||
release_window_starts = case when b.release_window_started_at = ramp.window_start then b.release_window_starts + 1 else 1 end,
|
||||
release_window_started_at = ramp.window_start, updated_at = now()
|
||||
from ramp
|
||||
where b.company_id = ramp.company_id and b.state = 'half_open'
|
||||
and (b.release_window_started_at is distinct from ramp.window_start or b.release_window_starts < ramp.rate)
|
||||
returning b.release_window_starts as starts, ramp.window_start + ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval as window_ends_at`)];
|
||||
if (granted) return true;
|
||||
const step = rampStep(entry.snapshot.stateChangedAt ?? entry.snapshot.dbNow, new Date(Date.now() + entry.skewMs));
|
||||
const windowEnds = (entry.snapshot.stateChangedAt?.getTime() ?? 0) + (step + 1) * AUTOMATIC_RUN_BRAKE.releaseIntervalMs;
|
||||
entry.heldUntil = Math.max(Date.now() + 1_000, windowEnds - entry.skewMs);
|
||||
return false;
|
||||
}
|
||||
|
||||
async function holdAutomaticRunStart(db: Db, run: AutomaticRunFacts, options: { personOrigin?: boolean } = {}): Promise<boolean> {
|
||||
if (!isAutomaticRun(run)) return false;
|
||||
const cached = companies.get(run.companyId);
|
||||
if (cached && cached.snapshot.state === "closed" && Date.now() - cached.readAt < AUTOMATIC_RUN_BRAKE.cacheTtlMs) return false;
|
||||
if (options.personOrigin ?? await isPersonOrigin(db, run)) return false;
|
||||
if (cached && cached.heldUntil > Date.now()) return true;
|
||||
const entry = await current(db, run.companyId);
|
||||
if (entry.snapshot.state === "closed") return false;
|
||||
if (entry.heldUntil > Date.now()) return true;
|
||||
const allowed = entry.snapshot.state === "open" ? await grantProbe(db, entry, run) : await grantRelease(db, entry, run.companyId);
|
||||
return !allowed;
|
||||
}
|
||||
|
||||
return {
|
||||
read,
|
||||
refresh: async (db: Db, companyId: string) => (await refresh(db, companyId)).snapshot,
|
||||
creationHeldUntil,
|
||||
reserveCreation,
|
||||
holdAutomaticRunStart,
|
||||
cacheSizes: () => ({ companies: companies.size, runs: personOrigin.size }),
|
||||
reset: () => { companies.clear(); personOrigin.clear(); },
|
||||
};
|
||||
}
|
||||
|
||||
export const automaticRunBrake = createAutomaticRunBrake();
|
||||
|
||||
export function automaticRunCreationHeldUntil(db: Db, companyId: string, work: AutomaticWork) {
|
||||
return automaticRunBrake.creationHeldUntil(db, companyId, work);
|
||||
}
|
||||
|
||||
export function reserveAutomaticRunCreation(db: Db, companyId: string, work: AutomaticWork) {
|
||||
return automaticRunBrake.reserveCreation(db, companyId, work);
|
||||
}
|
||||
|
||||
export function automaticWorkStatusText(snapshot: AutomaticRunBrakeSnapshot) {
|
||||
if (snapshot.state === "closed") return "Automatic work is running normally.";
|
||||
const since = (snapshot.pausedSince ?? snapshot.stateChangedAt)?.toISOString() ?? "recently";
|
||||
if (snapshot.state === "half_open") return `Automatic work is resuming gradually since ${snapshot.stateChangedAt?.toISOString() ?? since} because recent runs started normally again.`;
|
||||
const reason = snapshot.reason as { trigger?: string; recentAutomaticRuns?: number; failedBeforeStarting?: number };
|
||||
const because = reason.trigger === "sandbox_refusals"
|
||||
? `the last ${AUTOMATIC_RUN_BRAKE.consecutiveProviderRefusals} automatic runs could not get a sandbox`
|
||||
: `${reason.failedBeforeStarting ?? 0} of ${reason.recentAutomaticRuns ?? 0} recent automatic runs failed before starting`;
|
||||
const next = snapshot.nextProbeAt ?? snapshot.nextCreationAt;
|
||||
return `Automatic work is paused for this company since ${since} because ${because}${next ? `; next try at ${next.toISOString()}` : ""}.`;
|
||||
}
|
||||
|
||||
export async function readAutomaticWorkStatus(db: Db, companyId: string) {
|
||||
const snapshot = await automaticRunBrake.read(db, companyId);
|
||||
return {
|
||||
state: snapshot.state,
|
||||
pausedSince: snapshot.pausedSince,
|
||||
nextTryAt: snapshot.state === "open" ? snapshot.nextProbeAt ?? snapshot.nextCreationAt : null,
|
||||
recentAutomaticRuns: snapshot.evidence.counted,
|
||||
failedBeforeStarting: snapshot.evidence.failedBeforeStarting,
|
||||
summary: automaticWorkStatusText(snapshot),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { issueRecoveryActions, type Db } from "@paperclipai/db";
|
||||
|
||||
/** Records a delivery attempt for a pending continuation, so the next pass tries the records not yet tried first. */
|
||||
export function recordContinuationDeliveryAttempt(db: Db, action: Pick<typeof issueRecoveryActions.$inferSelect, "companyId" | "id">) {
|
||||
return db.update(issueRecoveryActions)
|
||||
.set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify(
|
||||
{ continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` })
|
||||
.where(and(
|
||||
eq(issueRecoveryActions.companyId, action.companyId),
|
||||
eq(issueRecoveryActions.id, action.id),
|
||||
eq(issueRecoveryActions.status, "resolved"),
|
||||
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`,
|
||||
));
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { releaseAncestorWorkspaceBinding } from "../execution-workspaces.js";
|
||||
import { issueService } from "../issues.js";
|
||||
import { REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE } from "../remote-workspace-writer-hold.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { agentRecoveryQueueIsFull } from "../recovery-queue-admission.js";
|
||||
|
||||
export const LEGACY_WORKSPACE_REHOME_SCHEMA = "paperclip.legacy-workspace-rehome.v1";
|
||||
export const LEGACY_WORKSPACE_REHOME_OUTCOME = "legacy_shared_workspace_rehomed";
|
||||
@@ -14,6 +15,7 @@ const BOARD_ESCALATION_POLICY = "board_escalation_no_takeover_v1";
|
||||
const PAGE_SIZE = 25;
|
||||
const REHOME_WAKE_RETRY_BASE_MS = 5 * 60_000;
|
||||
const REHOME_WAKE_RETRY_MAX_MS = 60 * 60_000;
|
||||
const RECOVERY_QUEUE_TURN_MS = 60_000;
|
||||
// A block set by the stranded escalation lands right after its failed run.
|
||||
const BLOCK_OWNERSHIP_SLACK_MS = 60_000;
|
||||
|
||||
@@ -123,6 +125,15 @@ export async function deliverPendingRecoveryWakes(db: Db, wake: RehomeWakeup, sc
|
||||
eq(agentWakeupRequests.companyId, row.companyId), sql`${agentWakeupRequests.payload}->>'issueId' = ${row.issueId}`,
|
||||
sql`${agentWakeupRequests.payload}->>'recoveryActionId' = ${row.actionId}`, isNotNull(agentWakeupRequests.runId))).limit(1) : [];
|
||||
let accepted = !owed || Boolean(delivered);
|
||||
// A few recovery runs per agent at a time: this one waits its turn, without
|
||||
// counting as a refusal, and is offered again on the next pass.
|
||||
if (!accepted && await agentRecoveryQueueIsFull(db, row.companyId, row.agentId!)) {
|
||||
await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence},
|
||||
array[${kind.key}::text, 'wakeRetryAt'], ${JSON.stringify(Date.now() + RECOVERY_QUEUE_TURN_MS)}::jsonb)` })
|
||||
.where(and(eq(issueRecoveryActions.companyId, row.companyId), eq(issueRecoveryActions.id, row.actionId),
|
||||
sql`${wakeField(kind.key, "wake")} = 'pending'`));
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
// Only a created run settles the wake; a refusal (for example a paused
|
||||
// company or disabled on-demand wakes) returns null and is retried.
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { sql, type SQLWrapper } from "drizzle-orm";
|
||||
import { OLDEST_DISPATCH_SHARE, QUEUED_RUN_AGING_MS } from "../heartbeat/queued-run-dispatch-order.js";
|
||||
import { personOriginRunSql } from "../heartbeat/queued-run-line.js";
|
||||
|
||||
/**
|
||||
* The assigned-work sweep's test for a queued run the dispatcher keeps passing
|
||||
* over: its agent's oldest queued run, past the aging bound, with more than
|
||||
* OLDEST_DISPATCH_SHARE later starts that the dispatcher ranked no higher.
|
||||
*/
|
||||
export type StalledRunColumns = { id: SQLWrapper; companyId: SQLWrapper; agentId: SQLWrapper; status: SQLWrapper; createdAt: SQLWrapper };
|
||||
|
||||
const UUID_TEXT = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$";
|
||||
|
||||
/**
|
||||
* A run people wait on, as the dispatcher ranks it: a person asked for it
|
||||
* (itself or through its source chain, the dispatcher's own rule) or its task
|
||||
* was created by a person. `alias` names a heartbeat_runs row in the enclosing query.
|
||||
*/
|
||||
function personWaitingRunSql(alias: string) {
|
||||
const run = (column: string) => sql.raw(`${alias}.${column}`);
|
||||
return sql`(${personOriginRunSql(sql`${run("company_id")}`, sql`${run("id")}`)}
|
||||
or exists (select 1 from issues person_task where person_task.company_id = ${run("company_id")}
|
||||
and person_task.id = case when ${run("context_snapshot")}->>'issueId' ~* ${UUID_TEXT}
|
||||
then (${run("context_snapshot")}->>'issueId')::uuid end
|
||||
and person_task.created_by_user_id is not null))`;
|
||||
}
|
||||
|
||||
export function queuedRunStalledSql(run: StalledRunColumns, at: Date) {
|
||||
const agedBefore = new Date(at.getTime() - QUEUED_RUN_AGING_MS).toISOString();
|
||||
return sql<boolean>`(${run.status} = 'queued' and ${run.createdAt} <= ${agedBefore}::timestamptz
|
||||
and not exists (select 1 from heartbeat_runs older where older.company_id = ${run.companyId}
|
||||
and older.agent_id = ${run.agentId} and older.status = 'queued' and older.created_at < ${run.createdAt})
|
||||
and (select count(*) from (select 1 from heartbeat_runs started where started.company_id = ${run.companyId}
|
||||
and started.agent_id = ${run.agentId} and started.id <> ${run.id}
|
||||
-- Work people wait on goes ahead of a run they do not by design; it does not pass that run over.
|
||||
and not (${personWaitingRunSql("started")} and not exists (select 1 from heartbeat_runs waiting
|
||||
where waiting.id = ${run.id} and ${personWaitingRunSql("waiting")}))
|
||||
and started.started_at > greatest(${run.createdAt} + ${`${QUEUED_RUN_AGING_MS} milliseconds`}::interval,
|
||||
(select max(coalesce(prior.started_at, prior.finished_at)) from heartbeat_runs prior
|
||||
where prior.company_id = ${run.companyId} and prior.agent_id = ${run.agentId} and prior.created_at < ${run.createdAt}
|
||||
and (prior.started_at >= ${run.createdAt} or (prior.started_at is null and prior.finished_at >= ${run.createdAt}))))
|
||||
limit ${OLDEST_DISPATCH_SHARE + 1}) passed_over) > ${OLDEST_DISPATCH_SHARE})`;
|
||||
}
|
||||
|
||||
export const aliasedRun = (alias: string): StalledRunColumns => ({ id: sql.raw(`${alias}.id`), companyId: sql.raw(`${alias}.company_id`),
|
||||
agentId: sql.raw(`${alias}.agent_id`), status: sql.raw(`${alias}.status`), createdAt: sql.raw(`${alias}.created_at`) });
|
||||
@@ -4,13 +4,15 @@ import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRAN
|
||||
import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js";
|
||||
|
||||
export { ASSIGNED_WORK_CONTINUED_ACTION, SCHEDULER_TRANSIENT_RETRY_REASON };
|
||||
export const PAUSED_FOR_AUTOMATIC_WORK = "pausedForAutomaticWork";
|
||||
|
||||
export async function taskAutomaticAttempts(db: Db, companyId: string, issueId: string, at: Date) {
|
||||
const since = new Date(at.getTime() - CONTINUATION_CAP_WINDOW_MS);
|
||||
const [continuations, transientRetries] = await Promise.all([
|
||||
db.select({ createdAt: activityLog.createdAt, outcome: sql<string | null>`${activityLog.details}->>'outcome'` }).from(activityLog).where(and(
|
||||
eq(activityLog.companyId, companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId),
|
||||
eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since)))
|
||||
eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since),
|
||||
sql`coalesce(${activityLog.details}->>${PAUSED_FOR_AUTOMATIC_WORK}, '') <> 'true'`))
|
||||
.orderBy(desc(activityLog.createdAt)).limit(64),
|
||||
db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
|
||||
|
||||
Loaded 100 of 139 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user