From 0111689d0aa559405e41dd2cefe87e8b889a0ad0 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:36:46 -0500 Subject: [PATCH 01/10] test(ui): cover monitor-only task policies (#15587) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Operators open tasks to read progress and edit reviewers and approvers. > - Stored execution policies can contain a monitor and omit the stages array. > - These policies previously crashed the task properties panel. > - The policy reader now applies schema defaults through the fix in #15593. > - This pull request adds regression coverage for monitor-only policies with external service metadata. > - The tests check that the panel renders and participant edits keep the existing monitor. ## Linked Issues or Issue Description **What happened?** A task with a monitor-only execution policy could crash with `Cannot read properties of undefined (reading 'find')`. The stored policy omitted `stages`. **Expected behavior** The task shows no reviewers or approvers. Adding both kinds of participant keeps the monitor and its external service metadata. **Steps to reproduce** 1. Load a task whose execution policy contains an external service monitor and omits `stages`. 2. Render its properties panel and inspect both participant controls. 3. Add reviewers and approvers. Check that the resulting policy keeps the monitor. Refs #15593, which provides the policy normalization used by these tests. ## What Changed - Test that both stage types return empty participant lists for a monitor-only policy. - Test adding reviewers and approvers while keeping the external service monitor. - Test retaining a monitor when no review or approval stages exist. - Render the properties panel with omitted stages and check its participant and monitor controls. ## Verification - The helper and properties panel suites pass on current master with these tests: 108 tests. - `pnpm check:token-gates` passes. - All CI gates pass on the updated PR commit, including build, types, Runner verification, full test suites, browser suites, security scans, and canary checks. - Greptile gives commit `2634bd1d` 5/5 with no actionable findings or unresolved review threads. - The full local workspace suite was not repeated for this test-only rebase. CI verifies the full suite, build, types, Runner checks, browser suites, security scans, and canary run. ## Risks - Low risk. This change adds tests against the existing policy behavior. - The fixture uses example external references and test participant IDs. - Existing documentation remains accurate for this regression coverage. ## Model Used - OpenAI Codex, based on GPT-6, with tool use and code execution. The exact serving model ID and context window were not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-Authored-By: Paperclip --- ui/src/components/IssueProperties.test.tsx | 24 +++++++++++ ui/src/lib/issue-execution-policy.test.ts | 46 ++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/ui/src/components/IssueProperties.test.tsx b/ui/src/components/IssueProperties.test.tsx index dd19e8712b6..6fd44f52b1e 100644 --- a/ui/src/components/IssueProperties.test.tsx +++ b/ui/src/components/IssueProperties.test.tsx @@ -2728,6 +2728,30 @@ describe("IssueProperties", () => { act(() => root.unmount()); }); + it("renders reviewer and approver controls for a monitor-only execution policy", async () => { + const policy = { + monitor: { + nextCheckAt: "2026-10-08T16:00:00.000Z", + notes: "Check pull request", + scheduledBy: "assignee", + kind: "external_service", + serviceName: "github-pr", + }, + } as IssueExecutionPolicy; + const root = renderProperties(container, { + issue: createIssue({ executionPolicy: policy }), + childIssues: [], + onUpdate: vi.fn(), + }); + await flush(); + + expect(findRowTrigger(container, "Reviewers")?.textContent).toContain("None"); + expect(findRowTrigger(container, "Approvers")?.textContent).toContain("None"); + expect(container.querySelector('[data-testid="monitor-row-trigger"]')).not.toBeNull(); + + act(() => root.unmount()); + }); + it("shows a run review action after reviewers are configured and starts execution explicitly when clicked", async () => { const onUpdate = vi.fn(); const root = renderProperties(container, { diff --git a/ui/src/lib/issue-execution-policy.test.ts b/ui/src/lib/issue-execution-policy.test.ts index 4b16adb2efb..6b94e331304 100644 --- a/ui/src/lib/issue-execution-policy.test.ts +++ b/ui/src/lib/issue-execution-policy.test.ts @@ -36,11 +36,57 @@ describe("reading execution policy defaults", () => { }); }); +// Persisted monitor-only policies can omit fields supplied by schema defaults. +const monitorOnlyPolicy = { + monitor: { + nextCheckAt: "2026-10-08T16:00:00.000Z", + notes: "Check pull request", + scheduledBy: "assignee", + kind: "external_service", + serviceName: "github-pr", + externalRef: "https://github.com/example/repo/pull/1", + maxAttempts: 5, + }, +} as IssueExecutionPolicy; + +describe("stageParticipantValues", () => { + it.each(["review", "approval"] as const)("reads %s participants from a monitor-only policy", (stageType) => { + expect(issueExecutionPolicySchema.safeParse(monitorOnlyPolicy).success).toBe(true); + expect(stageParticipantValues(monitorOnlyPolicy, stageType)).toEqual([]); + }); +}); + describe("buildExecutionPolicy", () => { afterEach(() => { vi.unstubAllGlobals(); }); + it("adds review and approval stages to a monitor-only policy without losing its monitor", () => { + const policy = buildExecutionPolicy({ + existingPolicy: monitorOnlyPolicy, + reviewerValues: [`agent:${AGENT_ID}`], + approverValues: ["user:local-board"], + }); + + expect(issueExecutionPolicySchema.safeParse(policy).success).toBe(true); + expect(policy?.monitor).toEqual(monitorOnlyPolicy.monitor); + expect(stageParticipantValues(policy, "review")).toEqual([`agent:${AGENT_ID}`]); + expect(stageParticipantValues(policy, "approval")).toEqual(["user:local-board"]); + }); + + it("keeps a monitor-only policy when there are no review or approval stages", () => { + expect(buildExecutionPolicy({ + existingPolicy: monitorOnlyPolicy, + reviewerValues: [], + approverValues: [], + })).toEqual({ + mode: "normal", + commentRequired: true, + stages: [], + monitor: monitorOnlyPolicy.monitor, + }); + }); + it("generates schema-valid UUIDs when crypto.randomUUID is unavailable", () => { vi.stubGlobal("crypto", { getRandomValues: (bytes: Uint8Array) => { From 53105d5830f7f3051a265bdd17847170b72788dc Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 8 Oct 2026 12:37:06 -0700 Subject: [PATCH 02/10] feat(apps): add Gauge connection (#15596) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Agents reach external services through the Apps catalog. Each catalog entry is a reviewed `AppDefinition` that connects a provider's hosted MCP server to Paperclip's shared vault, grants, policies, gateway, and audit trail. > - Gauge (`withgauge.com`) measures how AI answer engines mention and cite a brand. Its hosted MCP server also gives SEO, analytics and ads reports, and runs a content pipeline that can publish to a connected CMS. > - Gauge is not in the catalog. Operators must use the generic "Connect your own MCP server" flow. That flow has no branding, no organization guidance, and no warning about live CMS publishing. > - The connector playbook supports this provider with existing definition fields. The server supports dynamic client registration with a reviewed `mcp` scope, and it accepts an organization API key as a bearer header. > - This pull request adds the Gauge definition, its artwork, the research and permission-review ledger rows, documentation, and deterministic tests. > - The benefit is a branded, governed Gauge connection with browser sign-in, an API-key option, and a clear publish warning. ## Linked Issues or Issue Description **Problem or motivation** Marketing and growth teams use Gauge to track their brand in AI answers and to run content workflows. They want their Paperclip agents to read visibility, keyword and traffic data, and to prepare content. Gauge is not in the Apps catalog. Operators must paste the MCP URL into the generic remote-MCP flow, which gives no branding, no method guidance, and no warning that content tools can publish live. **Proposed solution** Add a catalog-only Gauge connection built from the connector playbook. Browser sign-in uses Gauge's dynamic client registration and requests only the reviewed `mcp` scope. The user selects one Gauge organization on the consent screen. An optional method sends a customer-created organization API key as an `Authorization: Bearer` header. Both methods warn the operator to set publish actions to Ask first. Every discovered tool stays governed by the normal per-action policies. **Alternatives considered** A plugin was not needed because no custom UI, tables, workers, or webhooks are involved. The identity scopes that Gauge also advertises (`openid`, `profile`, `email`, `organizations`) are not requested, because Gauge selects the organization on its consent screen. A Gauge-specific `classifyRisk` rule was not added, because the tool names cannot be seen without an account. The generic rule already classifies `publish`, `create` and `update` tools as writes. **Roadmap alignment** This extends the existing self-serve remote-MCP connection catalog and does not overlap planned core work. ## What Changed - Added the `gauge` provider to `scripts/ingest-app-definitions.mjs` (category `analytics`, API-key placement, guidance, warnings, description) and regenerated `packages/shared/src/app-definitions/gauge.json` and the generated registry. - Added the Gauge row to the self-serve MCP research ledger with `dcr_or_api_key` auth and risk tier S3. - Added permission reviews for `gauge/mcp-oauth` (explicit scope `mcp`, from Gauge's live authorization-server metadata) and `gauge/mcp-api-key` (provider key), with evidence links. - Added Gauge's mark (`ui/public/brands/apps/gauge.png`, the avatar of Gauge's official GitHub organization) and the brand manifest entry. - Added gallery copy for the Gauge card. - Added `doc/connections/GAUGE.md` (endpoints, scopes, administrator setup, capabilities and policy, manifest, brand provenance, validation hook) and linked it from the connections README and the permission audit. - Tests: Gauge definition shape, store visibility and artwork, URL recognition, reviewed scope, bearer-header placement, the connect form's sign-in default and API-key gating, and the pinned catalog counts. ## Verification - `pnpm exec vitest run packages/shared/src/app-definitions.test.ts packages/shared/src/app-definitions-url.test.ts ui/src/pages/apps/AppsConnect.test.tsx ui/src/pages/apps/Browse.test.tsx ui/src/lib/app-brand-assets.test.ts ui/src/pages/apps/AppLogo.brand-assets.test.tsx server/src/__tests__/tool-access-service.test.ts` — 738 passed. - `node scripts/check-app-brand-assets.mjs` and `node --test scripts/app-brand-validation.test.mjs` — passed. - `pnpm --filter @paperclipai/shared typecheck`, `pnpm --filter @paperclipai/server typecheck`, `pnpm --filter @paperclipai/ui typecheck` — clean. - `node scripts/ingest-app-definitions.mjs --definitions-only` produces no Gauge drift. - Manual: on a local instance, open Apps → Browse and confirm the Gauge card and icon. Open `/apps/connect?source=gauge` and confirm that sign-in is the default and that the API-key method is under Advanced. - Live metadata probe on 2026-10-08: an unauthenticated `initialize` on `https://app.withgauge.com/mcp` returns 401 with `resource_metadata`. Both `.well-known` documents return the recorded endpoints and scopes. Dynamic client registration succeeds. The authorize endpoint accepts `scope=mcp` and rejects an unknown scope with HTTP 400. ## Risks - Low risk to existing providers: the change is additive catalog data plus tests. The generated registry only gains one import. - Gauge content tools can publish to a connected CMS, including live, and a bulk keyword update replaces each prompt's keyword list. Both methods warn the operator, and the API-key helper text tells operators to set publish actions to Ask first. - Gauge API keys are not scoped and reach the whole organization. Paperclip cannot narrow an issued key. - The permission-review ledger records live proof for both methods as not run. The lifecycle checklist in `doc/connections/GAUGE.md` still needs a documented pass with a Gauge organization. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`, 1M context) in Claude Code, with tool use (shell, file editing, web research). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- doc/connections/CONNECTOR-PERMISSION-AUDIT.md | 3 +- doc/connections/GAUGE.md | 189 ++++++++++++++++++ doc/connections/README.md | 2 +- .../tool-method-permission-reviews.json | 34 ++++ .../shared/src/app-definitions-url.test.ts | 1 + .../shared/src/app-definitions.generated.ts | 39 ++-- packages/shared/src/app-definitions.test.ts | 60 +++++- .../shared/src/app-definitions/gauge.json | 94 +++++++++ .../shared/src/self-serve-mcp-research.json | 3 +- scripts/ingest-app-definitions.mjs | 32 ++- .../src/__tests__/tool-access-service.test.ts | 2 +- ui/public/brands/apps/gauge.png | Bin 0 -> 28340 bytes ui/public/brands/apps/manifest.json | 6 + ui/src/lib/app-gallery-copy.ts | 4 + ui/src/pages/apps/AppsConnect.test.tsx | 42 ++++ 15 files changed, 482 insertions(+), 29 deletions(-) create mode 100644 doc/connections/GAUGE.md create mode 100644 packages/shared/src/app-definitions/gauge.json create mode 100644 ui/public/brands/apps/gauge.png diff --git a/doc/connections/CONNECTOR-PERMISSION-AUDIT.md b/doc/connections/CONNECTOR-PERMISSION-AUDIT.md index 57f72332831..5f0844ae68b 100644 --- a/doc/connections/CONNECTOR-PERMISSION-AUDIT.md +++ b/doc/connections/CONNECTOR-PERMISSION-AUDIT.md @@ -9,7 +9,8 @@ separate contracts and were not changed. Methods reviewed after this audit carry their own `reviewedAt` date in the same ledger; the counts above are not restated. Later additions: [Neon](./NEON.md) (`mcp-oauth`, `mcp-api-key`; 2026-10-02), [Superagent](./SUPERAGENT.md) -(`mcp-api-key`; 2026-10-06). +(`mcp-api-key`; 2026-10-06), [Gauge](./GAUGE.md) +(`mcp-oauth`, `mcp-api-key`; 2026-10-08). ## Shared credential failure diff --git a/doc/connections/GAUGE.md b/doc/connections/GAUGE.md new file mode 100644 index 00000000000..39a2d89447b --- /dev/null +++ b/doc/connections/GAUGE.md @@ -0,0 +1,189 @@ +# Gauge + +Updated: 2026-10-08. Status: catalog definition reviewed against official +documentation and live provider metadata; live account qualification +outstanding. + +Gauge (`withgauge.com`) measures how AI answer engines mention and cite a +brand, combines that with SEO, analytics and ads data, and runs a content +pipeline that can publish to a connected CMS. It appears in Apps and uses +Paperclip's shared remote-MCP connection, vault, catalog, grants, policies, +gateway, and audit trail. It is a resource connection, not Paperclip sign-in. +No plugin, database migration, or provider-specific runtime code is required. + +Paperclip connects to Gauge's hosted MCP server at +`https://app.withgauge.com/mcp` (Streamable HTTP) with two methods: + +- `mcp-oauth`: browser sign-in through Gauge's dynamic client registration. + The user chooses one Gauge organization on the consent screen. +- `mcp-api-key`: an organization API key stored as a Paperclip secret and + sent as an `Authorization: Bearer ` header. + +This curated connection is the polished route: it provides branding, method +guidance, and a publish warning. The same URL can also be connected from +**Connect your own MCP server**. See +[Connecting any remote MCP server](./GENERIC-REMOTE-MCP.md). + +## Service involvement + +Gauge hosts the MCP resource and the authorization server. Paperclip stores +the OAuth tokens or the key as secret references, discovers tools with +`tools/list`, and sends the credential on every governed call. No +Paperclip-operated vendor relay is involved; cloud and self-hosted instances +use the same path. + +## Endpoints + +| Purpose | URL | +| --- | --- | +| MCP resource | `https://app.withgauge.com/mcp` | +| Protected-resource metadata | `https://app.withgauge.com/.well-known/oauth-protected-resource` | +| Authorization-server metadata | `https://app.withgauge.com/.well-known/oauth-authorization-server` | +| Authorize | `https://app.withgauge.com/mcp/oauth/authorize` | +| Token | `https://app.withgauge.com/mcp/oauth/token` | +| Dynamic client registration | `https://app.withgauge.com/mcp/oauth/register` | + +The authorization server supports `authorization_code` and `refresh_token`, +PKCE `S256`, and public clients (`token_endpoint_auth_method: none`). It +advertises the scopes `mcp`, `openid`, `profile`, `email` and +`organizations`, and rejects unknown scopes with HTTP 400. + +## Scopes + +Paperclip requests only `mcp`, the scope that grants MCP access. The identity +scopes are not needed: Gauge selects the organization on its consent screen, +not through a scope. This is recorded as an explicit review in +[`tool-method-permission-reviews.json`](./tool-method-permission-reviews.json). + +## Administrator setup + +Browser sign-in: + +1. In Paperclip, open **Apps → Browse → Gauge** and click + **Continue to sign in**. +2. Sign in to Gauge, choose the organization, and approve access. + +API key: + +1. In Gauge, open **Settings → Integrations → API Keys** and click + **Generate API Key**. Use a separate key for Paperclip so it can be deleted + independently. +2. In Paperclip, open **Apps → Browse → Gauge**, open **Advanced**, choose + **Use an API key**, paste the key, and click **Connect**. + +After either method, set publish actions to **Ask first** or **Off** on the +connection's **Permissions** screen if agents run unattended. + +To disconnect, remove the connection in Paperclip. For an API key, also +delete the key in Gauge under **Settings → Integrations → API Keys**, which +revokes every connection that uses it. + +## Capabilities and policy + +Gauge documents about 25 tools in these groups: + +- AI visibility and citations: mentions, citations, answers, visibility and + performance trends, and the sources and pages behind them. +- Topics and prompts: browse tracked prompts and answers; create, generate, + move and tag prompts; bulk keyword updates, which replace each prompt's + full keyword list. +- SEO and keyword reporting: organic rankings, volume, difficulty, and the + keywords a domain or page ranks for. +- GA4, Search Console and PostHog reports, where those integrations are + connected in Gauge. +- Google Ads and OpenAI Ads reporting. +- Content pipeline: brief, research, outline, write, review, images, and + publish to a connected CMS as a draft, to staging, or live. +- Memory files and saved skills: list, read, create and edit. +- Action Center: list, create, assign, set due dates and update status. +- Ask Gauge: natural-language questions answered from the organization's + data. + +Tool names were not captured during this review because it needed a signed-in +account. The generic `classifyRisk` rules apply: `publish`, `create`, +`update`, `set` and similar verbs are writes, and `delete` or `remove` is +destructive. Bulk keyword replacement and live CMS publishing are the +highest-impact actions. The method warnings and the API-key helper tell +operators to put publish actions behind Ask first. If live discovery shows +mutations with read-like names, add a reviewed `gauge` rule to +`classifyRisk`, as Superagent has. + +## Transport and auth + +- Transport: `mcp_remote`, Streamable HTTP (SSE is also documented). +- OAuth: DCR, public client, PKCE, scope `mcp`. Connects as the approving + user, limited to the chosen organization. +- API key: `Authorization: Bearer `. Keys are bound to one + organization, are not scoped, and carry no user identity. Requests are + attributed to the organization. Paperclip cannot narrow an issued key. + +## Resource filters + +None. Gauge documents no query or header options that narrow the hosted +server. Each connection is already limited to one organization; narrow further +with per-action policies. + +## Manifest + +| Field | Value | +| --- | --- | +| Slug | `gauge` | +| Category | `analytics` | +| Methods | `mcp-oauth` (`dcr`, S3, scope `mcp`); `mcp-api-key` (`customer`, S3) | +| Server URL | `https://app.withgauge.com/mcp` | +| Credential | `authorization`, password, required | +| Key placement | header `Authorization`, prefix `Bearer ` | +| Console links | docs `https://docs.withgauge.com/help/mcp` | + +The definition is generated from the `gauge` row in +`packages/shared/src/self-serve-mcp-research.json` and the `gauge` branch of +`specialMethodsFor` in `scripts/ingest-app-definitions.mjs`. + +## Wizard path + +`/apps/connect?source=gauge` defaults to **Sign in with Gauge**. Both methods +share one `write` capability profile, so there is no capability picker, but +the access step shows its description, including the publish warning, before +sign-in. Under +**Advanced**, **Use an API key** shows the key field; **Connect** stays +disabled until a key is entered. A successful sign-in or key check and tool +discovery lead to the connection's Permissions screen. + +## Governance defaults + +- Default profile and bindings: the standard connection profile; every + discovered action starts Allowed under the current product default. +- Policies: operators narrow publish and bulk-edit tools to Ask first or Off + on the Permissions screen. +- Risk: the generic `classifyRisk` rules. +- Quarantine rules: the shared defaults; no Gauge-specific exceptions. + +## Brand provenance + +Gauge's site ships only a 32×32 favicon, below the catalog's 128px minimum. +The same orange "G" gauge mark is the avatar of Gauge's official GitHub +organization (`withgauge`, whose profile links `https://withgauge.com/`), +published as a 460×460 transparent PNG. That file is used unchanged in both +themes. + +| File | Source | SHA-256 | +| --- | --- | --- | +| `ui/public/brands/apps/gauge.png` (460×460) | `https://avatars.githubusercontent.com/u/213101673?v=4&s=512` | `bf1f63a5d30b7f6af1064b74f5f66bf5ea6454d6036a068bf8b0c926fa3f37a6` | + +## Validation hook + +- Environment: definition review on 2026-10-08 against `origin/master`; no + Gauge account was used. +- Metadata probe: an unauthenticated `initialize` on `/mcp` returned HTTP 401 + with `resource_metadata`; both `.well-known` documents returned the + endpoints and scopes above; dynamic client registration succeeded for a + loopback redirect; the authorize endpoint accepted `scope=mcp` (redirect to + Gauge sign-in) and rejected an unknown scope with HTTP 400. +- Deterministic tests: manifest shape, store visibility, artwork, URL + recognition, reviewed scope, bearer-header placement, and the connect + form's sign-in default, the publish warning before sign-in, and API-key + gating. +- Connect evidence, catalog evidence, allowed read, governed write, denied + case, revoke, audit: not run. Both methods still need the full live + lifecycle with a Gauge organization before this connection is considered + qualified. diff --git a/doc/connections/README.md b/doc/connections/README.md index 5f5b4d09741..829ff427b52 100644 --- a/doc/connections/README.md +++ b/doc/connections/README.md @@ -13,7 +13,7 @@ Runtime authentication: [AI Connections](./AI-CONNECTIONS.md). Long-term memory: [Experimental memory connectors](./MEMORY.md). Provider notes: [Google Workspace](./GOOGLE-WORKSPACE.md), -[Gmail](./GMAIL.md), [Asana](./ASANA.md), [PostHog](./POSTHOG.md), [Neon](./NEON.md), [Superagent](./SUPERAGENT.md), +[Gmail](./GMAIL.md), [Asana](./ASANA.md), [PostHog](./POSTHOG.md), [Neon](./NEON.md), [Superagent](./SUPERAGENT.md), [Gauge](./GAUGE.md), [AgentMail](./AGENTMAIL.md), [Telem.AI](./TELEM.md), [iMessage Photon](./IMESSAGE-PHOTON.md), and [Enterpret](./ENTERPRET.md). Optional credential custody: [Vercel Connect](./VERCEL-CONNECT.md). diff --git a/doc/connections/tool-method-permission-reviews.json b/doc/connections/tool-method-permission-reviews.json index f92216c57bf..a3618c46d06 100644 --- a/doc/connections/tool-method-permission-reviews.json +++ b/doc/connections/tool-method-permission-reviews.json @@ -507,6 +507,40 @@ "liveProof": "not-run", "keyPermissions": "Use an API key for an account with write access to the meetings your agents need to share, rename, move or turn into soundbites. Paperclip cannot increase the key\u2019s permissions." }, + { + "app": "gauge", + "method": "mcp-oauth", + "auth": "oauth", + "policy": "explicit", + "requestedScopes": [ + "mcp" + ], + "capability": "write", + "supportedActions": "Read AI visibility, mentions, citations, answers and competitor data; browse and curate tracked topics and prompts (create, generate, move, tag, and replace keyword lists); pull SEO keyword, GA4, Search Console, PostHog and Google Ads/OpenAI Ads reports; run the content pipeline from brief to article and publish to a connected CMS as a draft, to staging or live; read and edit organization memory files and skills; create, assign and update Action Center items; ask Ask Gauge questions. Access is limited to the organization chosen during consent.", + "evidence": [ + "https://docs.withgauge.com/help/mcp", + "https://app.withgauge.com/.well-known/oauth-protected-resource", + "https://app.withgauge.com/.well-known/oauth-authorization-server" + ], + "reviewedAt": "2026-10-08", + "liveProof": "not-run" + }, + { + "app": "gauge", + "method": "mcp-api-key", + "auth": "api_key", + "policy": "provider-key", + "requestedScopes": [], + "capability": "provider-controlled", + "supportedActions": "Read AI visibility, mentions, citations, answers and competitor data; browse and curate tracked topics and prompts (create, generate, move, tag, and replace keyword lists); pull SEO keyword, GA4, Search Console, PostHog and Google Ads/OpenAI Ads reports; run the content pipeline from brief to article and publish to a connected CMS as a draft, to staging or live; read and edit organization memory files and skills; create, assign and update Action Center items; ask Ask Gauge questions.", + "evidence": [ + "https://docs.withgauge.com/help/mcp", + "https://app.withgauge.com/.well-known/oauth-protected-resource" + ], + "reviewedAt": "2026-10-08", + "liveProof": "not-run", + "keyPermissions": "Gauge API keys are not scoped: a key reaches every MCP tool in the organization that created it, including publishing to a connected CMS, and requests are attributed to the organization rather than a user. Generate a separate key for Paperclip so it can be deleted independently, and set publish actions to Ask first before agents run unattended. Paperclip cannot narrow an existing key." + }, { "app": "github", "method": "managed", diff --git a/packages/shared/src/app-definitions-url.test.ts b/packages/shared/src/app-definitions-url.test.ts index 153a509f844..1d9711a5d8b 100644 --- a/packages/shared/src/app-definitions-url.test.ts +++ b/packages/shared/src/app-definitions-url.test.ts @@ -12,6 +12,7 @@ describe("tool app gallery URL matching", () => { expect(getAppDefinitionForUrl("https://docs.google.com/spreadsheets/d/sheet_123/edit")?.slug).toBe("google-sheets"); expect(getAppDefinitionForUrl("https://gmailmcp.googleapis.com/mcp/v1")?.slug).toBe("gmail"); expect(getAppDefinitionForUrl("https://mcp.neon.tech/mcp")?.slug).toBe("neon"); + expect(getAppDefinitionForUrl("https://app.withgauge.com/mcp")?.slug).toBe("gauge"); expect(getAppDefinitionForUrl("https://www.superagent.sh/mcp")?.slug).toBe("superagent"); }); diff --git a/packages/shared/src/app-definitions.generated.ts b/packages/shared/src/app-definitions.generated.ts index 5a6482dbec6..a3d7fa86773 100644 --- a/packages/shared/src/app-definitions.generated.ts +++ b/packages/shared/src/app-definitions.generated.ts @@ -71,23 +71,24 @@ import a69 from "./app-definitions/honcho.json" with { type: "json" }; import a70 from "./app-definitions/neon.json" with { type: "json" }; import a71 from "./app-definitions/superagent.json" with { type: "json" }; import a72 from "./app-definitions/telem.json" with { type: "json" }; -import a73 from "./app-definitions/gmail.json" with { type: "json" }; -import a74 from "./app-definitions/google-drive.json" with { type: "json" }; -import a75 from "./app-definitions/google-docs.json" with { type: "json" }; -import a76 from "./app-definitions/google-sheets.json" with { type: "json" }; -import a77 from "./app-definitions/google-slides.json" with { type: "json" }; -import a78 from "./app-definitions/google-calendar.json" with { type: "json" }; -import a79 from "./app-definitions/google-chat.json" with { type: "json" }; -import a80 from "./app-definitions/google-people.json" with { type: "json" }; -import a81 from "./app-definitions/google-workspace-search.json" with { type: "json" }; -import a82 from "./app-definitions/openai.json" with { type: "json" }; -import a83 from "./app-definitions/openrouter.json" with { type: "json" }; -import a84 from "./app-definitions/xai.json" with { type: "json" }; -import a85 from "./app-definitions/google.json" with { type: "json" }; -import a86 from "./app-definitions/bedrock.json" with { type: "json" }; -import a87 from "./app-definitions/responses-api.json" with { type: "json" }; -import a88 from "./app-definitions/messages-api.json" with { type: "json" }; -import a89 from "./app-definitions/chat-completions-api.json" with { type: "json" }; -import a90 from "./app-definitions/local.json" with { type: "json" }; +import a73 from "./app-definitions/gauge.json" with { type: "json" }; +import a74 from "./app-definitions/gmail.json" with { type: "json" }; +import a75 from "./app-definitions/google-drive.json" with { type: "json" }; +import a76 from "./app-definitions/google-docs.json" with { type: "json" }; +import a77 from "./app-definitions/google-sheets.json" with { type: "json" }; +import a78 from "./app-definitions/google-slides.json" with { type: "json" }; +import a79 from "./app-definitions/google-calendar.json" with { type: "json" }; +import a80 from "./app-definitions/google-chat.json" with { type: "json" }; +import a81 from "./app-definitions/google-people.json" with { type: "json" }; +import a82 from "./app-definitions/google-workspace-search.json" with { type: "json" }; +import a83 from "./app-definitions/openai.json" with { type: "json" }; +import a84 from "./app-definitions/openrouter.json" with { type: "json" }; +import a85 from "./app-definitions/xai.json" with { type: "json" }; +import a86 from "./app-definitions/google.json" with { type: "json" }; +import a87 from "./app-definitions/bedrock.json" with { type: "json" }; +import a88 from "./app-definitions/responses-api.json" with { type: "json" }; +import a89 from "./app-definitions/messages-api.json" with { type: "json" }; +import a90 from "./app-definitions/chat-completions-api.json" with { type: "json" }; +import a91 from "./app-definitions/local.json" with { type: "json" }; import type { AppDefinition } from "./types/app-definition.js"; -export const APP_DEFINITIONS=[a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19,a20,a21,a22,a23,a24,a25,a26,a27,a28,a29,a30,a31,a32,a33,a34,a35,a36,a37,a38,a39,a40,a41,a42,a43,a44,a45,a46,a47,a48,a49,a50,a51,a52,a53,a54,a55,a56,a57,a58,a59,a60,a61,a62,a63,a64,a65,a66,a67,a68,a69,a70,a71,a72,a73,a74,a75,a76,a77,a78,a79,a80,a81,a82,a83,a84,a85,a86,a87,a88,a89,a90] as AppDefinition[]; +export const APP_DEFINITIONS=[a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19,a20,a21,a22,a23,a24,a25,a26,a27,a28,a29,a30,a31,a32,a33,a34,a35,a36,a37,a38,a39,a40,a41,a42,a43,a44,a45,a46,a47,a48,a49,a50,a51,a52,a53,a54,a55,a56,a57,a58,a59,a60,a61,a62,a63,a64,a65,a66,a67,a68,a69,a70,a71,a72,a73,a74,a75,a76,a77,a78,a79,a80,a81,a82,a83,a84,a85,a86,a87,a88,a89,a90,a91] as AppDefinition[]; diff --git a/packages/shared/src/app-definitions.test.ts b/packages/shared/src/app-definitions.test.ts index 3167197d008..3687d047dea 100644 --- a/packages/shared/src/app-definitions.test.ts +++ b/packages/shared/src/app-definitions.test.ts @@ -273,7 +273,7 @@ describe("AppDefinition catalog", () => { "google-workspace-search", ]), ); - expect(SELF_SERVE_MCP_CANDIDATES).toHaveLength(52); + expect(SELF_SERVE_MCP_CANDIDATES).toHaveLength(53); expect(BLOCKED_MCP_PROVIDERS.map((entry) => entry.slug)).toEqual([ "g2", "vercel", @@ -437,15 +437,15 @@ describe("AppDefinition catalog", () => { expect(channel("slack")?.guidanceMd).toContain("reactions"); expect(channel("slack")?.guidanceMd).toContain("direct messages"); }); - it("keeps a complete, unique, dated evidence ledger for all 55 researched MCP providers", () => { + it("keeps a complete, unique, dated evidence ledger for all 56 researched MCP providers", () => { // Ledger-wide date reflects the last full re-verification (2026-08-26); // later provider additions carry their own research evidence, but // bumping the shared date would overstate freshness for the other providers. expect(SELF_SERVE_MCP_RESEARCH.verifiedAt).toBe("2026-08-26"); - expect(SELF_SERVE_MCP_RESEARCH.entries).toHaveLength(55); + expect(SELF_SERVE_MCP_RESEARCH.entries).toHaveLength(56); expect( new Set(SELF_SERVE_MCP_RESEARCH.entries.map((entry) => entry.slug)), - ).toHaveProperty("size", 55); + ).toHaveProperty("size", 56); for (const entry of SELF_SERVE_MCP_RESEARCH.entries) { expect(new URL(entry.docsUrl).protocol).toBe("https:"); expect(new URL(entry.serverUrl).protocol).toBe("https:"); @@ -845,7 +845,7 @@ describe("AppDefinition catalog", () => { "ticktick", "xero", ]); - expect(APP_STORE_DEFINITIONS).toHaveLength(68); + expect(APP_STORE_DEFINITIONS).toHaveLength(69); const connectableSlugs = new Set( CONNECTABLE_APP_DEFINITIONS.map((entry) => entry.slug), ); @@ -1130,6 +1130,56 @@ describe("AppDefinition catalog", () => { expect(method.warnings?.length).toBe(2); } }); + it("connects Gauge's hosted server with browser sign-in or an organization API key", () => { + const gauge = APP_DEFINITIONS.find((app) => app.slug === "gauge")!; + expect(gauge).toMatchObject({ + name: "Gauge", + categories: ["analytics"], + urlPatterns: ["https://app.withgauge.com/*"], + docsUrl: "https://docs.withgauge.com/help/mcp", + branding: { logoUrl: "/brands/apps/gauge.png" }, + redirectConstraints: "https-or-loopback-http", + }); + expect(APP_STORE_DEFINITIONS.some((app) => app.slug === "gauge")).toBe(true); + expect(gauge.methods.map((method) => method.key)).toEqual(["mcp-oauth", "mcp-api-key"]); + const [oauth, apiKey] = gauge.methods; + expect(oauth).toMatchObject({ + transport: "mcp_remote", + auth: "oauth", + ownershipModes: ["dcr"], + riskTier: "S3", + // Gauge advertises identity scopes too; only the reviewed MCP scope is + // requested, and the organization is chosen on Gauge's consent screen. + defaults: { serverUrl: "https://app.withgauge.com/mcp", scopesHint: ["mcp"] }, + }); + expect(apiKey).toMatchObject({ + transport: "mcp_remote", + auth: "api_key", + ownershipModes: ["customer"], + riskTier: "S3", + defaults: { serverUrl: "https://app.withgauge.com/mcp" }, + keyPlacement: { location: "header", name: "Authorization", prefix: "Bearer " }, + }); + expect(apiKey!.credentialFields).toEqual([ + expect.objectContaining({ key: "authorization", type: "password", secret: true, required: true }), + ]); + // The key form renders credential helper text, not method warnings, so the + // publish advice must live in the helper. + expect(apiKey!.credentialFields?.[0]?.helperMd).toContain("set publish actions to Ask first"); + for (const method of gauge.methods) { + // One shared profile: no capability picker, but the access step shows + // its description (with the publish warning) before sign-in. + expect(method.capabilityProfile).toMatchObject({ + key: "write", + description: expect.stringContaining("publish to your connected CMS"), + }); + expect(method.tenantFields ?? []).toEqual([]); + expect(method.warnings).toEqual([ + expect.stringContaining("one Gauge organization"), + expect.stringContaining("publish to your connected CMS"), + ]); + } + }); it("connects Superagent's hosted server with an organization API key only", () => { const superagent = APP_DEFINITIONS.find((app) => app.slug === "superagent")!; expect(superagent).toMatchObject({ diff --git a/packages/shared/src/app-definitions/gauge.json b/packages/shared/src/app-definitions/gauge.json new file mode 100644 index 00000000000..da9d1f2bd15 --- /dev/null +++ b/packages/shared/src/app-definitions/gauge.json @@ -0,0 +1,94 @@ +{ + "schemaVersion": 1, + "slug": "gauge", + "name": "Gauge", + "description": "Track how AI answers mention your brand, research keywords and traffic, and run content workflows.", + "categories": [ + "analytics" + ], + "featured": false, + "branding": { + "logoUrl": "/brands/apps/gauge.png" + }, + "urlPatterns": [ + "https://app.withgauge.com/*" + ], + "docsUrl": "https://docs.withgauge.com/help/mcp", + "redirectConstraints": "https-or-loopback-http", + "methods": [ + { + "key": "mcp-oauth", + "transport": "mcp_remote", + "auth": "oauth", + "ownershipModes": [ + "dcr" + ], + "whenToUse": "Use browser sign-in for the provider-hosted MCP server.", + "defaults": { + "serverUrl": "https://app.withgauge.com/mcp", + "scopesHint": [ + "mcp" + ] + }, + "guidanceMd": "Connect Gauge in the browser and choose the organization this connection may use. Write tools start enabled and remain governed by Paperclip's action policies.", + "riskTier": "S3", + "label": "Sign in with Gauge", + "consoleLinks": { + "docs": "https://docs.withgauge.com/help/mcp" + }, + "warnings": [ + "A Gauge account. Each connection reaches one Gauge organization: sign-in uses the organization chosen during consent, and an API key uses the organization that created it.", + "Gauge content tools can publish to your connected CMS, including live. Set publish actions to Ask first before agents run unattended." + ], + "capabilityProfile": { + "key": "write", + "label": "Read and write", + "description": "Read AI visibility, SEO and traffic data, and run content workflows. Gauge content tools can publish to your connected CMS, including live. Set publish actions to Ask first before agents run unattended." + } + }, + { + "key": "mcp-api-key", + "transport": "mcp_remote", + "auth": "api_key", + "ownershipModes": [ + "customer" + ], + "whenToUse": "Use a Gauge organization API key when browser sign-in is not suitable.", + "defaults": { + "serverUrl": "https://app.withgauge.com/mcp" + }, + "guidanceMd": "In Gauge, open Settings → Integrations → API Keys, generate a key for Paperclip, and paste it below.", + "riskTier": "S3", + "label": "Use an API key", + "credentialFields": [ + { + "key": "authorization", + "label": "Gauge API key", + "type": "password", + "required": true, + "placeholder": "Paste your Gauge API key", + "secret": true, + "helperMd": "Gauge API keys are not scoped: a key reaches every MCP tool in the organization that created it, including publishing to a connected CMS, and requests are attributed to the organization rather than a user. Generate a separate key for Paperclip so it can be deleted independently, and set publish actions to Ask first before agents run unattended. Paperclip cannot narrow an existing key." + } + ], + "keyPlacement": { + "location": "header", + "name": "Authorization", + "prefix": "Bearer " + }, + "consoleLinks": { + "keys": "https://docs.withgauge.com/help/mcp", + "docs": "https://docs.withgauge.com/help/mcp" + }, + "warnings": [ + "A Gauge account. Each connection reaches one Gauge organization: sign-in uses the organization chosen during consent, and an API key uses the organization that created it.", + "Gauge content tools can publish to your connected CMS, including live. Set publish actions to Ask first before agents run unattended." + ], + "capabilityProfile": { + "key": "write", + "label": "Read and write", + "description": "Read AI visibility, SEO and traffic data, and run content workflows. Gauge content tools can publish to your connected CMS, including live. Set publish actions to Ask first before agents run unattended." + } + } + ] +} diff --git a/packages/shared/src/self-serve-mcp-research.json b/packages/shared/src/self-serve-mcp-research.json index 83b4601523e..c721d572f36 100644 --- a/packages/shared/src/self-serve-mcp-research.json +++ b/packages/shared/src/self-serve-mcp-research.json @@ -60,6 +60,7 @@ {"slug": "honcho", "name": "Honcho", "wave": 3, "status": "self_serve", "docsUrl": "https://honcho.dev/docs/v3/guides/integrations/mcp", "serverUrl": "https://mcp.honcho.dev", "authMode": "api_key", "prerequisite": "An API key from the Honcho dashboard. Memory is stored in your Honcho account; select workspace and peer identifiers when calling tools.", "riskTier": "S3"}, { "slug": "neon", "name": "Neon", "wave": 4, "status": "self_serve", "docsUrl": "https://neon.com/docs/ai/neon-mcp-server", "serverUrl": "https://mcp.neon.tech/mcp", "authMode": "dcr_or_api_key", "prerequisite": "A Neon account. The hosted server grants broad project and database management, so use a development project and review write actions before connecting production data.", "riskTier": "S4" }, { "slug": "superagent", "name": "Superagent", "wave": 4, "status": "self_serve", "docsUrl": "https://www.superagent.sh/docs/mcp", "serverUrl": "https://www.superagent.sh/mcp", "authMode": "api_key", "prerequisite": "Superagent report and triage tools consume organization credits, and deleting a finding is permanent. Set those actions to Ask first before agents run unattended.", "riskTier": "S4" }, - { "slug": "telem", "name": "Telem.AI", "wave": 4, "status": "self_serve", "docsUrl": "https://docs.telem.ai", "serverUrl": "https://mcp.telem.ai/mcp", "authMode": "api_key", "prerequisite": "A Telem API key. Create one in the Telem console at app.telem.ai.", "riskTier": "S2" } + { "slug": "telem", "name": "Telem.AI", "wave": 4, "status": "self_serve", "docsUrl": "https://docs.telem.ai", "serverUrl": "https://mcp.telem.ai/mcp", "authMode": "api_key", "prerequisite": "A Telem API key. Create one in the Telem console at app.telem.ai.", "riskTier": "S2" }, + { "slug": "gauge", "name": "Gauge", "wave": 4, "status": "self_serve", "docsUrl": "https://docs.withgauge.com/help/mcp", "serverUrl": "https://app.withgauge.com/mcp", "authMode": "dcr_or_api_key", "prerequisite": "A Gauge account. Each connection reaches one Gauge organization: sign-in uses the organization chosen during consent, and an API key uses the organization that created it.", "riskTier": "S3" } ] } diff --git a/scripts/ingest-app-definitions.mjs b/scripts/ingest-app-definitions.mjs index 42dbb080689..9191eafa9dc 100644 --- a/scripts/ingest-app-definitions.mjs +++ b/scripts/ingest-app-definitions.mjs @@ -1017,6 +1017,7 @@ const categoryBySlug = { egnyte: "content", embat: "commerce", fireflies: "productivity", + gauge: "analytics", "hugging-face": "ai", jira: "productivity", kernel: "developer", @@ -1100,6 +1101,7 @@ const apiKeySpec = { prefix: "Bearer ", placeholder: "Paste your Coda API token", }, + gauge: { name: "Authorization", prefix: "Bearer ", placeholder: "Paste your Gauge API key" }, kernel: { name: "X-API-Key", prefix: null, @@ -1229,6 +1231,34 @@ const specialMethodsFor = (entry) => { }] } : {}), }), ]; + // Gauge's sign-in picks one organization during consent; an API key is bound + // to the organization that created it and carries no user identity. + if (entry.slug === "gauge") { + const warning = + "Gauge content tools can publish to your connected CMS, including live. Set publish actions to Ask first before agents run unattended."; + // The access step renders the capability profile before sign-in, so the + // publish warning is visible on the default browser path too. + const capabilityProfile = { + key: "write", + label: "Read and write", + description: `Read AI visibility, SEO and traffic data, and run content workflows. ${warning}`, + }; + return [ + oauthMethodFor(entry, "mcp-oauth", entry.serverUrl, { + guidanceMd: + "Connect Gauge in the browser and choose the organization this connection may use. Write tools start enabled and remain governed by Paperclip's action policies.", + warnings: [entry.prerequisite, warning], + capabilityProfile, + }), + apiKeyMethodFor(entry, "mcp-api-key", entry.serverUrl, { + whenToUse: "Use a Gauge organization API key when browser sign-in is not suitable.", + guidanceMd: + "In Gauge, open Settings → Integrations → API Keys, generate a key for Paperclip, and paste it below.", + warnings: [entry.prerequisite, warning], + capabilityProfile, + }), + ]; + } // Superagent's hosted server advertises protected-resource metadata, but its // authorization server publishes no OAuth metadata, so organization API keys // are the only working credential. @@ -1750,7 +1780,7 @@ for (const entry of researchManifest.entries) { schemaVersion: 1, slug: entry.slug, name: entry.name, - description: ({ neon: "Manage Postgres projects and branches, run SQL, and inspect schemas in Neon.", superagent: "Review security findings, start red-team reports, and score content and packages before agents trust them.", mem0: "Remember preferences, conversations, events, and agent state.", zep: "Retrieve temporal graph memory and authorized business context.", supermemory: "Search and save shared memories, documents, and profiles.", honcho: "Remember conversations and retrieve context about peers.", telem: "Search the web and read pages across many search providers with one API key." })[entry.slug] ?? (entry.slug === "fireflies" + description: ({ neon: "Manage Postgres projects and branches, run SQL, and inspect schemas in Neon.", gauge: "Track how AI answers mention your brand, research keywords and traffic, and run content workflows.", superagent: "Review security findings, start red-team reports, and score content and packages before agents trust them.", mem0: "Remember preferences, conversations, events, and agent state.", zep: "Retrieve temporal graph memory and authorized business context.", supermemory: "Search and save shared memories, documents, and profiles.", honcho: "Remember conversations and retrieve context about peers.", telem: "Search the web and read pages across many search providers with one API key." })[entry.slug] ?? (entry.slug === "fireflies" ? "Search meeting transcripts, read summaries and action items, and connect meeting-ready routines." : `Connect ${entry.name}'s provider-hosted MCP server.`), categories: [categoryBySlug[entry.slug] ?? "other"], diff --git a/server/src/__tests__/tool-access-service.test.ts b/server/src/__tests__/tool-access-service.test.ts index ba944f45d45..5da1230161d 100644 --- a/server/src/__tests__/tool-access-service.test.ts +++ b/server/src/__tests__/tool-access-service.test.ts @@ -5214,7 +5214,7 @@ describeEmbeddedPostgres("tool access service", () => { "telem", ]), ); - expect(res.body.apps).toHaveLength(68); + expect(res.body.apps).toHaveLength(69); for (const slug of ["openrouter", "bedrock", "responses-api", "messages-api", "chat-completions-api", "local"]) { expect(res.body.apps.find((app: { slug: string }) => app.slug === slug).tags).toContain("model-provider"); } diff --git a/ui/public/brands/apps/gauge.png b/ui/public/brands/apps/gauge.png new file mode 100644 index 0000000000000000000000000000000000000000..6e27551c5e4cdd18b181fcd77d2bd0d173a86ec2 GIT binary patch literal 28340 zcmd3N^LL!x_jZHER^v3b?Z!?T+sVXktfsMTYhv5BZQHiK^L*Za;{9RPz1GYe?6Y;Q zy{|j;x4Z-*94_3qZ{HB5Bt@0JeFGc+?*jt``bJ|dCFt8XTTdxbAr-fb^9^V({DFUqH702W@YH>E@h;PcW%nN@g$xtMZ$pZ*6!sP5=QQW|DIVp*M{hgMBAK*wa z3S$b=itHskU{(NA? zzh5>hU;o#53|w2GXwB*}jt|XK#5pAhcjjV)*s>J!uQgZU7?b@gmL!&rnPa#)G@ro{ z+-T@sZ47;hH%O=w!Ri6^sA$m0)E#)tTo6p%FJ4+YJtYvnuo@OtIrKV_`0Ue}rZ;AZ zvx)^&n9o%?m|au5ugSls9?PAdHwZ)k`?)PH|0ZaKf$L8M1D5o*#V*P|);cd4Il6E; zMF;L-7-CdCb2;M*p0|7{C;@{GW!<~%|Ax}x3O)vo1&?nIXO6XsyZ!dcqJIdw1>gd?~3R_;v zJ03Mpe>Btzkp2>3`lZ{amDJuE18hXdFhEpkBGT+tDd2~djg6Yem@X5lAoTBa=e_OH z7&BM{mg{qfAf^418fF^jLXcqXUXuRFgPv%U3kObszB4XfMN5MBv3>P%^z_AT$WhmI z)Hnln(A<@MY3^jd*EKcGL>BHx@Z}%+A?V3}*z{(6{xj-dHFJ3lhEqBnR{K3icxQ#F zCID;e(5%*cb%<>uy(w3nrmv0ULed~irP0Cqm}Ywgu6$$#gS}_C7v}rZMSl)hK6aZK zZ684&ip|`Z`qC!rA5Q>5&+*)>#4G1Tc6hB(wha6+){`&dw-u_<+HP(W3G1ODp2Fui zy2I$JiQco?mqx1L(srHB=LtcLCDaVOr(dS#>IHNl`A5eSs5QtN_BqRQtG`gBH+f>f z)Zp9j`b77B<5~|oCK6pqxZtV$YdPS`!=mF9$l|nV ze;mcCe@Tti`QL9M?HvNCfAyh z|5R&yq$ifJcxr!iMWAc!+Kvb7y`0%}jc^vFU;RmXj&zrV_q@RE=@fU7k2CSclg`;i z*nJLEf8jVTGEE0REd$J+I^P8@nKtSwp>l_|&mrAn#O< zBI{!BXDEv@>hf7RPeBDLh{qsU*KUM~#Ck+%BcZ=tvKadcr@hBShs)8MMU5Jq7%Qh_ zW=@PNPaai#DooF}an5i%tzRcNK*VBz;%_pKT>uSMg5ZUMkQgd7cAwrScn~1yc&_Y+ zw`#!Oxv<$<22~G-^+GH-*?@oVvZIbp%W5xNT(iA-Lr+ICGIyJvBJQX93bXYyCd9A` zJ2)T%`JFFRK`V9maSWD#h7Q?*?58HhVYk@RhFdQBb*Nw3; zae*2^82S0~8?Ky6JG-6ppa%sVKUmj)B0)U;=3(qYqk`hx*(or2{fNY03H8Zqcp~_s zal&-%*ds&6g%p=NrqHILovk!v2|xItFm;^wED#ZX#VQqsC}mw3ciSN!Kte1~1e=|b@O?a~Kf58Z zXiENJgq5A42g>%8zQnD+Q7UP0B!vQ#?jU5abGol8N$O^PrGcdQ5;@}DoqP;KgDH1a zC&0C;GT#3lpW1FE$k4JwAaJ=543yXc%6p{c@q+xkIgKUo*Ck(Vk=;PbxNb_uOm{9_ zeicUm6HA+FztkIqdifDf^&+cFFB4%Yw!GZW@Gd@!vwsQMHHdvxE5TMSo++*0;nxkd+%wSA<-nD#lSZYzRQA@~k%O|>V?^%Y zSBB=9_1N3unO3DV!pOFfd;?~HiMth0j&VpV_x{FiW@?C2^1g}R2Npz4o-k^-$(RPd z>X*2@cpr=2$189e^HM*C&W9F#!hKpJKg*rYCQVtQCAC_;6{hHlQl&b;&siNHDdn&S zIrS?6pBCipUQ@>0n)Dj6= zSL)8cwg7XOAlvl5PI>sEa~JS%%hl(L3x(_R$n5D^W*u5atbX5^EWhJ=2bfaoM3bu4 zuvoeGf z$I+7X4HKkHU9s<<{+`Nihg|Zo4fb?M-VVdDgoVe%7#x~xb;_~Xf}0N5cdAfd>cwuC zW8WHB&CniN{dzsoQuxzx*-V@>YI09{E?45y~pL2_Ai96vosOsjumJdPSj3^AS0 zX11Sn z$Q5i`TR{RM)pkMrBk-r7{~ho%l6`oRia!n#Fg1;n0IWsZng@r%uVak|PsIl=)&_Gt zM+6k2d?hY&1;9dKO5n2vbIbw9J8G>#n=U-g5r*Sp z$DaGuN$&c4L^da(dw@qOc0#AgtOD-Z%kQx8+mdb7Y4120c3)+Tl4gG_EWGp%*R|rp8K6X!Gx8M#SOH+vHw#g=c(& z;sp@WEG!;0u;yw?MP*yCF!jRxaMA|KsgSYP%^Li2fm&t_HWnMI5TO`Lm|}br;?-f} z0^0tJ^9PH8^iTuM)A~GxzMiF%d>PqWiL^Wo!SKdR1Acz`#)9O&CuXn6XwNJn!lpNZ zAP!#yv~6pOKP@!s=`xRO0Tnf^lw4^o8U=vG;P&?<4c_R;BTcP$O-=%s265a!UZ^$6 zZVMCPF0hDBW0s|k&H(61x3JZyFZ32I!Eqn5AH2Xqv|@eu{b|< z;8-1PpTCrNebv+83vN7n;jXPmz;kzE=uv*zx^ChMI@_sgcy_X#LmIW}@^N zv?SpIj*FNz+IVSS^rz21h@LomdUGt{93vyi@%^Qxzd6rkM*4+h$8Hr5M6v(HUde{k zqY!V`Gf`mgZyuJlxUFttxQyo|W&z^j((U3(%ln&T5Z4Im5M2R>4z+WVok?lZ#Is?G zRBe7fxw4blT0#rGbtn8{mt#^gohkdfP`k%QP@Ke zi?YUjzGv{cVLWYm)4!=DjCOG2j3meLttK}^cwgN_Oa*H@Bd(QlcqPTT;t3ZpUnp~x zCdPF9lj>meb=#+mSUtWnHKCRomgD076I6KW3@JkU>QHC_>;m*1PCE9$#BQ!zZGd0Y zGr5_gCU5$1nR+!9WUWKQ{iV~1fs7I)^xpmj^`tkT*oL@g?#pUyf~Z04L1FW|H8o#; zBlXtdqvfjA_qNH`Yrt|PWfc_YHZAn=KBgcqprWRoWtAaLB*HOR;Ai%5%73x)o@@0l zMiUGusu3}aaYbrMqsDIEM5EuxHqd&3h z4H+sAqQq3+M?nHP#}inDqqF-NS+=2yA_b&_3<6QM_BJuBK?!JQB@FMAEis=W5&gbZ zmp>5_U{o@bKQr|eu?wxRw~kMw3QRSkXS@$Y5hQVmw%0n&HfFxKiBDD1Jh8>3K$8V$ ztr?H?rdo4?Q&i9XRJ}%1wwNN&BQU(;u*96o^)JRR~ zBPzEPAb5^b{+I>G)=586KOZKe=9Qv@9fy2-Cg|+lQ5=(~i&fTvsYzW=cfoTvhI_uq zjy7lx0PISFra`ou(a;|sh&lwj$QR>D6%Luelo8iQU&F6MP8RT>GhXA(MoZg}1D}I% zr&RV_xy#r(ymn$q4J0{rP7gq#9_;y{(^I2N^L%y=&-GMXlkn#~G629C+kHQL-(&&% zkwn>9xyRHNFcbltIOBjNRtg6nvn=px69)3Tfx8Ow2j&v^Q9nJ}{v;}OU0>i4*Xg^h z5$%TxmWA3H1C*qYf9Bb-8NqIcp zT&u%21%)IKI{|8_ej6Ez%cKic=V{o%RskJ$;yJGSelcho-w`@Ineo7gGJ0q~6NqBO zdZJ;zIsy4EfW&WYRheCjIlzKS)d`xriJ0w8zwcIzwk73TooSOfaX(;It-NR@OmN?R z`{2!?JRze$84}=lT+YBO0<4)eag2YQqsyfZSe9MwC<(rarrAl5@IAE^rO(BoL`%_~ zqo>>^ue#01FOXM;sWDyWnZOdq=g11b65QPqR{rA$3HYJgBv|;x6ZwRj@TcY_{(l6rSkWIwPi=Nqlhr?d8esTlU>d8dY|hv*0-J1k78& z5p$%2q00=KLzNME20C5o_0rxQqQ(*emZS`9{XlM!B^Yje2cB`p7H>=)fljWxmZ8zW zA8h@{$mIYs5wv1I-~>jd@ZT~K!@-oV#(0_2p|x8fDgQV_T=%-{l%T@3U6F$MQ-o8O;BI<|)2y7vbAtDHLq~!-i9-cBz8verf zO#8EF5jQpPoI(ydcbL_-q}9=%tFwc=(UUuzGL1VQQ-VXW)e(%a<6jeqt<*WtO%50w zrlap}&(y&=dOfjy67~n`N1tog)?T=OSQ>0r*8itJ4VqGfb1b*7*y!eCF+r!&QZlMG zf5=0MV(^A1A%qnhDv7;jyoOJOnrW3XZp-7Jl`N{4X;BW&2MAep5PSu6?%C9$3Z|<| zeNMe#0-tAe3_b-(E35#?soFM4gHJA1orujd_4CI&ez%{~rndrm160_xkXe>hmLxj9B9i z*_@>9P%!$_6cws-S|qtn(13MIT3i$dN(hgx+ppQnF4`qqWY8)jA`GpODvxI7)megQ z00URXv#ky1G*Fma!#YqanUk$v4mEaupAil9{!D{nbG{@BXHNpi6-UzR6_2o?3e_Qv zeWW&A`qJQT7*jVXEjJG(cTg*3&1+I5%hB{+fz3bfm}D{i+j`EE?MIOu7D5ytFieOS z+No+f>^Qla#h=Qyvo3wYm+-{|)N5~&?2dU;M@EW4ip^84Cge5f&CJ4CHVm1SyV5kRCgifoJ27^8-SSVq-O+4b7S~}A#+v>9oiEWx`er}M@qIl{ z_WhKMGciAuDqQn^_fY0@0`PIzLWpba8EQnDvF>1<-s=k4aj{AJb%@_ zoG;1n75Xb7FY;Gludfe4E(9|Dh`lzshU-622>4^F9_-5YyG^^lR~5Mix2YltMhv+H zHm%`rCej!+?Jv^lXeUhN)EKM?^~YP2xR<>%PQnE94Zt z`3rxXPj>gtBz#X>kzP4Jw6pK4=jUrOsX?k4;5nnakWJ#hN21(pN#HE_L*{WiTqa+Y=Y4Tx%;DD=!uiTr^Wi^*X3KnHjhFrS zUkaUOrOd|HU2Lg)Z}&&yYDzir`*ZfSsjR-#5>c%S6?KlgIBD>VxhD2t>a)GDQE~v} zO93tlh_z>pGKldi%}I4e?*a7|p-rOo^eKiHX!=-+b9GCPdE5?5BkIL}KRX49etnic zSTA7!?h72nA5Mle&f8WM&8yB|_HQg0Z8)V#f&!$?e=PT&KlogyH?kM3hRCOv41=&$q;`fp-#RLuuwAnWXnTJAh%ALkEeOO6tah}18KQ( zxZ*D)5Df54v47zuIsCTp{UrNqoMWsA-ZXi;^DusT>S>GW%m1LGes)DB$k==Fs1?=7 z=2L}t?=3zD2y=Jr@nRr!fFIlYNYH$ugsU~q#W^s^*nj)j8?eYu+S1$2Eshk`XfWru zJAB%?p>}1qa>kl8y9+7i5=`EH>+Zod-wVcM(Pa~WfjUMg*oxR+4bg+CIvKEmA zV$x>;j(v9}LeGrFlzV-1ROJ0V#OZzBX@=6Vks>jbbfeRKf5x@#S8G#v_w+NIu5dRK z>09H}qj_s+X>a(MvohgoTjqY4TF*{_*PA>4<6EwelX0>$l56+%I%CKGQQtDvF;fU1 z2&zijU|BgcUS_kWOE9Jll->$!ee%Uu8;$)yMEMfGYGcQ7y_8Ywij9a^WLY3O?9L&c z$_&jsUs7W^F0RYCK{DIu`p0P!r{H*I;a4(FM3v*GEtimAb3^$Cy{*SCpcQfG!yhBF za8n+BG$m;4>S)IwC3w7mU#P^-miWd%bsF$YrqnO@#6K~VO=ot(@3j6Z)dJ?6zwB>r z0r7$^bK-iQdw!`s(DDfbQE>Sbj1xJh*8NYy%yBYv>ozPQnykxYD!jP_lHG^In{ms( zj0HnEB<{jFog>ln!JI^6+;Z$-u`Fx<#lmg|qbl3?^>JU{9VYm|%s$B0=uD+$gc~X! zoXZ93;lb+VIaAdu9&fk7tm|Mx;;6bke_ER?sg!Pj(-IOQ3cIgg_qE!X$)7lCf`do8 zGHkd_KA&mkC%cpS^|E)iwtePoO@Ihemrf zA>@_A?)KXobck6P{<#pK@X$`I-84B1TXVd#BI~`~_LMlxosJv#n1wBI8fNro0TI7X z>GG#^))L|=i`*H_fJF!6kzbZd(aGmO;yjnXQ=2aduKqrse-5n1$381DL;)=njJh+) z(>Bo6R$RS?ZPBJ?`gcKvxNWT+x-`qb&0d7qyV@Ms=XkLcB_=_4flEq@2p$t!X@Km3 zyD$72a7;8nR_?tC6qyLC>*t~KK^SkIB_AMZ1=AFXKVHw zfgG5Xq5eN#-lPjC45#mgGz8H4S~>}WMteMCf(&~Bcb~&7+-AxOtg+59sx8g@z8NNCpb_US8~Iy6q}VOO-C!p6+2}*0rOm z+MedDEw>$cyG3g3qn@U|^m!^Nvnld5I;Gq+Of9u~Yd z8`Ji*ch0N(BvvJyS~p@p*L;%Dl1ga3!S@AQ6l(3Vew{os^IcgPYgggT)3>(B*WI9b zm>u3AFZ4d%e69#e&#jhfR)n?qOs?_a=m}qBDNSc4O=gH*KRfJ4pA3z3YE2???Vn(z zmV5|)Jn_hI2P>;+K?B^m{Z@*u@79?(%gjA&Efzpmr*wMr*yB8T+EIQG`yz%He2U@OJr~sbOK^Hmit|xAU8o7#R7N zD;*`$@PSELB$zy;Oj$vv8IFYip4p8=ncM!()XQCj!m}C{xg^nu*R2h;^PSHdN@0mBH4Ms)>WIL zbsfSa(*yQi>XGhaRCt7_UAi*HJw8P?8ewk48(0sqEzmwFq5ze1DE)aEESv2H{z zGi4`tc$vus{pVz9*GlajJLFHx>)vzHN`H8;c9cpJzRxPM2$VPnrlm;f$v`MqFuAw# zQ+VoV3{nLfD8|fn@g#TNRI00h?%5i>rct|K7i;5Bg}D9$X{Kjx&FtyM$i9I;czTsW zi1Oq;f`XI@4|ZN*is-8j=!XKrnkHEw+P-i4T4ExD6jxObH?B-GoFWhi@rK3H zA24_3x`x&oS`l9=KWC7yIbGa*7{%F4wx^P0%{HND%d{KZOiIRjh&s|1vog22z3kD9 zGa+5ybk*9WZ}#$xuh~PHS*Uy^!@1rYat^p00)kVv1X=UN(>bTXrGKj<0h{cwVaV3I z{{8NvjmjV9ImJ^H*KYAvXBRXPD9W#Mt8_}|nNWWcC|5!@cf|zT2Ln5Hg`FM)$I4z7 zLpwowANDjgl{Hw(e(jm>Yp<~o>I~MaIS%}b{`pB*#{oC?$HM?UbSFxDohSam*TcVP zWREh|x|Cq9(?(4gFV8^>UgP>1@BQ^j{|*>n`YMxbFhi6v9D>kivz#y-`OK@gXpt86 z#H&Vp2Jq^}F)m^?E2o2}*xe8OzT5@sP8o-5@_0bLr2~Xt+XkN70#7o2J@q*!kwg}X zAS-7oOEGw-go#LLRs6uCtYKWqoUn8UBKgd)GP^s!71PNN*RXTFn=S_f6&5Nq$G!L2Cp+`gmP{1? zWsx3urnDvPwmoRbA1{m#j8cr9J(AIK5)i!8jBYKFl3 z!TB54UeuEY(bNFqVwRM=5b@+yPD`ouV;Kf)F0YK2wyohWs$Tl~T0wcQ#TXZaWg#qq zXRslLjiePNo>|sHFN4psX#Dm$xX)5dA8v0>1v<7ROL^+fh~WL*9~L=xJ+i61RQXZb zq1A#xRGNbYdgBFuVV?%=5oC@BPyO$QFl9(m9DK&rR)zxaf zf3uFC4HmHZSMi>)1zyJ}b{QoiImBiTtqtaIP_ltsgc=VbL)xXS^Sq`5p~}N$wCRdzQG~oU9+FSJg=a=ylcQg^nzv>Ap`jt zlU+xu9t6AUry9jEmyOB#+y|(3aZW~}q^Asd_axT7#yNUELyB$GFk`m&mc4d%Z)}}C zuNg*^}a#n>ujmU+08?X*&?U6Q0F9SgPYw7f0`D<@kNn$LU@ zo{3E)U$#>radeGWQimZ(wh+XsdZ2Mj-xo@8T8Ts38tMphiiywd1*x7@!`k9_hhovq zAG}bdA%^q=%>XJ5F!XDFC{99PH}A*Ccx*Ljf@GY%t zOfN||B#VLE65Fnez}Fp1`|M!fQ09sS)q1)$X7wPv`7o%fT69!80EJ(7_K6AFPw$Rx zFGFim9SKLTsYv3tXeR!q=cR5G_l`H=n^(g)K8A2e!IhyNN8tP3;~v!)i?~6Y;I?ws z;NZ|(W8eHpbSpg8PeTwDZe*)ZMGi5d5qCws=$OTaTCA<7neF^2Iaj}~n3Rf~E)a!e z1){DQ`(rNjeAmt(Znmha7Vjv-B1I>pCgF#kaRJ!`y#) z{n?^Yb1GiJXB>jFQ?hPJ4ox`3B!0YuqaiO)X{fqPNor#nRsz>8xzP!r#)sR*5fBXH?Wq}61jyy&!ZoY?L$PH-pOhp z?*>UurwK8q!ylR_)Dn>dBIxW3=-+NDgX@&n z;r1}8?grfJlFSE|ffT`xa@)#$GO95j$lClghP5dD-_Ku33GqGxd`a!^ z_~K{gN>Co7P8hM~s0v>fGAmaux3SxL0}fAc=`yix*@5anhcm?}1(&+t~tT#*Lil5P$d5YFfhAs(DY<)AgK)tcg9 zj3NHLI22&t*K=h{MGE3b)`Z5D26J~M8q8Tz7{!ws;t0x@yPo8KAE+2DK?Hf=n(F_qjMce}-fBCuWM>Unsae9d|LQ z_kxa~QHKQ*xR!1m9%pF4`SaM>e>M>PEncdj(A{IwF%^?G07W+zL8KOl=CyP+6jDyr zGHmqSA^&}u$fvjN=p(*~_g2{h#OtgHjLQq=wiR>?wjN@@26vQ+CgK(CTgPl?GHbEk zAP3Eoh~d{_i%4E>Oqq$#zga)i9EH5l*uK^>9KCdbDjcG6==`gptLxA40he_;mJBR_MeLhv?IxGSEPdlVyZ{mE@Mn&`2o;O!$9ocl;3Oer)9iX_kQxas1ZN>&NBb@$Ybip+SF(t;7QQ z)$k*L&nRxy8fgH^N&g)2N;<`F4URNLdHAH{3&|hSLfQeVOPhP2$6hPFGN4V4j1dC0 zYQy~MnZ{?$PYehBqRn+l_2oSyzh9B`)d+*Klinkfs&R_nAk+L=mMxFHz1h&BsoBlE zT1VmIe^O4=< z5R>U(b)+^oRbvJ?CK*px&=bXGkC#bpG>Y&a_$hE!N@>Ehyvgs&16Zu`F_>1k?%@Zb z?Y0pp!UeQb;C0qNJG|&?zf9G|Y$6_O44OXMd7$(3nj~O|XA{l*G@V%jUFvx8X=7Ku zt3kO;pkdkiBA*@9fqeCHPt~CGb!zybp{pZ-c!ePZfZ}5arr*fm>Y0423DQ63jb$Ox zf_Z#;**?@3rudHe#E-yKFQzJQE;Wx7DIl6kA>S|zDyQn!#u1uN*Bu!XZFd+C0x4FC zWB8UM9Odj_vb&fUjbZAa`P6tA9mSFB2)dP4`HwT#t~9cdOsbN#HUwL zeDNrM!y*dyPzt0r@Z8SS`^VkolU}fN{7pjbLFi1YDtLwBC_$hjO3xYcQL&-vyF5F~ z-{Nv3a?y4%n^#lFjdM$x`_O&=rEPE8+3P!@+>i!>i@1x!u#Qth*x7;TM3pt5s`Mif=#26a=CL zWe)>l^IOE1lxXZ|1CJ@7YDJ!1E9~`8$lrNn+zg_?%2t>VM5tzlt_s%M(t>v z71a{*7EF)u}+VijUiXq)!qALywPa;Q3wfymgkGz zz9!&OZBn>aj7tzE3sQWJkh>y)diYBSu))^v2hd3aMqCDD^s5U_1y&|$d*&02?)@ZK zcB_r#ryiIw@#rn6MR%!SutnUJm3QxGXP{M{@TGIug8wZmyZ?*M6kwaFHq5SL{_6T~0ZnH2qPJf&7te2+`gHK=x?Zz>S| zQ-x^wa-DT9Sf7zzVp^~iiIzvn3I*r*d&oxOCrB&i29;0;RH@hK&@N2MfME(@Iy$1Y zOmB$*P10 zoFMEzu^i4n7VOla*~7HqV*$#*I~B``*la;y%4lBv6!!+?2bU4OKDFjTKg`YNMSbtS zN9u8sPn5_!8g1uv>J+K5D(ziv&=^vSvM@cLQx(FaXBfg>oHU3k4iXR1Q^)r7)0_S- z54Chf)SQvKtn79@r}7c!*KR}i+AnNF84-1_Me!g(Uc?fk7D19kJjAln_o>*ljF*VA z-5%WCz{f+(yoMth@5pC$x|U|)3ql-)JsEJPH;9`HIy z7K7g%?l=bZdNNtFawVb<3-|_9^6s6osZ5_xiQ~e1u0&6k%7`O_O~NDA`o@Jv+Hx3R z-Y~Pz5OCrmAsQS`hDAJUuZoFv>_B2|s>8+3)BT*2x99g2wu6g(q`#mPc7Wl2>2Oe4{}SOD&N-SVw05gU zk0fiB3GzSA6wCxyJ|XEVrI7tY3FUQMsXgMAVj-L{f;d$IJfiRk!&T_I$UN!{YQGiL zn(B<2AD&3K?TVR&2m^4j?+Mxtv76ylZy3OuKuhf!6>z6Jt_$bCK7|_4BQ6^e#nM zz~|lxi$W(Ub2I}LyOdp8Q2_0dU9PyKq8_f+t{Ka&w>n{9Y^)jr6%99BinaAW5bY|&EtU`{30>+WZmpZ(=bq+d z9uK;bg2){}NoNoTZE`Dd7ZwLUz2X^jBI3GLQ8?tD)Q?(8OBM6<9uS0?@e0I1<0oAEZDs+IisjklgOJ!Gr-97{aN@vDWMC#j}wF0_|%I zSigLdd-H;6r9dzqYt~~%cwxWSqA^u_VEG#KzAlO$&-4reE60kxP)YS{6dV$0xG6IY zR6{Y}F-CKCu|m87Sff;GPU!Gj$foJP5GaWVx%pBZNz=VR(TYiec{) z{Fb-#K|C_N-8CSHy|cWB?2aFLVQ3lQ$CH3CY|HGg6f|DlH3SJv53i|y z7fUb{A~GB5ojntWCFV0(M<|iG-=7J1WJ>p1*1ACq%KgO@J0y<>mz|Goz3MR-V{J;V zQ%xJ#2F)cv8?K*95hOFE6hb&T4P=9tql%>bFnL}l1kk4ykH>BZ2mK71abf!K)U?e! z*hCmT8@QOneOxjGIf8I=gc{pCrCSDlZZQ1o`fdnMME?Pn@};X9nzh^;A<(iVt#*Ap z5_C2w;7@hAbqgoo@AQIVKAz^d@b0wh_W zhM3vLCG7{qgdtdqyw~0NgCs%6Z;PPOLP~YhO0dx*n;(KuITM5)As6%qw5P8g189hO z%GYXG#PQUc%Q}F2IdA6h9uUv8deJhBPs_KNx4A2LCES{&d9j`uGo6kc;z>B<$^rg` zy|F7sHMH_X;SYWG2o)gFuXr84?N=^A`lcQysl3XYm8QzrC%!zy5Opwu`xt$|vt7dA zJjd+)@s_#NTesJaKHDOvF>UJe9Hv0;y@EgCVNi8>M_O%;%MWrG_YEpy(OVWQAkZ&S zy*>TeGCZIP%V~kNo}sJ=ir%kf0KdMF`k@ua!GK{U_#Akal18+9RqAKcVH<-&(n9S| za9w8tt)FIhg{cf=77!eZ>@Dzk8kTvB$}7kNx0=kD*9A|E~m(rcx~T9kR#dS(GQ^_lTxpVBN@_9NrZ?B-TvH<#eU1`iw{E(IdBt(e zpn$FL+WkjHf??fX=e9-fKSR`=ng93kS>C^7u&Zd0maJ!HW0j6D;t~jcVP$v@mVcby zf5pr>ujvRAyk^Cv^}i7{wUI${`cHs>*?9VR`E-%{$k=(r#1lf@T-AgP|FjDkL#asU zKJ@gQj!8$7V6%G11{WCMJl+z95s%uh*eOof7OR%r)ogsRWGWb{qcSP0%|M zExSV{rD`!r$}A|ZfCQMlTM(vBq$r_XwFST#;{8US7v^;(&THj=9p zXGMyHAx9x*mn)bX!U^)jNi5r< z)8D;us;v`w>COh8P8g(rdw%~Yn5<2A7SOTGa}i)oraKH1TBhy* zL`!i%8Trj$6>P!cU9KDkh>ptUpbO+c4f5$o;~%W=ikIJytn*g<*D*{>W!gHg>B2aa zbB)TL+FoNIkSu^=RP$;}_?It_EVq{B-Sp*LBn6D6rzRX@kfpr1LE}gjozHVu(@Mh( z-_@P8Wx9<78ntINiUeLy~?T8}6e+an#%ctZ^CC5X*tr4c}1Rw3NmKbHt(Vod4 z!``3`A3d?Voq9L}y{J|{Dm_n_RxX2NQmcSPeyj)QFJ@zdKDWuq_veGMJeAw2Z$jO+ zSITnt=RcWEGs`a$D4PG^H?Z0!rOT^=m+TC7>fhD_3k-D zxZ7UrC9>?p>*qc==ghdnvWcXI3$m*-c4P-lFTE>tb3T zv+=AD7v51U=gVQkz-Ho<`{VpvJ>gySlLqEQZ*02Fj{)0K!~1Cxs0|l{547u!H0{q{ zZUY`k>2!G`zu!}%Wa$_&qo@rhBL?{9QIkl%XMlGC#Rp$8UTj~Ik68*WO6{DiBKg~p z_})m={Wa^O>>{Iis#arc>$CX?jG}K$H@+Bd+6(J59#ESY%61syPae(5IrpSm3@tL0 z1iZdj8CqYQlU)8|^TMU5ix#)U`iFi89`gkH1AKr3P5dlO#$A8Zg4(jqo5XD!Z%~0h zDMx3J<;i)$B<5K43Lb&UshijbndWaTWSnrnA_^6P*5scp_>Z+(k|zvrut)hHo?Tj`~QSDY?mPGU1FgF+)vC%|Rat_L+)9 zNqW{9?hnP|kkj*R@zI7DJm22Q@~g9xg~t2$Y>TdSlwF|)#^EeX+wEegVVQ1X6PLwSOJwe$-41-j-4d-DUCHh zq}&p+5+Xq6-@goWo;OKw`-u`fF1D0+U0_z~Y7R|k%uAZv^0>~S5-H}pIe%zPx;Ouw z;>9BW)yq;?+zV#%^@l|e7Ta+@3X2j>g1S=;OwDrxV+x|KJ|J*#*GA;l=5I!60=FU(R4{^GiUb#Bm|d_T{$U;Z5~;M4^vXUI{L8i9 z(+gWHw4Q4CN3w!O?Ki#kh5+Z4z;lDub$({ktGTO_;yN~3OwOzi?zyVw;K5VNDsA${ z7~a4fOS>q%v;jXQPaH}(5~0(tYnG;sXA<}hLWi4InaE6&$sbmlrf=qPYZmWR*H
?dC{NmBl-_s^dH-EZ$S%&Ph)`qY$isVHE2yR!q!im(xBs{3~7( z3lY@F=ajD#mPL;ZjJTgnYj#LJi!>8&(;rqjkJwP)EfFhrg079$d3L?FcDK37<{po# zmzu39WpY?HddNj&kyi_vn{QwXwILqTPF*u~Z%VkMfQY3?DcYdosiCqJv5^|Z;3Yqt zy$DeFI=T*GMoYQb2PO2){FNr~CJNS>4p~=*u@Wb9fmhVaQslclblxqGTdH2mT}8ku;`X!PDPUd`$jWX^1Q#4(8iyXcv)nv$nZ6BF zbt;ih8sLp1X4-g4Ihrt$%Acy(L3hZ*OoMv-7(Sd1gH!NG{S$Q9XLqw%ny4cdgLz*) z%|>8I4dMz=FSc1*>_MtdU1~Mngb|4~OWjpDWywT6C6h@Rv~q^~r9lz3L)E)&xMjl}_dsH=*N^sff`92U zlU6}-L-o2tH;k_G^r95PvGwkVWadWe{DQM}>a#?W>?&C;XPpTx&|KkwF92xJEpCQg ze#7UZQNR+nXh@q5sWEd4XdofsZx(Pv=*)%eZYq7F_I#GP+*Gr+x=(c@>ls55>I(nc z&gCmiEy%MgI2IJWn%wMwI&lWciBIt6{r1VsgLzLBakO!Y3cpQme5ptS|Pza`H1;0Awl&}S!+x7D1csLHe6j9W3LA%DetWSQnO_pduFOnD39c5?Q2Hu0|J^odo}%@AyyG%RCj zO>E+NmKdb)W_$Br_Wr_MgB_?iPOTvP7}H1LXSJlcd_sw|#6s*S&L zkET3#^ml2M=4f;sC|uV6O!yI-(YP>*k-hS5Z=9L`)EB=k;RvCAlVgmAvNEO6^mO2K z-nZLxYGbrTCR=!hesL3Z9Ky3gSOtyOF)2L=EtY4NEV29T9{5rJ)m%7cWaL)zocZP{ zQp4-@AaszTZ?WTMOq<}<(}#MSha{vU@bO}s_LGX2#cL`l%UA-_(b5U%t5`2ZswF;1 z`8Sn0?>}TS=#0XIxhy1RmKi7O@RUdI*S5$w;(}of%jN{Xq=CAfVJ5FLv-s$BAr?O| zv>HlNJPbeIOp^4_%5vJ1D9ievy9QR|O3KZ6;pz-pq?=tun-1pQ z+k*4bcD!;21gCNWI6EiB2SOGmQ>(#~ALdvBX(BaYL7e*pY93x$HiYs)%rg;>uiQR+ z;DP22%rQNY7Kg0aw7m06Gzy8YsG|<9LtZ|}?lwemE4vL7X1d|AR=1yvQ4W|SYy8r` zI~$E$Oc^(r=#CVnI;|DT9IV%5re|=I4@KmG>a+b}Bl5no( z1!PP6Lbxb{L*|`>($)el=GS$j zcrn+5B|7B@uTp!dV2AP$jAnHS6~v~&+k2+bE5OMfY}hFks$dNQMD-l46!VtMgcWGY z3jHZhBCW}la)XRsaf82RCDfEUZ4sOvo_x)MOz;#5xzz7{bGtPd1g)-BtP+`*VZW(* zO77Bc%#5ixRFCHG3Li)jMj2pv(mkhSVHcv-({0Iw3_R&0DsQbz)hkK2bOLai^d{*)ndK{v8({esbxzYCSBK#x%XagxQG&-hP}raASD$@$BM#pfg^I(6 zxQU{P)3T{n$u<0@^>infu0GCSgTc|6dez{15VXaSBpJQ^`LE`V6}-@gZ*>su$?$28 zi!t>spkdvO37D2lkc1z?9alY6(T+;IBu)2dgiOH9Qjnq^F- zfw;hgv0MoA@=lm&5i8RwUT$qX0ND{lTwJ#tGAQ)HjFKyW*#mS9V3kfc( zBJ_zILNg3_LwG&EgA4B0rGlY4hb0({RYpQeTdl>|2a3#UlzGES3VS$#UU;loqHwX6 zuDOn2_nLcC*DJ?lzzLNR8nSzeGvjr!gD~PMhxygln1_vO&MS3)|GO0ZKWP_vW_`gu z+-p;kF|4sDax+|KAJJ4p(gX!vhtn7h0f&&sUFa`qbBOE5+b~+Y5iGYP?FlyZmQ4R# zC*uz);oQrYzir)G>Lo^;^4jWKdTM~%Dl}E1jX#^7aT&_@VUkDnS7hoSfhuzA^~Y+H zuX4@KBPO_)y5*!c-Pop0Dx; zptJ{ge^>$UuKyt#l5gW^J|*?kjSVmsh5z7g6YW`DFNzeLrFL`<9VgkeOzr=6vceqso-*$vOgMIcGd_9 z2j|gH354ek^Cc>oGOQD807_;@)%Yt_8&Gv%~3 zAp@XHHba&O7v-EZ5Ie*qU{a`d`jox+CiCv04`x`(e8k2JC@i?%79^TxOfV~tJjPqC zWa|x*blJnlRVfa)Q(mX)8?hT6*^cS7onuxv&2982 zX6M{Aks@Hb*)PeF3YP1{;c`ewIfP( zBW^ldS+BCx) zldx2G@dlFLNE-Ru=VXIV7+6{&H$f38X|{3MWXfjo8C=mLBPYiz%XOYyMlWTL)D$*$l(+juc#?;gt@FDCZ*-=Eo*M+YUD-Z% z1J+aHWYaTx=DM*wIhzqZX;N`JF&D$$@HWZy7$Fu=Y{jR1O~YvIN!k-cn`uI1zFU9x zn}0d%@x#PiZnXS%2-4*%3a8Ccvz8y!Qg1%uPVzVN-M{FXSrUHFkFb>+lO0{$v|-Dz z$j!E_icy53oRFk18LAB~Jzcup4iX`}a|08GUe1`5 zj5{DklBjx%jrKVMCiYG;(AXt5Mx(KTtt@f&@uf8+BOTMl#byu{yR9|%UZ;GGx1Q6i zhhP<`0zpqI2lHp4ojB|X6|NNqpB;{01$!Bd-?LL9~x~u zE&IWA9Y>URyRD(}Ro?c7HM;g*XR}f?e03K%d#b%_MR@or{xSe-6mjH`O&$Y5cSZ|! zt#Z-l6oQ8DyLTn8BCr%J)o#CzTy57TZoFY@JemnMqd%3re0S@{xVeRbWr25Dx$)^H z@;F--Hr(j!{u+#_2)q&d9jpe4BNj>|A4-f22a$mS_>XvA9ylF;85F^p4Q~271oD%o zw)01PeDF;s}$cDJVy6)dmu`A0+r9i54V~a)eDeR^16?@MDAFSGCV83VdW}a9r!ru)i|6o`cK< zzXDqQ1K!Q^MTA_oizSTP9ydI6Xy-WDVf^45a(}p}CcL-@fBq6ozh-yqDCW`CTim1P z+|Ya9;htOU@AKQJ-r%*Ui~FgL=l0j*J^jH{-x`Ru9~d9@ZGhW$nR13cg=Z>UX$s`l z-(c9jk_OK?T+X@JMgIc8$W=}$HLUIRLO08p!sHcr4M|3wr`#x@7zadMWuLs}S5a)g0Q3arLpxFa|qvB{dSwC$IgIm3OSECadpoC_~zHkt1a6S)Fr z)L!mn1uNNXA8+6JS7arQ)O)rG@yivmc0?~Odu4%U{O%Tm&!aC;9yBNa8{$>0 zAs_H#W=+u!>x~OCa*Xv1lMUYA@q%Egziy?m1H3JqyH_KCGe@y$);X|nS-p6Qs&J?o z^7C_Vyz5VkQAZy_$cgNiY?Uz1{^HsVg|BubHylbs-Bs`WeJF1+G-F36LG6&4GjCPo z5$Q~vM<9&1aVr15D*;~V<~6;vl69PfN~o2LWDeHvZz#grZt5HW7Z@{f{d#|_3@qdZ zYxrBS28+A|$wi4HLGsH8>)X?Zv%c4Z zeU=;s?8t^p)Z!mR5^qewv~KJYwK-Y8Lvt$JlyOy@kYtIgKdHh-QoJ^{DGhF5eG*Xu8!&gcQ3C9o#iei zuG&Mxe2?8-EGDBZ63Gl(vw~XBOK#6hi+yJS5?J-LJC?MDnLqe4fRv`}n?3Q?>BWM- zZix0IZYg?S+{?s#-ZA=9Grp`!9h5>n%`t4WW@^PM=gIQGgXzb;c!C!P^u_Fi<7cQ zG5(X4@%-eO+JuiP;%xS>EUkLHSTOU1O2~2qY3_q4I*h+$$c`?LzQ}p0Eur}QXy_{D zl{#o=dIvPenOEW%=X4*LEyYrJ;%HzvoE^*wE4~q3YN@ii@LXy3TerS?K4yF5(!kNePGK>A)eUYFAqT# zcSCB%KA7}v7;ectuzWj0K;%6wqF#t-2@c9F&D$98XP?s1yG=Yeg`i5`X-j4+yY6v4 z2JKqMPho-&<1X1XVmm9WGJh)KO&qKl9#<`m*1kRQQ#_@^SWaxT%}+~VTtD9h(+{HN zFxp!0zaWVDF$^Xgb0fd@q%kSS3Uz2*% z&D#ERq@CcVK%^x2Zk?`i{Vs$k=rwu|cF01K^aZ?;mMFz-hug^*zh&H~;GSHXy5+v) znRS?fNYO(@#=DGnbEz6g8>8;?lOMJ>Th4n{4Fnhw^~tT_Ojcd|+Yy!~3h>ZWK}cqqC8gma z<4l-WXZs4nj_x(4kFeOH`Cf1wR8l=dRW)ueQ4%FGWM2f zTQoq-y-*3Lng6o1l7HE;URC926Zp3eQa)?ww#^@E{AiwP<{Vj_e3Ni|dR>iS{ni@NKQaI5Ha{$FW zqd5*iG<=AQ%eqRE*U+$G@pSiTv2!pPJLSa8CDR#mh-3H~xm!^uz~eRKyZ+>koEbV^ z(if^kG9$|0yRqMG92!z#szgRA0;FD!J0YTJmr{udMy&tYxZyOa)lRvuM_38345a>j z&HVSK8wy&Xi|!K$q;-1~$ol-cZ5xhFJ}M+SRu0~K^mlOn-%67 zQ!2$nqo%7Rz}mcy6DL^;GgYS!B{6Gr5_7qisBn>{&4_s-xuGBXH2L!hUuLk1@$*bj z)i&?I>Je$)TJ+0h;?wbqg%rCVvKdMGtXRbYf7}Osr7q~0{}A2#d6hu!g6YL+{O50_ zO{jwAoMc2ITJ+Rs1mO*#a3|qfBYby*`lV86Lw;*mw$RbskU>g`3~pY`2Ou(kOK5jt zo~upg9%zJtbNM4Kb1zhfnj8uqIdEd|D-Ffm7LZNdM#S*SFTY?pR9!lnZ?L7Tj-qVD zsOX@k*)_K9aHXh54im{8UZARIeg8D`rzTePG zc=k(bFJZl(%mz{r9S72X*XEVfX-+sZ36PK?;stge8Sm=IiU2JZ<>~?lBYc6Ynq>bG zn=+n*0mkP-b>Mq54cH{(k2+cmCs;1Ow!5e9ALT-+Mf0nF>zuAou+1D}o0v0ovI&-E zbF6qz{hu@eq-6pqhOan%XBoE4Xy130Gzp8FEVG_(!Q>cI8F z{u+qc!Cfh33>U=_^U zXOFtgOdLm=?bFbov;ma!jY~))T$!b$S>6bzME3>W2ezoz^?tyhhhv}=9h%CuIe#q#4b(Jh>z7wr;86vG&fzLL}#q&{!c72XJOh~ z>BSYiQp^Kf*2T@pbJ{$>x{c$!u#Oql-c8jfkdB-yk`i^;L6D_g9y7Z{L&q+I_Je+Q zACthsE7p`T>7Q}2jTQK`lS67f1g{Vju`^_SZN_>nnzXZu)-D^Yq1jRliD1M${S$SCK z+o#?fKHF3|-THC-Me#}W1TRN_MvDhwCG#LDrvIc~fk&c14Ljm2!Of z3$c8uOAMrZJ4;$lnKSeR4R=q%DOEI`M59jXZ<0C?Bj|vnRcgE@huU~2E=HJbyvAWp z_=4k(SX=U_+>E^gIG4llq^(nA4M@WC{vug$nJ@=CYm}rN}bQ~R5Ti)Z7A4&Ya#se1;6l*|jk-1a>XAjU z|N1-~Gu1@&M8dwP7{r2_3ua0fEos?d&QFehJrH{s|s3ci38PJ47TQ zrVPsr%kH@)p9(9FWlezq{2TXqUF?gQ*1;E*1v)l=I7W`sKalkHl8CfZ_{aM9Pk^0I z_SF5L-|TTb2us%%4jvh-T`yE`&oF^y#g1k;Lr_ZY<6bLy`%_*7fPBn?C(C42de{g~ zbnAjh!BaI3qz3i%S)*OAG@R{qTD?5PbX>{SD&h+Z;*Po8(b2A?5#bP~{-j1YrVjfc zzMjqMBuhWnpbz_v3^>0%;C}A`f%63uSZaHMic{{sO4O2ZXC+ExTP#6#WKpJv73j8M z2VGs`lirej3;v@ya9Efs?!gpYo(HBnvE0Rc{r!Sgh|nVCo}w@S5Q$L|IUv*C9z+-% zY=whP=vY|L0M(#lOIAdfCBNkqeK^U)cs#In=3Ws3*tJMVTce#+5lJnZCy&Y`v`-Xo zn*CSFM(T!L{&U^x4YBSK7fbxwoWLiW$Uf}C2NQJo$FZDI_7S2Mw}pulLs+NNHSqAg zrG4`Ogqy}eUHf8U+v9G3_1Pg}Pdmf7*OfJPQqx^FU zq?m|@prifMhR3O@KhfyY*_&`Sszq12WKIGEYh5SmQc7H+e{zq_6{hP=wFWSb4yOJt z*!)kPHKwyEY{<}qW}K4RWLQ1aQeZknXH70&&-LeeFPOk->#6Xb8R*Kj_wW>fc{E5osGJF{rUu&EMdpF&?j{Yj&K2Tn&SHuBUt`tO-5vFtZ+WPmoe z9Z@kUD=HR=s$_y3hmjW&E{$PBFP{gKzXu++U&!+&rIVM~>mOcpJ+Bibi2N0J-|9_pe_>7;Z%W$zvkCj+uYp0?Z*%gas+Sir_)rQ0eO5#YPlr&3_55j(R z1`fTH-OM-ScoS+^T;-6@KP$;3!WXxXr;6!so{Bo!c{xW5!bXBKEeL`-R0oj?_tJ~3~eTbPs2pOtP1-cvQvxQ zg5=0$9ITuF{3Li3xrc66&h<&FE^91ie=@Y#e}pazg~Wk z{@Mt>d>IDkl5ZQ!#~x1zqw}DuMrxQp+1Vv<-?>!!+6M`%5IR$DJgh}xY^iy@m#wXT zyVF|v_7H((F@1qNyDfPCeO|qr)s4BotL{Jl9}R{Sr>lCCH!+B^1kMkChiPbJ{t^NA zW)jkHi1Q}15IBWD((fR)7(f6&KTTX1qg6kL#(mn=Z4^6StX>Oxq_kt_)bQGpg($9P zEQ1H;0LXYO9Vv-lQ%#m@zh#C?QM-xp(9rTExwqnSJ89~!_j`mP{&{M{zU*noUet-k zpchw#r`pCy%?ddL(`TY+mx!@uC2fBew|}sX5l_VN975^5#RdE>r5;pa`P(R1yEom8 zzU*-(HfBwzMVEKYH6St3;X1;6kM5=I3R@JG(CMc)tX9>`GND(|k`)0grAf)qrNd72 zPP_i03hdt71?G=sF42a{ut;6F&1tCjbb*k=x}I5v;)n;>JCVXk(PT_#qZJfEpLLtkbmxtt=i1AV-fjb2@69oj`ZL(?|dX~D-d(bsS@AHOxv^VcDa z!RKvNpwS69c3kUKPkN=-;cu4uv@TjhTycBGw8?FuL4i!W>*8!Xi9{B;$KGEp<3bSJ zXCSw!OERC5!Sy6h=n=7W{L!A{AM&Z4{ufr74sqLU(`o#;JWoiHmTHwgx;0*yu4F@( zG=6}uihLHFBjalH6O}ueSOIg(AXosgar{60qA7t3BIUOF+d$VrZ$?y3FjND zt4k^P))$vDh5!%h3t{ptCVexy)DI=D=%qYh%v}DW^O`^4{2EJ58QRiYOm0@rgsl;V zP&FKaPHRHF0HvtP6MD3{t%8>7Fvy{RDLaR`-RV28*iKA`ZGti1zYP1m;&2-d2Rp8C z42JV_oo6%AoaZ#B0DcnKv4m8-=BE$OwgD^B!=_El-TMXzKWJhZ^hkl#g5BaDWei@X za7kQ(LYBOnj?dK4wnVhYBXtT&)aiw4!oY14mXIxw`gz?;H(Tcdp%$NV2AbQQfOBoi z-+A$6=Zsj!V4YvCqZ_o@Ve=cx>9{o5vao+e(9Ytbv4Ou{o^@xYdLT7MA!Y8?x}2bN zipl&=D0hA`*lO=sx5d5;j@_L~F(CtZ&D$ ze(r+DsEAuDBj+u)AaWJ0o?ZGWmkTGa2Me(xHDbN07X>K)mTLyG zPsuC%%S^yG@QO=k$c`=CgP`p2xzuYR_Hjut{P1}(CwQ~@ea9&{|N5Ze3a>8s;AT)X zSmN~hFojp1=`U4umkvmQ+lA#hZ=UiDO8%Fsp`aD3MbM)nY7ez<_s)spb|5Rb<$8%D5^kc= z&)iv#mLhp{VA6|57^N9~C&PM~!l1zl@|kLIiq5fl&ND>;5YGugYe-V>4cBo3O1{_Q z_~K2~2pf~O8o zvi{7UT0;T*_dvvn;1oNO$hHCu<{c2cqj>ClWi6%!w@LdXeNAJS%mClQ0p9;}NSprn we|^(r3ICs)+L!-ll=lDc&Ns<-@7~^gZ=XxUdPmLRZglUI = { tagline: "Explore product usage, errors, flags, and experiments.", short: "Sign in with PostHog. Project pinning and access controls are optional.", }, + gauge: { + tagline: "Track your brand in AI answers and run content workflows.", + short: "Sign in with Gauge and pick an organization. API keys are optional.", + }, neon: { tagline: "Manage Postgres projects, branches, and queries.", short: "Sign in with Neon. Project pinning and read-only mode are optional.", diff --git a/ui/src/pages/apps/AppsConnect.test.tsx b/ui/src/pages/apps/AppsConnect.test.tsx index eea8a1fc6fc..14c409e923f 100644 --- a/ui/src/pages/apps/AppsConnect.test.tsx +++ b/ui/src/pages/apps/AppsConnect.test.tsx @@ -65,6 +65,7 @@ const BOX = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "box")!; const POSTHOG = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "posthog")!; const NEON = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "neon")!; const SUPERAGENT = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "superagent")!; +const GAUGE = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "gauge")!; const POSTMAN = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "postman")!; const SHOPIFY = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "shopify")!; const GOOGLE_SHEETS = CONNECTABLE_APP_DEFINITIONS.find((app) => app.slug === "google-sheets")!; @@ -2056,6 +2057,47 @@ describe("AppsConnect — Connect with a link (M4 frame)", () => { })); }); + it("defaults Gauge to browser sign-in and connects an organization API key once entered", async () => { + mockParams.appKey = "gauge"; + listGalleryMock.mockResolvedValueOnce({ apps: [GAUGE] }); + await render(); + // The publish warning must be visible on the default sign-in path, before + // the operator opens Advanced or switches to the API-key method. + expect(container.textContent).toContain("can publish to your connected CMS, including live"); + expect(buttonByText("Continue to sign in")?.disabled).toBe(false); + await openAccessAdvanced(); + + expect(radioContaining("Sign in with Gauge")?.getAttribute("aria-checked")).toBe("true"); + expect(buttonByText("Continue to sign in")?.disabled).toBe(false); + + await act(async () => { + radioContaining("Use an API key")?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + await flushReact(); + + expect(container.textContent).toContain("set publish actions to Ask first"); + const keyInput = container.querySelector('input[type="password"]'); + expect(keyInput).toBeTruthy(); + expect(buttonByText("Connect")?.disabled).toBe(true); + + await act(async () => { + setInputValue(keyInput!, "gauge-test-key"); + }); + await flushReact(); + const submit = buttonByText("Connect"); + expect(submit?.disabled).toBe(false); + await act(async () => { + submit?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + await flushReact(); + + expect(connectAppMock).toHaveBeenCalledWith("company-1", expect.objectContaining({ + galleryKey: "gauge", + connectionMethodKey: "mcp-api-key", + credentialValues: { "credentials.authorization": "gauge-test-key" }, + })); + }); + it("enables Neon's Connect button only once the API key is entered, with pin and read-only optional", async () => { mockParams.appKey = "neon"; listGalleryMock.mockResolvedValueOnce({ apps: [NEON] }); From 8e59efc50b162126f33816841e06509634eba65c Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 8 Oct 2026 12:37:55 -0700 Subject: [PATCH 03/10] fix(ui): keep sidebar plugin outlets quiet when the server is unreachable (#15588) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The board UI sidebar shows plugin contributions: plugin nav slots, plugin launchers, and the plugin panel at the bottom > - These outlets read `GET /plugins/ui-contributions` through React Query > - When the server is unavailable, the refetch fails. Each outlet then replaces its content with a red "Plugin extensions unavailable: Failed to fetch" or "Plugin launchers unavailable" box > - React Query still has the last good data, so the error box removes useful content and adds noise in a persistent part of the UI > - This pull request adds an option to hide the outlet error, and the sidebar uses it > - The benefit is a quiet sidebar during a server outage. The sidebar keeps the last loaded plugin items. Other pages still show the error inline ## Linked Issues or Issue Description No public issue exists. I searched open and closed issues and PRs for related work and found none. **What happened?** When the server is unavailable, the sidebar shows red error boxes in place of plugin items. The text is "Plugin extensions unavailable: Failed to fetch" or "Plugin launchers unavailable: Failed to fetch". **Expected behavior** The sidebar does not show errors while the server is unavailable. It keeps the last loaded content. **Steps to reproduce** 1. Start Paperclip with at least one plugin that contributes a `sidebar` slot, a `sidebar` launcher, or a `sidebarPanel` slot. 2. Open the board UI and let the sidebar load. 3. Stop the server. 4. Wait for the next refetch of `/plugins/ui-contributions`, or focus the window. 5. See the red error boxes in the sidebar. **Paperclip version or commit** `master` at 0ac194450a48a407450921a16c3ef8684dcb85ca **Deployment mode** Not deployment-specific. This is UI behavior. ## What Changed - `PluginSlotOutlet` (`ui/src/plugins/slots.tsx`) and `PluginLauncherOutlet` (`ui/src/plugins/launchers.tsx`) accept a new `errorBehavior: "inline" | "hidden"` prop. The default is `"inline"`, so current behavior does not change. - With `"hidden"`, the outlet does not render the error box. It continues to render the last loaded contributions. - `Sidebar.tsx` and `Sidebar.production.tsx` pass `errorBehavior="hidden"` to the three sidebar outlets. - New test `ui/src/plugins/outlet-errors.test.tsx`. It checks that the inline default still shows the error. It also checks that hidden mode keeps the cached `sidebar` slot, `sidebarPanel` slot, and `sidebar` launcher rendered after a failed refetch. A mutation check confirmed the test fails if hidden mode drops the slots or shows the launcher error. ## Verification - `cd ui && pnpm vitest run src/plugins/ src/components/Sidebar.test.tsx`: 55/55 tests pass. - `cd ui && pnpm typecheck`: clean. - `pnpm check:token-gates`: clean. - Browser check on a local `local_trusted` instance with the kitchen-sink example plugin installed (it contributes a `sidebar` slot, a `sidebar` launcher, and a `sidebarPanel` slot). I loaded the board, stopped the server, waited past the 30s stale time, focused the window, and let the retries finish. I took the "before" screenshot with the `master` versions of the four changed UI files, and the "after" screenshot with this branch. - Not run locally: the full `pnpm -r typecheck`, `pnpm test:run`, and `pnpm build`. CI runs them. **Screenshots (sidebar only):** | Server running | Server stopped, before (`master`) | Server stopped, after (this PR) | | --- | --- | --- | | Sidebar with plugin items loaded | Sidebar with three red plugin error boxes | Sidebar keeps Kitchen Sink item and panel, no errors | Main-content plugin outlets, like the dashboard widget, still show the inline error while the server is down. That is intentional, because this PR changes only the sidebar. ## Risks - Low risk. The new prop is opt-in, and the default is unchanged. - If the first load fails in the sidebar, the outlet shows nothing. It does not show an error. Users can still see the error on other plugin surfaces. - This change does not change request timing or retries. Background refetches still run while the server is unavailable. Only the sidebar display changes. ## Model Used - Claude Opus 5.5 (Anthropic), model ID `claude-opus-5-5`, run as an agent in Claude Code with tool use (file edit, shell, test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Paperclip Co-authored-by: Claude Opus 5.5 --- ui/src/components/Sidebar.production.tsx | 3 + ui/src/components/Sidebar.tsx | 3 + ui/src/plugins/launchers.tsx | 9 ++- ui/src/plugins/outlet-errors.test.tsx | 85 ++++++++++++++++++++++++ ui/src/plugins/slots.tsx | 9 ++- 5 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 ui/src/plugins/outlet-errors.test.tsx diff --git a/ui/src/components/Sidebar.production.tsx b/ui/src/components/Sidebar.production.tsx index 3bcc09b1eab..423c102ccda 100644 --- a/ui/src/components/Sidebar.production.tsx +++ b/ui/src/components/Sidebar.production.tsx @@ -209,6 +209,7 @@ export function Sidebar() { ) : null} ) : null} Plugin launchers unavailable: {errorMessage} diff --git a/ui/src/plugins/outlet-errors.test.tsx b/ui/src/plugins/outlet-errors.test.tsx new file mode 100644 index 00000000000..90c6ce9343c --- /dev/null +++ b/ui/src/plugins/outlet-errors.test.tsx @@ -0,0 +1,85 @@ +// @vitest-environment jsdom +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { MemoryRouter } from "react-router-dom"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import type { PluginUiContribution } from "@/api/plugins"; +import { pluginsApi } from "@/api/plugins"; +import { queryKeys } from "@/lib/queryKeys"; +import { PluginLauncherOutlet, PluginLauncherProvider } from "./launchers"; +import { _resetPluginModuleLoader, PluginSlotOutlet, registerPluginReactComponent } from "./slots"; + +vi.mock("@/context/CompanyContext", () => ({ useCompany: () => ({ selectedCompany: null }) })); + +const contribution: PluginUiContribution = { + pluginId: "fixture", pluginKey: "fixture.sidebar", displayName: "Fixture", version: "1", + uiEntryFile: "index.js", + slots: [ + { type: "sidebar", id: "nav", displayName: "Nav", exportName: "SidebarItem" }, + { type: "sidebarPanel", id: "panel", displayName: "Panel", exportName: "SidebarPanel" }, + ], + launchers: [{ + id: "open", displayName: "Fixture launcher", placementZone: "sidebar", + action: { type: "navigate", target: "/fixture" }, + }], +}; +const context = { companyId: "company-1", companyPrefix: "PAP" }; +let root: Root; +let container: HTMLDivElement; +let client: QueryClient; + +async function render(errorBehavior?: "inline" | "hidden") { + await act(async () => root.render( + + + + + + + + + , + )); +} + +beforeEach(() => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + // Module and session fetches stay pending; components are registered directly. + vi.stubGlobal("__paperclipPluginBridge__", {}); + vi.stubGlobal("fetch", vi.fn(() => new Promise(() => {}))); + client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: Infinity } } }); + container = document.createElement("div"); document.body.append(container); root = createRoot(container); + vi.spyOn(pluginsApi, "listUiContributions").mockRejectedValue(new TypeError("Failed to fetch")); +}); +afterEach(async () => { + await act(async () => root.unmount()); container.remove(); client.clear(); _resetPluginModuleLoader(); + vi.unstubAllGlobals(); vi.restoreAllMocks(); +}); + +it("shows inline errors by default when contributions fail to load", async () => { + await render(); + await vi.waitFor(() => expect(container.textContent).toContain("unavailable")); + expect(container.textContent).toContain("Plugin extensions unavailable: Failed to fetch"); + expect(container.textContent).toContain("Plugin launchers unavailable: Failed to fetch"); +}); + +it("hides errors and keeps the last loaded slots and launchers when errorBehavior is hidden", async () => { + registerPluginReactComponent("fixture.sidebar", "SidebarItem", () => Fixture sidebar item); + registerPluginReactComponent("fixture.sidebar", "SidebarPanel", () => Fixture sidebar panel); + client.setQueryData(queryKeys.plugins.uiContributions, [contribution]); + await render("hidden"); + expect(container.textContent).toContain("Fixture sidebar item"); + expect(container.textContent).toContain("Fixture sidebar panel"); + + await act(async () => { + await client.refetchQueries({ queryKey: queryKeys.plugins.uiContributions }); + // React Query notifies observers on a timer; let the error render land. + await new Promise((resolve) => setTimeout(resolve, 0)); + }); + expect(client.getQueryState(queryKeys.plugins.uiContributions)?.status).toBe("error"); + expect(container.textContent).not.toContain("unavailable"); + expect(container.textContent).toContain("Fixture sidebar item"); + expect(container.textContent).toContain("Fixture sidebar panel"); + expect(container.textContent).toContain("Fixture launcher"); +}); diff --git a/ui/src/plugins/slots.tsx b/ui/src/plugins/slots.tsx index a86c5a7d722..5ea4d700b6f 100644 --- a/ui/src/plugins/slots.tsx +++ b/ui/src/plugins/slots.tsx @@ -905,6 +905,12 @@ type PluginSlotOutletProps = { itemClassName?: string; errorClassName?: string; missingBehavior?: "hidden" | "placeholder"; + /** + * `hidden` suppresses the inline error and keeps rendering the last loaded + * slots, so ambient chrome (the sidebar) stays quiet while the server is + * unreachable. + */ + errorBehavior?: "inline" | "hidden"; }; export function PluginSlotOutlet({ @@ -915,6 +921,7 @@ export function PluginSlotOutlet({ itemClassName, errorClassName, missingBehavior = "hidden", + errorBehavior = "inline", }: PluginSlotOutletProps) { const { slots, errorMessage } = usePluginSlots({ slotTypes, @@ -922,7 +929,7 @@ export function PluginSlotOutlet({ companyId: context.companyId, }); - if (errorMessage) { + if (errorMessage && errorBehavior === "inline") { return (
Plugin extensions unavailable: {errorMessage} From 4fe45bf8fe46c88f5e3130424fdae097f8573906 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:53:48 -0500 Subject: [PATCH 04/10] refactor(heartbeat): extract run retrieval and session state (#15601) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Heartbeat runs retain task sessions, usage totals, and bounded run data. > - After the preparation extraction, the main service still has over 24,000 lines. > - Run reads and session operations share one database and have no dispatch dependencies. > - This pull request moves that group into one module and preserves its callers. > - The benefit is a smaller run engine and one place to maintain saved run and session state. ## Linked Issues or Issue Description **What existing behavior does this improve?** It improves the structure and test coverage of heartbeat run retrieval and session state. **Current behavior** `server/src/services/heartbeat.ts` has 24,033 lines on the merged base. It mixes run projections, session queries, resume rules, compaction, and usage helpers with execution and recovery. **Proposed behavior** Move that group into `server/src/services/heartbeat/run-state.ts`. The main file loses 1,770 lines and ends at 22,263 lines. Keep current behavior, public exports, and service methods. This continues #15568, #15573, #15578, and #15591. **Reason and benefit** One domain extraction makes useful progress toward a few manageable files. Run reads and saved session state can be reviewed together without searching the run engine. A search found no duplicate open extraction. Related session changes include #15142, #14661, and #14670; this PR does not apply their proposed behavior changes. **Breaking changes** None. Existing public helpers retain their identity. Service method names and results stay the same. ## What Changed - Move bounded run projections, encoding checks, and run/event reads into `heartbeat/run-state.ts`. - Move task session persistence, explicit resumes, reset rules, compaction, and usage/billing helpers into the same module. - Bind database operations through `createHeartbeatRunState(db)`. Construction does no database work. The encoding cache stays local to each service instance. - Keep execution order, status transitions, session-goal recovery, cost accounting writes, dispatch, and cancellation in `heartbeat.ts`. - Preserve all 140 exports. A syntax-tree comparison confirms that 43 moved function bodies, 22 declarations, and nine service method bodies are unchanged. The copied private string helper is unchanged. Surrounding orchestration is unchanged apart from the factory binding. - Add eight tests for legacy export identity, database binding, encoding cache isolation, scoped resumes, cumulative usage, compaction, and stale or cancelled conversation session writes. - Document the module boundary in `doc/DEVELOPING.md`. ## Verification - Before and after extraction, eight existing suites pass: 235 tests. They cover workspace/session helpers, runtime state, cost accounting, ledger attribution, task reset, timer reset, run lists, and run privacy. - The new module suite passes: eight tests, including five real PostgreSQL cases. Total focused coverage: 243 tests across nine suites. - Commands: `pnpm exec vitest run server/src/services/heartbeat/run-state.test.ts` and `pnpm exec vitest run server/src/__tests__/heartbeat-runtime-state.test.ts server/src/__tests__/heartbeat-cost-accounting.test.ts server/src/__tests__/heartbeat-ledger-billing-code.test.ts server/src/__tests__/heartbeat-task-session-reset.test.ts server/src/__tests__/heartbeat-workspace-session.test.ts server/src/__tests__/heartbeat-timer-wake-session-reset-pf4.test.ts server/src/__tests__/heartbeat-list.test.ts server/src/__tests__/heartbeat-run-privacy-routes.test.ts`. - `pnpm --filter @paperclipai/server typecheck` passes. - `pnpm -r typecheck` and `pnpm build` pass. - The full local `pnpm test:run` did not complete and was stopped with SIGINT after it reported an OAuth scope health-check failure and a workspace-runtime timeout. All six OAuth variants and the workspace case pass in isolation. A single rerun of the 413-case tool-access suite passed the OAuth case but hit a different Notion callback timeout; that Notion case also passes in isolation. The full local suite is not claimed as passing. - Greptile is 5/5 with no findings or inline comments on head `2ca0908cfc112dd6ae26e2565cb3611b5bec52ba`. Its exact-head check completed successfully. - CI remains blocked on hosted-runner assignment after more than ten minutes. [`ci / Select trusted runner`](https://github.com/paperclipai/paperclip/actions/runs/37828454507/job/113487287178) is queued for `ubuntu-latest` with no runner assigned. All seven completed review/security checks pass; two optional Storybook checks are skipped. The full CI matrix has not started. It must complete before merge. ## Risks - Wiring errors could bind reads to the wrong database or share the encoding cache. Direct factory tests cover database and cache isolation. Real PostgreSQL suites cover query and transaction behavior. - Session writes must retain their issue row locks and generation fences. The original function bodies are unchanged. New database tests reject stale and cancelled conversation writes and clears. - Existing SQL_ASCII safeguards and bounded projections remain in place. No schema, API contract, migration, lockfile, or workflow changes are included. ## Model Used OpenAI GPT-6 via Codex. The exact serving model ID and context window were not exposed in this session. Used reasoning, repository inspection, shell tools, and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- doc/DEVELOPING.md | 9 + server/src/services/heartbeat.ts | 1918 +--------------- .../src/services/heartbeat/run-state.test.ts | 222 ++ server/src/services/heartbeat/run-state.ts | 1923 +++++++++++++++++ 4 files changed, 2228 insertions(+), 1844 deletions(-) create mode 100644 server/src/services/heartbeat/run-state.test.ts create mode 100644 server/src/services/heartbeat/run-state.ts diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 24fbd4ce96e..33b50a28b12 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1245,6 +1245,15 @@ dispatch, retries, cancellation, and execution order, and re-exports the existin public helpers and configuration-incomplete error class. Keep preparation policy changes in this module and its tests. +Run retrieval and session state are in `server/src/services/heartbeat/run-state.ts`. +It owns bounded run projections, database encoding checks, task session reads and +writes, explicit resumes, session compaction, and usage/billing helpers. +`createHeartbeatRunState(db)` binds these operations without doing database work +during construction. The encoding-check cache belongs to each factory instance. +`heartbeat.ts` keeps run execution, session-goal recovery, cost accounting writes, +and status transitions, and re-exports the existing public helpers. Keep session +policy changes separate from run orchestration changes. + ## Wake Context Delivery Built-in adapters deliver wake context through the run prompt, including structured diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 7c62ed73347..a29bf8b4454 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,41 @@ +import { + type RunSessionOutcome, + EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, + deriveTaskKeyWithHeartbeatFallback, + type UsageTotals, + getAdapterSessionCodec, + shouldResetTaskSessionForWake, + normalizeSessionParams, + normalizeResumeParamsForAdapter, + truncateDisplayId, + resolveLedgerScopeForRun, + describeSessionResetReason, + isCanonicalSessionIdForAdapter, + requiresCanonicalSessionIds, + resolveNextSessionState, + normalizeUsageTotals, + resolveCacheAdjustedCostUsd, + resolveLedgerBiller, + resolveLedgerCostStatus, + normalizeLedgerBillingType, + createHeartbeatRunState, +} from "./heartbeat/run-state.js"; +export { + summarizeHeartbeatRunContextSnapshot, + summarizeHeartbeatRunListResultJson, + normalizeBilledCostCents, + resolveLedgerCostStatus, + resolveCacheAdjustedCostUsd, + resolveLedgerScopeForRun, + buildExplicitResumeSessionOverride, + normalizeAdapterRunUsage, + parseSessionCompactionPolicy, + deriveTaskKeyWithHeartbeatFallback, + shouldResetTaskSessionForWake, + describeSessionResetReason, + normalizeSessionParams, + resolveNextSessionState, +} from "./heartbeat/run-state.js"; import { readGitConnectionFailure } from "./git-connection-failure.js"; import { CONFIGURATION_INCOMPLETE_FAILURE_CODE, @@ -158,7 +196,7 @@ import { activeIssueInteractionCondition } from "./issue-question-context.js"; import { createAgentIdentityRedactor } from "./agent-identity-redaction.js"; import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-identity.js"; import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils"; -import { retryIdempotentDatabaseOperation } from "../database-retry.js"; + import { prepareConnectionInstructionDelivery } from "./connection-instructions.js"; import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/assigned-mcp-tools.js"; import { externalObjectService } from "./external-objects.js"; @@ -295,8 +333,6 @@ import { ISSUE_DISPOSITION_REPAIR_RETRY_REASON, PROVIDER_QUOTA_MONITOR_SERVICE_NAME, isEnvironmentDriverSupportedForAdapter, - type BillingType, - type CostStatus, type EnvironmentLeaseStatus, type ExecutionWorkspace, type HeartbeatRunStatusPhase, @@ -446,8 +482,6 @@ import type { AdapterExecutionResult, AdapterInvocationMeta, AdapterRuntimeEvent, - AdapterSessionCodec, - UsageSummary, } from "../adapters/index.js"; import { createLocalAgentJwt } from "../agent-auth-jwt.js"; import { createRuntimeToolsToken } from "../runtime-tools-token.js"; @@ -498,16 +532,12 @@ import { buildHeartbeatRunIssueComment, CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON, findHeartbeatRunCompletionComment, - HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS, - HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS, - HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES, hasAcceptedSemanticResult, isExternalChatPresentationContext, mergeHeartbeatRunResultJson, readCompletedAssistantMessageCandidate, resolveHeartbeatRunResponse, selectHeartbeatRunFinalAgentMessage, - summarizeRunErrorForModel, type RunPresentationDecision, } from "./heartbeat-run-summary.js"; import { @@ -723,11 +753,8 @@ import { import { redactEventPayload, redactSensitiveText } from "../redaction.js"; import { createRunSecretRedactionRegistry } from "./run-secret-redaction.js"; import { - hasSessionCompactionThresholds, resolvePaperclipRunnerIdleTimeoutMs, - resolveSessionCompactionPolicy, type RuntimeStatusUpdate, - type SessionCompactionPolicy, } from "@paperclipai/adapter-utils"; import { readPaperclipSkillSyncPreference, @@ -748,7 +775,7 @@ import { } from "./environment-runtime.js"; import { skillVersionSelectionMap } from "./runtime-skill-selections.js"; import { environmentRunOrchestrator } from "./environment-run-orchestrator.js"; -import { isUnsafeSessionWorkspaceCwd } from "./session-workspace-cwd.js"; + import { clearHeartbeatRunRuntimeStatus, getHeartbeatRunRuntimeStatus, @@ -964,8 +991,7 @@ const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed"; const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete"; const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery"; -const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = - "execution_review_participant_recovery"; + const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery"; @@ -1540,322 +1566,7 @@ export function isConfigurationIncompleteFailedRun( ); } -const heartbeatRunProcessGroupIdColumn = - heartbeatRuns.processGroupId ?? sql`NULL`.as("processGroupId"); -const heartbeatRunListColumns = { - id: heartbeatRuns.id, - responsibleUserId: heartbeatRuns.responsibleUserId, - companyId: heartbeatRuns.companyId, - agentId: heartbeatRuns.agentId, - scopeKind: heartbeatRuns.scopeKind, - issueId: heartbeatRuns.issueId, - invocationSource: heartbeatRuns.invocationSource, - triggerDetail: heartbeatRuns.triggerDetail, - status: heartbeatRuns.status, - inputTokens: sql`(${heartbeatRuns.usageJson} ->> 'inputTokens')::numeric`.as("inputTokens"), - cachedInputTokens: sql`(${heartbeatRuns.usageJson} ->> 'cachedInputTokens')::numeric`.as("cachedInputTokens"), - outputTokens: sql`(${heartbeatRuns.usageJson} ->> 'outputTokens')::numeric`.as("outputTokens"), - totalTokens: sql`(${heartbeatRuns.usageJson} ->> 'totalTokens')::numeric`.as("totalTokens"), - costUsd: sql`coalesce( - (${heartbeatRuns.resultJson} ->> 'costUsd')::numeric, - (${heartbeatRuns.resultJson} ->> 'cost_usd')::numeric, - (${heartbeatRuns.resultJson} ->> 'total_cost_usd')::numeric - )`.as("costUsd"), - startedAt: heartbeatRuns.startedAt, - finishedAt: heartbeatRuns.finishedAt, - error: heartbeatRuns.error, - wakeupRequestId: heartbeatRuns.wakeupRequestId, - exitCode: heartbeatRuns.exitCode, - signal: heartbeatRuns.signal, - usageJson: heartbeatRuns.usageJson, - sessionIdBefore: heartbeatRuns.sessionIdBefore, - sessionIdAfter: heartbeatRuns.sessionIdAfter, - logStore: heartbeatRuns.logStore, - logRef: heartbeatRuns.logRef, - logBytes: heartbeatRuns.logBytes, - logSha256: heartbeatRuns.logSha256, - logCompressed: heartbeatRuns.logCompressed, - stdoutExcerpt: sql`NULL`.as("stdoutExcerpt"), - stderrExcerpt: sql`NULL`.as("stderrExcerpt"), - errorCode: heartbeatRuns.errorCode, - externalRunId: heartbeatRuns.externalRunId, - processPid: heartbeatRuns.processPid, - processGroupId: heartbeatRunProcessGroupIdColumn, - processStartedAt: heartbeatRuns.processStartedAt, - lastOutputAt: heartbeatRuns.lastOutputAt, - lastOutputSeq: heartbeatRuns.lastOutputSeq, - lastOutputStream: heartbeatRuns.lastOutputStream, - lastOutputBytes: heartbeatRuns.lastOutputBytes, - retryOfRunId: heartbeatRuns.retryOfRunId, - processLossRetryCount: heartbeatRuns.processLossRetryCount, - scheduledRetryAt: heartbeatRuns.scheduledRetryAt, - scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt, - scheduledRetryReason: heartbeatRuns.scheduledRetryReason, - livenessState: heartbeatRuns.livenessState, - livenessReason: heartbeatRuns.livenessReason, - continuationAttempt: heartbeatRuns.continuationAttempt, - lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt, - nextAction: heartbeatRuns.nextAction, - createdAt: heartbeatRuns.createdAt, - updatedAt: heartbeatRuns.updatedAt, -} as const; - -const heartbeatRunSummaryListColumns = { - ...heartbeatRunListColumns, - usageJson: sql | null>`NULL`.as("usageJson"), - sessionIdBefore: sql`NULL`.as("sessionIdBefore"), - sessionIdAfter: sql`NULL`.as("sessionIdAfter"), - logStore: sql`NULL`.as("logStore"), - logRef: sql`NULL`.as("logRef"), - logSha256: sql`NULL`.as("logSha256"), - externalRunId: sql`NULL`.as("externalRunId"), - processPid: sql`NULL`.as("processPid"), - processGroupId: sql`NULL`.as("processGroupId"), - resultJson: sql | null>`NULL`.as("resultJson"), -} as const; - -const heartbeatRunListContextColumns = { - contextIssueId: sql`coalesce( - ${heartbeatRuns.issueId}::text, - ${heartbeatRuns.contextSnapshot} ->> 'issueId' - )`.as("contextIssueId"), - contextTaskId: sql`${heartbeatRuns.contextSnapshot} ->> 'taskId'`.as("contextTaskId"), - contextTaskKey: sql`${heartbeatRuns.contextSnapshot} ->> 'taskKey'`.as("contextTaskKey"), - contextCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), - contextWakeCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as("contextWakeCommentId"), - contextWakeReason: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeReason'`.as("contextWakeReason"), - contextWakeSource: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeSource'`.as("contextWakeSource"), - contextWakeTriggerDetail: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeTriggerDetail'`.as("contextWakeTriggerDetail"), -} as const; - -const heartbeatRunListResultColumns = { - resultSummary: sql< - string | null - >`left(${heartbeatRuns.resultJson} ->> 'summary', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( - "resultSummary", - ), - resultResult: sql< - string | null - >`left(${heartbeatRuns.resultJson} ->> 'result', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( - "resultResult", - ), - resultMessage: sql< - string | null - >`left(${heartbeatRuns.resultJson} ->> 'message', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( - "resultMessage", - ), - resultError: sql< - string | null - >`left(${heartbeatRuns.resultJson} ->> 'error', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( - "resultError", - ), - resultTotalCostUsd: sql< - string | null - >`${heartbeatRuns.resultJson} ->> 'total_cost_usd'`.as("resultTotalCostUsd"), - resultCostUsd: sql< - string | null - >`${heartbeatRuns.resultJson} ->> 'cost_usd'`.as("resultCostUsd"), - resultCostUsdCamel: sql< - string | null - >`${heartbeatRuns.resultJson} ->> 'costUsd'`.as("resultCostUsdCamel"), -} as const; - -// Reserve at most 9 KiB for diagnostics in the reduced result. An oversized -// multibyte field uses a conservative four-byte-per-character prefix, with a -// visible pointer to the full (adapter-bounded) run error and transcript. -const diagnosticRetrievalTitleBytes = 1024; -const diagnosticRetrievalDetailsBytes = 8192; -function boundedRunDiagnosticText(field: "title" | "details", maxBytes: number) { - const value = sql`${heartbeatRuns.resultJson} #>> ARRAY['terminalSessionFailure', ${field}]`; - return sql`case when octet_length(${value}) <= ${maxBytes} then ${value} - else left(${value}, ${Math.floor((maxBytes - 100) / 4)}) - || E'\\n[truncated for run retrieval; full text in run error/transcript]' end`; -} - -const heartbeatRunSafeResultJsonColumn = sql | null>` - case - when ${heartbeatRuns.resultJson} is null then null - when pg_column_size(${heartbeatRuns.resultJson}) <= ${HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES} - then ${heartbeatRuns.resultJson} - else jsonb_strip_nulls( - jsonb_build_object( - 'summary', left(${heartbeatRuns.resultJson} ->> 'summary', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), - 'result', left(${heartbeatRuns.resultJson} ->> 'result', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), - 'message', left(${heartbeatRuns.resultJson} ->> 'message', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), - 'error', left(${heartbeatRuns.resultJson} ->> 'error', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), - 'stdout', left(${heartbeatRuns.resultJson} ->> 'stdout', ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS}), - 'stderr', left(${heartbeatRuns.resultJson} ->> 'stderr', ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS}), - 'terminalSessionFailure', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'terminalSessionFailure') = 'object' - then jsonb_strip_nulls(jsonb_build_object( - 'category', left(${heartbeatRuns.resultJson} #>> '{terminalSessionFailure,category}', 32), - 'title', ${boundedRunDiagnosticText("title", diagnosticRetrievalTitleBytes)}, - 'details', ${boundedRunDiagnosticText("details", diagnosticRetrievalDetailsBytes)}, - 'retrievalTruncated', case when - octet_length(${heartbeatRuns.resultJson} #>> '{terminalSessionFailure,title}') > ${diagnosticRetrievalTitleBytes} - or octet_length(${heartbeatRuns.resultJson} #>> '{terminalSessionFailure,details}') > ${diagnosticRetrievalDetailsBytes} - then to_jsonb(true) end, - 'truncatedFields', case when ${heartbeatRuns.resultJson} #> '{terminalSessionFailure,truncatedFields}' - in ('["title"]'::jsonb, '["details"]'::jsonb, '["title","details"]'::jsonb) - then ${heartbeatRuns.resultJson} #> '{terminalSessionFailure,truncatedFields}' end - )) end, - 'instructionSave', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'instructionSave') = 'object' - then jsonb_strip_nulls(jsonb_build_object( - 'state', left(${heartbeatRuns.resultJson} #>> '{instructionSave,state}', 32), - 'contract', left(${heartbeatRuns.resultJson} #>> '{instructionSave,contract}', 32), - 'entryFile', left(${heartbeatRuns.resultJson} #>> '{instructionSave,entryFile}', 512), - 'errorCode', left(${heartbeatRuns.resultJson} #>> '{instructionSave,errorCode}', 128), - 'errorMessage', left(${heartbeatRuns.resultJson} #>> '{instructionSave,errorMessage}', 1024), - 'storageWarning', left(${heartbeatRuns.resultJson} #>> '{instructionSave,storageWarning}', 1024) - )) end, - 'workspaceRestoreRecovery', case when ${heartbeatRuns.resultJson} #>> '{workspaceRestoreRecovery,schema}' = 'paperclip.workspace-restore-recovery.v1' - then jsonb_build_object('schema', 'paperclip.workspace-restore-recovery.v1') end, - 'workspaceRestoreFailure', case when ${heartbeatRuns.resultJson} ->> 'workspaceRestoreFailure' - in ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed') - then ${heartbeatRuns.resultJson} -> 'workspaceRestoreFailure' end, - 'cancellation', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'cancellation') = 'object' - then jsonb_strip_nulls(jsonb_build_object( - 'source', case when ${heartbeatRuns.resultJson} #>> '{cancellation,source}' - in ('operator', 'queued_message', 'shutdown', 'provider', 'transport', 'control_plane', 'unknown') - then ${heartbeatRuns.resultJson} #> '{cancellation,source}' end, - 'expected', case when jsonb_typeof(${heartbeatRuns.resultJson} #> '{cancellation,expected}') = 'boolean' - then ${heartbeatRuns.resultJson} #> '{cancellation,expected}' end, - 'initiator', jsonb_strip_nulls(jsonb_build_object( - 'type', case when ${heartbeatRuns.resultJson} #>> '{cancellation,initiator,type}' in ('user', 'agent', 'system', 'provider') - then ${heartbeatRuns.resultJson} #> '{cancellation,initiator,type}' end, - 'id', left(${heartbeatRuns.resultJson} #>> '{cancellation,initiator,id}', 128) - )), - 'reason', left(${heartbeatRuns.resultJson} #>> '{cancellation,reason}', 512), - 'recordedAt', left(${heartbeatRuns.resultJson} #>> '{cancellation,recordedAt}', 64) - )) end, - 'acpToolInventoryComplete', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'acpToolInventoryComplete') = 'boolean' - then ${heartbeatRuns.resultJson} -> 'acpToolInventoryComplete' end, - 'acpPendingToolCount', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'acpPendingToolCount') = 'number' - and length(${heartbeatRuns.resultJson} ->> 'acpPendingToolCount') < 16 - then ${heartbeatRuns.resultJson} -> 'acpPendingToolCount' end, - 'errorFamily', left(${heartbeatRuns.resultJson} ->> 'errorFamily', 32), - 'finalResponseRecorded', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'finalResponseRecorded') = 'boolean' - then ${heartbeatRuns.resultJson} -> 'finalResponseRecorded' end, - 'executionBeforeRestore', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'executionBeforeRestore') = 'object' - then jsonb_strip_nulls(jsonb_build_object( - 'errorCode', left(${heartbeatRuns.resultJson} #>> '{executionBeforeRestore,errorCode}', 128), - 'exitCode', case when jsonb_typeof(${heartbeatRuns.resultJson} #> '{executionBeforeRestore,exitCode}') = 'number' - and length(${heartbeatRuns.resultJson} #>> '{executionBeforeRestore,exitCode}') < 16 - then ${heartbeatRuns.resultJson} #> '{executionBeforeRestore,exitCode}' end, - 'signal', left(${heartbeatRuns.resultJson} #>> '{executionBeforeRestore,signal}', 50), - 'timedOut', case when jsonb_typeof(${heartbeatRuns.resultJson} #> '{executionBeforeRestore,timedOut}') = 'boolean' - then ${heartbeatRuns.resultJson} #> '{executionBeforeRestore,timedOut}' end - )) end, - 'stdoutTruncated', case - when length(${heartbeatRuns.resultJson} ->> 'stdout') > ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS} - then to_jsonb(true) - else null - end, - 'stderrTruncated', case - when length(${heartbeatRuns.resultJson} ->> 'stderr') > ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS} - then to_jsonb(true) - else null - end, - 'costUsd', coalesce( - ${heartbeatRuns.resultJson} -> 'costUsd', - ${heartbeatRuns.resultJson} -> 'cost_usd', - ${heartbeatRuns.resultJson} -> 'total_cost_usd' - ), - 'cost_usd', coalesce( - ${heartbeatRuns.resultJson} -> 'cost_usd', - ${heartbeatRuns.resultJson} -> 'costUsd', - ${heartbeatRuns.resultJson} -> 'total_cost_usd' - ), - 'total_cost_usd', coalesce( - ${heartbeatRuns.resultJson} -> 'total_cost_usd', - ${heartbeatRuns.resultJson} -> 'cost_usd', - ${heartbeatRuns.resultJson} -> 'costUsd' - ), - 'truncated', true, - 'truncationReason', 'oversized_result_json', - 'originalSizeBytes', pg_column_size(${heartbeatRuns.resultJson}) - ) - ) - end -`.as("resultJson"); - -// Execution admission needs retained server receipts even when presentation -// projection omits resultJson (SQL_ASCII or oversized provider output). Select -// only the evidence used by eligibility; never retrieve provider diagnostics. -const heartbeatRunExecutionEvidenceColumn = sql | null>` - case when ${heartbeatRuns.resultJson} is null then null else jsonb_build_object( - 'startupCancellation', ${heartbeatRuns.resultJson} -> 'startupCancellation', - 'startupPreparationSettledAt', ${heartbeatRuns.resultJson} -> 'startupPreparationSettledAt', - 'stopReason', ${heartbeatRuns.resultJson} -> 'stopReason', - 'timeoutSource', ${heartbeatRuns.resultJson} -> 'timeoutSource', - 'workspaceRestoreFailure', ${heartbeatRuns.resultJson} -> 'workspaceRestoreFailure', - 'workspaceRestoreRecovery', ${heartbeatRuns.resultJson} -> 'workspaceRestoreRecovery', - 'executionCancellation', ${heartbeatRuns.resultJson} -> 'executionCancellation', - 'nativeCancellation', ${heartbeatRuns.resultJson} -> 'nativeCancellation', - 'cancelledByActorType', ${heartbeatRuns.resultJson} -> 'cancelledByActorType', - 'cancelledByUserId', ${heartbeatRuns.resultJson} -> 'cancelledByUserId', - 'conversationContinuation', ${heartbeatRuns.resultJson} -> 'conversationContinuation', - 'cancellation', ${heartbeatRuns.resultJson} -> 'cancellation', - 'acpToolInventoryComplete', ${heartbeatRuns.resultJson} -> 'acpToolInventoryComplete', - 'acpPendingToolCount', ${heartbeatRuns.resultJson} -> 'acpPendingToolCount' - ) end -`.as("resultJson"); - -const heartbeatRunSafeColumns = { - ...getTableColumns(heartbeatRuns), - processGroupId: heartbeatRunProcessGroupIdColumn, - resultJson: heartbeatRunSafeResultJsonColumn, -} as const; - -const heartbeatRunSqlAsciiSafeColumns = { - ...getTableColumns(heartbeatRuns), - processGroupId: heartbeatRunProcessGroupIdColumn, - error: sql`NULL`.as("error"), - resultJson: sql | null>`NULL`.as("resultJson"), - stdoutExcerpt: sql`NULL`.as("stdoutExcerpt"), - stderrExcerpt: sql`NULL`.as("stderrExcerpt"), -} as const; - -const heartbeatRunLogAccessColumns = { - id: heartbeatRuns.id, - companyId: heartbeatRuns.companyId, - scopeKind: heartbeatRuns.scopeKind, - issueId: heartbeatRuns.issueId, - logStore: heartbeatRuns.logStore, - logRef: heartbeatRuns.logRef, -} as const; - -const heartbeatRunIssueSummaryColumns = { - id: heartbeatRuns.id, - runtimeMode: heartbeatRuns.runtimeMode, - status: heartbeatRuns.status, - invocationSource: heartbeatRuns.invocationSource, - triggerDetail: heartbeatRuns.triggerDetail, - contextCommentId: sql< - string | null - >`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), - contextWakeCommentId: sql< - string | null - >`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as( - "contextWakeCommentId", - ), - startedAt: heartbeatRuns.startedAt, - finishedAt: heartbeatRuns.finishedAt, - createdAt: heartbeatRuns.createdAt, - agentId: heartbeatRuns.agentId, - logBytes: heartbeatRuns.logBytes, - processStartedAt: heartbeatRuns.processStartedAt, - livenessState: heartbeatRuns.livenessState, - livenessReason: heartbeatRuns.livenessReason, - continuationAttempt: heartbeatRuns.continuationAttempt, - lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt, - nextAction: heartbeatRuns.nextAction, - lastOutputAt: heartbeatRuns.lastOutputAt, - lastOutputSeq: heartbeatRuns.lastOutputSeq, - lastOutputStream: heartbeatRuns.lastOutputStream, - lastOutputBytes: heartbeatRuns.lastOutputBytes, - issueId: heartbeatRuns.issueId, -} as const; function normalizeMaxConcurrentRuns(value: unknown) { const parsed = Math.floor( @@ -1868,19 +1579,6 @@ function normalizeMaxConcurrentRuns(value: unknown) { ); } -type UsageTotals = { - inputTokens: number; - cachedInputTokens: number; - outputTokens: number; -}; - -type SessionCompactionDecision = { - rotate: boolean; - reason: string | null; - handoffMarkdown: string | null; - previousRunId: string | null; -}; - interface ParsedIssueAssigneeAdapterOverrides { adapterConfig: Record | null; useProjectWorkspace: boolean | null; @@ -1920,353 +1618,6 @@ function parseNativeSessionGoalControl( } -export function summarizeHeartbeatRunContextSnapshot( - contextSnapshot: Record | null | undefined, -): Record | null { - const summary: Record = {}; - const allowedKeys = [ - "issueId", - "taskId", - "taskKey", - "commentId", - "wakeCommentId", - "wakeReason", - "wakeSource", - "wakeTriggerDetail", - ] as const; - - for (const key of allowedKeys) { - const value = readNonEmptyString(contextSnapshot?.[key]); - if (value) summary[key] = value; - } - - return Object.keys(summary).length > 0 ? summary : null; -} - -export function summarizeHeartbeatRunListResultJson(input: { - summary?: string | null; - result?: string | null; - message?: string | null; - error?: string | null; - totalCostUsd?: string | null; - costUsd?: string | null; - costUsdCamel?: string | null; -}): Record | null { - const summary: Record = {}; - for (const [key, value] of [ - ["summary", input.summary], - ["result", input.result], - ["message", input.message], - ["error", input.error], - ] as const) { - const normalized = readNonEmptyString(value); - if (normalized) summary[key] = normalized; - } - - for (const [key, value] of [ - ["total_cost_usd", input.totalCostUsd], - ["cost_usd", input.costUsd], - ["costUsd", input.costUsdCamel], - ] as const) { - const normalized = readNonEmptyString(value); - if (!normalized) continue; - const parsed = Number(normalized); - if (Number.isFinite(parsed)) summary[key] = parsed; - } - - return Object.keys(summary).length > 0 ? summary : null; -} - -function normalizeLedgerBillingType(value: unknown): BillingType { - const raw = readNonEmptyString(value); - switch (raw) { - case "api": - case "metered_api": - return "metered_api"; - case "subscription": - case "subscription_included": - return "subscription_included"; - case "subscription_overage": - return "subscription_overage"; - case "credits": - return "credits"; - case "fixed": - return "fixed"; - default: - return "unknown"; - } -} - -function resolveLedgerBiller(result: AdapterExecutionResult): string { - return ( - readNonEmptyString(result.biller) ?? - readNonEmptyString(result.provider) ?? - "unknown" - ); -} - -export function normalizeBilledCostCents( - costUsd: number | null | undefined, - billingType: BillingType, -): number { - if (billingType === "subscription_included") return 0; - if (typeof costUsd !== "number" || !Number.isFinite(costUsd)) return 0; - return Math.max(0, Number((costUsd * 100).toFixed(7))); -} - -export function resolveLedgerCostStatus(input: { - costUsd: number | null | undefined; - billingType?: BillingType; - inputTokens: number; - cachedInputTokens: number; - outputTokens: number; -}): CostStatus { - if (input.billingType === "subscription_included") return "reported"; - // A paused turn can have neither a token receipt nor a cost receipt. Zero - // normalized counters do not establish that its billed cost was zero. - return typeof input.costUsd === "number" && - Number.isFinite(input.costUsd) && - input.costUsd >= 0 - ? "reported" - : "unpriced"; -} - -export function resolveCacheAdjustedCostUsd(input: { - costUsd?: number | null; - cacheAdjustedCostUsd?: number | null; -}) { - const explicit = input.cacheAdjustedCostUsd; - if ( - typeof explicit === "number" && - Number.isFinite(explicit) && - explicit >= 0 - ) { - return explicit; - } - const reported = input.costUsd; - if ( - typeof reported === "number" && - Number.isFinite(reported) && - reported >= 0 - ) { - return reported; - } - return null; -} - -export async function resolveLedgerScopeForRun( - db: Db, - companyId: string, - run: typeof heartbeatRuns.$inferSelect, -) { - const context = parseObject(run.contextSnapshot); - const contextIssueId = readNonEmptyString(context.issueId); - const contextProjectId = readNonEmptyString(context.projectId); - - if (!contextIssueId) { - return { - issueId: null, - projectId: contextProjectId, - billingCode: null, - }; - } - - const issue = await db - .select({ - id: issues.id, - projectId: issues.projectId, - billingCode: issues.billingCode, - }) - .from(issues) - .where(and(eq(issues.id, contextIssueId), eq(issues.companyId, companyId))) - .then((rows) => rows[0] ?? null); - - return { - issueId: issue?.id ?? null, - projectId: issue?.projectId ?? contextProjectId, - billingCode: issue?.billingCode ?? null, - }; -} - -type ResumeSessionRow = { - sessionParamsJson: Record | null; - sessionDisplayId: string | null; - lastRunId: string | null; -}; - -export function buildExplicitResumeSessionOverride(input: { - adapterType?: string | null; - resumeFromRunId: string; - resumeRunSessionIdBefore: string | null; - resumeRunSessionIdAfter: string | null; - resumeRunSessionParams?: Record | null; - taskSession: ResumeSessionRow | null; - sessionCodec: AdapterSessionCodec; -}) { - const resumeRunSessionIdAfter = truncateDisplayId( - input.resumeRunSessionIdAfter, - ); - const resumeRunSessionIdBefore = truncateDisplayId( - input.resumeRunSessionIdBefore, - ); - const desiredDisplayId = requiresCanonicalSessionIds(input.adapterType) - ? isCanonicalSessionIdForAdapter(input.adapterType, resumeRunSessionIdAfter) - ? resumeRunSessionIdAfter - : isCanonicalSessionIdForAdapter( - input.adapterType, - resumeRunSessionIdBefore, - ) - ? resumeRunSessionIdBefore - : null - : (resumeRunSessionIdAfter ?? resumeRunSessionIdBefore); - const runSessionParams = requiresCanonicalSessionIds(input.adapterType) - ? normalizeResumeParamsForAdapter( - input.adapterType, - input.sessionCodec.deserialize(input.resumeRunSessionParams ?? null), - ) - : null; - const runSessionDisplayId = truncateDisplayId( - readNonEmptyString(runSessionParams?.sessionId), - ); - const taskSessionParams = normalizeResumeParamsForAdapter( - input.adapterType, - input.sessionCodec.deserialize( - input.taskSession?.sessionParamsJson ?? null, - ), - ); - const taskSessionRawDisplayId = input.taskSession?.sessionDisplayId ?? null; - const taskSessionDisplayId = truncateDisplayId( - requiresCanonicalSessionIds(input.adapterType) - ? (readNonEmptyString(taskSessionParams?.sessionId) ?? - (isCanonicalSessionIdForAdapter( - input.adapterType, - taskSessionRawDisplayId, - ) - ? taskSessionRawDisplayId - : null)) - : (taskSessionRawDisplayId ?? - (input.sessionCodec.getDisplayId - ? input.sessionCodec.getDisplayId(taskSessionParams) - : null) ?? - readNonEmptyString(taskSessionParams?.sessionId)), - ); - const canReuseTaskSessionParams = - input.taskSession != null && - (!requiresCanonicalSessionIds(input.adapterType) || - taskSessionParams != null) && - (input.taskSession.lastRunId === input.resumeFromRunId || - (!!desiredDisplayId && taskSessionDisplayId === desiredDisplayId)); - const sessionParams = canReuseTaskSessionParams - ? taskSessionParams - : runSessionParams - ? runSessionParams - : desiredDisplayId - ? { sessionId: desiredDisplayId } - : null; - const sessionDisplayId = canReuseTaskSessionParams - ? taskSessionDisplayId - : runSessionParams - ? runSessionDisplayId - : desiredDisplayId; - - if (!sessionDisplayId && !sessionParams) return null; - return { - sessionDisplayId, - sessionParams, - }; -} - -function normalizeUsageTotals( - usage: UsageSummary | null | undefined, -): UsageTotals | null { - if (!usage) return null; - return { - inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))), - cachedInputTokens: Math.max( - 0, - Math.floor(asNumber(usage.cachedInputTokens, 0)), - ), - outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))), - }; -} - -function readRawUsageTotals(usageJson: unknown): UsageTotals | null { - const parsed = parseObject(usageJson); - if (Object.keys(parsed).length === 0) return null; - - const inputTokens = Math.max( - 0, - Math.floor( - asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0)), - ), - ); - const cachedInputTokens = Math.max( - 0, - Math.floor( - asNumber( - parsed.rawCachedInputTokens, - asNumber(parsed.cachedInputTokens, 0), - ), - ), - ); - const outputTokens = Math.max( - 0, - Math.floor( - asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0)), - ), - ); - - if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) { - return null; - } - - return { - inputTokens, - cachedInputTokens, - outputTokens, - }; -} - -export function normalizeAdapterRunUsage( - current: UsageTotals | null, - previous: UsageTotals | null, - usageBasis?: "per_run" | "session_cumulative" | null, -): UsageTotals | null { - if (!current) return null; - if (!previous || usageBasis !== "session_cumulative") return { ...current }; - - const inputTokens = - current.inputTokens >= previous.inputTokens - ? current.inputTokens - previous.inputTokens - : current.inputTokens; - const cachedInputTokens = - current.cachedInputTokens >= previous.cachedInputTokens - ? current.cachedInputTokens - previous.cachedInputTokens - : current.cachedInputTokens; - const outputTokens = - current.outputTokens >= previous.outputTokens - ? current.outputTokens - previous.outputTokens - : current.outputTokens; - - return { - inputTokens: Math.max(0, inputTokens), - cachedInputTokens: Math.max(0, cachedInputTokens), - outputTokens: Math.max(0, outputTokens), - }; -} - -function formatCount(value: number | null | undefined) { - if (typeof value !== "number" || !Number.isFinite(value)) return "0"; - return value.toLocaleString("en-US"); -} - -export function parseSessionCompactionPolicy( - agent: typeof agents.$inferSelect, -): SessionCompactionPolicy { - return resolveSessionCompactionPolicy(agent.adapterType, agent.runtimeConfig) - .policy; -} - function parseIssueAssigneeAdapterOverrides( raw: unknown, ): ParsedIssueAssigneeAdapterOverrides | null { @@ -2285,60 +1636,6 @@ function parseIssueAssigneeAdapterOverrides( }; } -/** - * Synthetic task key for timer/heartbeat wakes that have no issue context. - * This allows timer wakes to participate in the `agentTaskSessions` system - * and benefit from robust session resume, instead of relying solely on the - * simpler `agentRuntimeState.sessionId` fallback. - */ -const HEARTBEAT_TASK_KEY = "__heartbeat__"; - -/** - * Extended task key derivation that falls back to a stable synthetic key - * for timer/heartbeat wakes. The synthetic key keeps the - * `agentTaskSessions` row addressable across heartbeats so the row can be - * cleared and re-keyed deterministically. Unscoped exploratory timer wakes - * still start fresh to avoid accumulating low-value inbox scans, while timer - * wakes scoped to a real issue reuse that issue's task session. - * - * The synthetic key is only used when: - * - No explicit task/issue key exists in the context - * - The wake source is "timer" (scheduled heartbeat) - */ -export function deriveTaskKeyWithHeartbeatFallback( - contextSnapshot: Record | null | undefined, - payload: Record | null | undefined, -) { - const explicit = deriveTaskKey(contextSnapshot, payload); - if (explicit) return explicit; - - const wakeSource = readNonEmptyString(contextSnapshot?.wakeSource); - if (wakeSource === "timer") return HEARTBEAT_TASK_KEY; - - return null; -} - -export function shouldResetTaskSessionForWake( - contextSnapshot: Record | null | undefined, -) { - if (contextSnapshot?.forceFreshSession === true) return true; - - const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); - if ( - wakeReason === "issue_assigned" || - wakeReason === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON || - wakeReason === "execution_approval_requested" || - // PF-4: unscoped timer wakes are exploratory ("any new work?") and should - // not accumulate low-value inbox scans. Issue-scoped timer wakes are - // continuation work, so reuse their task session to avoid paying the full - // session-start and re-orientation cost on every heartbeat. - (wakeReason === "heartbeat_timer" && !deriveTaskKey(contextSnapshot, null)) - ) { - return true; - } - return false; -} - function shouldRequireIssueCommentForWake( contextSnapshot: Record | null | undefined, ) { @@ -2419,30 +1716,6 @@ export function filterZombieCoalesceTarget< return target && isZombieRun(target, tracked) ? null : target; } -export function describeSessionResetReason( - contextSnapshot: Record | null | undefined, -) { - if (contextSnapshot?.forceFreshSession === true) - return "forceFreshSession was requested"; - - const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); - if (wakeReason === "issue_assigned") return "wake reason is issue_assigned"; - if (wakeReason === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON) { - return `wake reason is ${EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON}`; - } - if (wakeReason === "execution_approval_requested") - return "wake reason is execution_approval_requested"; - // PF-4: paired with shouldResetTaskSessionForWake — keep the reason wording - // explicit so run logs make session reuse/reset behavior legible. - if ( - wakeReason === "heartbeat_timer" && - !deriveTaskKey(contextSnapshot, null) - ) { - return "wake reason is heartbeat_timer (unscoped timer wake starts fresh)"; - } - return null; -} - /** * Failure signatures from sandbox→host git workspace reconciliation. These * describe the state of the SHARED workspace (divergent histories written by @@ -2982,51 +2255,12 @@ function mergeHotRestartAdoptionResultJson( }; } -function truncateDisplayId(value: string | null | undefined, max = 128) { - if (!value) return null; - return value.length > max ? value.slice(0, max) : value; -} - function normalizeAgentNameKey(value: string | null | undefined) { if (typeof value !== "string") return null; const normalized = value.trim().toLowerCase(); return normalized.length > 0 ? normalized : null; } -const defaultSessionCodec: AdapterSessionCodec = { - deserialize(raw: unknown) { - const asObj = parseObject(raw); - if (Object.keys(asObj).length > 0) return asObj; - const sessionId = readNonEmptyString( - (raw as Record | null)?.sessionId, - ); - if (sessionId) return { sessionId }; - return null; - }, - serialize(params: Record | null) { - if (!params || Object.keys(params).length === 0) return null; - return params; - }, - getDisplayId(params: Record | null) { - return readNonEmptyString(params?.sessionId); - }, -}; - -function getAdapterSessionCodec(adapterType: string) { - const adapter = getServerAdapter(adapterType); - return adapter.sessionCodec ?? defaultSessionCodec; -} - -export function normalizeSessionParams( - params: Record | null | undefined, -) { - if (!params) return null; - return Object.keys(params).length > 0 ? params : null; -} - -type RunSessionOutcome = - "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out"; - type SkillTestHeartbeatCompletion = { outcome: "failed" | "cancelled"; error: string | null; @@ -3061,196 +2295,6 @@ export function resolveSkillTestRunCompletionForHeartbeatOutcome( return null; } -const HERMES_ADAPTER_TYPE = "hermes_local"; -const HERMES_SESSION_ID_REGEX = - /^(?:\d{8}_\d{6}_[A-Za-z0-9_-]{4,}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$/; - -function requiresCanonicalSessionIds(adapterType: string | null | undefined) { - return adapterType === HERMES_ADAPTER_TYPE; -} - -function isCanonicalSessionIdForAdapter( - adapterType: string | null | undefined, - sessionId: string | null | undefined, -) { - if (!sessionId) return false; - if (!requiresCanonicalSessionIds(adapterType)) return true; - return HERMES_SESSION_ID_REGEX.test(sessionId); -} - -function normalizeResumeParamsForAdapter( - adapterType: string | null | undefined, - params: Record | null | undefined, -) { - const normalized = normalizeSessionParams(params); - if (!normalized) return null; - if (!requiresCanonicalSessionIds(adapterType)) return normalized; - const sessionId = readNonEmptyString(normalized.sessionId); - return isCanonicalSessionIdForAdapter(adapterType, sessionId) - ? normalized - : null; -} - -export function resolveNextSessionState(input: { - adapterType?: string | null; - codec: AdapterSessionCodec; - adapterResult: AdapterExecutionResult; - outcome: RunSessionOutcome; - previousParams: Record | null; - previousDisplayId: string | null; - previousLegacySessionId: string | null; -}) { - const { - adapterType, - codec, - adapterResult, - previousParams, - previousDisplayId, - previousLegacySessionId, - } = input; - - if (adapterResult.clearSession) { - return { - params: null as Record | null, - displayId: null as string | null, - legacySessionId: null as string | null, - }; - } - - if (!requiresCanonicalSessionIds(adapterType)) { - const explicitParams = adapterResult.sessionParams; - const hasExplicitParams = adapterResult.sessionParams !== undefined; - const hasExplicitSessionId = adapterResult.sessionId !== undefined; - const explicitSessionId = readNonEmptyString(adapterResult.sessionId); - const hasExplicitDisplay = adapterResult.sessionDisplayId !== undefined; - const explicitDisplayId = readNonEmptyString( - adapterResult.sessionDisplayId, - ); - const shouldUsePrevious = - !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay; - - const candidateParams = hasExplicitParams - ? explicitParams - : hasExplicitSessionId - ? explicitSessionId - ? { sessionId: explicitSessionId } - : null - : previousParams; - - const serialized = normalizeSessionParams( - codec.serialize(normalizeSessionParams(candidateParams) ?? null), - ); - const deserialized = normalizeSessionParams(codec.deserialize(serialized)); - - const displayId = truncateDisplayId( - explicitDisplayId ?? - (codec.getDisplayId ? codec.getDisplayId(deserialized) : null) ?? - readNonEmptyString(deserialized?.sessionId) ?? - (shouldUsePrevious ? previousDisplayId : null) ?? - explicitSessionId ?? - (shouldUsePrevious ? previousLegacySessionId : null), - ); - - const legacySessionId = - explicitSessionId ?? - readNonEmptyString(deserialized?.sessionId) ?? - displayId ?? - (shouldUsePrevious ? previousLegacySessionId : null); - - return { - params: serialized, - displayId, - legacySessionId, - }; - } - - const previousSerializedParams = normalizeResumeParamsForAdapter( - adapterType, - codec.serialize( - normalizeResumeParamsForAdapter(adapterType, previousParams), - ), - ); - const validPreviousDisplayId = isCanonicalSessionIdForAdapter( - adapterType, - previousDisplayId, - ) - ? previousDisplayId - : null; - const validPreviousLegacySessionId = isCanonicalSessionIdForAdapter( - adapterType, - previousLegacySessionId, - ) - ? previousLegacySessionId - : null; - const previousState = () => { - const displayId = truncateDisplayId( - readNonEmptyString(previousSerializedParams?.sessionId) ?? - validPreviousDisplayId ?? - validPreviousLegacySessionId, - ); - return { - params: previousSerializedParams, - displayId, - legacySessionId: - readNonEmptyString(previousSerializedParams?.sessionId) ?? - displayId ?? - validPreviousLegacySessionId, - }; - }; - - if (input.outcome !== "succeeded") { - return previousState(); - } - - const explicitParams = adapterResult.sessionParams; - const hasExplicitParams = adapterResult.sessionParams !== undefined; - const explicitSessionId = readNonEmptyString(adapterResult.sessionId); - const validExplicitSessionId = isCanonicalSessionIdForAdapter( - adapterType, - explicitSessionId, - ) - ? explicitSessionId - : null; - const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId); - const validExplicitDisplayId = isCanonicalSessionIdForAdapter( - adapterType, - explicitDisplayId, - ) - ? explicitDisplayId - : null; - const explicitSerializedParams = hasExplicitParams - ? normalizeResumeParamsForAdapter( - adapterType, - codec.serialize(normalizeSessionParams(explicitParams) ?? null), - ) - : null; - const explicitCanonicalSessionId = - readNonEmptyString(explicitSerializedParams?.sessionId) ?? - validExplicitSessionId ?? - validExplicitDisplayId; - - if (!explicitCanonicalSessionId) { - return previousState(); - } - - const serialized = normalizeResumeParamsForAdapter( - adapterType, - codec.serialize({ sessionId: explicitCanonicalSessionId }), - ); - const displayId = truncateDisplayId( - readNonEmptyString(serialized?.sessionId) ?? - (codec.getDisplayId ? codec.getDisplayId(serialized) : null) ?? - explicitCanonicalSessionId, - ); - const legacySessionId = - readNonEmptyString(serialized?.sessionId) ?? explicitCanonicalSessionId; - - return { - params: serialized, - displayId, - legacySessionId, - }; -} export type HeartbeatEnvironmentRuntime = ReturnType< typeof environmentRuntimeService @@ -3446,6 +2490,30 @@ export function heartbeatService( db: Db, options: HeartbeatServiceOptions = {}, ) { + const { + getAgent, + resolveSessionBeforeForWakeup, + getRun, + ensureRuntimeState, + getTaskSession, + getLatestAgentConfigRevision, + evaluateSessionCompaction, + upsertTaskSession, + resolveNormalizedUsageForSession, + clearTaskSessions, + resolveExplicitResumeSessionOverride, + hasResolvablePriorSessionWorkspaceForWake, + getRunLogAccess, + listRuns, + getRuntimeStateWithSessions, + listTaskSessions, + resetRuntimeSession, + listEvents, + getRetryExhaustedReason, + getRunIssueSummary, + getActiveRunForAgent, + getActiveRunIssueSummaryForAgent, + } = createHeartbeatRunState(db); const { toAgentOrgRow, listCompanyAgentOrgRows, @@ -4077,8 +3145,6 @@ export function heartbeatService( } const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup }); - let unsafeTextProjectionPromise: Promise | null = null; - async function completeSkillTestRunForHeartbeatOutcome(input: { run: typeof heartbeatRuns.$inferSelect; issueId: string | null; @@ -4578,66 +3644,12 @@ export function heartbeatService( } } - async function hasUnsafeTextProjectionDatabase() { - if (!unsafeTextProjectionPromise) { - unsafeTextProjectionPromise = db - .execute( - sql`select current_setting('server_encoding') as server_encoding`, - ) - .then((rows) => { - const first = Array.isArray(rows) ? rows[0] : null; - const serverEncoding = - typeof first === "object" && first !== null - ? (first as Record).server_encoding - : null; - return ( - typeof serverEncoding === "string" && - serverEncoding.toUpperCase() === "SQL_ASCII" - ); - }) - .catch((err) => { - logger.warn( - { err }, - "failed to inspect database server encoding; using conservative heartbeat result projection", - ); - return true; - }); - } - return unsafeTextProjectionPromise; - } - - async function getAgent(agentId: string) { - return db - .select() - .from(agents) - .where(eq(agents.id, agentId)) - .then((rows) => rows[0] ?? null); - } - async function getAgentInvokability( agent: typeof agents.$inferSelect | null | undefined, ) { return evaluateAgentInvokabilityFromDb(db, agent); } - async function getRun( - runId: string, - opts?: { unsafeFullResultJson?: boolean; includeExecutionEvidence?: boolean }, - ) { - const safeForLegacyEncoding = - !opts?.unsafeFullResultJson && (await hasUnsafeTextProjectionDatabase()); - const columns = opts?.unsafeFullResultJson - ? getTableColumns(heartbeatRuns) - : safeForLegacyEncoding ? heartbeatRunSqlAsciiSafeColumns : heartbeatRunSafeColumns; - return db - .select(opts?.includeExecutionEvidence - ? { ...columns, resultJson: heartbeatRunExecutionEvidenceColumn } - : columns) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null); - } - async function recordCurrentHeartbeatRunRuntimeProgress( run: Pick< typeof heartbeatRuns.$inferSelect, @@ -4660,91 +3672,6 @@ export function heartbeatService( return recordHeartbeatRunRuntimeProgress(currentRun, update, issueId); } - async function getRunLogAccess(runId: string) { - return db - .select(heartbeatRunLogAccessColumns) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null); - } - - async function getRuntimeState(agentId: string) { - return db - .select() - .from(agentRuntimeState) - .where(eq(agentRuntimeState.agentId, agentId)) - .then((rows) => rows[0] ?? null); - } - - async function getLatestAgentConfigRevision( - companyId: string, - agentId: string, - ) { - return db - .select({ - id: agentConfigRevisions.id, - changedKeys: agentConfigRevisions.changedKeys, - createdAt: agentConfigRevisions.createdAt, - }) - .from(agentConfigRevisions) - .where( - and( - eq(agentConfigRevisions.companyId, companyId), - eq(agentConfigRevisions.agentId, agentId), - ), - ) - .orderBy( - desc(agentConfigRevisions.createdAt), - desc(agentConfigRevisions.id), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - } - - async function getTaskSession( - companyId: string, - agentId: string, - adapterType: string, - taskKey: string, - ) { - return db - .select() - .from(agentTaskSessions) - .where( - and( - eq(agentTaskSessions.companyId, companyId), - eq(agentTaskSessions.agentId, agentId), - eq(agentTaskSessions.adapterType, adapterType), - eq(agentTaskSessions.taskKey, taskKey), - ), - ) - .then((rows) => rows[0] ?? null); - } - - async function getLatestRunForSession( - agentId: string, - sessionId: string, - opts?: { excludeRunId?: string | null }, - ) { - const conditions = [ - eq(heartbeatRuns.agentId, agentId), - eq(heartbeatRuns.sessionIdAfter, sessionId), - ]; - if (opts?.excludeRunId) { - conditions.push(sql`${heartbeatRuns.id} <> ${opts.excludeRunId}`); - } - return db - .select({ - id: heartbeatRuns.id, - usageJson: heartbeatRuns.usageJson, - }) - .from(heartbeatRuns) - .where(and(...conditions)) - .orderBy(desc(heartbeatRuns.createdAt)) - .limit(1) - .then((rows) => rows[0] ?? null); - } - const issueMonitorDispatchColumns = { id: issues.id, companyId: issues.companyId, @@ -5611,470 +4538,11 @@ export function heartbeatService( }; } - async function getOldestRunForSession(agentId: string, sessionId: string) { - return db - .select({ - id: heartbeatRuns.id, - createdAt: heartbeatRuns.createdAt, - }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.agentId, agentId), - eq(heartbeatRuns.sessionIdAfter, sessionId), - ), - ) - .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) - .limit(1) - .then((rows) => rows[0] ?? null); - } - - async function resolveNormalizedUsageForSession(input: { - agentId: string; - runId: string; - sessionId: string | null; - rawUsage: UsageTotals | null; - usageBasis?: "per_run" | "session_cumulative" | null; - }) { - const { agentId, runId, sessionId, rawUsage, usageBasis } = input; - // Adapters that declare per-run usage (e.g. the ACPX lane reports each - // turn's tokens, not session totals) must not be session-delta'd, or - // consecutive runs would be undercounted. - if (!sessionId || !rawUsage || usageBasis !== "session_cumulative") { - return { - normalizedUsage: rawUsage, - previousRawUsage: null as UsageTotals | null, - derivedFromSessionTotals: false, - }; - } - - const previousRun = await getLatestRunForSession(agentId, sessionId, { - excludeRunId: runId, - }); - const previousRawUsage = readRawUsageTotals(previousRun?.usageJson); - return { - normalizedUsage: normalizeAdapterRunUsage(rawUsage, previousRawUsage, usageBasis), - previousRawUsage, - derivedFromSessionTotals: previousRawUsage !== null, - }; - } - - async function evaluateSessionCompaction(input: { - agent: typeof agents.$inferSelect; - sessionId: string | null; - issueId: string | null; - continuationSummaryBody?: string | null; - }): Promise { - const { agent, sessionId, issueId } = input; - if (!sessionId) { - return { - rotate: false, - reason: null, - handoffMarkdown: null, - previousRunId: null, - }; - } - - const policy = parseSessionCompactionPolicy(agent); - if (!policy.enabled || !hasSessionCompactionThresholds(policy)) { - return { - rotate: false, - reason: null, - handoffMarkdown: null, - previousRunId: null, - }; - } - - const fetchLimit = Math.max( - policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, - 4, - ); - const runs = await db - .select({ - id: heartbeatRuns.id, - createdAt: heartbeatRuns.createdAt, - usageJson: heartbeatRuns.usageJson, - error: heartbeatRuns.error, - terminalFailureCategory: sql`case - when jsonb_typeof(${heartbeatRuns.resultJson} -> 'terminalSessionFailure') = 'object' - then coalesce(left(${heartbeatRuns.resultJson} #>> '{terminalSessionFailure,category}', 32), 'unknown') end`, - ...heartbeatRunListResultColumns, - }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.agentId, agent.id), - eq(heartbeatRuns.sessionIdAfter, sessionId), - ), - ) - .orderBy(desc(heartbeatRuns.createdAt)) - .limit(fetchLimit); - - if (runs.length === 0) { - return { - rotate: false, - reason: null, - handoffMarkdown: null, - previousRunId: null, - }; - } - - const latestRun = runs[0] ?? null; - const oldestRun = - policy.maxSessionAgeHours > 0 - ? await getOldestRunForSession(agent.id, sessionId) - : (runs[runs.length - 1] ?? latestRun); - const latestRawUsage = readRawUsageTotals(latestRun?.usageJson); - // Historical Codex/Gemini raw input includes cache reads. Only add the - // separate cache counter when the writer explicitly saved exclusive input. - const latestRawInputTokens = (latestRawUsage?.inputTokens ?? 0) + - (latestRun?.usageJson?.rawInputIncludesCached === false ? latestRawUsage?.cachedInputTokens ?? 0 : 0); - const sessionAgeHours = - latestRun && oldestRun - ? Math.max( - 0, - (new Date(latestRun.createdAt).getTime() - - new Date(oldestRun.createdAt).getTime()) / - (1000 * 60 * 60), - ) - : 0; - - let reason: string | null = null; - if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) { - reason = `session exceeded ${policy.maxSessionRuns} runs`; - } else if ( - policy.maxRawInputTokens > 0 && - latestRawUsage && - latestRawInputTokens >= policy.maxRawInputTokens - ) { - reason = - `session raw input reached ${formatCount(latestRawInputTokens)} tokens ` + - `(threshold ${formatCount(policy.maxRawInputTokens)})`; - } else if ( - policy.maxSessionAgeHours > 0 && - sessionAgeHours >= policy.maxSessionAgeHours - ) { - reason = `session age reached ${Math.floor(sessionAgeHours)} hours`; - } - - if (!reason || !latestRun) { - return { - rotate: false, - reason: null, - handoffMarkdown: null, - previousRunId: latestRun?.id ?? null, - }; - } - - const latestSummary = summarizeHeartbeatRunListResultJson({ - summary: latestRun?.resultSummary, - result: latestRun?.resultResult, - message: latestRun?.resultMessage, - error: latestRun?.resultError, - totalCostUsd: latestRun?.resultTotalCostUsd, - costUsd: latestRun?.resultCostUsd, - costUsdCamel: latestRun?.resultCostUsdCamel, - }); - const latestTextSummary = - readNonEmptyString(latestSummary?.summary) ?? - readNonEmptyString(latestSummary?.result) ?? - readNonEmptyString(latestSummary?.message) ?? - readNonEmptyString(summarizeRunErrorForModel(latestRun.error, latestRun.terminalFailureCategory)); - - const handoffMarkdown = [ - "Paperclip session handoff:", - `- Previous session: ${sessionId}`, - issueId ? `- Issue: ${issueId}` : "", - `- Rotation reason: ${reason}`, - latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "", - input.continuationSummaryBody - ? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}` - : "", - "Continue from the current task state. Rebuild only the minimum context you need.", - ] - .filter(Boolean) - .join("\n"); - - return { - rotate: true, - reason, - handoffMarkdown, - previousRunId: latestRun.id, - }; - } - - async function resolveSessionBeforeForWakeup( - agent: typeof agents.$inferSelect, - taskKey: string | null, - ) { - if (taskKey) { - const codec = getAdapterSessionCodec(agent.adapterType); - const existingTaskSession = await getTaskSession( - agent.companyId, - agent.id, - agent.adapterType, - taskKey, - ); - const parsedParams = normalizeSessionParams( - codec.deserialize(existingTaskSession?.sessionParamsJson ?? null), - ); - return truncateDisplayId( - existingTaskSession?.sessionDisplayId ?? - (codec.getDisplayId ? codec.getDisplayId(parsedParams) : null) ?? - readNonEmptyString(parsedParams?.sessionId), - ); - } - - const runtimeForRun = await getRuntimeState(agent.id); - return runtimeForRun?.sessionId ?? null; - } - - async function hasResolvableSessionWorkspaceCwd( - sessionParams: Record | null | undefined, - ) { - const cwd = readNonEmptyString(sessionParams?.cwd); - if (!cwd || isUnsafeSessionWorkspaceCwd(cwd)) return false; - return fs - .stat(cwd) - .then((stats) => stats.isDirectory()) - .catch(() => false); - } - - async function hasResolvablePriorSessionWorkspaceForWake(input: { - agent: typeof agents.$inferSelect; - contextSnapshot: Record; - taskKey: string | null; - explicitResumeSession: Awaited< - ReturnType - > | null; - }) { - if ( - await hasResolvableSessionWorkspaceCwd( - input.explicitResumeSession?.sessionParams, - ) - ) - return true; - if (shouldResetTaskSessionForWake(input.contextSnapshot)) return false; - if (!input.taskKey) return false; - - const codec = getAdapterSessionCodec(input.agent.adapterType); - const taskSession = await getTaskSession( - input.agent.companyId, - input.agent.id, - input.agent.adapterType, - input.taskKey, - ); - const taskSessionParams = normalizeResumeParamsForAdapter( - input.agent.adapterType, - codec.deserialize(taskSession?.sessionParamsJson ?? null), - ); - return hasResolvableSessionWorkspaceCwd(taskSessionParams); - } - - async function resolveExplicitResumeSessionOverride( - agent: typeof agents.$inferSelect, - payload: Record | null, - taskKey: string | null, - ) { - const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId); - if (!resumeFromRunId) return null; - - const resumeRun = await db - .select({ - id: heartbeatRuns.id, - contextSnapshot: heartbeatRuns.contextSnapshot, - resultJson: heartbeatRuns.resultJson, - sessionIdBefore: heartbeatRuns.sessionIdBefore, - sessionIdAfter: heartbeatRuns.sessionIdAfter, - }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.id, resumeFromRunId), - eq(heartbeatRuns.companyId, agent.companyId), - eq(heartbeatRuns.agentId, agent.id), - ), - ) - .then((rows) => rows[0] ?? null); - if (!resumeRun) return null; - - const resumeContext = parseObject(resumeRun.contextSnapshot); - const resumeTaskKey = deriveTaskKey(resumeContext, null) ?? taskKey; - const resumeTaskSession = resumeTaskKey - ? await getTaskSession( - agent.companyId, - agent.id, - agent.adapterType, - resumeTaskKey, - ) - : null; - const sessionCodec = getAdapterSessionCodec(agent.adapterType); - const resumeRunResult = parseObject(resumeRun.resultJson); - const resumeRunSessionId = requiresCanonicalSessionIds(agent.adapterType) - ? (readNonEmptyString(resumeRunResult.sessionId) ?? - readNonEmptyString(resumeRunResult.session_id)) - : null; - const sessionOverride = buildExplicitResumeSessionOverride({ - adapterType: agent.adapterType, - resumeFromRunId, - resumeRunSessionIdBefore: resumeRun.sessionIdBefore, - resumeRunSessionIdAfter: resumeRun.sessionIdAfter, - resumeRunSessionParams: resumeRunSessionId - ? { sessionId: resumeRunSessionId } - : null, - taskSession: resumeTaskSession, - sessionCodec, - }); - if (!sessionOverride) return null; - - return { - resumeFromRunId, - taskKey: resumeTaskKey, - issueId: readNonEmptyString(resumeContext.issueId), - taskId: - readNonEmptyString(resumeContext.taskId) ?? - readNonEmptyString(resumeContext.issueId), - sessionDisplayId: sessionOverride.sessionDisplayId, - sessionParams: sessionOverride.sessionParams, - }; - } - const { resolveReusedGitWorkspaceAnchor, resolveWorkspaceForRun, } = createHeartbeatWorkspaceResolver(db); - async function upsertTaskSession(input: { - companyId: string; - agentId: string; - adapterType: string; - taskKey: string; - sessionParamsJson: Record | null; - sessionDisplayId: string | null; - lastRunId: string | null; - lastError: string | null; - }) { - return db.transaction(async (tx) => { - const [issue] = await tx.select().from(issues).where(and(sql`${issues.id}::text = ${input.taskKey}`, eq(issues.companyId, input.companyId))).for("update"); - if (isConversation(issue)) { - const [run] = input.lastRunId ? await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.lastRunId)) : []; - if (run?.status === "cancelled" || run?.contextSnapshot?.conversationSessionGeneration !== issue.conversationSessionGeneration) return null; - } - const existing = await tx.select().from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, input.companyId), eq(agentTaskSessions.agentId, input.agentId), eq(agentTaskSessions.adapterType, input.adapterType), eq(agentTaskSessions.taskKey, input.taskKey))).then((rows) => rows[0] ?? null); - if (existing) { - return tx - .update(agentTaskSessions) - .set({ - sessionParamsJson: input.sessionParamsJson, - sessionDisplayId: input.sessionDisplayId, - lastRunId: input.lastRunId, - lastError: input.lastError, - updatedAt: new Date(), - }) - .where(eq(agentTaskSessions.id, existing.id)) - .returning() - .then((rows) => rows[0] ?? null); - } - - return tx - .insert(agentTaskSessions) - .values({ - companyId: input.companyId, - agentId: input.agentId, - adapterType: input.adapterType, - taskKey: input.taskKey, - sessionParamsJson: input.sessionParamsJson, - sessionDisplayId: input.sessionDisplayId, - lastRunId: input.lastRunId, - lastError: input.lastError, - }) - .returning() - .then((rows) => rows[0] ?? null); - }); - } - - async function clearTaskSessions( - companyId: string, - agentId: string, - opts?: { - taskKey?: string | null; - adapterType?: string | null; - expectedRunId?: string; - includeIssueAliases?: boolean; - }, - ) { - const conditions = [ - eq(agentTaskSessions.companyId, companyId), - eq(agentTaskSessions.agentId, agentId), - ]; - if (opts?.taskKey) { - const exactTaskKey = eq(agentTaskSessions.taskKey, opts.taskKey); - if (opts.includeIssueAliases) { - const selectedIssue = isUuidLike(opts.taskKey) - ? eq(issues.id, opts.taskKey) - : eq(issues.identifier, opts.taskKey.toUpperCase()); - // Operator task resets accept the UUID sent by run detail and the - // identifier used by some saved sessions. Resolve only from the current - // same-company issue row, in this DELETE's snapshot; arbitrary custom - // keys retain exact-match behavior and run/model context grants no alias. - conditions.push( - or( - exactTaskKey, - sql`exists ( - select 1 from ${issues} - where ${issues.companyId} = ${companyId} - and ${selectedIssue} - and (${agentTaskSessions.taskKey} = ${issues.id}::text - or ${agentTaskSessions.taskKey} = ${issues.identifier}) - )`, - )!, - ); - } else { - conditions.push(exactTaskKey); - } - } - if (opts?.adapterType) { - conditions.push(eq(agentTaskSessions.adapterType, opts.adapterType)); - } - - return db.transaction(async (tx) => { - if (opts?.taskKey && opts.expectedRunId) { - const [issue] = await tx.select().from(issues).where(sql`${issues.id}::text = ${opts.taskKey}`).for("update"); - if (isConversation(issue)) { - const [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, opts.expectedRunId)); - if (run?.status === "cancelled" || run?.contextSnapshot?.conversationSessionGeneration !== issue.conversationSessionGeneration) return 0; - } - } - return tx.delete(agentTaskSessions).where(and(...conditions)).returning().then((rows) => rows.length); - }); - } - - async function ensureRuntimeState(agent: typeof agents.$inferSelect) { - const existing = await getRuntimeState(agent.id); - if (existing) return existing; - - const inserted = await db - .insert(agentRuntimeState) - .values({ - agentId: agent.id, - companyId: agent.companyId, - adapterType: agent.adapterType, - stateJson: {}, - }) - .onConflictDoNothing({ - target: agentRuntimeState.agentId, - }) - .returning() - .then((rows) => rows[0] ?? null); - if (inserted) return inserted; - - const ensured = await getRuntimeState(agent.id); - if (!ensured) { - throw new Error(`Failed to ensure runtime state for agent ${agent.id}`); - } - return ensured; - } - // Emits agent.task_run for a run write that just reached a terminal // status, unless the write only re-set a status the run already had (a // status-preserving patch, such as a livenessReason update on a run that @@ -23534,102 +22002,7 @@ export function heartbeatService( await new Promise((resolve) => setTimeout(resolve, intervalMs)); } }, - list: async ( - companyId: string, - agentId?: string, - limit?: number, - options: { summary?: boolean } = {}, - ) => { - const safeForLegacyEncoding = await hasUnsafeTextProjectionDatabase(); - const summary = options.summary === true; - const rows = await retryIdempotentDatabaseOperation(async () => { - const query = db - .select( - summary - ? { - ...heartbeatRunSummaryListColumns, - ...heartbeatRunListContextColumns, - } - : safeForLegacyEncoding - ? { - ...heartbeatRunListColumns, - error: sql`NULL`.as("error"), - ...heartbeatRunListContextColumns, - } - : { - ...heartbeatRunListColumns, - ...heartbeatRunListContextColumns, - ...heartbeatRunListResultColumns, - }, - ) - .from(heartbeatRuns) - .where( - agentId - ? and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, agentId), - ) - : eq(heartbeatRuns.companyId, companyId), - ) - .orderBy(desc(heartbeatRuns.createdAt)); - - return limit ? await query.limit(limit) : await query; - }); - return rows.map((row) => { - const { - contextIssueId, - contextTaskId, - contextTaskKey, - contextCommentId, - contextWakeCommentId, - contextWakeReason, - contextWakeSource, - contextWakeTriggerDetail, - resultSummary, - resultResult, - resultMessage, - resultError, - resultTotalCostUsd, - resultCostUsd, - resultCostUsdCamel, - ...rest - } = row as typeof row & { - resultSummary?: string | null; - resultResult?: string | null; - resultMessage?: string | null; - resultError?: string | null; - resultTotalCostUsd?: string | null; - resultCostUsd?: string | null; - resultCostUsdCamel?: string | null; - }; - - return { - ...rest, - contextSnapshot: summarizeHeartbeatRunContextSnapshot({ - issueId: contextIssueId, - taskId: contextTaskId, - taskKey: contextTaskKey, - commentId: contextCommentId, - wakeCommentId: contextWakeCommentId, - wakeReason: contextWakeReason, - wakeSource: contextWakeSource, - wakeTriggerDetail: contextWakeTriggerDetail, - }), - resultJson: - safeForLegacyEncoding || summary - ? null - : summarizeHeartbeatRunListResultJson({ - summary: resultSummary, - result: resultResult, - message: resultMessage, - error: resultError, - totalCostUsd: resultTotalCostUsd, - costUsd: resultCostUsd, - costUsdCamel: resultCostUsdCamel, - }), - }; - }); - }, + list: listRuns, getRun, @@ -23639,125 +22012,15 @@ export function heartbeatService( getRunLogAccess, - getRuntimeState: async (agentId: string) => { - const state = await getRuntimeState(agentId); - const agent = await getAgent(agentId); - if (!agent) return null; - const ensured = state ?? (await ensureRuntimeState(agent)); - const latestTaskSession = await db - .select() - .from(agentTaskSessions) - .where( - and( - eq(agentTaskSessions.companyId, agent.companyId), - eq(agentTaskSessions.agentId, agent.id), - ), - ) - .orderBy(desc(agentTaskSessions.updatedAt)) - .limit(1) - .then((rows) => rows[0] ?? null); - return { - ...ensured, - sessionDisplayId: - latestTaskSession?.sessionDisplayId ?? ensured.sessionId, - sessionParamsJson: latestTaskSession?.sessionParamsJson ?? null, - }; - }, + getRuntimeState: getRuntimeStateWithSessions, - listTaskSessions: async (agentId: string) => { - const agent = await getAgent(agentId); - if (!agent) throw notFound("Agent not found"); + listTaskSessions, - return db - .select() - .from(agentTaskSessions) - .where( - and( - eq(agentTaskSessions.companyId, agent.companyId), - eq(agentTaskSessions.agentId, agentId), - ), - ) - .orderBy( - desc(agentTaskSessions.updatedAt), - desc(agentTaskSessions.createdAt), - ); - }, + resetRuntimeSession, - resetRuntimeSession: async ( - agentId: string, - opts?: { taskKey?: string | null }, - ) => { - const agent = await getAgent(agentId); - if (!agent) throw notFound("Agent not found"); - await ensureRuntimeState(agent); - const taskKey = readNonEmptyString(opts?.taskKey); - const clearedTaskSessions = await clearTaskSessions( - agent.companyId, - agent.id, - taskKey - ? { - taskKey, - adapterType: agent.adapterType, - includeIssueAliases: true, - } - : undefined, - ); - const runtimePatch: Partial = { - sessionId: null, - lastError: null, - updatedAt: new Date(), - }; - if (!taskKey) { - runtimePatch.stateJson = {}; - } + listEvents, - const updated = await db - .update(agentRuntimeState) - .set(runtimePatch) - .where(eq(agentRuntimeState.agentId, agentId)) - .returning() - .then((rows) => rows[0] ?? null); - - if (!updated) return null; - return { - ...updated, - sessionDisplayId: null, - sessionParamsJson: null, - clearedTaskSessions, - }; - }, - - listEvents: (runId: string, afterSeq = 0, limit = 200) => - db - .select() - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.runId, runId), - gt(heartbeatRunEvents.seq, afterSeq), - ), - ) - .orderBy(asc(heartbeatRunEvents.seq)) - .limit(Math.max(1, Math.min(limit, 1000))), - - getRetryExhaustedReason: async (runId: string) => { - const row = await db - .select({ - message: heartbeatRunEvents.message, - }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.runId, runId), - eq(heartbeatRunEvents.eventType, "lifecycle"), - sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, - ), - ) - .orderBy(desc(heartbeatRunEvents.id)) - .limit(1) - .then((rows) => rows[0] ?? null); - return row?.message ?? null; - }, + getRetryExhaustedReason, readLog: async ( runOrLookup: @@ -23991,43 +22254,10 @@ export function heartbeatService( cancelBudgetScopeWork, - getRunIssueSummary: async (runId: string) => { - const [run] = await db - .select(heartbeatRunIssueSummaryColumns) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .limit(1); - return run ?? null; - }, + getRunIssueSummary, - getActiveRunForAgent: async (agentId: string) => { - const [run] = await db - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.agentId, agentId), - eq(heartbeatRuns.status, "running"), - ), - ) - .orderBy(desc(heartbeatRuns.startedAt)) - .limit(1); - return run ?? null; - }, + getActiveRunForAgent, - getActiveRunIssueSummaryForAgent: async (agentId: string) => { - const [run] = await db - .select(heartbeatRunIssueSummaryColumns) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.agentId, agentId), - eq(heartbeatRuns.status, "running"), - ), - ) - .orderBy(desc(heartbeatRuns.startedAt)) - .limit(1); - return run ?? null; - }, + getActiveRunIssueSummaryForAgent, }; } diff --git a/server/src/services/heartbeat/run-state.test.ts b/server/src/services/heartbeat/run-state.test.ts new file mode 100644 index 00000000000..c9bb5e50011 --- /dev/null +++ b/server/src/services/heartbeat/run-state.test.ts @@ -0,0 +1,222 @@ +import { randomUUID } from "node:crypto"; +import type { Db } from "@paperclipai/db"; +import { + agents, + agentTaskSessions, + companies, + createDb, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import { eq, type SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "../../__tests__/helpers/embedded-postgres.js"; +import * as legacy from "../heartbeat.js"; +import * as extracted from "./run-state.js"; + +// SQL_ASCII needs a separate database cluster. Inspect its selected SQL here; +// the PostgreSQL tests below exercise the session queries and transactions. +function readingDatabase(serverEncoding: string, ...results: unknown[][]) { + const scopes: unknown[][] = []; + const columns: Record[] = []; + const execute = vi.fn(async () => [{ server_encoding: serverEncoding }]); + const select = vi.fn((projection: Record = {}) => { + columns.push(projection); + const rows = results.shift() ?? []; + const query = { + from: () => query, + where: (condition: SQL) => { + scopes.push(new PgDialect().sqlToQuery(condition).params); + return query; + }, + orderBy: () => query, + limit: () => query, + then: (resolve: (rows: unknown[]) => unknown) => Promise.resolve(rows).then(resolve), + }; + return query; + }); + return { db: { execute, select } as unknown as Db, execute, select, columns, scopes }; +} + +describe("heartbeat run state module", () => { + it("preserves the legacy public helper identities", () => { + expect(legacy.resolveNextSessionState).toBe(extracted.resolveNextSessionState); + expect(legacy.buildExplicitResumeSessionOverride).toBe(extracted.buildExplicitResumeSessionOverride); + expect(legacy.normalizeAdapterRunUsage).toBe(extracted.normalizeAdapterRunUsage); + expect(legacy.resolveLedgerScopeForRun).toBe(extracted.resolveLedgerScopeForRun); + expect(legacy.summarizeHeartbeatRunListResultJson).toBe(extracted.summarizeHeartbeatRunListResultJson); + }); + + it("constructs without queries and binds session lookups to each database", async () => { + const first = readingDatabase("UTF8", [{ sessionDisplayId: "first" }]); + const second = readingDatabase("UTF8", [{ sessionDisplayId: "second" }]); + const firstState = extracted.createHeartbeatRunState(first.db); + const secondState = extracted.createHeartbeatRunState(second.db); + expect(first.select).not.toHaveBeenCalled(); + expect(first.execute).not.toHaveBeenCalled(); + expect(second.select).not.toHaveBeenCalled(); + expect(second.execute).not.toHaveBeenCalled(); + const [one, two] = await Promise.all([ + firstState.getTaskSession("company-one", "agent-one", "codex_local", "task-one"), + secondState.getTaskSession("company-two", "agent-two", "claude_local", "task-two"), + ]); + expect(one?.sessionDisplayId).toBe("first"); + expect(two?.sessionDisplayId).toBe("second"); + expect(first.scopes[0]).toEqual(["company-one", "agent-one", "codex_local", "task-one"]); + expect(second.scopes[0]).toEqual(["company-two", "agent-two", "claude_local", "task-two"]); + }); + + it("keeps encoding caches local to each factory and preserves full-result reads", async () => { + const ascii = readingDatabase("SQL_ASCII"); + const utf8 = readingDatabase("UTF8"); + const asciiState = extracted.createHeartbeatRunState(ascii.db); + const utf8State = extracted.createHeartbeatRunState(utf8.db); + await Promise.all([asciiState.getRun("ascii-one"), asciiState.getRun("ascii-two"), utf8State.getRun("utf8")]); + const projectedSql = (column: unknown) => new PgDialect().sqlToQuery((column as SQL.Aliased).sql).sql; + expect(ascii.execute).toHaveBeenCalledTimes(1); + expect(utf8.execute).toHaveBeenCalledTimes(1); + expect(projectedSql(ascii.columns[0].resultJson)).toBe("NULL"); + expect(projectedSql(utf8.columns[0].resultJson)).toContain("pg_column_size"); + await asciiState.getRun("evidence", { includeExecutionEvidence: true }); + expect(projectedSql(ascii.columns[2].resultJson)).toContain("startupPreparationSettledAt"); + const raw = readingDatabase("SQL_ASCII"); + await extracted.createHeartbeatRunState(raw.db).getRun("raw", { unsafeFullResultJson: true }); + expect(raw.execute).not.toHaveBeenCalled(); + expect(raw.columns[0].resultJson).toBe(heartbeatRuns.resultJson); + }); +}); + +const postgresSupport = await getEmbeddedPostgresTestSupport(); +const describePostgres = postgresSupport.supported ? describe : describe.skip; +if (!postgresSupport.supported) { + console.warn(`Skipping run-state PostgreSQL tests: ${postgresSupport.reason ?? "unsupported host"}`); +} + +describePostgres("heartbeat run state database wiring", () => { + let database: Awaited>; + let db: Db; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("heartbeat-run-state-"); + db = createDb(database.connectionString); + }, 20_000); + + afterAll(async () => { + await database?.cleanup(); + }); + + async function fixture() { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Run state", issuePrefix: `S${companyId.slice(0, 8)}` }); + const [agent] = await db.insert(agents).values({ + companyId, name: "Session agent", adapterType: "codex_local", + }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "Session task", assigneeAgentId: agent.id }).returning(); + return { companyId, agent, issue, state: extracted.createHeartbeatRunState(db) }; + } + + it("resumes only the selected agent's run and task session", async () => { + const { companyId, agent, issue, state } = await fixture(); + const foreign = await fixture(); + const [run] = await db.insert(heartbeatRuns).values({ + companyId, agentId: agent.id, invocationSource: "assignment", status: "failed", + contextSnapshot: { issueId: issue.id }, sessionIdAfter: "our-session", + }).returning(); + const [foreignRun] = await db.insert(heartbeatRuns).values({ + companyId: foreign.companyId, agentId: foreign.agent.id, invocationSource: "assignment", + contextSnapshot: { issueId: foreign.issue.id }, sessionIdAfter: "foreign-session", + }).returning(); + await state.upsertTaskSession({ + companyId, agentId: agent.id, adapterType: agent.adapterType, taskKey: issue.id, + sessionParamsJson: { sessionId: "our-session", cwd: "/saved-workspace" }, + sessionDisplayId: "our-session", lastRunId: run.id, lastError: null, + }); + expect(await state.resolveExplicitResumeSessionOverride(agent, { resumeFromRunId: foreignRun.id }, issue.id)).toBeNull(); + expect(await state.resolveExplicitResumeSessionOverride(agent, { resumeFromRunId: run.id }, issue.id)).toMatchObject({ + resumeFromRunId: run.id, issueId: issue.id, taskKey: issue.id, + sessionDisplayId: "our-session", sessionParams: { sessionId: "our-session", cwd: "/saved-workspace" }, + }); + expect(await state.resolveSessionBeforeForWakeup(agent, issue.id)).toBe("our-session"); + expect(await state.getRuntimeStateWithSessions(agent.id)).toMatchObject({ + companyId, sessionDisplayId: "our-session", sessionParamsJson: { sessionId: "our-session" }, + }); + expect((await state.listRuns(companyId)).map((row) => row.id)).toEqual([run.id]); + }); + + it("subtracts prior cumulative usage while excluding the current run", async () => { + const { companyId, agent, state } = await fixture(); + await db.insert(heartbeatRuns).values({ + companyId, agentId: agent.id, invocationSource: "assignment", sessionIdAfter: "cumulative", + createdAt: new Date("2026-01-01T00:00:00Z"), + usageJson: { rawInputTokens: 800, rawCachedInputTokens: 100, rawOutputTokens: 80 }, + }); + const [current] = await db.insert(heartbeatRuns).values({ + companyId, agentId: agent.id, invocationSource: "assignment", sessionIdAfter: "cumulative", + createdAt: new Date("2026-01-02T00:00:00Z"), + usageJson: { rawInputTokens: 1000, rawCachedInputTokens: 125, rawOutputTokens: 100 }, + }).returning(); + const rawUsage = { inputTokens: 1000, cachedInputTokens: 125, outputTokens: 100 }; + const input = { agentId: agent.id, runId: current.id, sessionId: "cumulative", rawUsage }; + expect(await state.resolveNormalizedUsageForSession({ ...input, usageBasis: "session_cumulative" })).toEqual({ + normalizedUsage: { inputTokens: 200, cachedInputTokens: 25, outputTokens: 20 }, + previousRawUsage: { inputTokens: 800, cachedInputTokens: 100, outputTokens: 80 }, + derivedFromSessionTotals: true, + }); + expect(await state.resolveNormalizedUsageForSession({ ...input, usageBasis: "per_run" })).toEqual({ + normalizedUsage: rawUsage, previousRawUsage: null, derivedFromSessionTotals: false, + }); + }); + + it("uses the selected session's usage and summary for compaction", async () => { + const { companyId, agent, issue, state } = await fixture(); + const [run] = await db.insert(heartbeatRuns).values({ + companyId, agentId: agent.id, invocationSource: "assignment", sessionIdAfter: "compact", + usageJson: { rawInputTokens: 90, rawCachedInputTokens: 20, rawInputIncludesCached: false }, + resultJson: { summary: "Continue our task", providerPrivateData: "omit this" }, + }).returning(); + const foreign = await fixture(); + await db.insert(heartbeatRuns).values({ + companyId: foreign.companyId, agentId: foreign.agent.id, invocationSource: "assignment", + sessionIdAfter: "compact", resultJson: { summary: "Foreign task" }, + }); + const decision = await state.evaluateSessionCompaction({ + agent: { ...agent, runtimeConfig: { heartbeat: { sessionCompaction: { maxRawInputTokens: 100 } } } }, + sessionId: "compact", issueId: issue.id, + }); + expect(decision).toMatchObject({ rotate: true, reason: "session raw input reached 110 tokens (threshold 100)", previousRunId: run.id }); + expect(decision.handoffMarkdown).toContain("Continue our task"); + expect(decision.handoffMarkdown).not.toContain("Foreign task"); + expect(decision.handoffMarkdown).not.toContain("omit this"); + }); + + it.each(["superseded", "cancelled"] as const)("fences %s conversation session writes and clears", async (scenario) => { + const { companyId, agent, issue, state } = await fixture(); + await db.update(issues).set({ + conversationAgentId: agent.id, conversationUserId: "test-user", conversationState: "active", + conversationSessionGeneration: 2, + }).where(eq(issues.id, issue.id)); + const [run] = await db.insert(heartbeatRuns).values({ + companyId, agentId: agent.id, invocationSource: "assignment", status: "succeeded", + contextSnapshot: { conversationSessionGeneration: 2 }, + }).returning(); + const input = { + companyId, agentId: agent.id, adapterType: agent.adapterType, taskKey: issue.id, + sessionParamsJson: { sessionId: "retained" }, sessionDisplayId: "retained", lastRunId: run.id, lastError: null, + }; + const saved = await state.upsertTaskSession(input); + expect(saved?.sessionDisplayId).toBe("retained"); + if (scenario === "superseded") { + await db.update(issues).set({ conversationSessionGeneration: 3 }).where(eq(issues.id, issue.id)); + } else { + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, run.id)); + } + expect(await state.upsertTaskSession({ ...input, sessionDisplayId: "stale" })).toBeNull(); + expect(await state.clearTaskSessions(companyId, agent.id, { taskKey: issue.id, expectedRunId: run.id })).toBe(0); + const [retained] = await db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, saved!.id)); + expect(retained.sessionDisplayId).toBe("retained"); + }); +}); diff --git a/server/src/services/heartbeat/run-state.ts b/server/src/services/heartbeat/run-state.ts new file mode 100644 index 00000000000..75961481cad --- /dev/null +++ b/server/src/services/heartbeat/run-state.ts @@ -0,0 +1,1923 @@ +import { deriveTaskKey } from "./run-preparation.js"; +import { retryIdempotentDatabaseOperation } from "../../database-retry.js"; +import { isConversation } from "../agent-conversations.js"; +import fs from "node:fs/promises"; +import { + and, + asc, + desc, + eq, + getTableColumns, + gt, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + type BillingType, + type CostStatus, +} from "@paperclipai/shared"; +import { + agents, + agentConfigRevisions, + agentRuntimeState, + agentTaskSessions, + heartbeatRunEvents, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import { notFound } from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { getServerAdapter } from "../../adapters/index.js"; +import type { + AdapterExecutionResult, + AdapterSessionCodec, + UsageSummary, +} from "../../adapters/index.js"; +import { + parseObject, + asNumber, +} from "../../adapters/utils.js"; +import { + HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS, + HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS, + HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES, + summarizeRunErrorForModel, +} from "../heartbeat-run-summary.js"; +import { + hasSessionCompactionThresholds, + resolveSessionCompactionPolicy, + type SessionCompactionPolicy, +} from "@paperclipai/adapter-utils"; +import { isUuidLike } from "@paperclipai/shared"; +import { isUnsafeSessionWorkspaceCwd } from "../session-workspace-cwd.js"; + +export const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = + "execution_review_participant_recovery"; + +const heartbeatRunProcessGroupIdColumn = + heartbeatRuns.processGroupId ?? sql`NULL`.as("processGroupId"); + +const heartbeatRunListColumns = { + id: heartbeatRuns.id, + responsibleUserId: heartbeatRuns.responsibleUserId, + companyId: heartbeatRuns.companyId, + agentId: heartbeatRuns.agentId, + scopeKind: heartbeatRuns.scopeKind, + issueId: heartbeatRuns.issueId, + invocationSource: heartbeatRuns.invocationSource, + triggerDetail: heartbeatRuns.triggerDetail, + status: heartbeatRuns.status, + inputTokens: sql`(${heartbeatRuns.usageJson} ->> 'inputTokens')::numeric`.as("inputTokens"), + cachedInputTokens: sql`(${heartbeatRuns.usageJson} ->> 'cachedInputTokens')::numeric`.as("cachedInputTokens"), + outputTokens: sql`(${heartbeatRuns.usageJson} ->> 'outputTokens')::numeric`.as("outputTokens"), + totalTokens: sql`(${heartbeatRuns.usageJson} ->> 'totalTokens')::numeric`.as("totalTokens"), + costUsd: sql`coalesce( + (${heartbeatRuns.resultJson} ->> 'costUsd')::numeric, + (${heartbeatRuns.resultJson} ->> 'cost_usd')::numeric, + (${heartbeatRuns.resultJson} ->> 'total_cost_usd')::numeric + )`.as("costUsd"), + startedAt: heartbeatRuns.startedAt, + finishedAt: heartbeatRuns.finishedAt, + error: heartbeatRuns.error, + wakeupRequestId: heartbeatRuns.wakeupRequestId, + exitCode: heartbeatRuns.exitCode, + signal: heartbeatRuns.signal, + usageJson: heartbeatRuns.usageJson, + sessionIdBefore: heartbeatRuns.sessionIdBefore, + sessionIdAfter: heartbeatRuns.sessionIdAfter, + logStore: heartbeatRuns.logStore, + logRef: heartbeatRuns.logRef, + logBytes: heartbeatRuns.logBytes, + logSha256: heartbeatRuns.logSha256, + logCompressed: heartbeatRuns.logCompressed, + stdoutExcerpt: sql`NULL`.as("stdoutExcerpt"), + stderrExcerpt: sql`NULL`.as("stderrExcerpt"), + errorCode: heartbeatRuns.errorCode, + externalRunId: heartbeatRuns.externalRunId, + processPid: heartbeatRuns.processPid, + processGroupId: heartbeatRunProcessGroupIdColumn, + processStartedAt: heartbeatRuns.processStartedAt, + lastOutputAt: heartbeatRuns.lastOutputAt, + lastOutputSeq: heartbeatRuns.lastOutputSeq, + lastOutputStream: heartbeatRuns.lastOutputStream, + lastOutputBytes: heartbeatRuns.lastOutputBytes, + retryOfRunId: heartbeatRuns.retryOfRunId, + processLossRetryCount: heartbeatRuns.processLossRetryCount, + scheduledRetryAt: heartbeatRuns.scheduledRetryAt, + scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt, + scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + livenessState: heartbeatRuns.livenessState, + livenessReason: heartbeatRuns.livenessReason, + continuationAttempt: heartbeatRuns.continuationAttempt, + lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt, + nextAction: heartbeatRuns.nextAction, + createdAt: heartbeatRuns.createdAt, + updatedAt: heartbeatRuns.updatedAt, +} as const; + +const heartbeatRunSummaryListColumns = { + ...heartbeatRunListColumns, + usageJson: sql | null>`NULL`.as("usageJson"), + sessionIdBefore: sql`NULL`.as("sessionIdBefore"), + sessionIdAfter: sql`NULL`.as("sessionIdAfter"), + logStore: sql`NULL`.as("logStore"), + logRef: sql`NULL`.as("logRef"), + logSha256: sql`NULL`.as("logSha256"), + externalRunId: sql`NULL`.as("externalRunId"), + processPid: sql`NULL`.as("processPid"), + processGroupId: sql`NULL`.as("processGroupId"), + resultJson: sql | null>`NULL`.as("resultJson"), +} as const; + +const heartbeatRunListContextColumns = { + contextIssueId: sql`coalesce( + ${heartbeatRuns.issueId}::text, + ${heartbeatRuns.contextSnapshot} ->> 'issueId' + )`.as("contextIssueId"), + contextTaskId: sql`${heartbeatRuns.contextSnapshot} ->> 'taskId'`.as("contextTaskId"), + contextTaskKey: sql`${heartbeatRuns.contextSnapshot} ->> 'taskKey'`.as("contextTaskKey"), + contextCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), + contextWakeCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as("contextWakeCommentId"), + contextWakeReason: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeReason'`.as("contextWakeReason"), + contextWakeSource: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeSource'`.as("contextWakeSource"), + contextWakeTriggerDetail: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeTriggerDetail'`.as("contextWakeTriggerDetail"), +} as const; + +const heartbeatRunListResultColumns = { + resultSummary: sql< + string | null + >`left(${heartbeatRuns.resultJson} ->> 'summary', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( + "resultSummary", + ), + resultResult: sql< + string | null + >`left(${heartbeatRuns.resultJson} ->> 'result', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( + "resultResult", + ), + resultMessage: sql< + string | null + >`left(${heartbeatRuns.resultJson} ->> 'message', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( + "resultMessage", + ), + resultError: sql< + string | null + >`left(${heartbeatRuns.resultJson} ->> 'error', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( + "resultError", + ), + resultTotalCostUsd: sql< + string | null + >`${heartbeatRuns.resultJson} ->> 'total_cost_usd'`.as("resultTotalCostUsd"), + resultCostUsd: sql< + string | null + >`${heartbeatRuns.resultJson} ->> 'cost_usd'`.as("resultCostUsd"), + resultCostUsdCamel: sql< + string | null + >`${heartbeatRuns.resultJson} ->> 'costUsd'`.as("resultCostUsdCamel"), +} as const; + +// Reserve at most 9 KiB for diagnostics in the reduced result. An oversized +// multibyte field uses a conservative four-byte-per-character prefix, with a +// visible pointer to the full (adapter-bounded) run error and transcript. +const diagnosticRetrievalTitleBytes = 1024; + +const diagnosticRetrievalDetailsBytes = 8192; + +function boundedRunDiagnosticText(field: "title" | "details", maxBytes: number) { + const value = sql`${heartbeatRuns.resultJson} #>> ARRAY['terminalSessionFailure', ${field}]`; + return sql`case when octet_length(${value}) <= ${maxBytes} then ${value} + else left(${value}, ${Math.floor((maxBytes - 100) / 4)}) + || E'\\n[truncated for run retrieval; full text in run error/transcript]' end`; +} + +const heartbeatRunSafeResultJsonColumn = sql | null>` + case + when ${heartbeatRuns.resultJson} is null then null + when pg_column_size(${heartbeatRuns.resultJson}) <= ${HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES} + then ${heartbeatRuns.resultJson} + else jsonb_strip_nulls( + jsonb_build_object( + 'summary', left(${heartbeatRuns.resultJson} ->> 'summary', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), + 'result', left(${heartbeatRuns.resultJson} ->> 'result', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), + 'message', left(${heartbeatRuns.resultJson} ->> 'message', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), + 'error', left(${heartbeatRuns.resultJson} ->> 'error', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS}), + 'stdout', left(${heartbeatRuns.resultJson} ->> 'stdout', ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS}), + 'stderr', left(${heartbeatRuns.resultJson} ->> 'stderr', ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS}), + 'terminalSessionFailure', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'terminalSessionFailure') = 'object' + then jsonb_strip_nulls(jsonb_build_object( + 'category', left(${heartbeatRuns.resultJson} #>> '{terminalSessionFailure,category}', 32), + 'title', ${boundedRunDiagnosticText("title", diagnosticRetrievalTitleBytes)}, + 'details', ${boundedRunDiagnosticText("details", diagnosticRetrievalDetailsBytes)}, + 'retrievalTruncated', case when + octet_length(${heartbeatRuns.resultJson} #>> '{terminalSessionFailure,title}') > ${diagnosticRetrievalTitleBytes} + or octet_length(${heartbeatRuns.resultJson} #>> '{terminalSessionFailure,details}') > ${diagnosticRetrievalDetailsBytes} + then to_jsonb(true) end, + 'truncatedFields', case when ${heartbeatRuns.resultJson} #> '{terminalSessionFailure,truncatedFields}' + in ('["title"]'::jsonb, '["details"]'::jsonb, '["title","details"]'::jsonb) + then ${heartbeatRuns.resultJson} #> '{terminalSessionFailure,truncatedFields}' end + )) end, + 'instructionSave', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'instructionSave') = 'object' + then jsonb_strip_nulls(jsonb_build_object( + 'state', left(${heartbeatRuns.resultJson} #>> '{instructionSave,state}', 32), + 'contract', left(${heartbeatRuns.resultJson} #>> '{instructionSave,contract}', 32), + 'entryFile', left(${heartbeatRuns.resultJson} #>> '{instructionSave,entryFile}', 512), + 'errorCode', left(${heartbeatRuns.resultJson} #>> '{instructionSave,errorCode}', 128), + 'errorMessage', left(${heartbeatRuns.resultJson} #>> '{instructionSave,errorMessage}', 1024), + 'storageWarning', left(${heartbeatRuns.resultJson} #>> '{instructionSave,storageWarning}', 1024) + )) end, + 'workspaceRestoreRecovery', case when ${heartbeatRuns.resultJson} #>> '{workspaceRestoreRecovery,schema}' = 'paperclip.workspace-restore-recovery.v1' + then jsonb_build_object('schema', 'paperclip.workspace-restore-recovery.v1') end, + 'workspaceRestoreFailure', case when ${heartbeatRuns.resultJson} ->> 'workspaceRestoreFailure' + in ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed') + then ${heartbeatRuns.resultJson} -> 'workspaceRestoreFailure' end, + 'cancellation', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'cancellation') = 'object' + then jsonb_strip_nulls(jsonb_build_object( + 'source', case when ${heartbeatRuns.resultJson} #>> '{cancellation,source}' + in ('operator', 'queued_message', 'shutdown', 'provider', 'transport', 'control_plane', 'unknown') + then ${heartbeatRuns.resultJson} #> '{cancellation,source}' end, + 'expected', case when jsonb_typeof(${heartbeatRuns.resultJson} #> '{cancellation,expected}') = 'boolean' + then ${heartbeatRuns.resultJson} #> '{cancellation,expected}' end, + 'initiator', jsonb_strip_nulls(jsonb_build_object( + 'type', case when ${heartbeatRuns.resultJson} #>> '{cancellation,initiator,type}' in ('user', 'agent', 'system', 'provider') + then ${heartbeatRuns.resultJson} #> '{cancellation,initiator,type}' end, + 'id', left(${heartbeatRuns.resultJson} #>> '{cancellation,initiator,id}', 128) + )), + 'reason', left(${heartbeatRuns.resultJson} #>> '{cancellation,reason}', 512), + 'recordedAt', left(${heartbeatRuns.resultJson} #>> '{cancellation,recordedAt}', 64) + )) end, + 'acpToolInventoryComplete', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'acpToolInventoryComplete') = 'boolean' + then ${heartbeatRuns.resultJson} -> 'acpToolInventoryComplete' end, + 'acpPendingToolCount', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'acpPendingToolCount') = 'number' + and length(${heartbeatRuns.resultJson} ->> 'acpPendingToolCount') < 16 + then ${heartbeatRuns.resultJson} -> 'acpPendingToolCount' end, + 'errorFamily', left(${heartbeatRuns.resultJson} ->> 'errorFamily', 32), + 'finalResponseRecorded', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'finalResponseRecorded') = 'boolean' + then ${heartbeatRuns.resultJson} -> 'finalResponseRecorded' end, + 'executionBeforeRestore', case when jsonb_typeof(${heartbeatRuns.resultJson} -> 'executionBeforeRestore') = 'object' + then jsonb_strip_nulls(jsonb_build_object( + 'errorCode', left(${heartbeatRuns.resultJson} #>> '{executionBeforeRestore,errorCode}', 128), + 'exitCode', case when jsonb_typeof(${heartbeatRuns.resultJson} #> '{executionBeforeRestore,exitCode}') = 'number' + and length(${heartbeatRuns.resultJson} #>> '{executionBeforeRestore,exitCode}') < 16 + then ${heartbeatRuns.resultJson} #> '{executionBeforeRestore,exitCode}' end, + 'signal', left(${heartbeatRuns.resultJson} #>> '{executionBeforeRestore,signal}', 50), + 'timedOut', case when jsonb_typeof(${heartbeatRuns.resultJson} #> '{executionBeforeRestore,timedOut}') = 'boolean' + then ${heartbeatRuns.resultJson} #> '{executionBeforeRestore,timedOut}' end + )) end, + 'stdoutTruncated', case + when length(${heartbeatRuns.resultJson} ->> 'stdout') > ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS} + then to_jsonb(true) + else null + end, + 'stderrTruncated', case + when length(${heartbeatRuns.resultJson} ->> 'stderr') > ${HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS} + then to_jsonb(true) + else null + end, + 'costUsd', coalesce( + ${heartbeatRuns.resultJson} -> 'costUsd', + ${heartbeatRuns.resultJson} -> 'cost_usd', + ${heartbeatRuns.resultJson} -> 'total_cost_usd' + ), + 'cost_usd', coalesce( + ${heartbeatRuns.resultJson} -> 'cost_usd', + ${heartbeatRuns.resultJson} -> 'costUsd', + ${heartbeatRuns.resultJson} -> 'total_cost_usd' + ), + 'total_cost_usd', coalesce( + ${heartbeatRuns.resultJson} -> 'total_cost_usd', + ${heartbeatRuns.resultJson} -> 'cost_usd', + ${heartbeatRuns.resultJson} -> 'costUsd' + ), + 'truncated', true, + 'truncationReason', 'oversized_result_json', + 'originalSizeBytes', pg_column_size(${heartbeatRuns.resultJson}) + ) + ) + end +`.as("resultJson"); + +// Execution admission needs retained server receipts even when presentation +// projection omits resultJson (SQL_ASCII or oversized provider output). Select +// only the evidence used by eligibility; never retrieve provider diagnostics. +const heartbeatRunExecutionEvidenceColumn = sql | null>` + case when ${heartbeatRuns.resultJson} is null then null else jsonb_build_object( + 'startupCancellation', ${heartbeatRuns.resultJson} -> 'startupCancellation', + 'startupPreparationSettledAt', ${heartbeatRuns.resultJson} -> 'startupPreparationSettledAt', + 'stopReason', ${heartbeatRuns.resultJson} -> 'stopReason', + 'timeoutSource', ${heartbeatRuns.resultJson} -> 'timeoutSource', + 'workspaceRestoreFailure', ${heartbeatRuns.resultJson} -> 'workspaceRestoreFailure', + 'workspaceRestoreRecovery', ${heartbeatRuns.resultJson} -> 'workspaceRestoreRecovery', + 'executionCancellation', ${heartbeatRuns.resultJson} -> 'executionCancellation', + 'nativeCancellation', ${heartbeatRuns.resultJson} -> 'nativeCancellation', + 'cancelledByActorType', ${heartbeatRuns.resultJson} -> 'cancelledByActorType', + 'cancelledByUserId', ${heartbeatRuns.resultJson} -> 'cancelledByUserId', + 'conversationContinuation', ${heartbeatRuns.resultJson} -> 'conversationContinuation', + 'cancellation', ${heartbeatRuns.resultJson} -> 'cancellation', + 'acpToolInventoryComplete', ${heartbeatRuns.resultJson} -> 'acpToolInventoryComplete', + 'acpPendingToolCount', ${heartbeatRuns.resultJson} -> 'acpPendingToolCount' + ) end +`.as("resultJson"); + +const heartbeatRunSafeColumns = { + ...getTableColumns(heartbeatRuns), + processGroupId: heartbeatRunProcessGroupIdColumn, + resultJson: heartbeatRunSafeResultJsonColumn, +} as const; + +const heartbeatRunSqlAsciiSafeColumns = { + ...getTableColumns(heartbeatRuns), + processGroupId: heartbeatRunProcessGroupIdColumn, + error: sql`NULL`.as("error"), + resultJson: sql | null>`NULL`.as("resultJson"), + stdoutExcerpt: sql`NULL`.as("stdoutExcerpt"), + stderrExcerpt: sql`NULL`.as("stderrExcerpt"), +} as const; + +const heartbeatRunLogAccessColumns = { + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + scopeKind: heartbeatRuns.scopeKind, + issueId: heartbeatRuns.issueId, + logStore: heartbeatRuns.logStore, + logRef: heartbeatRuns.logRef, +} as const; + +const heartbeatRunIssueSummaryColumns = { + id: heartbeatRuns.id, + runtimeMode: heartbeatRuns.runtimeMode, + status: heartbeatRuns.status, + invocationSource: heartbeatRuns.invocationSource, + triggerDetail: heartbeatRuns.triggerDetail, + contextCommentId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), + contextWakeCommentId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as( + "contextWakeCommentId", + ), + startedAt: heartbeatRuns.startedAt, + finishedAt: heartbeatRuns.finishedAt, + createdAt: heartbeatRuns.createdAt, + agentId: heartbeatRuns.agentId, + logBytes: heartbeatRuns.logBytes, + processStartedAt: heartbeatRuns.processStartedAt, + livenessState: heartbeatRuns.livenessState, + livenessReason: heartbeatRuns.livenessReason, + continuationAttempt: heartbeatRuns.continuationAttempt, + lastUsefulActionAt: heartbeatRuns.lastUsefulActionAt, + nextAction: heartbeatRuns.nextAction, + lastOutputAt: heartbeatRuns.lastOutputAt, + lastOutputSeq: heartbeatRuns.lastOutputSeq, + lastOutputStream: heartbeatRuns.lastOutputStream, + lastOutputBytes: heartbeatRuns.lastOutputBytes, + issueId: heartbeatRuns.issueId, +} as const; + +export type UsageTotals = { + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; +}; + +type SessionCompactionDecision = { + rotate: boolean; + reason: string | null; + handoffMarkdown: string | null; + previousRunId: string | null; +}; + +export function summarizeHeartbeatRunContextSnapshot( + contextSnapshot: Record | null | undefined, +): Record | null { + const summary: Record = {}; + const allowedKeys = [ + "issueId", + "taskId", + "taskKey", + "commentId", + "wakeCommentId", + "wakeReason", + "wakeSource", + "wakeTriggerDetail", + ] as const; + + for (const key of allowedKeys) { + const value = readNonEmptyString(contextSnapshot?.[key]); + if (value) summary[key] = value; + } + + return Object.keys(summary).length > 0 ? summary : null; +} + +export function summarizeHeartbeatRunListResultJson(input: { + summary?: string | null; + result?: string | null; + message?: string | null; + error?: string | null; + totalCostUsd?: string | null; + costUsd?: string | null; + costUsdCamel?: string | null; +}): Record | null { + const summary: Record = {}; + for (const [key, value] of [ + ["summary", input.summary], + ["result", input.result], + ["message", input.message], + ["error", input.error], + ] as const) { + const normalized = readNonEmptyString(value); + if (normalized) summary[key] = normalized; + } + + for (const [key, value] of [ + ["total_cost_usd", input.totalCostUsd], + ["cost_usd", input.costUsd], + ["costUsd", input.costUsdCamel], + ] as const) { + const normalized = readNonEmptyString(value); + if (!normalized) continue; + const parsed = Number(normalized); + if (Number.isFinite(parsed)) summary[key] = parsed; + } + + return Object.keys(summary).length > 0 ? summary : null; +} + +export function normalizeLedgerBillingType(value: unknown): BillingType { + const raw = readNonEmptyString(value); + switch (raw) { + case "api": + case "metered_api": + return "metered_api"; + case "subscription": + case "subscription_included": + return "subscription_included"; + case "subscription_overage": + return "subscription_overage"; + case "credits": + return "credits"; + case "fixed": + return "fixed"; + default: + return "unknown"; + } +} + +export function resolveLedgerBiller(result: AdapterExecutionResult): string { + return ( + readNonEmptyString(result.biller) ?? + readNonEmptyString(result.provider) ?? + "unknown" + ); +} + +export function normalizeBilledCostCents( + costUsd: number | null | undefined, + billingType: BillingType, +): number { + if (billingType === "subscription_included") return 0; + if (typeof costUsd !== "number" || !Number.isFinite(costUsd)) return 0; + return Math.max(0, Number((costUsd * 100).toFixed(7))); +} + +export function resolveLedgerCostStatus(input: { + costUsd: number | null | undefined; + billingType?: BillingType; + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; +}): CostStatus { + if (input.billingType === "subscription_included") return "reported"; + // A paused turn can have neither a token receipt nor a cost receipt. Zero + // normalized counters do not establish that its billed cost was zero. + return typeof input.costUsd === "number" && + Number.isFinite(input.costUsd) && + input.costUsd >= 0 + ? "reported" + : "unpriced"; +} + +export function resolveCacheAdjustedCostUsd(input: { + costUsd?: number | null; + cacheAdjustedCostUsd?: number | null; +}) { + const explicit = input.cacheAdjustedCostUsd; + if ( + typeof explicit === "number" && + Number.isFinite(explicit) && + explicit >= 0 + ) { + return explicit; + } + const reported = input.costUsd; + if ( + typeof reported === "number" && + Number.isFinite(reported) && + reported >= 0 + ) { + return reported; + } + return null; +} + +export async function resolveLedgerScopeForRun( + db: Db, + companyId: string, + run: typeof heartbeatRuns.$inferSelect, +) { + const context = parseObject(run.contextSnapshot); + const contextIssueId = readNonEmptyString(context.issueId); + const contextProjectId = readNonEmptyString(context.projectId); + + if (!contextIssueId) { + return { + issueId: null, + projectId: contextProjectId, + billingCode: null, + }; + } + + const issue = await db + .select({ + id: issues.id, + projectId: issues.projectId, + billingCode: issues.billingCode, + }) + .from(issues) + .where(and(eq(issues.id, contextIssueId), eq(issues.companyId, companyId))) + .then((rows) => rows[0] ?? null); + + return { + issueId: issue?.id ?? null, + projectId: issue?.projectId ?? contextProjectId, + billingCode: issue?.billingCode ?? null, + }; +} + +type ResumeSessionRow = { + sessionParamsJson: Record | null; + sessionDisplayId: string | null; + lastRunId: string | null; +}; + +export function buildExplicitResumeSessionOverride(input: { + adapterType?: string | null; + resumeFromRunId: string; + resumeRunSessionIdBefore: string | null; + resumeRunSessionIdAfter: string | null; + resumeRunSessionParams?: Record | null; + taskSession: ResumeSessionRow | null; + sessionCodec: AdapterSessionCodec; +}) { + const resumeRunSessionIdAfter = truncateDisplayId( + input.resumeRunSessionIdAfter, + ); + const resumeRunSessionIdBefore = truncateDisplayId( + input.resumeRunSessionIdBefore, + ); + const desiredDisplayId = requiresCanonicalSessionIds(input.adapterType) + ? isCanonicalSessionIdForAdapter(input.adapterType, resumeRunSessionIdAfter) + ? resumeRunSessionIdAfter + : isCanonicalSessionIdForAdapter( + input.adapterType, + resumeRunSessionIdBefore, + ) + ? resumeRunSessionIdBefore + : null + : (resumeRunSessionIdAfter ?? resumeRunSessionIdBefore); + const runSessionParams = requiresCanonicalSessionIds(input.adapterType) + ? normalizeResumeParamsForAdapter( + input.adapterType, + input.sessionCodec.deserialize(input.resumeRunSessionParams ?? null), + ) + : null; + const runSessionDisplayId = truncateDisplayId( + readNonEmptyString(runSessionParams?.sessionId), + ); + const taskSessionParams = normalizeResumeParamsForAdapter( + input.adapterType, + input.sessionCodec.deserialize( + input.taskSession?.sessionParamsJson ?? null, + ), + ); + const taskSessionRawDisplayId = input.taskSession?.sessionDisplayId ?? null; + const taskSessionDisplayId = truncateDisplayId( + requiresCanonicalSessionIds(input.adapterType) + ? (readNonEmptyString(taskSessionParams?.sessionId) ?? + (isCanonicalSessionIdForAdapter( + input.adapterType, + taskSessionRawDisplayId, + ) + ? taskSessionRawDisplayId + : null)) + : (taskSessionRawDisplayId ?? + (input.sessionCodec.getDisplayId + ? input.sessionCodec.getDisplayId(taskSessionParams) + : null) ?? + readNonEmptyString(taskSessionParams?.sessionId)), + ); + const canReuseTaskSessionParams = + input.taskSession != null && + (!requiresCanonicalSessionIds(input.adapterType) || + taskSessionParams != null) && + (input.taskSession.lastRunId === input.resumeFromRunId || + (!!desiredDisplayId && taskSessionDisplayId === desiredDisplayId)); + const sessionParams = canReuseTaskSessionParams + ? taskSessionParams + : runSessionParams + ? runSessionParams + : desiredDisplayId + ? { sessionId: desiredDisplayId } + : null; + const sessionDisplayId = canReuseTaskSessionParams + ? taskSessionDisplayId + : runSessionParams + ? runSessionDisplayId + : desiredDisplayId; + + if (!sessionDisplayId && !sessionParams) return null; + return { + sessionDisplayId, + sessionParams, + }; +} + +export function normalizeUsageTotals( + usage: UsageSummary | null | undefined, +): UsageTotals | null { + if (!usage) return null; + return { + inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))), + cachedInputTokens: Math.max( + 0, + Math.floor(asNumber(usage.cachedInputTokens, 0)), + ), + outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))), + }; +} + +function readRawUsageTotals(usageJson: unknown): UsageTotals | null { + const parsed = parseObject(usageJson); + if (Object.keys(parsed).length === 0) return null; + + const inputTokens = Math.max( + 0, + Math.floor( + asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0)), + ), + ); + const cachedInputTokens = Math.max( + 0, + Math.floor( + asNumber( + parsed.rawCachedInputTokens, + asNumber(parsed.cachedInputTokens, 0), + ), + ), + ); + const outputTokens = Math.max( + 0, + Math.floor( + asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0)), + ), + ); + + if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) { + return null; + } + + return { + inputTokens, + cachedInputTokens, + outputTokens, + }; +} + +export function normalizeAdapterRunUsage( + current: UsageTotals | null, + previous: UsageTotals | null, + usageBasis?: "per_run" | "session_cumulative" | null, +): UsageTotals | null { + if (!current) return null; + if (!previous || usageBasis !== "session_cumulative") return { ...current }; + + const inputTokens = + current.inputTokens >= previous.inputTokens + ? current.inputTokens - previous.inputTokens + : current.inputTokens; + const cachedInputTokens = + current.cachedInputTokens >= previous.cachedInputTokens + ? current.cachedInputTokens - previous.cachedInputTokens + : current.cachedInputTokens; + const outputTokens = + current.outputTokens >= previous.outputTokens + ? current.outputTokens - previous.outputTokens + : current.outputTokens; + + return { + inputTokens: Math.max(0, inputTokens), + cachedInputTokens: Math.max(0, cachedInputTokens), + outputTokens: Math.max(0, outputTokens), + }; +} + +function formatCount(value: number | null | undefined) { + if (typeof value !== "number" || !Number.isFinite(value)) return "0"; + return value.toLocaleString("en-US"); +} + +export function parseSessionCompactionPolicy( + agent: typeof agents.$inferSelect, +): SessionCompactionPolicy { + return resolveSessionCompactionPolicy(agent.adapterType, agent.runtimeConfig) + .policy; +} + +/** + * Synthetic task key for timer/heartbeat wakes that have no issue context. + * This allows timer wakes to participate in the `agentTaskSessions` system + * and benefit from robust session resume, instead of relying solely on the + * simpler `agentRuntimeState.sessionId` fallback. + */ +const HEARTBEAT_TASK_KEY = "__heartbeat__"; + +/** + * Extended task key derivation that falls back to a stable synthetic key + * for timer/heartbeat wakes. The synthetic key keeps the + * `agentTaskSessions` row addressable across heartbeats so the row can be + * cleared and re-keyed deterministically. Unscoped exploratory timer wakes + * still start fresh to avoid accumulating low-value inbox scans, while timer + * wakes scoped to a real issue reuse that issue's task session. + * + * The synthetic key is only used when: + * - No explicit task/issue key exists in the context + * - The wake source is "timer" (scheduled heartbeat) + */ +export function deriveTaskKeyWithHeartbeatFallback( + contextSnapshot: Record | null | undefined, + payload: Record | null | undefined, +) { + const explicit = deriveTaskKey(contextSnapshot, payload); + if (explicit) return explicit; + + const wakeSource = readNonEmptyString(contextSnapshot?.wakeSource); + if (wakeSource === "timer") return HEARTBEAT_TASK_KEY; + + return null; +} + +export function shouldResetTaskSessionForWake( + contextSnapshot: Record | null | undefined, +) { + if (contextSnapshot?.forceFreshSession === true) return true; + + const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); + if ( + wakeReason === "issue_assigned" || + wakeReason === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON || + wakeReason === "execution_approval_requested" || + // PF-4: unscoped timer wakes are exploratory ("any new work?") and should + // not accumulate low-value inbox scans. Issue-scoped timer wakes are + // continuation work, so reuse their task session to avoid paying the full + // session-start and re-orientation cost on every heartbeat. + (wakeReason === "heartbeat_timer" && !deriveTaskKey(contextSnapshot, null)) + ) { + return true; + } + return false; +} + +export function describeSessionResetReason( + contextSnapshot: Record | null | undefined, +) { + if (contextSnapshot?.forceFreshSession === true) + return "forceFreshSession was requested"; + + const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); + if (wakeReason === "issue_assigned") return "wake reason is issue_assigned"; + if (wakeReason === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON) { + return `wake reason is ${EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON}`; + } + if (wakeReason === "execution_approval_requested") + return "wake reason is execution_approval_requested"; + // PF-4: paired with shouldResetTaskSessionForWake — keep the reason wording + // explicit so run logs make session reuse/reset behavior legible. + if ( + wakeReason === "heartbeat_timer" && + !deriveTaskKey(contextSnapshot, null) + ) { + return "wake reason is heartbeat_timer (unscoped timer wake starts fresh)"; + } + return null; +} + +export function truncateDisplayId(value: string | null | undefined, max = 128) { + if (!value) return null; + return value.length > max ? value.slice(0, max) : value; +} + +const defaultSessionCodec: AdapterSessionCodec = { + deserialize(raw: unknown) { + const asObj = parseObject(raw); + if (Object.keys(asObj).length > 0) return asObj; + const sessionId = readNonEmptyString( + (raw as Record | null)?.sessionId, + ); + if (sessionId) return { sessionId }; + return null; + }, + serialize(params: Record | null) { + if (!params || Object.keys(params).length === 0) return null; + return params; + }, + getDisplayId(params: Record | null) { + return readNonEmptyString(params?.sessionId); + }, +}; + +export function getAdapterSessionCodec(adapterType: string) { + const adapter = getServerAdapter(adapterType); + return adapter.sessionCodec ?? defaultSessionCodec; +} + +export function normalizeSessionParams( + params: Record | null | undefined, +) { + if (!params) return null; + return Object.keys(params).length > 0 ? params : null; +} + +export type RunSessionOutcome = + "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out"; + +const HERMES_ADAPTER_TYPE = "hermes_local"; + +const HERMES_SESSION_ID_REGEX = + /^(?:\d{8}_\d{6}_[A-Za-z0-9_-]{4,}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$/; + +export function requiresCanonicalSessionIds(adapterType: string | null | undefined) { + return adapterType === HERMES_ADAPTER_TYPE; +} + +export function isCanonicalSessionIdForAdapter( + adapterType: string | null | undefined, + sessionId: string | null | undefined, +) { + if (!sessionId) return false; + if (!requiresCanonicalSessionIds(adapterType)) return true; + return HERMES_SESSION_ID_REGEX.test(sessionId); +} + +export function normalizeResumeParamsForAdapter( + adapterType: string | null | undefined, + params: Record | null | undefined, +) { + const normalized = normalizeSessionParams(params); + if (!normalized) return null; + if (!requiresCanonicalSessionIds(adapterType)) return normalized; + const sessionId = readNonEmptyString(normalized.sessionId); + return isCanonicalSessionIdForAdapter(adapterType, sessionId) + ? normalized + : null; +} + +export function resolveNextSessionState(input: { + adapterType?: string | null; + codec: AdapterSessionCodec; + adapterResult: AdapterExecutionResult; + outcome: RunSessionOutcome; + previousParams: Record | null; + previousDisplayId: string | null; + previousLegacySessionId: string | null; +}) { + const { + adapterType, + codec, + adapterResult, + previousParams, + previousDisplayId, + previousLegacySessionId, + } = input; + + if (adapterResult.clearSession) { + return { + params: null as Record | null, + displayId: null as string | null, + legacySessionId: null as string | null, + }; + } + + if (!requiresCanonicalSessionIds(adapterType)) { + const explicitParams = adapterResult.sessionParams; + const hasExplicitParams = adapterResult.sessionParams !== undefined; + const hasExplicitSessionId = adapterResult.sessionId !== undefined; + const explicitSessionId = readNonEmptyString(adapterResult.sessionId); + const hasExplicitDisplay = adapterResult.sessionDisplayId !== undefined; + const explicitDisplayId = readNonEmptyString( + adapterResult.sessionDisplayId, + ); + const shouldUsePrevious = + !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay; + + const candidateParams = hasExplicitParams + ? explicitParams + : hasExplicitSessionId + ? explicitSessionId + ? { sessionId: explicitSessionId } + : null + : previousParams; + + const serialized = normalizeSessionParams( + codec.serialize(normalizeSessionParams(candidateParams) ?? null), + ); + const deserialized = normalizeSessionParams(codec.deserialize(serialized)); + + const displayId = truncateDisplayId( + explicitDisplayId ?? + (codec.getDisplayId ? codec.getDisplayId(deserialized) : null) ?? + readNonEmptyString(deserialized?.sessionId) ?? + (shouldUsePrevious ? previousDisplayId : null) ?? + explicitSessionId ?? + (shouldUsePrevious ? previousLegacySessionId : null), + ); + + const legacySessionId = + explicitSessionId ?? + readNonEmptyString(deserialized?.sessionId) ?? + displayId ?? + (shouldUsePrevious ? previousLegacySessionId : null); + + return { + params: serialized, + displayId, + legacySessionId, + }; + } + + const previousSerializedParams = normalizeResumeParamsForAdapter( + adapterType, + codec.serialize( + normalizeResumeParamsForAdapter(adapterType, previousParams), + ), + ); + const validPreviousDisplayId = isCanonicalSessionIdForAdapter( + adapterType, + previousDisplayId, + ) + ? previousDisplayId + : null; + const validPreviousLegacySessionId = isCanonicalSessionIdForAdapter( + adapterType, + previousLegacySessionId, + ) + ? previousLegacySessionId + : null; + const previousState = () => { + const displayId = truncateDisplayId( + readNonEmptyString(previousSerializedParams?.sessionId) ?? + validPreviousDisplayId ?? + validPreviousLegacySessionId, + ); + return { + params: previousSerializedParams, + displayId, + legacySessionId: + readNonEmptyString(previousSerializedParams?.sessionId) ?? + displayId ?? + validPreviousLegacySessionId, + }; + }; + + if (input.outcome !== "succeeded") { + return previousState(); + } + + const explicitParams = adapterResult.sessionParams; + const hasExplicitParams = adapterResult.sessionParams !== undefined; + const explicitSessionId = readNonEmptyString(adapterResult.sessionId); + const validExplicitSessionId = isCanonicalSessionIdForAdapter( + adapterType, + explicitSessionId, + ) + ? explicitSessionId + : null; + const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId); + const validExplicitDisplayId = isCanonicalSessionIdForAdapter( + adapterType, + explicitDisplayId, + ) + ? explicitDisplayId + : null; + const explicitSerializedParams = hasExplicitParams + ? normalizeResumeParamsForAdapter( + adapterType, + codec.serialize(normalizeSessionParams(explicitParams) ?? null), + ) + : null; + const explicitCanonicalSessionId = + readNonEmptyString(explicitSerializedParams?.sessionId) ?? + validExplicitSessionId ?? + validExplicitDisplayId; + + if (!explicitCanonicalSessionId) { + return previousState(); + } + + const serialized = normalizeResumeParamsForAdapter( + adapterType, + codec.serialize({ sessionId: explicitCanonicalSessionId }), + ); + const displayId = truncateDisplayId( + readNonEmptyString(serialized?.sessionId) ?? + (codec.getDisplayId ? codec.getDisplayId(serialized) : null) ?? + explicitCanonicalSessionId, + ); + const legacySessionId = + readNonEmptyString(serialized?.sessionId) ?? explicitCanonicalSessionId; + + return { + params: serialized, + displayId, + legacySessionId, + }; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +/** Run retrieval and session persistence bound to one service database. */ +export function createHeartbeatRunState(db: Db) { + let unsafeTextProjectionPromise: Promise | null = null; + + async function hasUnsafeTextProjectionDatabase() { + if (!unsafeTextProjectionPromise) { + unsafeTextProjectionPromise = db + .execute( + sql`select current_setting('server_encoding') as server_encoding`, + ) + .then((rows) => { + const first = Array.isArray(rows) ? rows[0] : null; + const serverEncoding = + typeof first === "object" && first !== null + ? (first as Record).server_encoding + : null; + return ( + typeof serverEncoding === "string" && + serverEncoding.toUpperCase() === "SQL_ASCII" + ); + }) + .catch((err) => { + logger.warn( + { err }, + "failed to inspect database server encoding; using conservative heartbeat result projection", + ); + return true; + }); + } + return unsafeTextProjectionPromise; + } + + async function getAgent(agentId: string) { + return db + .select() + .from(agents) + .where(eq(agents.id, agentId)) + .then((rows) => rows[0] ?? null); + } + + async function getRun( + runId: string, + opts?: { unsafeFullResultJson?: boolean; includeExecutionEvidence?: boolean }, + ) { + const safeForLegacyEncoding = + !opts?.unsafeFullResultJson && (await hasUnsafeTextProjectionDatabase()); + const columns = opts?.unsafeFullResultJson + ? getTableColumns(heartbeatRuns) + : safeForLegacyEncoding ? heartbeatRunSqlAsciiSafeColumns : heartbeatRunSafeColumns; + return db + .select(opts?.includeExecutionEvidence + ? { ...columns, resultJson: heartbeatRunExecutionEvidenceColumn } + : columns) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + } + + async function getRunLogAccess(runId: string) { + return db + .select(heartbeatRunLogAccessColumns) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + } + + async function getRuntimeState(agentId: string) { + return db + .select() + .from(agentRuntimeState) + .where(eq(agentRuntimeState.agentId, agentId)) + .then((rows) => rows[0] ?? null); + } + + async function getLatestAgentConfigRevision( + companyId: string, + agentId: string, + ) { + return db + .select({ + id: agentConfigRevisions.id, + changedKeys: agentConfigRevisions.changedKeys, + createdAt: agentConfigRevisions.createdAt, + }) + .from(agentConfigRevisions) + .where( + and( + eq(agentConfigRevisions.companyId, companyId), + eq(agentConfigRevisions.agentId, agentId), + ), + ) + .orderBy( + desc(agentConfigRevisions.createdAt), + desc(agentConfigRevisions.id), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + async function getTaskSession( + companyId: string, + agentId: string, + adapterType: string, + taskKey: string, + ) { + return db + .select() + .from(agentTaskSessions) + .where( + and( + eq(agentTaskSessions.companyId, companyId), + eq(agentTaskSessions.agentId, agentId), + eq(agentTaskSessions.adapterType, adapterType), + eq(agentTaskSessions.taskKey, taskKey), + ), + ) + .then((rows) => rows[0] ?? null); + } + + async function getLatestRunForSession( + agentId: string, + sessionId: string, + opts?: { excludeRunId?: string | null }, + ) { + const conditions = [ + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.sessionIdAfter, sessionId), + ]; + if (opts?.excludeRunId) { + conditions.push(sql`${heartbeatRuns.id} <> ${opts.excludeRunId}`); + } + return db + .select({ + id: heartbeatRuns.id, + usageJson: heartbeatRuns.usageJson, + }) + .from(heartbeatRuns) + .where(and(...conditions)) + .orderBy(desc(heartbeatRuns.createdAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + async function getOldestRunForSession(agentId: string, sessionId: string) { + return db + .select({ + id: heartbeatRuns.id, + createdAt: heartbeatRuns.createdAt, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.sessionIdAfter, sessionId), + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + async function resolveNormalizedUsageForSession(input: { + agentId: string; + runId: string; + sessionId: string | null; + rawUsage: UsageTotals | null; + usageBasis?: "per_run" | "session_cumulative" | null; + }) { + const { agentId, runId, sessionId, rawUsage, usageBasis } = input; + // Adapters that declare per-run usage (e.g. the ACPX lane reports each + // turn's tokens, not session totals) must not be session-delta'd, or + // consecutive runs would be undercounted. + if (!sessionId || !rawUsage || usageBasis !== "session_cumulative") { + return { + normalizedUsage: rawUsage, + previousRawUsage: null as UsageTotals | null, + derivedFromSessionTotals: false, + }; + } + + const previousRun = await getLatestRunForSession(agentId, sessionId, { + excludeRunId: runId, + }); + const previousRawUsage = readRawUsageTotals(previousRun?.usageJson); + return { + normalizedUsage: normalizeAdapterRunUsage(rawUsage, previousRawUsage, usageBasis), + previousRawUsage, + derivedFromSessionTotals: previousRawUsage !== null, + }; + } + + async function evaluateSessionCompaction(input: { + agent: typeof agents.$inferSelect; + sessionId: string | null; + issueId: string | null; + continuationSummaryBody?: string | null; + }): Promise { + const { agent, sessionId, issueId } = input; + if (!sessionId) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: null, + }; + } + + const policy = parseSessionCompactionPolicy(agent); + if (!policy.enabled || !hasSessionCompactionThresholds(policy)) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: null, + }; + } + + const fetchLimit = Math.max( + policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, + 4, + ); + const runs = await db + .select({ + id: heartbeatRuns.id, + createdAt: heartbeatRuns.createdAt, + usageJson: heartbeatRuns.usageJson, + error: heartbeatRuns.error, + terminalFailureCategory: sql`case + when jsonb_typeof(${heartbeatRuns.resultJson} -> 'terminalSessionFailure') = 'object' + then coalesce(left(${heartbeatRuns.resultJson} #>> '{terminalSessionFailure,category}', 32), 'unknown') end`, + ...heartbeatRunListResultColumns, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agent.id), + eq(heartbeatRuns.sessionIdAfter, sessionId), + ), + ) + .orderBy(desc(heartbeatRuns.createdAt)) + .limit(fetchLimit); + + if (runs.length === 0) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: null, + }; + } + + const latestRun = runs[0] ?? null; + const oldestRun = + policy.maxSessionAgeHours > 0 + ? await getOldestRunForSession(agent.id, sessionId) + : (runs[runs.length - 1] ?? latestRun); + const latestRawUsage = readRawUsageTotals(latestRun?.usageJson); + // Historical Codex/Gemini raw input includes cache reads. Only add the + // separate cache counter when the writer explicitly saved exclusive input. + const latestRawInputTokens = (latestRawUsage?.inputTokens ?? 0) + + (latestRun?.usageJson?.rawInputIncludesCached === false ? latestRawUsage?.cachedInputTokens ?? 0 : 0); + const sessionAgeHours = + latestRun && oldestRun + ? Math.max( + 0, + (new Date(latestRun.createdAt).getTime() - + new Date(oldestRun.createdAt).getTime()) / + (1000 * 60 * 60), + ) + : 0; + + let reason: string | null = null; + if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) { + reason = `session exceeded ${policy.maxSessionRuns} runs`; + } else if ( + policy.maxRawInputTokens > 0 && + latestRawUsage && + latestRawInputTokens >= policy.maxRawInputTokens + ) { + reason = + `session raw input reached ${formatCount(latestRawInputTokens)} tokens ` + + `(threshold ${formatCount(policy.maxRawInputTokens)})`; + } else if ( + policy.maxSessionAgeHours > 0 && + sessionAgeHours >= policy.maxSessionAgeHours + ) { + reason = `session age reached ${Math.floor(sessionAgeHours)} hours`; + } + + if (!reason || !latestRun) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: latestRun?.id ?? null, + }; + } + + const latestSummary = summarizeHeartbeatRunListResultJson({ + summary: latestRun?.resultSummary, + result: latestRun?.resultResult, + message: latestRun?.resultMessage, + error: latestRun?.resultError, + totalCostUsd: latestRun?.resultTotalCostUsd, + costUsd: latestRun?.resultCostUsd, + costUsdCamel: latestRun?.resultCostUsdCamel, + }); + const latestTextSummary = + readNonEmptyString(latestSummary?.summary) ?? + readNonEmptyString(latestSummary?.result) ?? + readNonEmptyString(latestSummary?.message) ?? + readNonEmptyString(summarizeRunErrorForModel(latestRun.error, latestRun.terminalFailureCategory)); + + const handoffMarkdown = [ + "Paperclip session handoff:", + `- Previous session: ${sessionId}`, + issueId ? `- Issue: ${issueId}` : "", + `- Rotation reason: ${reason}`, + latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "", + input.continuationSummaryBody + ? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}` + : "", + "Continue from the current task state. Rebuild only the minimum context you need.", + ] + .filter(Boolean) + .join("\n"); + + return { + rotate: true, + reason, + handoffMarkdown, + previousRunId: latestRun.id, + }; + } + + async function resolveSessionBeforeForWakeup( + agent: typeof agents.$inferSelect, + taskKey: string | null, + ) { + if (taskKey) { + const codec = getAdapterSessionCodec(agent.adapterType); + const existingTaskSession = await getTaskSession( + agent.companyId, + agent.id, + agent.adapterType, + taskKey, + ); + const parsedParams = normalizeSessionParams( + codec.deserialize(existingTaskSession?.sessionParamsJson ?? null), + ); + return truncateDisplayId( + existingTaskSession?.sessionDisplayId ?? + (codec.getDisplayId ? codec.getDisplayId(parsedParams) : null) ?? + readNonEmptyString(parsedParams?.sessionId), + ); + } + + const runtimeForRun = await getRuntimeState(agent.id); + return runtimeForRun?.sessionId ?? null; + } + + async function hasResolvableSessionWorkspaceCwd( + sessionParams: Record | null | undefined, + ) { + const cwd = readNonEmptyString(sessionParams?.cwd); + if (!cwd || isUnsafeSessionWorkspaceCwd(cwd)) return false; + return fs + .stat(cwd) + .then((stats) => stats.isDirectory()) + .catch(() => false); + } + + async function hasResolvablePriorSessionWorkspaceForWake(input: { + agent: typeof agents.$inferSelect; + contextSnapshot: Record; + taskKey: string | null; + explicitResumeSession: Awaited< + ReturnType + > | null; + }) { + if ( + await hasResolvableSessionWorkspaceCwd( + input.explicitResumeSession?.sessionParams, + ) + ) + return true; + if (shouldResetTaskSessionForWake(input.contextSnapshot)) return false; + if (!input.taskKey) return false; + + const codec = getAdapterSessionCodec(input.agent.adapterType); + const taskSession = await getTaskSession( + input.agent.companyId, + input.agent.id, + input.agent.adapterType, + input.taskKey, + ); + const taskSessionParams = normalizeResumeParamsForAdapter( + input.agent.adapterType, + codec.deserialize(taskSession?.sessionParamsJson ?? null), + ); + return hasResolvableSessionWorkspaceCwd(taskSessionParams); + } + + async function resolveExplicitResumeSessionOverride( + agent: typeof agents.$inferSelect, + payload: Record | null, + taskKey: string | null, + ) { + const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId); + if (!resumeFromRunId) return null; + + const resumeRun = await db + .select({ + id: heartbeatRuns.id, + contextSnapshot: heartbeatRuns.contextSnapshot, + resultJson: heartbeatRuns.resultJson, + sessionIdBefore: heartbeatRuns.sessionIdBefore, + sessionIdAfter: heartbeatRuns.sessionIdAfter, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, resumeFromRunId), + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agent.id), + ), + ) + .then((rows) => rows[0] ?? null); + if (!resumeRun) return null; + + const resumeContext = parseObject(resumeRun.contextSnapshot); + const resumeTaskKey = deriveTaskKey(resumeContext, null) ?? taskKey; + const resumeTaskSession = resumeTaskKey + ? await getTaskSession( + agent.companyId, + agent.id, + agent.adapterType, + resumeTaskKey, + ) + : null; + const sessionCodec = getAdapterSessionCodec(agent.adapterType); + const resumeRunResult = parseObject(resumeRun.resultJson); + const resumeRunSessionId = requiresCanonicalSessionIds(agent.adapterType) + ? (readNonEmptyString(resumeRunResult.sessionId) ?? + readNonEmptyString(resumeRunResult.session_id)) + : null; + const sessionOverride = buildExplicitResumeSessionOverride({ + adapterType: agent.adapterType, + resumeFromRunId, + resumeRunSessionIdBefore: resumeRun.sessionIdBefore, + resumeRunSessionIdAfter: resumeRun.sessionIdAfter, + resumeRunSessionParams: resumeRunSessionId + ? { sessionId: resumeRunSessionId } + : null, + taskSession: resumeTaskSession, + sessionCodec, + }); + if (!sessionOverride) return null; + + return { + resumeFromRunId, + taskKey: resumeTaskKey, + issueId: readNonEmptyString(resumeContext.issueId), + taskId: + readNonEmptyString(resumeContext.taskId) ?? + readNonEmptyString(resumeContext.issueId), + sessionDisplayId: sessionOverride.sessionDisplayId, + sessionParams: sessionOverride.sessionParams, + }; + } + + async function upsertTaskSession(input: { + companyId: string; + agentId: string; + adapterType: string; + taskKey: string; + sessionParamsJson: Record | null; + sessionDisplayId: string | null; + lastRunId: string | null; + lastError: string | null; + }) { + return db.transaction(async (tx) => { + const [issue] = await tx.select().from(issues).where(and(sql`${issues.id}::text = ${input.taskKey}`, eq(issues.companyId, input.companyId))).for("update"); + if (isConversation(issue)) { + const [run] = input.lastRunId ? await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.lastRunId)) : []; + if (run?.status === "cancelled" || run?.contextSnapshot?.conversationSessionGeneration !== issue.conversationSessionGeneration) return null; + } + const existing = await tx.select().from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, input.companyId), eq(agentTaskSessions.agentId, input.agentId), eq(agentTaskSessions.adapterType, input.adapterType), eq(agentTaskSessions.taskKey, input.taskKey))).then((rows) => rows[0] ?? null); + if (existing) { + return tx + .update(agentTaskSessions) + .set({ + sessionParamsJson: input.sessionParamsJson, + sessionDisplayId: input.sessionDisplayId, + lastRunId: input.lastRunId, + lastError: input.lastError, + updatedAt: new Date(), + }) + .where(eq(agentTaskSessions.id, existing.id)) + .returning() + .then((rows) => rows[0] ?? null); + } + + return tx + .insert(agentTaskSessions) + .values({ + companyId: input.companyId, + agentId: input.agentId, + adapterType: input.adapterType, + taskKey: input.taskKey, + sessionParamsJson: input.sessionParamsJson, + sessionDisplayId: input.sessionDisplayId, + lastRunId: input.lastRunId, + lastError: input.lastError, + }) + .returning() + .then((rows) => rows[0] ?? null); + }); + } + + async function clearTaskSessions( + companyId: string, + agentId: string, + opts?: { + taskKey?: string | null; + adapterType?: string | null; + expectedRunId?: string; + includeIssueAliases?: boolean; + }, + ) { + const conditions = [ + eq(agentTaskSessions.companyId, companyId), + eq(agentTaskSessions.agentId, agentId), + ]; + if (opts?.taskKey) { + const exactTaskKey = eq(agentTaskSessions.taskKey, opts.taskKey); + if (opts.includeIssueAliases) { + const selectedIssue = isUuidLike(opts.taskKey) + ? eq(issues.id, opts.taskKey) + : eq(issues.identifier, opts.taskKey.toUpperCase()); + // Operator task resets accept the UUID sent by run detail and the + // identifier used by some saved sessions. Resolve only from the current + // same-company issue row, in this DELETE's snapshot; arbitrary custom + // keys retain exact-match behavior and run/model context grants no alias. + conditions.push( + or( + exactTaskKey, + sql`exists ( + select 1 from ${issues} + where ${issues.companyId} = ${companyId} + and ${selectedIssue} + and (${agentTaskSessions.taskKey} = ${issues.id}::text + or ${agentTaskSessions.taskKey} = ${issues.identifier}) + )`, + )!, + ); + } else { + conditions.push(exactTaskKey); + } + } + if (opts?.adapterType) { + conditions.push(eq(agentTaskSessions.adapterType, opts.adapterType)); + } + + return db.transaction(async (tx) => { + if (opts?.taskKey && opts.expectedRunId) { + const [issue] = await tx.select().from(issues).where(sql`${issues.id}::text = ${opts.taskKey}`).for("update"); + if (isConversation(issue)) { + const [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, opts.expectedRunId)); + if (run?.status === "cancelled" || run?.contextSnapshot?.conversationSessionGeneration !== issue.conversationSessionGeneration) return 0; + } + } + return tx.delete(agentTaskSessions).where(and(...conditions)).returning().then((rows) => rows.length); + }); + } + + async function ensureRuntimeState(agent: typeof agents.$inferSelect) { + const existing = await getRuntimeState(agent.id); + if (existing) return existing; + + const inserted = await db + .insert(agentRuntimeState) + .values({ + agentId: agent.id, + companyId: agent.companyId, + adapterType: agent.adapterType, + stateJson: {}, + }) + .onConflictDoNothing({ + target: agentRuntimeState.agentId, + }) + .returning() + .then((rows) => rows[0] ?? null); + if (inserted) return inserted; + + const ensured = await getRuntimeState(agent.id); + if (!ensured) { + throw new Error(`Failed to ensure runtime state for agent ${agent.id}`); + } + return ensured; + } + + return { + getAgent, + resolveSessionBeforeForWakeup, + getRun, + ensureRuntimeState, + getTaskSession, + getLatestAgentConfigRevision, + evaluateSessionCompaction, + upsertTaskSession, + resolveNormalizedUsageForSession, + clearTaskSessions, + resolveExplicitResumeSessionOverride, + hasResolvablePriorSessionWorkspaceForWake, + getRunLogAccess, + + listRuns: async ( + companyId: string, + agentId?: string, + limit?: number, + options: { summary?: boolean } = {}, + ) => { + const safeForLegacyEncoding = await hasUnsafeTextProjectionDatabase(); + const summary = options.summary === true; + const rows = await retryIdempotentDatabaseOperation(async () => { + const query = db + .select( + summary + ? { + ...heartbeatRunSummaryListColumns, + ...heartbeatRunListContextColumns, + } + : safeForLegacyEncoding + ? { + ...heartbeatRunListColumns, + error: sql`NULL`.as("error"), + ...heartbeatRunListContextColumns, + } + : { + ...heartbeatRunListColumns, + ...heartbeatRunListContextColumns, + ...heartbeatRunListResultColumns, + }, + ) + .from(heartbeatRuns) + .where( + agentId + ? and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, agentId), + ) + : eq(heartbeatRuns.companyId, companyId), + ) + .orderBy(desc(heartbeatRuns.createdAt)); + + return limit ? await query.limit(limit) : await query; + }); + return rows.map((row) => { + const { + contextIssueId, + contextTaskId, + contextTaskKey, + contextCommentId, + contextWakeCommentId, + contextWakeReason, + contextWakeSource, + contextWakeTriggerDetail, + resultSummary, + resultResult, + resultMessage, + resultError, + resultTotalCostUsd, + resultCostUsd, + resultCostUsdCamel, + ...rest + } = row as typeof row & { + resultSummary?: string | null; + resultResult?: string | null; + resultMessage?: string | null; + resultError?: string | null; + resultTotalCostUsd?: string | null; + resultCostUsd?: string | null; + resultCostUsdCamel?: string | null; + }; + + return { + ...rest, + contextSnapshot: summarizeHeartbeatRunContextSnapshot({ + issueId: contextIssueId, + taskId: contextTaskId, + taskKey: contextTaskKey, + commentId: contextCommentId, + wakeCommentId: contextWakeCommentId, + wakeReason: contextWakeReason, + wakeSource: contextWakeSource, + wakeTriggerDetail: contextWakeTriggerDetail, + }), + resultJson: + safeForLegacyEncoding || summary + ? null + : summarizeHeartbeatRunListResultJson({ + summary: resultSummary, + result: resultResult, + message: resultMessage, + error: resultError, + totalCostUsd: resultTotalCostUsd, + costUsd: resultCostUsd, + costUsdCamel: resultCostUsdCamel, + }), + }; + }); + }, + + getRuntimeStateWithSessions: async (agentId: string) => { + const state = await getRuntimeState(agentId); + const agent = await getAgent(agentId); + if (!agent) return null; + const ensured = state ?? (await ensureRuntimeState(agent)); + const latestTaskSession = await db + .select() + .from(agentTaskSessions) + .where( + and( + eq(agentTaskSessions.companyId, agent.companyId), + eq(agentTaskSessions.agentId, agent.id), + ), + ) + .orderBy(desc(agentTaskSessions.updatedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + return { + ...ensured, + sessionDisplayId: + latestTaskSession?.sessionDisplayId ?? ensured.sessionId, + sessionParamsJson: latestTaskSession?.sessionParamsJson ?? null, + }; + }, + + listTaskSessions: async (agentId: string) => { + const agent = await getAgent(agentId); + if (!agent) throw notFound("Agent not found"); + + return db + .select() + .from(agentTaskSessions) + .where( + and( + eq(agentTaskSessions.companyId, agent.companyId), + eq(agentTaskSessions.agentId, agentId), + ), + ) + .orderBy( + desc(agentTaskSessions.updatedAt), + desc(agentTaskSessions.createdAt), + ); + }, + + resetRuntimeSession: async ( + agentId: string, + opts?: { taskKey?: string | null }, + ) => { + const agent = await getAgent(agentId); + if (!agent) throw notFound("Agent not found"); + await ensureRuntimeState(agent); + const taskKey = readNonEmptyString(opts?.taskKey); + const clearedTaskSessions = await clearTaskSessions( + agent.companyId, + agent.id, + taskKey + ? { + taskKey, + adapterType: agent.adapterType, + includeIssueAliases: true, + } + : undefined, + ); + const runtimePatch: Partial = { + sessionId: null, + lastError: null, + updatedAt: new Date(), + }; + if (!taskKey) { + runtimePatch.stateJson = {}; + } + + const updated = await db + .update(agentRuntimeState) + .set(runtimePatch) + .where(eq(agentRuntimeState.agentId, agentId)) + .returning() + .then((rows) => rows[0] ?? null); + + if (!updated) return null; + return { + ...updated, + sessionDisplayId: null, + sessionParamsJson: null, + clearedTaskSessions, + }; + }, + + listEvents: (runId: string, afterSeq = 0, limit = 200) => + db + .select() + .from(heartbeatRunEvents) + .where( + and( + eq(heartbeatRunEvents.runId, runId), + gt(heartbeatRunEvents.seq, afterSeq), + ), + ) + .orderBy(asc(heartbeatRunEvents.seq)) + .limit(Math.max(1, Math.min(limit, 1000))), + + getRetryExhaustedReason: async (runId: string) => { + const row = await db + .select({ + message: heartbeatRunEvents.message, + }) + .from(heartbeatRunEvents) + .where( + and( + eq(heartbeatRunEvents.runId, runId), + eq(heartbeatRunEvents.eventType, "lifecycle"), + sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, + ), + ) + .orderBy(desc(heartbeatRunEvents.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + return row?.message ?? null; + }, + + getRunIssueSummary: async (runId: string) => { + const [run] = await db + .select(heartbeatRunIssueSummaryColumns) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .limit(1); + return run ?? null; + }, + + getActiveRunForAgent: async (agentId: string) => { + const [run] = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "running"), + ), + ) + .orderBy(desc(heartbeatRuns.startedAt)) + .limit(1); + return run ?? null; + }, + + getActiveRunIssueSummaryForAgent: async (agentId: string) => { + const [run] = await db + .select(heartbeatRunIssueSummaryColumns) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "running"), + ), + ) + .orderBy(desc(heartbeatRuns.startedAt)) + .limit(1); + return run ?? null; + }, + }; +} From 3d1d5294d994c2c787cabb28b48c6c8de40a7490 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 8 Oct 2026 12:57:32 -0700 Subject: [PATCH 05/10] Drain idle plugin workers before automatic sleep (#15599) ## Thinking Path > - Paperclip runs autonomous work through agents, schedules, and plugins. > - Automatic idle sleep must preserve accepted work and unfinished cleanup. > - Every enabled plugin currently blocks sleep, including unused providers. > - An enabled flag or package version cannot prove that a worker is idle. > - This change adds a live worker drain under the existing owned hold. > - Idle workers can allow sleep while unknown work stays protected. ## Linked Issues or Issue Description Refs #15522. Related: #15391 adds plugin readiness for agent admission; this change concerns instance sleep and does not replace that contract. **Subsystem affected** Plugin worker lifecycle and automatic idle sleep. **Problem or motivation** A workspace with no pending work cannot sleep when any plugin is enabled. Removing that check alone would lose accepted RPCs, background tasks, or cleanup after a caller timeout. **Proposed solution** Require a live `onIdleDrain` handshake from each worker. Close admission in both processes for the exact owner and expiry. Count accepted work until completion. Continue checking durable work separately. ## What Changed - Add bounded worker holds, exact-owner release, automatic expiry, and an abort signal for plugin-owned background work. - Count host and worker requests through their real completion receipts. Keep timed-out work counted. Check active notifications and terminal routes. - Accept enabled plugins only when their current workers provide matching runtime receipts. Missing workers, old SDKs, crashes, invalid replies, and unknown cleanup still prevent sleep. - Let unused Daytona workers opt in. Once a worker contacts the provider, it remains a blocker for that process lifetime. This restriction avoids treating its existing timeout and terminal-close behavior as a cleanup receipt. - Document the plugin author contract. No schema or user-facing API is added. ## Verification - All hosted CI checks passed on 6ffb682894, including all server, runner, workspace, build, typecheck, and end-to-end checks. One unrelated OpenCode transport test hit a five-second timeout and passed on rerun. - `pnpm -r typecheck` passed. - `pnpm build` passed. - Focused Vitest: 293 tests passed across worker admission, real child-process RPC, durable idle checks, and Daytona. - Adjacent worker manager and duplex tests: 119 passed. - Real child-process tests cover late worker completion, a host write after caller timeout, stale owners, closed admission, and release. - SDK tests cover hook failures, busy workers, expiry without another request, and late acknowledgements after expiry. - Initial long local run: 17,202 passed, 10 failed. Five failures involved parent-directory skill fixtures; four of those were already reproduced and passed in an isolated worktree. One HTTP test had a socket hang-up; all 66 tests in that file passed on rerun. The other four used new tests with runtime modules loaded before the follow-up edits; all pass in the fresh focused run above. A clean build and full-suite rerun at the final commit are running in an isolated worktree. No full-suite pass is claimed. - Follow-up: 44 focused tests passed, including repeated owned holds without an intervening normal request. Worker status changes now invalidate an in-progress scan. - Apex follow-up: 13 idle tests, 119 adjacent worker tests, 71 instance-settings route tests, and server typecheck passed. A real queued notification completes its write while preparation is in flight; synchronous and async session callback failures are logged. - Diff whitespace and added-line secret/private-reference scans passed. ## Risks - Plugins are trusted code. A hook must account for work outside SDK RPCs and keep it quiescent until its signal aborts. There is no package-name or manifest-only exemption. - This first change targets unused workers. Used Daytona workers remain awake until remote cleanup receipts are complete. Restarting does not bypass durable lease and recovery checks. - Unknown completion remains a blocker, including after a worker crash. This can retain cost but cannot authorize sleep from a timeout alone. - Enabled jobs, schedules, credentials, integrations, and other durable work remain blockers. Durable wake ownership is separate follow-up work. - Rollback restores the blanket plugin blocker. No data migration is required. ## Model Used OpenAI Codex, GPT-6. Used repository inspection, reasoning, code editing, and command execution. The runtime did not report an exact model revision or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either linked existing public issues or described the issue in this PR - [x] I have not referenced internal or instance-local issues or links - [x] My branch name describes the change and contains no internal identifiers - [x] Focused local tests pass; full CI is green (long local rerun status is recorded above) - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation - [x] I have considered and documented risks - [x] All Paperclip CI gates are green - [x] Greptile Apex is 5/5 on 6ffb682894 with no open findings - [x] I will address all review comments before requesting merge --------- Co-authored-by: Paperclip --- doc/plugins/PLUGIN_AUTHORING_GUIDE.md | 32 +++++ .../daytona/src/plugin.test.ts | 15 +++ .../sandbox-providers/daytona/src/plugin.ts | 11 ++ packages/plugins/sdk/src/define-plugin.ts | 11 ++ packages/plugins/sdk/src/idle-drain.test.ts | 55 ++++++++ packages/plugins/sdk/src/idle-drain.ts | 67 ++++++++++ packages/plugins/sdk/src/protocol.ts | 4 + packages/plugins/sdk/src/worker-rpc-host.ts | 51 ++++++-- .../__tests__/fixtures/plugin-worker-idle.cjs | 27 ++++ .../src/__tests__/idle-sleep-safety.test.ts | 16 +++ .../instance-settings-routes.test.ts | 33 ++++- .../src/__tests__/plugin-idle-manager.test.ts | 98 +++++++++++++++ server/src/__tests__/plugin-idle-rpc.test.ts | 89 +++++++++++++ server/src/app.ts | 2 +- server/src/routes/instance-settings.ts | 7 +- server/src/services/idle-sleep-safety.ts | 21 +++- server/src/services/plugin-worker-manager.ts | 117 +++++++++++++++++- 17 files changed, 636 insertions(+), 20 deletions(-) create mode 100644 packages/plugins/sdk/src/idle-drain.test.ts create mode 100644 packages/plugins/sdk/src/idle-drain.ts create mode 100644 server/src/__tests__/fixtures/plugin-worker-idle.cjs create mode 100644 server/src/__tests__/plugin-idle-manager.test.ts create mode 100644 server/src/__tests__/plugin-idle-rpc.test.ts diff --git a/doc/plugins/PLUGIN_AUTHORING_GUIDE.md b/doc/plugins/PLUGIN_AUTHORING_GUIDE.md index a3728720e1c..4030c0104d9 100644 --- a/doc/plugins/PLUGIN_AUTHORING_GUIDE.md +++ b/doc/plugins/PLUGIN_AUTHORING_GUIDE.md @@ -25,6 +25,38 @@ It is intentionally narrower than [PLUGIN_SPEC.md](./PLUGIN_SPEC.md). The spec i building custom versions. - `ctx.assets` is not supported in the current runtime. +## Idle sleep + +An enabled worker blocks automatic idle sleep unless it implements +`onIdleDrain(signal): Promise<"none" | "present" | "unknown">`. This is a live +runtime handshake, not a manifest or version exemption. Older SDKs remain +blockers. Manual shutdown keeps its existing behavior. + +The host checks the exact owned, expiring task-drain hold. It closes new worker +admission and checks outstanding RPCs in both directions and live terminal +routes. The SDK also counts accepted handlers and notifications until they +actually finish. Caller timeouts do not count as completion. An unconfirmed +write, missing worker, crash, invalid reply, or expired hold prevents sleep. + +The hook must return `none` only when plugin-owned timers, sockets, detached +operations, and cleanup are settled and cannot start work until `signal` +aborts. Do not cancel useful work to make a worker idle. Return `present` while +work remains and `unknown` when completion cannot be proved. The SDK closes +ordinary RPC admission while the hook runs and until the matching hold releases +or expires. Release aborts the signal; expiry does so without another request. +Hooks that suspend autonomous work must arrange its safe resumption on abort. + +The handshake does not exempt database work. Enabled jobs, deliveries, +schedules, external ingress, and other persisted work still block sleep until +they have their own safe wake contract. An installed plugin without a matching +live worker receipt also blocks sleep. + +Daytona currently opts in only before its worker has contacted the provider. +After provider access it remains a blocker for that process lifetime, even if +the visible operation finishes. Its timeout and terminal cleanup paths need +stronger completion receipts before that restriction can be relaxed. Restarting +does not bypass the separate persisted lease and recovery checks. + ## Durable resource lifecycle inbox Plugins with `events.subscribe` can read durable, company-scoped resource hooks diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index ac3a7c1dd0a..6022d9b3fe7 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -40,6 +40,21 @@ import { environmentCreationCleanupErrorData, readEnvironmentCreationCleanupErro import manifest from "./manifest.js"; import { parseTarVerboseListingLine, splitLinkEntryOnce } from "./file-sync.js"; +describe("Daytona idle eligibility", () => { + it("permits a fresh unused provider but remains conservative after provider access", async () => { + __resetDaytonaSandboxHandleCacheForTest(); + const signal = new AbortController().signal; + expect(await plugin.definition.onIdleDrain!(signal)).toBe("none"); + mockGet.mockRejectedValueOnce(new Error("provider unreachable")); + await plugin.definition.onEnvironmentProbe!({ + driverKey: "daytona", companyId: "fixture-company", environmentId: "fixture-environment", + config: { apiKey: "fixture-not-a-key" }, + }).catch(() => undefined); + expect(await plugin.definition.onIdleDrain!(signal)).toBe("present"); + __resetDaytonaSandboxHandleCacheForTest(); + }); +}); + function createMockSandbox(overrides: { id?: string; name?: string; diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 4dfa65c9d59..9c085d6a696 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -107,6 +107,10 @@ let timingNow: () => number = () => Date.now(); // read the tracer through `getPluginTracer()`. Before `setup` runs (or in a // test) the tracer is a no-op, so a span never throws. let pluginContext: PluginContext | null = null; +// Provider deadlines and terminal close failures do not prove remote work +// stopped. Until those paths provide complete cleanup receipts, only a worker +// which has never contacted the provider can opt into automatic idle sleep. +let providerUsed = false; /** * Return the plugin tracer. It is the injected `ctx.tracer` after `setup`, or a @@ -313,6 +317,7 @@ function resolveApiKey(config: DaytonaDriverConfig): string { } function createDaytonaClient(config: DaytonaDriverConfig): Daytona { + providerUsed = true; const clientConfig: DaytonaConfig = { apiKey: resolveApiKey(config), }; @@ -1570,6 +1575,7 @@ const sandboxHandleSessionStore = (() => { * Not used in production. */ export function __resetDaytonaSandboxHandleCacheForTest(): void { + providerUsed = false; sandboxHandleCache.reset(); sandboxHandleTeardownGates.reset(); sandboxHandleActivityGates.reset(); @@ -2281,6 +2287,11 @@ const plugin = definePlugin({ return { status: "ok", message: "Daytona sandbox provider plugin healthy" }; }, + async onIdleDrain() { + return providerUsed || daytonaLoginPtyByRoute.size > 0 || daytonaDuplexChannelByRoute.size > 0 + ? "present" : "none"; + }, + async onEnvironmentValidateConfig( params: PluginEnvironmentValidateConfigParams, ): Promise { diff --git a/packages/plugins/sdk/src/define-plugin.ts b/packages/plugins/sdk/src/define-plugin.ts index 947a67cbb55..12b763957c8 100644 --- a/packages/plugins/sdk/src/define-plugin.ts +++ b/packages/plugins/sdk/src/define-plugin.ts @@ -293,6 +293,17 @@ export interface PluginDefinition { */ onShutdown?(): Promise; + /** Prove plugin-owned background work is quiescent for idle sleep. + * The SDK first closes admission and checks accepted RPCs and notifications. + * Return `none` only when timers, sockets, detached operations and cleanup + * are settled and cannot start work until signal aborts. Do not cancel useful + * work to satisfy this check. Missing hooks, failures and uncertainty block + * sleep. The signal aborts on exact-owner release or bounded hold expiry. + * Plugins with autonomous work must keep returning `present` unless they + * can safely suspend and resume that work under this contract. + */ + onIdleDrain?(signal: AbortSignal): Promise<"none" | "present" | "unknown">; + /** * Called to validate the current plugin configuration. * diff --git a/packages/plugins/sdk/src/idle-drain.test.ts b/packages/plugins/sdk/src/idle-drain.test.ts new file mode 100644 index 00000000000..cc59f7881e7 --- /dev/null +++ b/packages/plugins/sdk/src/idle-drain.test.ts @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createPluginIdleDrain } from "./idle-drain.js"; + +describe("plugin idle admission", () => { + afterEach(() => vi.useRealTimers()); + const hold = () => ({ ownerId: "idle-owner", expiresAt: Date.now() + 10_000 }); + + it("keeps accepted work counted until actual settlement", async () => { + const gate = createPluginIdleDrain(); + const done = gate.begin(); + const inspect = vi.fn(async () => "none" as const); + expect(await gate.prepare(hold(), inspect)).toBe("present"); + expect(inspect).not.toHaveBeenCalled(); + done(); done(); + expect(await gate.prepare(hold(), inspect)).toBe("none"); + expect(() => gate.begin()).toThrow("held"); + gate.close(); + }); + + it("rejects legacy hooks, invalid deadlines, busy replies and hook errors", async () => { + const gate = createPluginIdleDrain(); + expect(await gate.prepare(hold())).toBe("present"); + expect(await gate.prepare({ ...hold(), expiresAt: Infinity }, async () => "none")).toBe("unknown"); + expect(await gate.prepare(hold(), async () => "present")).toBe("present"); + expect(await gate.prepare(hold(), async () => { throw new Error("private detail"); })).toBe("unknown"); + gate.begin()(); + }); + + it("releases only the matching owner and resumes on expiry without another request", async () => { + vi.useFakeTimers(); + const gate = createPluginIdleDrain(); + const lease = hold(); + let signal!: AbortSignal; + expect(await gate.prepare(lease, async (value) => { signal = value; return "none"; })).toBe("none"); + gate.release("stale-owner"); + expect(signal.aborted).toBe(false); + expect(await gate.prepare({ ...lease, ownerId: "other-owner" }, async () => "none")).toBe("unknown"); + await vi.advanceTimersByTimeAsync(10_000); + expect(signal.aborted).toBe(true); + gate.begin()(); + }); + + it("holds admission while the hook runs and rejects a late acknowledgement", async () => { + vi.useFakeTimers(); + const gate = createPluginIdleDrain(); + let resolve!: (state: "none") => void; + const pending = gate.prepare(hold(), () => new Promise((done) => { resolve = done; })); + expect(() => gate.begin()).toThrow("held"); + await vi.advanceTimersByTimeAsync(10_000); + expect(await gate.prepare(hold(), async () => "none")).toBe("present"); + resolve("none"); + expect(await pending).toBe("unknown"); + gate.begin()(); + }); +}); diff --git a/packages/plugins/sdk/src/idle-drain.ts b/packages/plugins/sdk/src/idle-drain.ts new file mode 100644 index 00000000000..ae6e4c6be46 --- /dev/null +++ b/packages/plugins/sdk/src/idle-drain.ts @@ -0,0 +1,67 @@ +export type PluginIdleHold = { ownerId: string; expiresAt: number }; +export type PluginIdleWork = "none" | "present" | "unknown"; + +/** Worker admission stays closed for the exact bounded hold. A caller timeout + * never releases a token: only the actual operation's settlement does. */ +export function createPluginIdleDrain() { + let active = 0; + let preparing = 0; + let hold: (PluginIdleHold & { controller: AbortController; ready: boolean }) | null = null; + let timer: ReturnType | undefined; + + function release(ownerId: string): void { + if (hold?.ownerId !== ownerId) return; + const prior = hold; + hold = null; + clearTimeout(timer); + prior.controller.abort(); + } + + function current() { + if (hold && hold.expiresAt <= Date.now()) release(hold.ownerId); + return hold; + } + + function begin(): () => void { + if (current()) throw new Error("Plugin is held for idle sleep"); + active++; + let done = false; + return () => { if (!done) { done = true; active--; } }; + } + + async function prepare( + requested: PluginIdleHold, + inspect?: (signal: AbortSignal) => Promise, + ): Promise { + if (!requested || typeof requested.ownerId !== "string" || !requested.ownerId || + !Number.isFinite(requested.expiresAt) || requested.expiresAt <= Date.now() || + requested.expiresAt > Date.now() + 300_000) return "unknown"; + if (!inspect) return "present"; + const prior = current(); + if (prior) return prior.ownerId === requested.ownerId && prior.expiresAt === requested.expiresAt && prior.ready + ? "none" : "unknown"; + if (active !== 0 || preparing !== 0) return "present"; + const candidate = { ...requested, controller: new AbortController(), ready: false }; + hold = candidate; + timer = setTimeout(() => release(candidate.ownerId), requested.expiresAt - Date.now()); + timer.unref?.(); + preparing++; + try { + const result = await inspect(candidate.controller.signal); + if (current() !== candidate || active !== 0) return "unknown"; + if (result !== "none") { + release(candidate.ownerId); + return result === "present" ? "present" : "unknown"; + } + candidate.ready = true; + return "none"; + } catch { + if (hold === candidate) release(candidate.ownerId); + return "unknown"; + } finally { + preparing--; + } + } + + return { begin, prepare, release, close: () => { if (hold) release(hold.ownerId); } }; +} diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index 76f0cd42284..7b2af372750 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -1330,6 +1330,8 @@ export interface HostToWorkerMethods { health: [params: Record, result: PluginHealthDiagnostics]; /** @see PLUGIN_SPEC.md §12.5 */ shutdown: [params: Record, result: void]; + prepareIdleSleep: [params: { ownerId: string; expiresAt: number }, result: { ownerId: string; expiresAt: number; backgroundWork: "none" | "present" | "unknown" }]; + releaseIdleSleep: [params: { ownerId: string }, result: void]; /** @see PLUGIN_SPEC.md §13.3 */ validateConfig: [params: ValidateConfigParams, result: PluginConfigValidationResult]; /** @see PLUGIN_SPEC.md §13.4 */ @@ -1471,6 +1473,8 @@ export const HOST_TO_WORKER_REQUIRED_METHODS: readonly HostToWorkerMethodName[] /** Optional methods the worker MAY implement. */ export const HOST_TO_WORKER_OPTIONAL_METHODS: readonly HostToWorkerMethodName[] = [ + "prepareIdleSleep", + "releaseIdleSleep", "validateConfig", "configChanged", "onEvent", diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index 786e71954fb..d6781c1b6db 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -1,6 +1,7 @@ import type { AiConnectionRouterRequest, AiConnectionRouterResult } from "@paperclipai/shared"; import { environmentCreationCleanupErrorData } from "./environment-creation-cleanup.js"; import { environmentSyncErrorData } from "./environment-sync-error.js"; +import { createPluginIdleDrain } from "./idle-drain.js"; /** * Worker-side RPC host — runs inside the child process spawned by the host. * @@ -335,6 +336,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost let running = true; let initialized = false; + const idleDrain = createPluginIdleDrain(); + const unsettledHostCalls = new Map void>(); let manifest: PaperclipPluginManifestV1 | null = null; let currentConfig: Record = {}; // The company whose config was last applied via configChanged. Used to fail @@ -397,6 +400,9 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost nextOutboundId = 1; } const id = nextOutboundId++; + // Keep the token after an RPC timeout. Only an actual host response + // proves that a host-side write has finished. + unsettledHostCalls.set(id, idleDrain.begin()); const timeout = timeoutMs ?? rpcTimeoutMs; let settled = false; @@ -439,6 +445,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost }; sendMessage(request); } catch (err) { + unsettledHostCalls.get(id)?.(); + unsettledHostCalls.delete(id); settle(reject, err instanceof Error ? err : new Error(String(err))); } }); @@ -1563,8 +1571,11 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost */ async function handleHostRequest(request: JsonRpcRequest): Promise { const { id, method, params } = request; - + let done: (() => void) | undefined; try { + if (method !== "prepareIdleSleep" && method !== "releaseIdleSleep" && method !== "shutdown") { + done = idleDrain.begin(); + } const invoke = () => dispatchMethod(method, params); const result = request.paperclipInvocation ? await invocationContextStorage.run(request.paperclipInvocation, invoke) @@ -1584,6 +1595,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost method === "environmentAcquireLease" || method === "environmentDestroyLease" ? environmentCreationCleanupErrorData(err) : method === "environmentSyncOut" ? environmentSyncErrorData(err) : undefined)); + } finally { + done?.(); } } @@ -1592,6 +1605,16 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost */ async function dispatchMethod(method: string, params: unknown): Promise { switch (method) { + case "prepareIdleSleep": { + const hold = params as { ownerId: string; expiresAt: number }; + const backgroundWork = !initialized || sessionEventCallbacks.size > 0 + ? "present" + : await idleDrain.prepare(hold, plugin.definition.onIdleDrain?.bind(plugin.definition)); + return { ...hold, backgroundWork }; + } + case "releaseIdleSleep": + idleDrain.release((params as { ownerId: string }).ownerId); + return; case "initialize": return handleInitialize(params as InitializeParams); @@ -1746,6 +1769,7 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost if (plugin.definition.onConfigChanged) supportedMethods.push("configChanged"); if (plugin.definition.onHealth) supportedMethods.push("health"); if (plugin.definition.onShutdown) supportedMethods.push("shutdown"); + if (plugin.definition.onIdleDrain) supportedMethods.push("prepareIdleSleep", "releaseIdleSleep"); if (plugin.definition.onApiRequest) supportedMethods.push("handleApiRequest"); if (plugin.definition.onDetectExternalObjects) supportedMethods.push("detectExternalObjects"); if (plugin.definition.onRouteAiConnection) supportedMethods.push("routeAiConnection"); @@ -2228,6 +2252,9 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost const id = response.id; if (id === null || id === undefined) return; + unsettledHostCalls.get(id)?.(); + unsettledHostCalls.delete(id); + const pending = pendingRequests.get(id); if (!pending) return; @@ -2284,16 +2311,23 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost } else if (isJsonRpcNotification(message)) { // Dispatch host→worker push notifications const notif = message as JsonRpcNotification & { method: string; params?: unknown }; - const runNotification = (fn: () => void | Promise) => { - if (notif.paperclipInvocation) { - return invocationContextStorage.run(notif.paperclipInvocation, fn); - } - return fn(); + const runNotification = async (fn: () => void | Promise) => { + const done = idleDrain.begin(); + try { + await (notif.paperclipInvocation ? invocationContextStorage.run(notif.paperclipInvocation, fn) : fn()); + } finally { done(); } }; - if (notif.method === "agents.sessions.event" && notif.params) { + if (notif.method === "releaseIdleSleep" && notif.params) { + idleDrain.release((notif.params as { ownerId: string }).ownerId); + } else if (notif.method === "agents.sessions.event" && notif.params) { const event = notif.params as AgentSessionEvent; const cb = sessionEventCallbacks.get(event.sessionId); - if (cb) cb(event); + if (cb) void runNotification(() => cb(event)).catch((err) => { + notifyHost("log", { + level: "error", + message: `Failed to handle session event notification: ${err instanceof Error ? err.message : String(err)}`, + }); + }); } else if (notif.method === "onEvent" && notif.params) { // Plugin event bus notifications — dispatch to registered event handlers Promise.resolve(runNotification(() => handleOnEvent(notif.params as OnEventParams))).catch((err) => { @@ -2312,6 +2346,7 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost function cleanup(): void { running = false; + idleDrain.close(); // Close readline if (readline) { diff --git a/server/src/__tests__/fixtures/plugin-worker-idle.cjs b/server/src/__tests__/fixtures/plugin-worker-idle.cjs new file mode 100644 index 00000000000..4b989b447ea --- /dev/null +++ b/server/src/__tests__/fixtures/plugin-worker-idle.cjs @@ -0,0 +1,27 @@ +// Use the real SDK so host tests cross a process and the NDJSON boundary. +void Promise.all([ + import("../../../../packages/plugins/sdk/src/define-plugin.ts"), + import("../../../../packages/plugins/sdk/src/worker-rpc-host.ts"), +]).then(([{ definePlugin }, { startWorkerRpcHost }]) => { + let context; + startWorkerRpcHost({ plugin: definePlugin({ + async setup(ctx) { + context = ctx; + ctx.events.on("fixture.write", async () => { + await ctx.state.set({ scopeKind: "instance", stateKey: "event" }, { value: 1 }); + }); + }, + async onIdleDrain() { return "none"; }, + async onHealth() { + if (process.env.IDLE_TEST_WRITE === "1") { + await context.state.set({ scopeKind: "instance", stateKey: "fixture" }, { value: 1 }); + } + return { status: "ok" }; + }, + async onEnvironmentProbe(params) { + if (params.config.crash) process.exit(1); + await new Promise((resolve) => setTimeout(resolve, params.config.delayMs)); + return { ok: true, summary: "fixture complete" }; + }, + }) }); +}); diff --git a/server/src/__tests__/idle-sleep-safety.test.ts b/server/src/__tests__/idle-sleep-safety.test.ts index 1579247a494..832673aca58 100644 --- a/server/src/__tests__/idle-sleep-safety.test.ts +++ b/server/src/__tests__/idle-sleep-safety.test.ts @@ -245,6 +245,22 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp expect(await read()).toEqual(none); }); + it("accepts only exact worker drain receipts and still checks durable work", async () => { + const [{ id }] = await db.execute<{ id: string }>(sql`INSERT INTO plugins (plugin_key, package_name, version, manifest_json) + VALUES ('demo.idle', 'demo-idle', '1.0.0', '{}'::jsonb) RETURNING id`); + const inspect = vi.fn(async () => ({ backgroundWork: "none" as const, pluginIds: [id!] })); + const scan = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, inspect); + expect(await scan()).toEqual(none); + expect(inspect).toHaveBeenCalledWith({ ownerId, expiresAt: owned().expiresAt!.getTime() }); + await db.execute(sql`INSERT INTO plugins (plugin_key, package_name, version, manifest_json) + VALUES ('demo.unknown', 'demo-unknown', '1.0.0', '{}'::jsonb)`); + expect(await scan()).toEqual(present); + await db.execute(sql`UPDATE plugins SET status = 'disabled' WHERE plugin_key = 'demo.unknown'`); + const { companyId } = await seed(); + await db.insert(issues).values({ companyId, title: "Still pending", status: "todo" }); + expect(await scan()).toEqual(present); + }); + it("fails closed when the installed schema is older than the report", async () => { await db.execute(sql`ALTER TABLE heartbeat_runs RENAME COLUMN cost_accounting_pending TO hidden_pending`); try { expect(await read()).toEqual(unknown); } diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.ts index 2483cd4a3e1..ffee8171aaf 100644 --- a/server/src/__tests__/instance-settings-routes.test.ts +++ b/server/src/__tests__/instance-settings-routes.test.ts @@ -2,6 +2,7 @@ import express from "express"; import request from "supertest"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { hoistModuleGraph } from "./helpers/hoist-module-graph.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; const mockInstanceSettingsService = vi.hoisted(() => ({ get: vi.fn(), @@ -76,7 +77,7 @@ describe("instance settings routes", () => { return { errorHandler, instanceSettingsRoutes }; }); - function createApp(actor: any) { + function createApp(actor: any, pluginWorkers?: PluginWorkerManager) { const { errorHandler, instanceSettingsRoutes } = routeModules.value; const app = express(); app.use(express.json()); @@ -84,7 +85,7 @@ describe("instance settings routes", () => { req.actor = actor; next(); }); - app.use("/api", instanceSettingsRoutes(mockDb as any)); + app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers)); app.use(errorHandler); return app; } @@ -972,7 +973,7 @@ describe("instance settings routes", () => { await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200); expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 }); await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200); - expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId); + expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, undefined, undefined); }); it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => { @@ -999,10 +1000,34 @@ describe("instance settings routes", () => { const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1"); expect(res.status).toBe(200); expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report }); - expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined); + expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, undefined, undefined); expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus); }); + it("uses the live worker manager for inspection and releases only the owned hold", async () => { + const inspect = vi.fn(async () => ({ backgroundWork: "none", pluginIds: [] })); + const release = vi.fn(); + const workers = { + inspectIdleSleep: inspect, + releaseIdleSleep: release, + } as unknown as PluginWorkerManager; + const ownerId = "fixture-idle-owner"; + const lease = { ownerId, expiresAt: Date.now() + 60_000 }; + mockHeartbeatService.getTaskDrainStatus.mockReturnValue({ ...idleStatus, draining: true, ownerId }); + mockReadIdleSleepSafety.mockImplementation(async (_db, _status, _now, owner, _local, inspectPlugins) => { + expect(owner).toBe(ownerId); + return { version: 1, backgroundWork: (await inspectPlugins(lease)).backgroundWork }; + }); + const app = createApp(adminActor, workers); + await request(app).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200); + expect(inspect).toHaveBeenCalledWith(lease); + expect(inspect.mock.contexts[0]).toBe(workers); + await request(app).delete("/api/instance/task-drain?ownerId=stale-owner").expect(409); + expect(release).not.toHaveBeenCalled(); + await request(app).delete(`/api/instance/task-drain?ownerId=${ownerId}`).expect(200); + expect(release).toHaveBeenCalledOnce(); + }); + it.each([ ["company member", nonAdminActor], ["agent", { type: "agent", agentId: "agent-1", companyId: "company-1" }], diff --git a/server/src/__tests__/plugin-idle-manager.test.ts b/server/src/__tests__/plugin-idle-manager.test.ts new file mode 100644 index 00000000000..1ea93bc172b --- /dev/null +++ b/server/src/__tests__/plugin-idle-manager.test.ts @@ -0,0 +1,98 @@ +import { fileURLToPath } from "node:url"; +import { createRequire } from "node:module"; +import { setTimeout as delay } from "node:timers/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js"; +import { startTaskDrain, stopTaskDrain } from "../services/task-admission.js"; + +const manifest = { id: "test.idle", apiVersion: 1 as const, version: "1.0.0", displayName: "Idle fixture", + description: "Idle fixture", author: "Paperclip", categories: ["automation" as const], capabilities: [], entrypoints: { worker: "worker.cjs" } }; +function worker(write = false, hostHandler = async () => undefined) { + return createPluginWorkerHandle("test.idle", { + entrypointPath: fileURLToPath(new URL("./fixtures/plugin-worker-idle.cjs", import.meta.url)), + execArgv: ["--import", createRequire(import.meta.url).resolve("tsx")], + manifest, config: {}, instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" }, apiVersion: 1, + env: { IDLE_TEST_WRITE: write ? "1" : "0" }, + hostHandlers: { "state.set": hostHandler, "events.subscribe": async () => ({}) }, + }); +} +function hold() { + const lease = startTaskDrain({ purpose: "idle", ttlMs: 30_000 }); + return { ownerId: lease.ownerId!, expiresAt: lease.expiresAt!.getTime() }; +} + +describe("plugin manager idle receipts", () => { + afterEach(() => stopTaskDrain()); + it("retains timed-out worker operations and admits work after the owned hold releases", async () => { + const handle = worker(); + await handle.start(); + try { + await expect(handle.call("environmentProbe", { driverKey: "fixture", companyId: "fixture", environmentId: "fixture", config: { delayMs: 200 } }, 10)).rejects.toThrow("timed out"); + const lease = hold(); + expect(await handle.prepareIdleSleep!(lease)).toBe("present"); + await expect.poll(() => handle.prepareIdleSleep!(lease)).toBe("none"); + await expect(handle.call("health", {})).rejects.toThrow("held"); + expect(await handle.prepareIdleSleep!({ ...lease, ownerId: "stale" })).toBe("unknown"); + stopTaskDrain(); handle.releaseIdleSleep!(); + expect(await handle.call("health", {})).toMatchObject({ status: "ok" }); + expect(await handle.prepareIdleSleep!(hold())).toBe("none"); + stopTaskDrain(); + // No intervening request or explicit worker release: the next controller + // attempt must discard the worker manager's previous ownership record. + expect(await handle.prepareIdleSleep!(hold())).toBe("none"); + } finally { stopTaskDrain(); await handle.stop(); } + }); + + it("keeps a host write counted after the caller stops waiting", async () => { + let finish!: () => void; + const handle = worker(true, () => new Promise((resolve) => { finish = resolve; })); + await handle.start(); + try { + await expect(handle.call("health", {}, 30)).rejects.toThrow("timed out"); + const lease = hold(); + expect(await handle.prepareIdleSleep!(lease)).toBe("present"); + finish(); + await expect.poll(() => handle.prepareIdleSleep!(lease)).toBe("none"); + } finally { stopTaskDrain(); await handle.stop(); } + }); + + it("lets a queued notification finish its write while idle preparation is in flight", async () => { + let finish!: () => void; + let writes = 0; + const handle = worker(false, () => new Promise((resolve) => { + writes++; + finish = resolve; + })); + await handle.start(); + try { + // Finish setup and its subscription receipt before queuing the event. + await handle.call("health", {}); + handle.notify("onEvent", { event: { eventType: "fixture.write" } }); + // No await: the host has set its hold before it can read the event's + // worker-to-host write. The worker must still be allowed to complete it. + const lease = hold(); + const preparation = handle.prepareIdleSleep!(lease); + expect(await preparation).not.toBe("none"); + await expect.poll(() => writes).toBe(1); + expect(await handle.prepareIdleSleep!(lease)).toBe("present"); + finish(); + await expect.poll(() => handle.prepareIdleSleep!(lease)).toBe("none"); + } finally { finish?.(); stopTaskDrain(); await handle.stop(); } + }); + + it("resumes crash recovery after an idle hold without discarding the restart", async () => { + const handle = worker(); + await handle.start(); + handle.on("crash", () => { hold(); }); + try { + await expect(handle.call("environmentProbe", { driverKey: "fixture", companyId: "fixture", environmentId: "fixture", config: { crash: true } })).rejects.toThrow(); + await delay(1_600); + expect(handle.status).toBe("backoff"); + expect(handle.diagnostics().nextRestartAt).not.toBeNull(); + stopTaskDrain(); + await expect.poll(() => handle.status, { timeout: 5_000 }).toBe("running"); + // Recovery restores service, but lost completion receipts remain unsafe. + expect(await handle.prepareIdleSleep!(hold())).toBe("unknown"); + } finally { stopTaskDrain(); await handle.stop(); } + }); +}); diff --git a/server/src/__tests__/plugin-idle-rpc.test.ts b/server/src/__tests__/plugin-idle-rpc.test.ts new file mode 100644 index 00000000000..c1e626c1b88 --- /dev/null +++ b/server/src/__tests__/plugin-idle-rpc.test.ts @@ -0,0 +1,89 @@ +import { PassThrough } from "node:stream"; +import { describe, expect, it } from "vitest"; +import { definePlugin, type PluginDefinition } from "../../../packages/plugins/sdk/src/define-plugin.js"; +import { startWorkerRpcHost } from "../../../packages/plugins/sdk/src/worker-rpc-host.js"; + +function fixture(definition: PluginDefinition, rpcTimeoutMs = 30_000) { + const stdin = new PassThrough(); + const stdout = new PassThrough(); + const worker = startWorkerRpcHost({ plugin: definePlugin(definition), stdin, stdout, rpcTimeoutMs }); + const messages: any[] = []; + stdout.on("data", (chunk) => { for (const line of String(chunk).trim().split("\n")) messages.push(JSON.parse(line)); }); + let id = 0; + async function call(method: string, params: unknown = {}) { + const callId = ++id; + stdin.write(JSON.stringify({ jsonrpc: "2.0", id: callId, method, params }) + "\n"); + await expect.poll(() => messages.find((message) => message.id === callId && !message.method)).toBeDefined(); + return messages.find((message) => message.id === callId && !message.method); + } + const initialize = () => call("initialize", { manifest: { id: "test.idle", apiVersion: 1 }, config: {}, apiVersion: 1 }); + return { call, initialize, messages, stdin, close: () => worker.stop() }; +} + +describe("plugin idle RPC", () => { + const hold = () => ({ ownerId: "idle-owner", expiresAt: Date.now() + 30_000 }); + + it("seals an idle worker, fences stale releases, and restores normal calls", async () => { + const f = fixture({ async setup() {}, async onIdleDrain() { return "none"; } }); + try { + expect((await f.initialize()).result.supportedMethods).toContain("prepareIdleSleep"); + const lease = hold(); + expect((await f.call("prepareIdleSleep", lease)).result).toEqual({ ...lease, backgroundWork: "none" }); + expect((await f.call("health")).error).toBeDefined(); + await f.call("releaseIdleSleep", { ownerId: "stale" }); + expect((await f.call("health")).error).toBeDefined(); + await f.call("releaseIdleSleep", lease); + expect((await f.call("health")).result.status).toBe("ok"); + } finally { f.close(); } + }); + + it("does not mistake a caller timeout for a completed worker handler", async () => { + let finish!: () => void; + const f = fixture({ async setup() {}, async onIdleDrain() { return "none"; }, + async onHealth() { await new Promise((resolve) => { finish = resolve; }); return { status: "ok" }; } }); + try { + await f.initialize(); + const accepted = f.call("health"); + await expect.poll(() => finish).toBeDefined(); + expect((await f.call("prepareIdleSleep", hold())).result.backgroundWork).toBe("present"); + finish(); await accepted; + expect((await f.call("prepareIdleSleep", hold())).result.backgroundWork).toBe("none"); + } finally { f.close(); } + }); + + it("retains timed-out worker-to-host writes until the host's late receipt", async () => { + let write!: () => Promise; + const f = fixture({ async setup(ctx) { write = () => ctx.state.set({ scopeKind: "instance", stateKey: "fixture" }, { value: 1 }); }, + async onIdleDrain() { return "none"; } }, 10); + try { + await f.initialize(); + await expect(write()).rejects.toThrow("timed out"); + const request = f.messages.find((message) => message.method === "state.set"); + expect((await f.call("prepareIdleSleep", hold())).result.backgroundWork).toBe("present"); + f.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result: null }) + "\n"); + expect((await f.call("prepareIdleSleep", hold())).result.backgroundWork).toBe("none"); + } finally { f.close(); } + }); + + it.each([false, true])("logs session callback failures (async: %s)", async (asynchronous) => { + let subscribe!: () => Promise; + const f = fixture({ async setup(ctx) { + subscribe = () => ctx.agents.sessions.sendMessage("fixture-session", "fixture-company", { + prompt: "fixture", + onEvent: asynchronous + ? async () => { throw new Error("fixture callback failure"); } + : () => { throw new Error("fixture callback failure"); }, + }); + } }); + try { + await f.initialize(); + const subscription = subscribe(); + const request = f.messages.find((message) => message.method === "agents.sessions.sendMessage"); + f.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result: {} }) + "\n"); + await subscription; + f.stdin.write(JSON.stringify({ jsonrpc: "2.0", method: "agents.sessions.event", params: { sessionId: "fixture-session" } }) + "\n"); + await expect.poll(() => f.messages.find((message) => message.method === "log" && + message.params.level === "error" && message.params.message.includes("fixture callback failure"))).toBeDefined(); + } finally { f.close(); } + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index 6c9dd7e32ec..7cdebae7f1d 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -834,7 +834,7 @@ export async function createApp( api.use(resourceMembershipRoutes(db)); api.use(primaryAgentRoutes(db)); api.use(inboxDismissalRoutes(db)); - api.use(instanceSettingsRoutes(db)); + api.use(instanceSettingsRoutes(db, workerManager)); if (opts.databaseBackupService) { api.use(instanceDatabaseBackupRoutes(opts.databaseBackupService)); } diff --git a/server/src/routes/instance-settings.ts b/server/src/routes/instance-settings.ts index 1e1fd26c41b..ef578bb7fda 100644 --- a/server/src/routes/instance-settings.ts +++ b/server/src/routes/instance-settings.ts @@ -15,6 +15,7 @@ import { } from "../services/cloud-instance.js"; import { getHiddenSettings } from "../services/settings-visibility.js"; import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { validate } from "../middleware/validate.js"; import { logger } from "../middleware/logger.js"; import { @@ -119,7 +120,7 @@ function withTaskDrainTransition(run: () => Promise): Promise { return turn; } -export function instanceSettingsRoutes(db: Db) { +export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManager) { const router = Router(); const svc = instanceSettingsService(db); const environments = environmentService(db); @@ -304,7 +305,8 @@ export function instanceSettingsRoutes(db: Db) { // members may read process counters, but not instance-wide work state. assertCanManageInstanceSettings(req); const idleSleepSafety = await readIdleSleepSafety(db, () => heartbeat.getTaskDrainStatus(), Date.now, - typeof req.query.ownerId === "string" ? req.query.ownerId : undefined); + typeof req.query.ownerId === "string" ? req.query.ownerId : undefined, undefined, + pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers)); res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety }); return; } @@ -418,6 +420,7 @@ export function instanceSettingsRoutes(db: Db) { ), ); heartbeat.stopTaskDrain(); + pluginWorkers?.releaseIdleSleep?.(); // See the POST handler above for why a publish failure here is // swallowed instead of failing the route: the audit record already // committed, so a publish failure here must not undo a drain-stop diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index 574b57c11f9..ad24e3037d1 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -17,6 +17,11 @@ export interface IdleSleepSafety { backgroundWork: "none" | "present" | "unknown"; } +export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) => Promise<{ + backgroundWork: "none" | "present" | "unknown"; + pluginIds: string[]; +}>; + // These are deliberately conservative. Completed agent runs and workspace // operations are history, not reasons to keep an otherwise idle instance up. // Other background features remain awake until their work and inbound events @@ -91,18 +96,28 @@ export async function readIdleSleepSafety( now: () => number = Date.now, ownerId?: string, inspectLocalWork: () => Promise = readIdleLocalWork, + inspectPlugins?: InspectIdlePlugins, ): Promise { const unknown: IdleSleepSafety = { version: 1, backgroundWork: "unknown" }; const before = getDrainStatus(); const localBefore = idleWorkSnapshot(); if (!sameQuietHold(before, before, now())) return unknown; try { + let quietPlugins: string[] = []; + if (inspectPlugins) { + if (!ownerId || ownerId !== before.ownerId || before.expiresAt === null || localBefore.active !== 0) return unknown; + const pluginWork = await inspectPlugins({ ownerId, expiresAt: before.expiresAt.getTime() }); + if (pluginWork.backgroundWork !== "none") return { version: 1, backgroundWork: pluginWork.backgroundWork }; + quietPlugins = pluginWork.pluginIds; + } const blocked = await db.transaction(async (tx) => { await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); const checks = WORK_CHECKS.map((query) => sql`EXISTS (${sql.raw(query)})`); - // Version labels and manifest declarations do not prove arbitrary worker - // code has no background activity. No plugin approvals are supported. - checks.push(sql`EXISTS (SELECT 1 FROM plugins WHERE status <> 'disabled')`); + // Only the current worker's owned runtime drain can exempt a plugin. + // Missing workers, old SDKs, and newly installed plugins still block. + checks.push(quietPlugins.length === 0 + ? sql`EXISTS (SELECT 1 FROM plugins WHERE status <> 'disabled')` + : sql`EXISTS (SELECT 1 FROM plugins WHERE status <> 'disabled' AND id NOT IN (${sql.join(quietPlugins.map((id) => sql`${id}::uuid`), sql`, `)}))`); const rows = await tx.execute<{ blocked: boolean }>(sql`SELECT ${sql.join(checks, sql` OR `)} AS blocked`); return rows.length === 1 && typeof rows[0]?.blocked === "boolean" ? rows[0].blocked : undefined; }, { isolationLevel: "repeatable read", accessMode: "read only" }); diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index af6721f3d4b..2d3f961d530 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -20,6 +20,7 @@ import { fork, type ChildProcess } from "node:child_process"; import { randomUUID } from "node:crypto"; +import { beginIdleTrackedWork, readTaskDrain, trackIdleWork } from "./task-admission.js"; import { EventEmitter } from "node:events"; import { createInterface, type Interface as ReadlineInterface } from "node:readline"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; @@ -676,6 +677,8 @@ interface ExecuteLogRoute { * with exponential backoff automatically when `autoRestart` is enabled. */ export interface PluginWorkerHandle { + prepareIdleSleep?(hold: { ownerId: string; expiresAt: number }): Promise<"none" | "present" | "unknown">; + releaseIdleSleep?(): void; /** The plugin ID this worker serves. */ readonly pluginId: string; @@ -793,6 +796,10 @@ export interface WorkerDiagnostics { * for starting/stopping all workers and routing RPC calls. */ export interface PluginWorkerManager { + inspectIdleSleep?(hold: { ownerId: string; expiresAt: number }): Promise<{ + backgroundWork: "none" | "present" | "unknown"; pluginIds: string[]; + }>; + releaseIdleSleep?(): void; /** * Register and start a worker for a plugin. * @@ -888,6 +895,30 @@ export function createPluginWorkerHandle( // Pending RPC requests awaiting a response const pendingRequests = new Map(); + // Unlike RPC promises, these survive timeouts. A late response is the + // completion receipt. A worker crash cannot manufacture one. + const unsettledCalls = new Map void>(); + let activeHostHandlers = 0; + let idleHold: { ownerId: string; expiresAt: number; ready: boolean } | null = null; + + function releaseIdleSleep(): void { + const prior = idleHold; + idleHold = null; + if (prior && childProcess?.stdin?.writable) { + try { + sendMessage({ jsonrpc: JSONRPC_VERSION, method: "releaseIdleSleep", params: { ownerId: prior.ownerId } }); + } catch { /* Worker-side expiry still bounds a lost release. */ } + } + } + + function idleSleepHeld(): boolean { + if (!idleHold) return false; + const drain = readTaskDrain(new Date()); + if (drain?.ownerId !== idleHold.ownerId || drain.expiresAt?.getTime() !== idleHold.expiresAt) { + releaseIdleSleep(); + } + return idleHold !== null; + } let nextRequestId = 1; const activeInvocations = new Map(); // Host-owned execute routes, keyed by the host-issued invocation id. Only an @@ -1020,6 +1051,8 @@ export function createPluginWorkerHandle( function setStatus(newStatus: WorkerStatus): void { const prev = status; if (prev === newStatus) return; + const done = beginIdleTrackedWork(); + done(); status = newStatus; log.debug({ from: prev, to: newStatus }, "worker status change"); emitter.emit("status", { pluginId, status: newStatus, previousStatus: prev }); @@ -1093,6 +1126,8 @@ export function createPluginWorkerHandle( return; } + unsettledCalls.get(id)?.(); + unsettledCalls.delete(id); const pending = pendingRequests.get(id); if (!pending) { log.warn({ id }, "received response for unknown request id"); @@ -2695,6 +2730,23 @@ export function createPluginWorkerHandle( * Handle a JSON-RPC request from the worker (worker→host call). */ async function handleWorkerRequest(request: JsonRpcRequest): Promise { + const done = beginIdleTrackedWork(); + activeHostHandlers++; + try { + // A queued notification may already be running in the worker while its + // prepare request is in flight. Let its writes finish until that worker + // has positively acknowledged that all accepted handlers have drained. + if (idleSleepHeld() && idleHold?.ready) { + try { + sendMessage(createErrorResponse(request.id, PLUGIN_RPC_ERROR_CODES.WORKER_UNAVAILABLE, "Plugin is held for idle sleep")); + } catch { /* Worker may have exited after sending its request. */ } + return; + } + await dispatchWorkerRequest(request); + } finally { activeHostHandlers--; done(); } + } + + async function dispatchWorkerRequest(request: JsonRpcRequest): Promise { const method = request.method as WorkerToHostMethodName; const handler = options.hostHandlers[method] as | ((params: unknown, context?: WorkerHostCallContext) => Promise) @@ -3072,6 +3124,12 @@ export function createPluginWorkerHandle( backoffTimer = setTimeout(async () => { backoffTimer = null; nextRestartAt = null; + // A temporary idle hold must not consume the crash-recovery attempt. + // Keep the bounded backoff until admission reopens. + if (readTaskDrain(new Date())?.ownerId) { + scheduleRestart(); + return; + } try { await startInternal(); } catch (err) { @@ -3293,6 +3351,11 @@ export function createPluginWorkerHandle( executeLogSink?: ExecuteLogSink, ): Promise { const rpcPromise = new Promise((resolve, reject) => { + const idleControl = method === "prepareIdleSleep" || method === "releaseIdleSleep"; + if (!idleControl && method !== "shutdown" && idleSleepHeld()) { + reject(new Error("Plugin is held for idle sleep")); + return; + } if (!childProcess?.stdin?.writable) { reject( new Error( @@ -3303,6 +3366,7 @@ export function createPluginWorkerHandle( } const id = nextRequestId++; + if (!idleControl) unsettledCalls.set(id, beginIdleTrackedWork()); const timeout = resolveRpcCallTimeoutMs(timeoutMs, rpcTimeoutMs); const invocationScope = deriveInvocationScope(method, params); const invocation = invocationScope ? registerInvocation(invocationScope) : null; @@ -3368,6 +3432,8 @@ export function createPluginWorkerHandle( } catch (err) { clearTimeout(timer); pendingRequests.delete(id); + unsettledCalls.get(id)?.(); + unsettledCalls.delete(id); clearInvocation(invocation); clearExecuteRoute(invocation?.id); reject( @@ -3394,6 +3460,38 @@ export function createPluginWorkerHandle( // ----------------------------------------------------------------------- const handle: PluginWorkerHandle = { + releaseIdleSleep, + async prepareIdleSleep(hold) { + const drain = readTaskDrain(new Date()); + if (drain?.ownerId !== hold.ownerId || drain.expiresAt?.getTime() !== hold.expiresAt || + hold.expiresAt <= Date.now()) return "unknown"; + // Clear a previous hold even when this idle worker received no ordinary + // call between two controller attempts. + idleSleepHeld(); + if (status !== "running" || totalCrashes > 0) return "unknown"; + if (!supportedMethods.includes("prepareIdleSleep")) return "present"; + if (unsettledCalls.size || activeHostHandlers || loginPtyRoutesByHostRouteId.size || + liveDuplexRoutes.size || openingDuplexRoutes.size || terminalDuplexRoutes.size) return "present"; + if (idleHold && (idleHold.ownerId !== hold.ownerId || idleHold.expiresAt !== hold.expiresAt)) return "unknown"; + const candidate = idleHold ??= { ...hold, ready: false }; + try { + const result = await callInternal("prepareIdleSleep", hold, Math.min(5_000, hold.expiresAt - Date.now())); + if (!idleSleepHeld() || idleHold !== candidate || status !== "running" || unsettledCalls.size || activeHostHandlers || + result.ownerId !== hold.ownerId || result.expiresAt !== hold.expiresAt) { + if (idleHold === candidate) releaseIdleSleep(); + return "unknown"; + } + if (result.backgroundWork === "none") { + candidate.ready = true; + return "none"; + } + releaseIdleSleep(); + return result.backgroundWork === "present" ? "present" : "unknown"; + } catch { + if (idleHold === candidate) releaseIdleSleep(); + return "unknown"; + } + }, get pluginId() { return pluginId; }, @@ -3459,6 +3557,7 @@ export function createPluginWorkerHandle( notify(method: string, params: unknown) { if (status !== "running") return; + if (idleSleepHeld()) throw new Error("Plugin is held for idle sleep"); const invocationScope = deriveInvocationScope(method, params); // Notifications have no response to settle on, so the invocation scope // is GC'd by TTL. Call-path invocations are registered without a TTL and @@ -3637,6 +3736,20 @@ export function createPluginWorkerManager( ); return { + async inspectIdleSleep(hold) { + if (startupLocks.size) return { backgroundWork: "unknown", pluginIds: [] }; + const snapshot = [...workers.entries()]; + const results = await Promise.all(snapshot.map(async ([, worker]) => + worker.prepareIdleSleep ? worker.prepareIdleSleep(hold) : "unknown")); + if (startupLocks.size || snapshot.length !== workers.size || snapshot.some(([id, worker]) => workers.get(id) !== worker || worker.status !== "running")) { + return { backgroundWork: "unknown", pluginIds: [] }; + } + const backgroundWork = results.includes("present") ? "present" : results.includes("unknown") ? "unknown" : "none"; + return { backgroundWork, pluginIds: backgroundWork === "none" ? snapshot.map(([id]) => id) : [] }; + }, + releaseIdleSleep() { + for (const worker of workers.values()) worker.releaseIdleSleep?.(); + }, async startWorker( pluginId: string, options: WorkerStartOptions, @@ -3690,7 +3803,7 @@ export function createPluginWorkerManager( log.info({ pluginId }, "starting plugin worker"); // Set the lock before awaiting start() to prevent concurrent spawns - const startPromise = handle.start().then(() => handle).finally(() => { + const startPromise = trackIdleWork(handle.start()).then(() => handle).finally(() => { startupLocks.delete(pluginId); }); startupLocks.set(pluginId, startPromise); @@ -3706,7 +3819,7 @@ export function createPluginWorkerManager( } log.info({ pluginId }, "stopping plugin worker"); - await handle.stop(); + await trackIdleWork(handle.stop()); workers.delete(pluginId); }, From 1881894973a2b25838d8abed9bd8aeebc3af4441 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 8 Oct 2026 13:13:16 -0700 Subject: [PATCH 06/10] fix: prevent inbox archive deadlocks during completion (#15615) Take a company-scoped parent lock before writing archive state and reuse the caller transaction. Preserve newer archive timestamps and attribution when a request waits, so completed tasks remain archived. Verified with real PostgreSQL race, rollback, company isolation, concurrency, timestamp and visibility tests, independent review, full CI and Greptile 5/5. Co-Authored-By: Paperclip --- doc/SPEC-implementation.md | 1 + .../__tests__/inbox-archive-routes.test.ts | 163 ++++++++++++++++++ server/src/services/issues.ts | 80 ++++++--- 3 files changed, 219 insertions(+), 25 deletions(-) diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index df952bf5250..f9f5bb5e0d9 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -1115,6 +1115,7 @@ Core authorization follows these rules: - A user may set inbox-agent policy to `disabled` or `allowlist`. Policy restrictions override the default-open path, and low-trust agents are denied. - An agent targeting any user other than its resolved responsible user requires either a materialized target-user policy that permits that agent (`open` or matching `allowlist`) or an explicit `inbox:manage` grant. The implicit default-open policy for a missing row remains responsible-user-only, so it never becomes a blanket cross-user grant. Grants may be unscoped or constrained by `scope.userIds` and act as administrative overrides, including over a disabled target-user policy. - Archive and unarchive operations are company-scoped, reversible, and activity logged with actor, agent, run, target user, target-resolution source, and policy mode. +- Concurrent archives keep the newest archive time and its actor attribution. An earlier request must not undo an archive written by human completion while it waited. Unarchive removes that state so a later archive starts fresh. - New qualifying issue activity may invalidate an archive so the item resurfaces; archival is not a substitute for resolving or closing work. - Viewing an issue may update its per-user read receipt, but read receipts alone do not enroll the issue in Mine. Mine participation begins with a user-authored comment, issue creation/assignment, or another audited user mutation; explicit product actions such as manually running a routine may record an audited inbox touch. diff --git a/server/src/__tests__/inbox-archive-routes.test.ts b/server/src/__tests__/inbox-archive-routes.test.ts index b33663e8cac..4b53f1f2736 100644 --- a/server/src/__tests__/inbox-archive-routes.test.ts +++ b/server/src/__tests__/inbox-archive-routes.test.ts @@ -20,6 +20,7 @@ import { import { LOW_TRUST_REVIEW_PRESET } from "@paperclipai/shared"; import { errorHandler } from "../middleware/index.js"; import { issueRoutes } from "../routes/issues.js"; +import { issueService } from "../services/issues.js"; import { subscribeCompanyLiveEvents } from "../services/live-events.js"; import { getEmbeddedPostgresTestSupport, @@ -194,6 +195,168 @@ describeEmbeddedPostgres("inbox archive routes", () => { .expect(({ body }) => expect(body).toEqual({ ok: true, userId: seeded.responsibleUserId })); }); + it.each([false, true])("allows manual archive during human completion (existing archive: %s)", async (alreadyArchived) => { + const seeded = await seed(); + const svc = issueService(db); + if (alreadyArchived) { + await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId); + } + const app = appFor({ + type: "board", + source: "session", + userId: seeded.responsibleUserId, + companyIds: [seeded.companyId], + memberships: [{ companyId: seeded.companyId, membershipRole: "operator", status: "active" }], + isInstanceAdmin: false, + }); + let manualArchive: Promise<{ status: number }> | undefined; + let completed: Awaited>; + let manualStatus: number | undefined; + try { + completed = await db.transaction(async (tx) => { + await tx.execute(sql`set local statement_timeout = '5s'`); + // Human completion locks the issue before it writes the archive. + await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, seeded.issueId)).for("update"); + const [{ pid }] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + manualArchive = request(app).post(`/api/issues/${seeded.issueId}/inbox-archive`).send({}).then((response) => response); + await expect.poll(async () => { + const blocked = await db.execute(sql` + select 1 from pg_stat_activity + where datname = current_database() and ${pid} = any(pg_blocking_pids(pid)) + `); + return blocked.length; + }, { timeout: 3_000 }).toBeGreaterThan(0); + + // The manual request is now waiting on our parent row. Previously it + // already owned the archive insert, so this completion formed a cycle. + return svc.update(seeded.issueId, { + status: "done", + actorUserId: seeded.responsibleUserId, + companyGuard: seeded.companyId, + }, tx, []); + }); + } finally { + // Do not leave an in-flight route writing into the next test's fixture. + if (manualArchive) manualStatus = (await manualArchive).status; + } + expect(manualStatus).toBe(200); + expect(completed!.status).toBe("done"); + const archives = await db.select().from(issueInboxArchives); + expect(archives).toHaveLength(1); + expect(archives[0]).toMatchObject({ + companyId: seeded.companyId, + issueId: seeded.issueId, + userId: seeded.responsibleUserId, + archivedByActorType: "user", + archivedByAgentId: null, + archivedByRunId: null, + }); + const audits = await db.select().from(activityLog).where(eq(activityLog.action, "issue.inbox_archived")); + expect(audits).toHaveLength(2); + expect(audits.some((audit) => audit.details?.source === "issue_status_done")).toBe(true); + expect(archives[0].archivedAt.getTime()).toBeGreaterThanOrEqual(completed!.completedAt!.getTime()); + // The status route records this activity after the completion transaction. + await db.insert(activityLog).values({ + companyId: seeded.companyId, + actorType: "user", + actorId: seeded.responsibleUserId, + action: "issue.updated", + entityType: "issue", + entityId: seeded.issueId, + details: { status: "done", _previous: { status: "todo" } }, + }); + await request(app).get(`/api/companies/${seeded.companyId}/issues`) + .query({ touchedByUserId: seeded.responsibleUserId, inboxArchivedByUserId: seeded.responsibleUserId, status: "done" }) + .expect(200) + .expect(({ body }) => expect(body.map((issue: { id: string }) => issue.id)).not.toContain(seeded.issueId)); + }); + + it("keeps caller rollback atomic without blocking a different user's archive", async () => { + const seeded = await seed(); + const svc = issueService(db); + const rollback = new Error("Roll back the first user's archive"); + await expect(db.transaction(async (tx) => { + await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, new Date(), undefined, tx); + // An exclusive parent lock would unnecessarily serialize these users. + await db.transaction(async (otherTx) => { + await otherTx.execute(sql`set local statement_timeout = '3s'`); + await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.targetUserId, new Date(), undefined, otherTx); + }); + throw rollback; + })).rejects.toBe(rollback); + const archives = await db.select().from(issueInboxArchives); + expect(archives).toHaveLength(1); + expect(archives[0].userId).toBe(seeded.targetUserId); + }); + + it.each([-1, 0, 1])("keeps the newest archive time and matching attribution (offset: %s)", async (offset) => { + const seeded = await seed(); + const svc = issueService(db); + const firstTime = new Date("2000-01-02T00:00:00.000Z"); + const nextTime = new Date(firstTime.getTime() + offset * 1_000); + const first = await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, firstTime, { + archivedByActorType: "agent", + archivedByAgentId: seeded.agentId, + archivedByRunId: seeded.runId, + }); + const next = await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, nextTime); + expect(next.id).toBe(first.id); + expect(next).toMatchObject(offset < 0 ? first : { + archivedAt: nextTime, + archivedByActorType: "user", + archivedByAgentId: null, + archivedByRunId: null, + }); + expect(await db.select().from(issueInboxArchives)).toEqual([next]); + + await svc.unarchiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId); + const afterUndo = await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, nextTime); + expect(afterUndo).toMatchObject({ archivedAt: nextTime, archivedByActorType: "user" }); + expect(afterUndo.id).not.toBe(first.id); + }); + + it("holds the issue company stable until its archive transaction ends", async () => { + const seeded = await seed(); + const otherCompany = await seed(); + const svc = issueService(db); + let move: Promise | undefined; + try { + await db.transaction(async (tx) => { + await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, new Date(), undefined, tx); + const [{ pid }] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + move = db.update(issues).set({ companyId: otherCompany.companyId }) + .where(eq(issues.id, seeded.issueId)).then((rows) => rows); + await expect.poll(async () => { + const blocked = await db.execute(sql` + select 1 from pg_stat_activity + where datname = current_database() and ${pid} = any(pg_blocking_pids(pid)) + `); + return blocked.length; + }, { timeout: 3_000 }).toBeGreaterThan(0); + const [stillOwned] = await tx.select().from(issues).where(eq(issues.id, seeded.issueId)); + expect(stillOwned.companyId).toBe(seeded.companyId); + }); + } finally { + await move; + } + const [moved] = await db.select().from(issues).where(eq(issues.id, seeded.issueId)); + expect(moved.companyId).toBe(otherCompany.companyId); + }); + + it("rejects missing and cross-company issues before archiving", async () => { + const seeded = await seed(); + const svc = issueService(db); + const otherCompany = await seed(); + for (const [companyId, issueId] of [ + [otherCompany.companyId, seeded.issueId], + [seeded.companyId, randomUUID()], + ]) { + await expect(svc.archiveInbox(companyId, issueId, seeded.responsibleUserId)) + .rejects.toMatchObject({ status: 404, message: "Issue not found" }); + } + expect(await db.select().from(issueInboxArchives)).toHaveLength(0); + }); + it("silently archives an issue for the board user who moves it to done", async () => { const seeded = await seed(); const app = appFor({ diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index f8c862ad91b..a4ef06d713d 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -25,6 +25,7 @@ import { isNull, like, lt, + lte, ne, notExists, notInArray, @@ -7951,35 +7952,64 @@ export function issueService(db: Db) { }, dbOrTx: any = db, ) { - const now = new Date(); - const [row] = await dbOrTx - .insert(issueInboxArchives) - .values({ - companyId, - issueId, - userId, - archivedByActorType: attribution?.archivedByActorType ?? "user", - archivedByAgentId: attribution?.archivedByAgentId ?? null, - archivedByRunId: attribution?.archivedByRunId ?? null, - archivedAt, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [ - issueInboxArchives.companyId, - issueInboxArchives.issueId, - issueInboxArchives.userId, - ], - set: { - archivedAt, + const runArchive = async (tx: typeof dbOrTx) => { + // Completion locks the issue before archiving. Take the FK's parent lock + // first too, or an insert can hold the archive key while waiting on that + // issue and deadlock with completion's archive UPSERT. SHARE also holds + // companyId stable; different users can still archive concurrently. + const [issue] = await tx + .select({ id: issues.id }) + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))) + .for("share"); + if (!issue) throw notFound("Issue not found"); + + const now = new Date(); + const [row] = await tx + .insert(issueInboxArchives) + .values({ + companyId, + issueId, + userId, archivedByActorType: attribution?.archivedByActorType ?? "user", archivedByAgentId: attribution?.archivedByAgentId ?? null, archivedByRunId: attribution?.archivedByRunId ?? null, + archivedAt, updatedAt: now, - }, - }) - .returning(); - return row; + }) + .onConflictDoUpdate({ + target: [ + issueInboxArchives.companyId, + issueInboxArchives.issueId, + issueInboxArchives.userId, + ], + set: { + archivedAt, + archivedByActorType: attribution?.archivedByActorType ?? "user", + archivedByAgentId: attribution?.archivedByAgentId ?? null, + archivedByRunId: attribution?.archivedByRunId ?? null, + updatedAt: now, + }, + // A request that waited behind completion must not replace the newer + // archive with its earlier request time and resurface the done task. + setWhere: lte(issueInboxArchives.archivedAt, archivedAt), + }) + .returning(); + if (row) return row; + // ON CONFLICT holds this row lock even when setWhere skips the update. + // Return the newer state, including its matching actor attribution. + const [existing] = await tx + .select() + .from(issueInboxArchives) + .where(and( + eq(issueInboxArchives.companyId, companyId), + eq(issueInboxArchives.issueId, issueId), + eq(issueInboxArchives.userId, userId), + )); + if (!existing) throw new Error("Inbox archive conflict row missing"); + return existing; + }; + return dbOrTx === db ? db.transaction(runArchive) : runArchive(dbOrTx); } const service = { From ed6abbf158b73086ca0237bec23ea72ec89e297f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:18:05 -0700 Subject: [PATCH 07/10] build(deps): bump actions/cache/restore from 5.1.0 to 6.1.0 (#15163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache/restore](https://github.com/actions/cache) from 5.1.0 to 6.1.0.
Release notes

Sourced from actions/cache/restore's releases.

v6.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v6...v6.1.0

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v6.0.0

Changelog

Sourced from actions/cache/restore's changelog.

Releases

How to prepare a release

[!NOTE] Relevant for maintainers with write access only.

  1. Switch to a new branch from main.
  2. Run npm test to ensure all tests are passing.
  3. Update the version in https://github.com/actions/cache/blob/main/package.json.
  4. Run npm run build to update the compiled files.
  5. Update this https://github.com/actions/cache/blob/main/RELEASES.md with the new version and changes in the ## Changelog section.
  6. Run licensed cache to update the license report.
  7. Run licensed status and resolve any warnings by updating the https://github.com/actions/cache/blob/main/.licensed.yml file with the exceptions.
  8. Commit your changes and push your branch upstream.
  9. Open a pull request against main and get it reviewed and merged.
  10. Draft a new release https://github.com/actions/cache/releases use the same version number used in package.json
    1. Create a new tag with the version number.
    2. Auto generate release notes and update them to match the changes you made in RELEASES.md.
    3. Toggle the set as the latest release option.
    4. Publish the release.
  11. Navigate to https://github.com/actions/cache/actions/workflows/release-new-action-version.yml
    1. There should be a workflow run queued with the same version number.
    2. Approve the run to publish the new version and update the major tags for this action.

Changelog

6.1.0

6.0.0

  • Updated @actions/cache to ^6.0.1, @actions/core to ^3.0.1, @actions/exec to ^3.0.0, @actions/io to ^3.0.2
  • Migrated to ESM module system
  • Upgraded Jest to v30 and test infrastructure to be ESM compatible

5.0.4

  • Bump minimatch to v3.1.5 (fixes ReDoS via globstar patterns)
  • Bump undici to v6.24.1 (WebSocket decompression bomb protection, header validation fixes)
  • Bump fast-xml-parser to v5.5.6

5.0.3

5.0.2

... (truncated)

Commits
  • 55cc834 Merge pull request #1768 from jasongin/readonly-cache
  • d8cd72f Bump @​actions/cache to v6.1.0 - handle cache write error due to RO token
  • 2c8a9bd Merge pull request #1760 from actions/samirat/esm_migration_and_package_update
  • e9b91fd Prettier fixes
  • e4884b8 Rebuild dist
  • 10baf01 Fixed licenses
  • e39b386 Fix test mock return order
  • b692820 PR feedback
  • 6074912 Rebuild dist bundles as ESM to match type:module
  • 5a912e8 Fix lint and jest issues
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache/restore&package-manager=github_actions&previous-version=5.1.0&new-version=6.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pr-trusted.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index b87e5115b9c..9b1d175b370 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -384,7 +384,7 @@ jobs: echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} @@ -609,7 +609,7 @@ jobs: echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} @@ -767,7 +767,7 @@ jobs: echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} @@ -905,7 +905,7 @@ jobs: echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} @@ -1078,7 +1078,7 @@ jobs: echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} @@ -1137,7 +1137,7 @@ jobs: echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} @@ -1327,7 +1327,7 @@ jobs: echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" - name: Restore pnpm store (read only) - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} From 2a5f65c9ae501b49b2a38ce7edd04209b806c9d0 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:38:48 -0500 Subject: [PATCH 08/10] refactor(server): extract heartbeat retry scheduling (#15617) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Heartbeat orchestration starts runs and recovers from supported failures. > - The heartbeat service still contains more than 22,000 lines after earlier extractions. > - Retry scheduling has a clear boundary, but it shares lifecycle callbacks with the service. > - This pull request moves retry scheduling into one module with explicit dependencies. > - It preserves the scheduling rules, database writes, and lock order. > - The benefit is a smaller service and focused tests for the retry boundary. ## Linked Issues or Issue Description **Current behavior** `server/src/services/heartbeat.ts` owns retry schedules, contention deferrals, due promotion, and retry-now requests. These operations sit among execution and recovery code. **Proposed behavior** Move these operations into `server/src/services/heartbeat/retries.ts`. Keep the public heartbeat API and existing helper exports. The service supplies its lifecycle callbacks and worktree cutoff. **Reason and benefit** Extract one complete area in a separate PR. This removes 1,641 lines from the main service without changing retry policy. Future retry changes can use a smaller module and direct tests. **Breaking changes** None. Existing imports, error-class identity, retry metadata, and API outcomes remain available. **Additional context** Refs #15601, the preceding run-state extraction. Related open retry changes: #14930, #13975, #15157, #12587, and #13276. This PR moves the existing implementation. Those policy changes remain separate. ## What Changed - Add `createHeartbeatRetries(db, dependencies)` for bounded retries, workspace and connection deferrals, shared-workspace holder checks, due promotion, and retry-now requests. - Keep status transitions, event publication, issue-lock release, plan-resume reporting, and worktree settings in the heartbeat service. - Preserve all 140 existing public exports. Re-export the same helpers, constants, and workspace-busy error class. - Add 14 tests for construction, callback order, cancellation races, concurrent scheduling, committed promotion, and retry-now scope. - Document the module boundary in `doc/DEVELOPING.md`. ## Verification - Existing focused suite before the move: 84 tests passed. - Expanded focused suite after the move: 243 tests passed across 11 files. Command: `pnpm exec vitest run server/src/services/heartbeat/retries.test.ts server/src/__tests__/heartbeat-retry-scheduling.test.ts server/src/__tests__/heartbeat-workspace-busy.test.ts server/src/__tests__/heartbeat-model-fallback-warning.test.ts server/src/__tests__/heartbeat-running-followup.test.ts server/src/__tests__/heartbeat-issue-rewake-throttle.test.ts server/src/__tests__/issue-scheduled-retry-routes.test.ts server/src/modules/run-dispatch` - After rebasing onto the latest master, 61 retry and plugin-idle tests passed across four files. - `pnpm -r typecheck` passed again on the rebased branch. - `pnpm build` passed again on the rebased branch. - `pnpm test:run` was started locally. The long serial run was stopped after full CI passed; it is not claimed as a complete local pass. - Full CI passed on commit `69e76f3eec75b0703570f1fce37ebfe9c63fa3ac`: 54 successful checks, two intentional Storybook skips, zero failures. This includes general tests, serialized server tests, browser tests, runner checks, typecheck, build, and canary verification. - The setup-timeout review fix passed all 14 module tests and the server typecheck. - Greptile completed a fresh review of commit `69e76f3eec75b0703570f1fce37ebfe9c63fa3ac`: 5/5, no open findings. - A structural comparison confirmed 22 unchanged moved functions and one unchanged three-line string reader, 16 unchanged declarations, one unchanged API body, and all 140 preserved exports. The remaining service body is unchanged after accounting for the new binding and removed declarations. ## Risks - A missing lifecycle callback or an eager call during construction could change retry behavior. The dependency contract and direct tests cover these boundaries. - Concurrent scheduling must keep the issue-first lock order and reuse one successor. The original transaction bodies are unchanged. PostgreSQL tests cover concurrent calls from separate factory instances. - Promotion must publish after commit and preserve worktree cutoffs. Direct tests cover both paths. - No schema, route, adapter, or retry-policy changes are included. ## Model Used OpenAI GPT-6 via Codex. The exact serving model ID and context window were not exposed in this session. Used reasoning, repository inspection, shell tools, and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/DEVELOPING.md | 11 +- server/src/services/heartbeat.ts | 1757 +--------------- server/src/services/heartbeat/retries.test.ts | 286 +++ server/src/services/heartbeat/retries.ts | 1853 +++++++++++++++++ 4 files changed, 2207 insertions(+), 1700 deletions(-) create mode 100644 server/src/services/heartbeat/retries.test.ts create mode 100644 server/src/services/heartbeat/retries.ts diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 33b50a28b12..709eb2cb930 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1241,7 +1241,7 @@ issue and wake context, responsible-user resolution, routine environment snapsho skill mentions, adapter environment configuration, and MCP/tool access setup. `createHeartbeatRunPreparation(db)` binds the context loaders to a service's database without doing database work during construction. `heartbeat.ts` keeps queueing, -dispatch, retries, cancellation, and execution order, and re-exports the existing +dispatch, cancellation, and execution order, and re-exports the existing public helpers and configuration-incomplete error class. Keep preparation policy changes in this module and its tests. @@ -1254,6 +1254,15 @@ during construction. The encoding-check cache belongs to each factory instance. and status transitions, and re-exports the existing public helpers. Keep session policy changes separate from run orchestration changes. +Retry scheduling is in `server/src/services/heartbeat/retries.ts`. It owns bounded +retry schedules, connection and workspace contention deferrals, shared-workspace +holder checks, due retry promotion, and retry-now requests. `createHeartbeatRetries` +binds these operations to the service database and explicit lifecycle callbacks +without doing work during construction. `heartbeat.ts` supplies status writes, +run events, issue-lock release, plan-resume reporting, and worktree cutoffs. It +re-exports the existing retry helpers and workspace-busy error class. Keep retry +policy changes separate from this extraction. + ## Wake Context Delivery Built-in adapters deliver wake context through the run prompt, including structured diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index a29bf8b4454..efc951317f4 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,25 @@ +import { + BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + normalizeAgentNameKey, + WorkspaceBusyDeferral, + readTransientRecoveryContractFromRun, + readHeartbeatRunErrorFamily, + isWorkspaceBusyDeferral, + isTransientWorkspaceGitScanCode, + createHeartbeatRetries, +} from "./heartbeat/retries.js"; +export { + BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS, + WORKSPACE_BUSY_RETRY_WAKE_REASON, + WORKSPACE_BUSY_ERROR_CODE, + WORKSPACE_BUSY_RETRY_BASE_DELAY_MS, + WORKSPACE_BUSY_RETRY_JITTER_MS, + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + type SharedWorkspaceHolder, + WorkspaceBusyDeferral, + computeWorkspaceBusyRetryDelayMs, + computeBoundedTransientHeartbeatRetrySchedule, +} from "./heartbeat/retries.js"; import { type RunSessionOutcome, EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, @@ -82,7 +104,6 @@ export { import { WORKSPACE_VALIDATION_FAILURE_CODE, isWorkspaceValidationFailedRun, - readWorkspaceValidationPayloadFromRun, resolveNativeRecoveryExecutionWorkspaceBinding, resolveExecutionWorkspaceReuseRequestForIssue, buildExecutionWorkspaceConfigSnapshot, @@ -240,7 +261,10 @@ import { legacyControllerBootId, legacyControllerClaim, renewLegacyControllerLea import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js"; import { hasRemoteTerminationReceipt, remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js"; import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js"; -import { admitExplicitNativeContinuation, admitExplicitContinuationRetry, undeliveredLegacyUserCommentIds } from "./explicit-native-continuation.js"; +import { + admitExplicitNativeContinuation, + undeliveredLegacyUserCommentIds, +} from "./explicit-native-continuation.js"; import { canRetryStoppedRun, isCancelledNativeStartup } from "./cancelled-native-startup.js"; import { connectionIntentService } from "./connection-intents.js"; import { @@ -271,16 +295,13 @@ import { registerAdapterExecutionControl, waitForAdapterStop, } from "./adapter-execution-control.js"; -import { executionFailureRetryCount, executionRetryAttemptCount, accountingForScheduledRetry } from "./execution-recovery-attempt.js"; +import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; import { buildExecutionContinuation, StaleExecutionContinuationError } from "./execution-continuation.js"; import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; -import { - isWorkspaceGitScanError, - WORKSPACE_GIT_SCAN_ERROR_CODES, -} from "./workspace-git-operation-scheduler.js"; +import { isWorkspaceGitScanError } from "./workspace-git-operation-scheduler.js"; import { initializeRunIdentity, @@ -969,23 +990,11 @@ export { } from "./recovery/service.js"; export const ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS = 60 * 1000; export const ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS = 5 * 1000; -export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS = [ - 30_000, 30_000, -] as const; -const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0; -const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; -const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; -function isTransientWorkspaceGitScanCode(code: string | null | undefined): boolean { - return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated; -} -const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = - BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; + export { INTERACTION_CONTINUATION_INFRA_RETRY_REASON, INTERACTION_CONTINUATION_INFRA_WAKE_REASON, }; -const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 2; - const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed"; const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete"; @@ -1042,36 +1051,8 @@ const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2; const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10; const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000; const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000; -const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = [ - "scheduled_retry", - "queued", - "running", -] as const; + export { WORKSPACE_BUSY_RETRY_REASON }; -export const WORKSPACE_BUSY_RETRY_WAKE_REASON = "workspace_busy_retry"; -export const WORKSPACE_BUSY_ERROR_CODE = "workspace_busy"; -export const WORKSPACE_BUSY_RETRY_BASE_DELAY_MS = 60 * 1000; -export const WORKSPACE_BUSY_RETRY_JITTER_MS = 60 * 1000; -// Preserve the one-hour shared-workspace holder cutoff independently of the -// informational output-silence warnings. Warning sooner must not let another -// run overtake a quiet holder and mutate its shared workspace. -export const WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS = 60 * 60 * 1000; -// Issue-level executionWorkspaceSettings.mode values that unambiguously opt an -// issue's runs out of the shared project workspace, and therefore out of -// shared-workspace serialization ("isolated" is the legacy alias -// parseIssueExecutionWorkspaceSettings normalizes to isolated_workspace). Any -// other value — including agent_default and an absent mode — may still resolve -// to the shared workspace and counts as a holder. -const ISOLATED_EXECUTION_WORKSPACE_MODES = [ - "isolated_workspace", - "operator_branch", - "isolated", -] as const; -type CodexTransientFallbackMode = - | "same_session" - | "safer_invocation" - | "fresh_session" - | "fresh_session_safer_invocation"; interface MaxTurnContinuationPolicy { enabled: boolean; @@ -1134,90 +1115,8 @@ function buildUnresolvedWorkspaceBaseRefResultJson( }; } -export interface SharedWorkspaceHolder { - runId: string; - agentId: string; - issueId: string; - issueIdentifier: string | null; -} - -// Pre-dispatch gate outcome: another running run currently holds the issue's -// shared project workspace. Not a failure — the run is parked as a bounded -// scheduled retry and re-attempted once the holder finishes, so two agents -// never mutate the same working tree concurrently. -export class WorkspaceBusyDeferral extends Error { - code = WORKSPACE_BUSY_ERROR_CODE; - holder: SharedWorkspaceHolder; - projectWorkspaceId: string; - deferralAttempt: number; - wasIssueAssignee: boolean; - - constructor(input: { - holder: SharedWorkspaceHolder; - projectWorkspaceId: string; - deferralAttempt: number; - wasIssueAssignee: boolean; - }) { - super( - `Shared project workspace is busy: run ${input.holder.runId} (issue ${ - input.holder.issueIdentifier ?? input.holder.issueId - }) is still running`, - ); - this.name = "WorkspaceBusyDeferral"; - this.holder = input.holder; - this.projectWorkspaceId = input.projectWorkspaceId; - this.deferralAttempt = input.deferralAttempt; - this.wasIssueAssignee = input.wasIssueAssignee; - } -} - -function isWorkspaceBusyDeferral( - error: unknown, -): error is WorkspaceBusyDeferral { - return error instanceof WorkspaceBusyDeferral; -} - -export function computeWorkspaceBusyRetryDelayMs( - random: () => number = Math.random, -) { - const jitter = Math.min(Math.max(random(), 0), 1); - return ( - WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + - Math.floor(jitter * WORKSPACE_BUSY_RETRY_JITTER_MS) - ); -} - export { isNonAssigneeWorkspaceBusyRetry }; -function resolveCodexTransientFallbackMode( - attempt: number, -): CodexTransientFallbackMode { - if (attempt <= 1) return "same_session"; - if (attempt === 2) return "safer_invocation"; - if (attempt === 3) return "fresh_session"; - return "fresh_session_safer_invocation"; -} - -function readHeartbeatRunErrorFamily( - run: Pick, -) { - const resultJson = parseObject(run.resultJson); - const persistedFamily = readNonEmptyString(resultJson.errorFamily); - if (persistedFamily) return persistedFamily; - - if (run.errorCode === "provider_quota") { - return "provider_quota"; - } - if ( - run.errorCode === "codex_transient_upstream" || - run.errorCode === "claude_transient_upstream" || - run.errorCode === "codex_harness_crash" - ) { - return "transient_upstream"; - } - return null; -} - function isMaxTurnExhaustionRun( run: Pick, ) { @@ -1228,63 +1127,6 @@ function isMaxTurnExhaustionRun( ); } -function readTransientRetryNotBeforeFromRun( - run: Pick, -) { - const resultJson = parseObject(run.resultJson); - const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore; - if (!( - typeof value === "string" || - typeof value === "number" || - value instanceof Date - )) { - return null; - } - const parsed = new Date(value); - return Number.isNaN(parsed.getTime()) ? null : parsed; -} - -function readTransientRecoveryContractFromRun( - run: Pick, -) { - const errorFamily = readHeartbeatRunErrorFamily(run); - return errorFamily === "transient_upstream" || - errorFamily === "provider_quota" - ? { - errorFamily, - retryNotBefore: readTransientRetryNotBeforeFromRun(run), - } - : null; -} - -function isSpawnLikeFailureMessage(value: unknown) { - if (typeof value !== "string") return false; - return /failed to start command|spawn\b|\bENOENT\b/i.test(value); -} - -// A sandbox provider plugin's worker can be briefly down during its own -// restart window (e.g. a rolling deploy of the plugin worker process). Lease -// acquisition fails immediately in that window, but the condition is -// transient and self-healing, so it must be treated as retryable -// infrastructure rather than a terminal setup failure. See -// resolveSandboxProviderPlugin's "worker_unavailable" message in -// environment-runtime.ts (":808"), e.g. 'Sandbox provider "kubernetes" is -// installed via plugin "acme.kubernetes-sandbox-provider", but its worker is -// not running.' -// -// This is anchored on both "is installed via plugin" and "but its worker is -// not running" so it does not also match plugin-environment-driver.ts's -// unrelated, permanent "provider not installed" message ('Sandbox provider -// "X" is not installed or its plugin worker is not running.'), which -// coincidentally contains the same "worker is not running" substring but -// describes a terminal condition that must not be retried. -function isSandboxProviderWorkerUnavailableFailureMessage(value: unknown) { - if (typeof value !== "string") return false; - return /sandbox provider .* is installed via plugin .* but its worker is not running/i.test( - value, - ); -} - // environment-runtime.ts's resolveSandboxProviderPlugin "not_ready" message, // e.g. 'Sandbox provider "kubernetes" is installed via plugin // "acme.kubernetes-sandbox-provider", but that plugin is currently error.' @@ -1312,33 +1154,6 @@ export function parseSandboxProviderPluginNotReadyFailureMessage( }; } -function isRetryableInteractionContinuationInfrastructureFailure( - run: Pick< - typeof heartbeatRuns.$inferSelect, - "error" | "errorCode" | "resultJson" - >, -) { - if ( - run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || - run.errorCode === "process_lost" - ) { - return true; - } - - if (run.errorCode !== "adapter_failed" && run.errorCode !== "setup_failed") - return false; - - const resultJson = parseObject(run.resultJson); - return ( - isSpawnLikeFailureMessage(run.error) || - isSpawnLikeFailureMessage(resultJson.errorMessage) || - isSpawnLikeFailureMessage(resultJson.message) || - isSandboxProviderWorkerUnavailableFailureMessage(run.error) || - isSandboxProviderWorkerUnavailableFailureMessage(resultJson.errorMessage) || - isSandboxProviderWorkerUnavailableFailureMessage(resultJson.message) - ); -} - function mergeAdapterRecoveryMetadata(input: { resultJson: Record | null | undefined; errorFamily?: string | null; @@ -1455,28 +1270,6 @@ export function getTaskDrainStatus(): { }; } - -export function computeBoundedTransientHeartbeatRetrySchedule( - attempt: number, - now = new Date(), - random: () => number = Math.random, -) { - if (!Number.isInteger(attempt) || attempt <= 0) return null; - const baseDelayMs = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS[attempt - 1]; - if (typeof baseDelayMs !== "number") return null; - const sample = Math.min(1, Math.max(0, random())); - const jitterMultiplier = - 1 + (sample * 2 - 1) * BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO; - const delayMs = Math.max(1_000, Math.round(baseDelayMs * jitterMultiplier)); - return { - attempt, - baseDelayMs, - delayMs, - dueAt: new Date(now.getTime() + delayMs), - maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, - }; -} - export function leaseReleaseStatusForRunStatus( status: string | null | undefined, ): Extract { @@ -2255,12 +2048,6 @@ function mergeHotRestartAdoptionResultJson( }; } -function normalizeAgentNameKey(value: string | null | undefined) { - if (typeof value !== "string") return null; - const normalized = value.trim().toLowerCase(); - return normalized.length > 0 ? normalized : null; -} - type SkillTestHeartbeatCompletion = { outcome: "failed" | "cancelled"; error: string | null; @@ -2640,6 +2427,33 @@ export function heartbeatService( BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, }); const runDispatch = createRunDispatch(db); + const { + scheduleBoundedRetryForRun, + timerClaimWasFirstHeartbeat, + scheduleInteractionContinuationInfrastructureRetryIfEligible, + findSharedWorkspaceHolder, + finalizeAiConnectionBusyDeferral, + finalizeWorkspaceBusyDeferral, + promoteDueScheduledRetries, + retryScheduledRetryNow, + scheduleBoundedRetry, + } = createHeartbeatRetries(db, { + appendRunEvent, + escalatePlanApprovalResumeFailureNeedsAttention, + getAgentInvokability, + runDispatch, + resolveSessionBeforeForWakeup, + resolveResponsibleUserIdForRunContext, + recordPlanApprovalResumeFailureRetry, + setRunStatusIfRunning, + setWakeupStatus, + getRun, + getAgent, + releaseIssueExecutionAndPromote, + finalizeAgentStatus, + getWorktreeExecutionCutoff, + applyRunDispatchPostCommitEffects, + }); // Applies the post-commit effects a run-dispatch operation returns, on a // best-effort basis, exactly as this service publishes them for every @@ -7089,1437 +6903,6 @@ export function heartbeatService( }; } - async function scheduleBoundedRetryForRun( - run: typeof heartbeatRuns.$inferSelect, - agent: typeof agents.$inferSelect, - opts?: { - now?: Date; - random?: () => number; - retryReason?: string; - wakeReason?: string; - maxAttempts?: number; - delayMs?: number; - }, - ) { - if (parseObject(agent.adapterConfig).provider === "openai_dot" - || parseObject(parseObject(parseObject(run.runnerProfileJson).nativeExecutionInput).provider).kind === "openai_dot") { - return { outcome: "not_scheduled" as const, reason: "Dot external execution must be reconciled before a new assignment; Paperclip cannot confirm its external stop.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; - } - if (run.errorCode === "provider_tool_definition_invalid") { - return { outcome: "not_scheduled" as const, - reason: "Repair the invalid tool definitions before starting a new attempt.", - issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; - } - if (Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds) && - run.contextSnapshot.chatCompletionDeliveryIds.some(id => typeof id === "string")) { - return { outcome: "not_scheduled" as const, reason: "The completion outbox owns this reply's retry budget and publication identity.", - errorCode: "chat_completion_outbox_owns_retry" as const, issueId: readNonEmptyString(run.contextSnapshot.issueId) }; - } - const now = opts?.now ?? new Date(); - const retryReason = - opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON; - const wakeReason = - opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; - const maxAttempts = Math.max( - 0, - Math.floor( - opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, - ), - ); - const consumedAttempts = executionRetryAttemptCount(run, retryReason); - const nextAttempt = consumedAttempts + 1; - const computedBaseSchedule = - opts?.delayMs != null - ? nextAttempt <= maxAttempts - ? { - attempt: nextAttempt, - baseDelayMs: Math.max(0, Math.floor(opts.delayMs)), - delayMs: Math.max(0, Math.floor(opts.delayMs)), - dueAt: new Date( - now.getTime() + Math.max(0, Math.floor(opts.delayMs)), - ), - maxAttempts, - } - : null - : nextAttempt <= maxAttempts - ? computeBoundedTransientHeartbeatRetrySchedule( - nextAttempt, - now, - opts?.random, - ) - : null; - const baseSchedule = computedBaseSchedule - ? { ...computedBaseSchedule, maxAttempts } - : null; - const transientRecovery = - retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON - ? readTransientRecoveryContractFromRun(run) - : null; - const codexTransientFallbackMode = - agent.adapterType === "codex_local" && - transientRecovery?.errorFamily === "transient_upstream" - ? resolveCodexTransientFallbackMode(nextAttempt) - : null; - const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null; - const contextSnapshot = parseObject(run.contextSnapshot); - // A retry inherits the durable authorization scope of its source run. Do - // not promote an untrusted or legacy contextSnapshot.issueId into a new - // issue binding: that could either violate the FK or misclassify history. - const issueId = run.scopeKind === "issue" ? run.issueId : null; - - if (!baseSchedule) { - const exhaustion = { - retryReason, - scheduledRetryAttempt: consumedAttempts, - maxAttempts, - }; - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: `Bounded retry exhausted after ${consumedAttempts} scheduled attempts; no further automatic retry will be queued`, - payload: exhaustion, - retryExhaustion: exhaustion, - }); - if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { - await escalatePlanApprovalResumeFailureNeedsAttention({ - run, - issueId, - attempt: Math.min( - consumedAttempts, - maxAttempts, - ), - maxAttempts, - }).catch((error) => { - logger.warn( - { err: error, runId: run.id, issueId }, - "failed to escalate exhausted plan-approval resume failure", - ); - }); - } - return { - outcome: "retry_exhausted" as const, - attempt: nextAttempt, - maxAttempts, - }; - } - - if (await legacyExecutionNeedsReconciliationWithEvidence(db, run)) { - return { - outcome: "not_scheduled" as const, - reason: - "Reconcile the previous execution before retrying; safe provider recovery is unavailable.", - errorCode: "legacy_execution_requires_reconciliation" as const, - issueId: readNonEmptyString(run.contextSnapshot?.issueId), - }; - } - if (retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON) { - const invokability = await getAgentInvokability(agent); - if (!invokability.invokable) { - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: - "Scheduled retry suppressed because the agent is not invokable", - payload: { - retryReason, - scheduledRetryAttempt: nextAttempt, - maxAttempts, - reason: invokability.reason, - invalidOrgChain: invokability.invalidOrgChain, - ...invokability.details, - }, - }); - return { - outcome: "not_scheduled" as const, - reason: - "Scheduled retry suppressed because the agent is not invokable", - errorCode: "agent_not_invokable" as const, - issueId, - }; - } - } - - const schedule = - transientRetryNotBefore && - transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime() - ? { - ...baseSchedule, - dueAt: transientRetryNotBefore, - delayMs: Math.max( - 0, - transientRetryNotBefore.getTime() - now.getTime(), - ), - } - : baseSchedule; - - const requiresIssueGate = - isTransientWorkspaceGitScanCode(run.errorCode) || - hasConversationContinuationPolicy(run.resultJson) || - (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) || - retryReason === MAX_TURN_CONTINUATION_RETRY_REASON || - retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON; - if (requiresIssueGate) { - const gate = await runDispatch.evaluateScheduledRetryGate({ - runId: run.id, - companyId: run.companyId, - retryReasonOverride: retryReason, - now, - }); - if (!gate.allowed) { - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: gate.reason, - payload: { - retryReason, - scheduledRetryAttempt: nextAttempt, - maxAttempts, - ...gate.details, - }, - }); - return { - outcome: "not_scheduled" as const, - reason: gate.reason, - errorCode: gate.errorCode, - issueId: gate.issueId, - }; - } - } - const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); - const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); - const interactionContinuationPayload = - retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON - ? { - mutation: "interaction", - interactionId: readNonEmptyString(contextSnapshot.interactionId), - interactionKind: readNonEmptyString( - contextSnapshot.interactionKind, - ), - interactionStatus: readNonEmptyString( - contextSnapshot.interactionStatus, - ), - continuationPolicy: readNonEmptyString( - contextSnapshot.continuationPolicy, - ), - } - : {}; - const workspaceValidationRetryPayload = - retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON && - isWorkspaceValidationFailedRun(run) - ? readWorkspaceValidationPayloadFromRun(run) - : null; - const shouldQuarantineWorkspaceForRetry = - workspaceValidationRetryPayload !== null && - Object.keys(workspaceValidationRetryPayload).length > 0; - const retryContextSnapshot: Record = withRecoveryContext( - { - ...contextSnapshot, - executionRetryAccounting: accountingForScheduledRetry(run, retryReason, schedule.attempt), - retryOfRunId: run.id, - wakeReason, - retryReason, - ...(retryReason === WORKSPACE_BUSY_RETRY_REASON - ? { - failureRetriesBeforeWorkspaceWait: - executionFailureRetryCount(run), - } - : {}), - ...((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) - ? { failureRetriesBeforeAiConnectionWait: executionFailureRetryCount(run) } - : {}), - ...(shouldQuarantineWorkspaceForRetry - ? { - workspaceValidationRecovery: { - strategy: "quarantine_failed_workspace_and_retry_clean", - sourceRunId: run.id, - reason: - readNonEmptyString(workspaceValidationRetryPayload?.reason) ?? - WORKSPACE_VALIDATION_FAILURE_CODE, - fingerprint: readNonEmptyString( - workspaceValidationRetryPayload?.fingerprint, - ), - failedExecutionWorkspaceId: readNonEmptyString( - workspaceValidationRetryPayload?.executionWorkspaceId, - ), - }, - } - : {}), - ...(transientRecovery - ? { errorFamily: transientRecovery.errorFamily } - : {}), - scheduledRetryAttempt: schedule.attempt, - scheduledRetryAt: schedule.dueAt.toISOString(), - ...(transientRetryNotBefore - ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } - : {}), - ...(transientRecovery?.errorFamily === "provider_quota" && - transientRetryNotBefore - ? { - providerQuotaRetryNotBefore: - transientRetryNotBefore.toISOString(), - } - : {}), - ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), - }, - "normal_model", - ); - const responsibleUserId = await resolveResponsibleUserIdForRunContext( - run, - retryContextSnapshot, - ); - const continuationRetryIdempotencyKey = - retryReason === MAX_TURN_CONTINUATION_RETRY_REASON - ? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` - : retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON - ? `interaction-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` - : null; - - type ScheduledRetryTransactionResult = - | { - outcome: "scheduled"; - run: typeof heartbeatRuns.$inferSelect; - reusedExisting: boolean; - } - | { - outcome: "not_scheduled"; - reason: string; - errorCode: - | "issue_not_found" - | "issue_reassigned" - | "issue_cancelled" - | "issue_terminal_status" - | "issue_not_in_progress" - | "continuation_user_authorization_missing" - | "issue_execution_lock_changed"; - issueId: string | null; - details: Record; - }; - - const scheduleResult = await db.transaction( - async (tx): Promise => { - // All automatic failure paths share the same predecessor claim. A - // duplicate monitor, restart sweep or wake must reuse its successor. - if ( - retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON && - retryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON - ) { - if (issueId) - await tx.execute( - sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, - ); - await tx.execute( - sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, - ); - const [existing] = await tx - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, run.companyId), - eq(heartbeatRuns.retryOfRunId, run.id), - ), - ) - .limit(1); - if (existing) - return { - outcome: "scheduled", - run: existing, - reusedExisting: true, - }; - } - if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { - if (issueId) { - await tx.execute( - sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, - ); - } else { - await tx.execute( - sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, - ); - } - - const existingContinuation = await tx - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, run.companyId), - eq(heartbeatRuns.retryOfRunId, run.id), - eq(heartbeatRuns.scheduledRetryReason, retryReason), - eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), - inArray(heartbeatRuns.status, [ - ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, - ]), - issueId - ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` - : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, - ), - ) - .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) - .limit(1) - .then((rows) => rows[0] ?? null); - - if (existingContinuation) { - if (existingContinuation.wakeupRequestId) { - const existingWakeup = await tx - .select({ coalescedCount: agentWakeupRequests.coalescedCount }) - .from(agentWakeupRequests) - .where( - eq( - agentWakeupRequests.id, - existingContinuation.wakeupRequestId, - ), - ) - .then((rows) => rows[0] ?? null); - - await tx - .update(agentWakeupRequests) - .set({ - coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, - updatedAt: now, - }) - .where( - eq( - agentWakeupRequests.id, - existingContinuation.wakeupRequestId, - ), - ); - } - - return { - outcome: "scheduled", - run: existingContinuation, - reusedExisting: true, - }; - } - } - - if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) { - if (issueId) { - await tx.execute( - sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, - ); - } else { - await tx.execute( - sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, - ); - } - - const existingContinuation = await tx - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, run.companyId), - eq(heartbeatRuns.retryOfRunId, run.id), - eq(heartbeatRuns.scheduledRetryReason, retryReason), - eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), - inArray(heartbeatRuns.status, [ - ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, - ]), - issueId - ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` - : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, - ), - ) - .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) - .limit(1) - .then((rows) => rows[0] ?? null); - - if (existingContinuation) { - if (existingContinuation.wakeupRequestId) { - const existingWakeup = await tx - .select({ coalescedCount: agentWakeupRequests.coalescedCount }) - .from(agentWakeupRequests) - .where( - eq( - agentWakeupRequests.id, - existingContinuation.wakeupRequestId, - ), - ) - .then((rows) => rows[0] ?? null); - - await tx - .update(agentWakeupRequests) - .set({ - coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, - updatedAt: now, - }) - .where( - eq( - agentWakeupRequests.id, - existingContinuation.wakeupRequestId, - ), - ); - } - - return { - outcome: "scheduled", - run: existingContinuation, - reusedExisting: true, - }; - } - - if (issueId) { - const lockedIssue = await tx - .select({ - id: issues.id, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - executionRunId: issues.executionRunId, - }) - .from(issues) - .where( - and( - eq(issues.id, issueId), - eq(issues.companyId, run.companyId), - ), - ) - .then((rows) => rows[0] ?? null); - - if (!lockedIssue) { - return { - outcome: "not_scheduled", - reason: - "Scheduled max-turn continuation suppressed because the target issue no longer exists", - errorCode: "issue_not_found", - issueId, - details: { issueId }, - }; - } - - if (lockedIssue.assigneeAgentId !== run.agentId) { - return { - outcome: "not_scheduled", - reason: - "Scheduled max-turn continuation suppressed because issue ownership changed", - errorCode: "issue_reassigned", - issueId, - details: { - issueId, - previousAssigneeAgentId: run.agentId, - currentAssigneeAgentId: lockedIssue.assigneeAgentId, - }, - }; - } - - if ( - lockedIssue.status === "cancelled" || - lockedIssue.status === "done" - ) { - return { - outcome: "not_scheduled", - reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`, - errorCode: - lockedIssue.status === "cancelled" - ? "issue_cancelled" - : "issue_terminal_status", - issueId, - details: { issueId, currentStatus: lockedIssue.status }, - }; - } - - if (lockedIssue.status !== "in_progress") { - return { - outcome: "not_scheduled", - reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`, - errorCode: "issue_not_in_progress", - issueId, - details: { - issueId, - currentStatus: lockedIssue.status, - requiredStatus: "in_progress", - }, - }; - } - - if (lockedIssue.executionRunId !== run.id) { - return { - outcome: "not_scheduled", - reason: - "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", - errorCode: "issue_execution_lock_changed", - issueId, - details: { - issueId, - expectedExecutionRunId: run.id, - currentExecutionRunId: lockedIssue.executionRunId, - }, - }; - } - } - } - - if ( - (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) && issueId && - !isNonAssigneeWorkspaceBusyRetry(retryReason, contextSnapshot) - ) { - // The issue row is locked above. Recheck after the preflight gate so - // cancellation or recovery cannot leave a successor without its lock. - const [lockedIssue] = await tx.select({ executionRunId: issues.executionRunId }) - .from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); - if (lockedIssue?.executionRunId !== run.id) { - return { - outcome: "not_scheduled", issueId, errorCode: "issue_execution_lock_changed", - reason: "Subscription retry suppressed because the task execution lock changed", - details: { issueId, expectedExecutionRunId: run.id, currentExecutionRunId: lockedIssue?.executionRunId ?? null }, - }; - } - } - - const scheduledRunId = randomUUID(); - if (contextSnapshot.explicitUserContinuation) { - const continuation = issueId && retryReason === "transient_failure" ? await admitExplicitContinuationRetry({ - db: tx as unknown as Db, companyId: run.companyId, issueId, agentId: run.agentId, - parentRunId: run.id, successorRunId: scheduledRunId, now, - }) : null; - if (!continuation) return { - outcome: "not_scheduled", issueId, - errorCode: "continuation_user_authorization_missing", - reason: "The automatic retry could not revalidate the original user continuation.", - details: {}, - }; - retryContextSnapshot.explicitUserContinuation = continuation; - retryContextSnapshot.previousRunId = continuation.previousRunId; - } - - const wakeupRequest = await tx - .insert(agentWakeupRequests) - .values({ - companyId: run.companyId, - agentId: run.agentId, - source: "automation", - triggerDetail: "system", - reason: wakeReason, - payload: withRecoveryContext( - { - ...(issueId ? { issueId } : {}), - retryOfRunId: run.id, - ...interactionContinuationPayload, - retryReason, - ...(transientRecovery - ? { errorFamily: transientRecovery.errorFamily } - : {}), - scheduledRetryAttempt: schedule.attempt, - scheduledRetryAt: schedule.dueAt.toISOString(), - ...(transientRetryNotBefore - ? { - transientRetryNotBefore: - transientRetryNotBefore.toISOString(), - } - : {}), - ...(transientRecovery?.errorFamily === "provider_quota" && - transientRetryNotBefore - ? { - providerQuotaRetryNotBefore: - transientRetryNotBefore.toISOString(), - } - : {}), - ...(codexTransientFallbackMode - ? { codexTransientFallbackMode } - : {}), - }, - "normal_model", - ), - status: "queued", - requestedByActorType: "system", - requestedByActorId: null, - idempotencyKey: continuationRetryIdempotencyKey, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - const scheduledRun = await tx - .insert(heartbeatRuns) - .values({ - id: scheduledRunId, - companyId: run.companyId, - agentId: run.agentId, - scopeKind: run.scopeKind, - issueId, - invocationSource: "automation", - triggerDetail: "system", - status: "scheduled_retry", - wakeupRequestId: wakeupRequest.id, - contextSnapshot: retryContextSnapshot, - ...(hasConversationContinuationPolicy(run.resultJson) - ? { resultJson: { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } } : {}), - responsibleUserId, - sessionIdBefore: sessionBefore, - retryOfRunId: run.id, - scheduledRetryAt: schedule.dueAt, - scheduledRetryAttempt: schedule.attempt, - scheduledRetryReason: retryReason, - continuationAttempt: readContinuationAttempt( - retryContextSnapshot.livenessContinuationAttempt, - ), - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - await tx - .update(agentWakeupRequests) - .set({ - runId: scheduledRun.id, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, wakeupRequest.id)); - - let detachWorkspaceFromIssue = false; - if (issueId && shouldQuarantineWorkspaceForRetry) { - const issueWorkspace = await tx - .select({ - id: issues.id, - companyId: issues.companyId, - executionWorkspaceId: issues.executionWorkspaceId, - }) - .from(issues) - .where( - and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), - ) - .for("update") - .then((rows) => rows[0] ?? null); - const failedExecutionWorkspaceId = - readNonEmptyString( - workspaceValidationRetryPayload?.executionWorkspaceId, - ) ?? readNonEmptyString(issueWorkspace?.executionWorkspaceId); - - if (issueWorkspace && failedExecutionWorkspaceId) { - const failedWorkspace = await tx - .select({ - id: executionWorkspaces.id, - companyId: executionWorkspaces.companyId, - sourceIssueId: executionWorkspaces.sourceIssueId, - status: executionWorkspaces.status, - metadata: executionWorkspaces.metadata, - }) - .from(executionWorkspaces) - .where( - and( - eq(executionWorkspaces.id, failedExecutionWorkspaceId), - eq(executionWorkspaces.companyId, run.companyId), - ), - ) - .for("update") - .then((rows) => rows[0] ?? null); - - const workspaceBelongsToIssue = failedWorkspace - ? failedWorkspace.sourceIssueId === issueId - : false; - - if ( - failedWorkspace && - workspaceBelongsToIssue && - issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId - ) { - const existingMetadata = parseObject(failedWorkspace.metadata); - const quarantine = { - reason: WORKSPACE_VALIDATION_FAILURE_CODE, - retryReason, - sourceRunId: run.id, - retryRunId: scheduledRun.id, - issueId, - sourceIssueId: failedWorkspace.sourceIssueId ?? null, - quarantinedAt: now.toISOString(), - workspaceValidation: workspaceValidationRetryPayload ?? {}, - }; - await tx - .update(executionWorkspaces) - .set({ - status: "archived", - closedAt: now, - cleanupEligibleAt: null, - cleanupReason: WORKSPACE_VALIDATION_FAILURE_CODE, - metadata: { - ...existingMetadata, - workspaceValidationQuarantine: quarantine, - }, - updatedAt: now, - }) - .where( - and( - eq(executionWorkspaces.id, failedWorkspace.id), - eq(executionWorkspaces.companyId, run.companyId), - ), - ); - - await logActivity(tx as unknown as Db, { - companyId: run.companyId, - actorType: "system", - actorId: "heartbeat", - agentId: run.agentId, - runId: run.id, - action: "execution_workspace.workspace_validation_quarantined", - entityType: "execution_workspace", - entityId: failedWorkspace.id, - details: quarantine, - }); - detachWorkspaceFromIssue = - issueWorkspace.executionWorkspaceId === - failedExecutionWorkspaceId; - } - } - } - - if (issueId) { - await tx - .update(issues) - .set({ - executionRunId: scheduledRun.id, - checkoutRunId: sql`case when ${issues.checkoutRunId} = ${run.id} then null else ${issues.checkoutRunId} end`, - executionAgentNameKey: normalizeAgentNameKey(agent.name), - executionLockedAt: now, - ...(detachWorkspaceFromIssue - ? { - executionWorkspaceId: null, - executionWorkspacePreference: null, - } - : {}), - updatedAt: now, - }) - .where( - and( - eq(issues.id, issueId), - eq(issues.companyId, run.companyId), - eq(issues.executionRunId, run.id), - ), - ); - } - - return { - outcome: "scheduled", - run: scheduledRun, - reusedExisting: false, - }; - }, - ); - - if (scheduleResult.outcome === "not_scheduled") { - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: scheduleResult.reason, - payload: { - retryReason, - scheduledRetryAttempt: nextAttempt, - maxAttempts, - ...scheduleResult.details, - }, - }); - return { - outcome: "not_scheduled" as const, - reason: scheduleResult.reason, - errorCode: scheduleResult.errorCode, - issueId: scheduleResult.issueId, - }; - } - - const retryRun = scheduleResult.run; - const dueAt = retryRun.scheduledRetryAt - ? new Date(retryRun.scheduledRetryAt) - : schedule.dueAt; - - if (scheduleResult.reusedExisting) { - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "info", - message: `Reused existing continuation retry ${retryRun.scheduledRetryAttempt}/${schedule.maxAttempts}`, - payload: { - retryRunId: retryRun.id, - retryReason, - idempotencyKey: continuationRetryIdempotencyKey, - scheduledRetryAttempt: retryRun.scheduledRetryAttempt, - scheduledRetryAt: dueAt.toISOString(), - }, - }); - - return { - outcome: "scheduled" as const, - run: retryRun, - dueAt, - attempt: retryRun.scheduledRetryAttempt, - maxAttempts: schedule.maxAttempts, - reusedExisting: true, - }; - } - - await appendRunEvent(run, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: `Scheduled bounded retry ${schedule.attempt}/${schedule.maxAttempts} for ${schedule.dueAt.toISOString()}`, - payload: { - retryRunId: retryRun.id, - retryReason, - ...(transientRecovery - ? { errorFamily: transientRecovery.errorFamily } - : {}), - scheduledRetryAttempt: schedule.attempt, - scheduledRetryAt: schedule.dueAt.toISOString(), - baseDelayMs: schedule.baseDelayMs, - delayMs: schedule.delayMs, - ...(transientRetryNotBefore - ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } - : {}), - ...(transientRecovery?.errorFamily === "provider_quota" && - transientRetryNotBefore - ? { - providerQuotaRetryNotBefore: - transientRetryNotBefore.toISOString(), - } - : {}), - ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), - }, - }); - - if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { - await recordPlanApprovalResumeFailureRetry({ - run, - issueId, - retryRunId: retryRun.id, - attempt: schedule.attempt, - maxAttempts: schedule.maxAttempts, - }).catch((error) => { - logger.warn( - { err: error, runId: run.id, issueId, retryRunId: retryRun.id }, - "failed to record plan-approval resume retry failure", - ); - }); - } - - return { - outcome: "scheduled" as const, - run: retryRun, - dueAt, - attempt: schedule.attempt, - maxAttempts: schedule.maxAttempts, - }; - } - - // Finds a running heartbeat run (other than the caller's) whose context - // issue shares the same project workspace, i.e. the run that currently - // "holds" the shared working tree. Runs that have been silent past - // WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS do not count — a zombie holder must - // not park other work forever. This cutoff is independent of informational - // silence warnings. When isolated workspaces are enabled, holders whose - // issue explicitly opted into an isolated workspace never touch the shared - // tree, so they are excluded; a NULL/agent_default mode may resolve to the - // shared tree and counts as a holder (over-serializing is the safe - // direction). When the isolated-workspaces experiment is off, every run - // resolves to the shared tree, so no holder is excluded. - async function findSharedWorkspaceHolder(input: { - companyId: string; - projectWorkspaceId: string; - excludeIssueId: string; - excludeRunId: string; - honorIsolatedWorkspaceModes: boolean; - now?: Date; - }): Promise { - const staleCutoff = new Date( - (input.now ?? new Date()).getTime() - - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, - ); - return await db - .select({ - runId: heartbeatRuns.id, - agentId: heartbeatRuns.agentId, - issueId: sql`${issues.id}::text`, - issueIdentifier: issues.identifier, - }) - .from(heartbeatRuns) - .innerJoin( - issues, - and( - eq(issues.companyId, heartbeatRuns.companyId), - sql`${issues.id}::text = ${heartbeatRuns.contextSnapshot} ->> 'issueId'`, - ), - ) - .where( - and( - eq(heartbeatRuns.companyId, input.companyId), - eq(heartbeatRuns.status, "running"), - ne(heartbeatRuns.id, input.excludeRunId), - // Last observed activity: output beats start beats creation. A run - // that started recently but has not written output yet is live. - sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) >= ${staleCutoff.toISOString()}::timestamptz`, - eq(issues.projectWorkspaceId, input.projectWorkspaceId), - ne(sql`${issues.id}::text`, input.excludeIssueId), - ...(input.honorIsolatedWorkspaceModes - ? [ - or( - // Covers both a NULL settings blob and a blob without a mode - // key; either may still resolve to the shared workspace. - sql`${issues.executionWorkspaceSettings} ->> 'mode' is null`, - notInArray( - sql`${issues.executionWorkspaceSettings} ->> 'mode'`, - [...ISOLATED_EXECUTION_WORKSPACE_MODES], - ), - ), - ] - : []), - ), - ) - .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) - .limit(1) - .then((rows) => rows[0] ?? null); - } - - // Credential rotation can briefly contend with a fresh runtime read. Keep - // the task on its automatic pre-provider retry path while the lock clears. - async function finalizeAiConnectionBusyDeferral( - run: typeof heartbeatRuns.$inferSelect, - error: HttpError, - wasIssueAssignee: boolean, - ) { - const now = new Date(); - const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { - error: error.message, errorCode: AI_CONNECTION_BUSY_RETRY_REASON, finishedAt: now, - resultJson: { - executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false }, - cancellation: { - source: "control_plane", - expected: true, - initiator: { type: "system" }, - reason: "Waiting for shared AI credentials", - recordedAt: now.toISOString(), - }, - }, - contextSnapshot: { - ...parseObject(run.contextSnapshot), - aiConnectionBusyDeferredWhileAssignee: wasIssueAssignee, - }, - }); - if (!cancelled.updated) return; - await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: error.message }).catch(() => undefined); - const cancelledRun = cancelled.run ?? await getRun(run.id); - const agent = await getAgent(run.agentId); - let scheduled = false; - try { - if (cancelledRun && agent) { - const retry = await scheduleBoundedRetryForRun(cancelledRun, agent, { - now, retryReason: AI_CONNECTION_BUSY_RETRY_REASON, wakeReason: "ai_connection_busy_retry", - maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, - delayMs: computeWorkspaceBusyRetryDelayMs(), - }); - scheduled = retry.outcome === "scheduled"; - await appendRunEvent(cancelledRun, { - eventType: "lifecycle", stream: "system", level: "info", - message: scheduled ? "Waiting for the shared AI subscription. This task will retry automatically." : "The AI subscription is busy; this task can no longer retry automatically.", - payload: { retryScheduled: scheduled }, - }); - } - } finally { - try { - if (cancelledRun && !scheduled) await releaseIssueExecutionAndPromote(cancelledRun); - } finally { - await finalizeAgentStatus(run.agentId, "cancelled", null, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) }); - } - } - } - - // Terminal handling for a WorkspaceBusyDeferral thrown by the pre-dispatch - // gate: cancel the run (contention is not a failure), schedule a - // workspace_busy retry, and leave the agent idle. The issue execution lock - // transfers to the scheduled retry run inside scheduleBoundedRetryForRun, so - // the issue keeps an active execution path and recovery leaves it alone. - // Deferral has no attempt ceiling — the retry keeps rescheduling while a - // live holder exists, and holder staleness (not a counter) is what prevents - // waiting on a zombie. If no retry could be scheduled (agent no longer - // invokable), the lock is released so the issue does not strand on a - // cancelled run. - async function finalizeWorkspaceBusyDeferral( - run: typeof heartbeatRuns.$inferSelect, - deferral: WorkspaceBusyDeferral, - ) { - const now = new Date(); - const cancelWrite = await setRunStatusIfRunning(run.id, "cancelled", { - error: deferral.message, - errorCode: WORKSPACE_BUSY_ERROR_CODE, - finishedAt: now, - resultJson: { - executionRecovery: { - kind: "workspace_wait", - providerWorkStarted: false, - }, - cancellation: { - source: "control_plane", - expected: true, - initiator: { type: "system" }, - reason: "Waiting for the shared project workspace", - recordedAt: now.toISOString(), - }, - workspaceBusy: { - projectWorkspaceId: deferral.projectWorkspaceId, - holderRunId: deferral.holder.runId, - holderIssueId: deferral.holder.issueId, - deferralAttempt: deferral.deferralAttempt, - }, - }, - // Recorded on the run (and inherited by the scheduled retry's context) - // so the retry promotion gate can tell a non-assignee wake — where an - // assignee mismatch is the expected state — from a reassignment race. - contextSnapshot: { - ...parseObject(run.contextSnapshot), - workspaceBusyDeferredWhileAssignee: deferral.wasIssueAssignee, - }, - }); - if (!cancelWrite.updated) { - logger.info( - { runId: run.id, currentStatus: cancelWrite.run?.status ?? null }, - "skipping workspace-busy deferral finalization because the run already left running state", - ); - return; - } - await setWakeupStatus(run.wakeupRequestId, "cancelled", { - finishedAt: now, - error: deferral.message, - }).catch(() => undefined); - - const cancelledRun = - cancelWrite.run ?? (await getRun(run.id).catch(() => null)); - const agentRow = await getAgent(run.agentId).catch(() => null); - let scheduleOutcome: string | null = null; - if (cancelledRun && agentRow) { - const scheduleResult = await scheduleBoundedRetryForRun( - cancelledRun, - agentRow, - { - now, - retryReason: WORKSPACE_BUSY_RETRY_REASON, - wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON, - // Always admit the next attempt: workspace-busy deferral is bounded by - // holder liveness, not by an attempt counter. - maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, - delayMs: computeWorkspaceBusyRetryDelayMs(), - }, - ).catch((scheduleErr) => { - logger.error( - { err: scheduleErr, runId: run.id }, - "failed to schedule workspace-busy retry after deferral", - ); - return null; - }); - scheduleOutcome = scheduleResult?.outcome ?? null; - } - - if (cancelledRun) { - await appendRunEvent(cancelledRun, { - eventType: "lifecycle", - stream: "system", - level: "info", - message: - scheduleOutcome === "scheduled" - ? `Deferred: ${deferral.message}. Retry ${deferral.deferralAttempt + 1} scheduled; the run waits for the workspace to free.` - : `Deferred: ${deferral.message}. No retry could be scheduled; releasing the issue for other runs.`, - payload: { - projectWorkspaceId: deferral.projectWorkspaceId, - holderRunId: deferral.holder.runId, - holderIssueId: deferral.holder.issueId, - deferralAttempt: deferral.deferralAttempt, - retryScheduled: scheduleOutcome === "scheduled", - }, - }).catch(() => undefined); - } - - if (cancelledRun && scheduleOutcome !== "scheduled") { - await releaseIssueExecutionAndPromote(cancelledRun).catch( - (releaseErr) => { - logger.error( - { err: releaseErr, runId: run.id }, - "failed to release issue execution after workspace-busy deferral", - ); - }, - ); - } - - await finalizeAgentStatus(run.agentId, "cancelled", null, { - wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - }).catch(() => undefined); - } - - async function scheduleInteractionContinuationInfrastructureRetryIfEligible( - run: typeof heartbeatRuns.$inferSelect, - agent: typeof agents.$inferSelect, - ) { - if (!run.wakeupRequestId) return null; - if (!isResolvedInteractionContinuationWakeContext(run.contextSnapshot)) - return null; - if (!isRetryableInteractionContinuationInfrastructureFailure(run)) { - const context = parseObject(run.contextSnapshot); - const issueId = readNonEmptyString(context.issueId); - await escalatePlanApprovalResumeFailureNeedsAttention({ - run, - issueId, - attempt: Math.min( - run.scheduledRetryAttempt ?? - INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, - INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, - ), - maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, - }).catch((error) => { - logger.warn( - { err: error, runId: run.id, issueId }, - "failed to escalate non-retryable plan-approval resume failure", - ); - }); - return null; - } - - return scheduleBoundedRetryForRun(run, agent, { - retryReason: INTERACTION_CONTINUATION_INFRA_RETRY_REASON, - wakeReason: INTERACTION_CONTINUATION_INFRA_WAKE_REASON, - maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, - }); - } - - async function promoteDueScheduledRetries(now = new Date()) { - const cutoff = await getWorktreeExecutionCutoff(); - const result = await runDispatch.promoteDueScheduledRetries({ - now, - cutoff, - }); - applyRunDispatchPostCommitEffects(result.postCommitEffects); - return { promoted: result.promoted, runIds: result.runIds }; - } - - async function getIssueRetryRun( - companyId: string, - issueId: string, - statuses: Array<"scheduled_retry" | "queued" | "running" | "cancelled">, - ) { - if (statuses.length === 0) return null; - return db - .select({ - run: heartbeatRuns, - agentName: agents.name, - }) - .from(heartbeatRuns) - .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) - .where( - and( - eq(heartbeatRuns.companyId, companyId), - inArray(heartbeatRuns.status, statuses), - sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, - sql`${heartbeatRuns.retryOfRunId} is not null`, - ), - ) - .orderBy( - desc(heartbeatRuns.updatedAt), - desc(heartbeatRuns.createdAt), - desc(heartbeatRuns.id), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - } - - function summarizeIssueScheduledRetryRun(row: { - run: typeof heartbeatRuns.$inferSelect; - agentName: string | null; - }) { - return { - runId: row.run.id, - status: row.run.status as - "scheduled_retry" | "queued" | "running" | "cancelled", - agentId: row.run.agentId, - agentName: row.agentName, - retryOfRunId: row.run.retryOfRunId, - scheduledRetryAt: row.run.scheduledRetryAt, - scheduledRetryAttempt: row.run.scheduledRetryAttempt, - scheduledRetryReason: row.run.scheduledRetryReason, - error: row.run.error, - errorCode: row.run.errorCode, - }; - } - - async function retryScheduledRetryNow(input: { - issueId: string; - actor?: { - actorType?: "user" | "agent" | "system"; - actorId?: string | null; - }; - now?: Date; - }) { - const now = input.now ?? new Date(); - const issue = await db - .select({ id: issues.id, companyId: issues.companyId }) - .from(issues) - .where(eq(issues.id, input.issueId)) - .then((rows) => rows[0] ?? null); - if (!issue) throw notFound("Issue not found"); - - const scheduled = await getIssueRetryRun(issue.companyId, issue.id, [ - "scheduled_retry", - ]); - if (!scheduled) { - const alreadyPromoted = await getIssueRetryRun( - issue.companyId, - issue.id, - ["queued", "running"], - ); - if (alreadyPromoted) { - return { - outcome: "already_promoted" as const, - message: "Scheduled retry was already promoted", - scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted), - }; - } - return { - outcome: "no_scheduled_retry" as const, - message: "No live scheduled retry exists for this issue", - scheduledRetry: null, - }; - } - - const contextSnapshot = { - ...parseObject(scheduled.run.contextSnapshot), - scheduledRetryAt: now.toISOString(), - retryNowRequestedAt: now.toISOString(), - retryNowRequestedByActorType: input.actor?.actorType ?? null, - retryNowRequestedByActorId: input.actor?.actorId ?? null, - }; - - const updated = await db.transaction(async (tx) => { - const row = await tx - .update(heartbeatRuns) - .set({ - scheduledRetryAt: now, - contextSnapshot, - updatedAt: now, - }) - .where( - and( - eq(heartbeatRuns.id, scheduled.run.id), - eq(heartbeatRuns.status, "scheduled_retry"), - ), - ) - .returning() - .then((rows) => rows[0] ?? null); - if (!row) return null; - - if (row.wakeupRequestId) { - const wakeupPayload = { - ...parseObject( - await tx - .select({ payload: agentWakeupRequests.payload }) - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.id, row.wakeupRequestId)) - .then((rows) => rows[0]?.payload ?? null), - ), - scheduledRetryAt: now.toISOString(), - retryNowRequestedAt: now.toISOString(), - }; - await tx - .update(agentWakeupRequests) - .set({ - payload: wakeupPayload, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, row.wakeupRequestId)); - } - - return row; - }); - - if (!updated) { - const alreadyPromoted = await getIssueRetryRun( - issue.companyId, - issue.id, - ["queued", "running"], - ); - if (alreadyPromoted) { - return { - outcome: "already_promoted" as const, - message: "Scheduled retry was already promoted", - scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted), - }; - } - return { - outcome: "no_scheduled_retry" as const, - message: "No live scheduled retry exists for this issue", - scheduledRetry: null, - }; - } - - await appendRunEvent(updated, { - eventType: "lifecycle", - stream: "system", - level: "info", - message: "Scheduled retry was requested to run now", - payload: { - issueId: issue.id, - scheduledRetryAttempt: updated.scheduledRetryAttempt, - scheduledRetryAt: updated.scheduledRetryAt - ? new Date(updated.scheduledRetryAt).toISOString() - : null, - scheduledRetryReason: updated.scheduledRetryReason, - requestedByActorType: input.actor?.actorType ?? null, - requestedByActorId: input.actor?.actorId ?? null, - }, - }); - - const promotion = await runDispatch.promoteScheduledRetry({ - runId: updated.id, - companyId: updated.companyId, - now, - }); - if (promotion.outcome === "promoted") { - applyRunDispatchPostCommitEffects(promotion.postCommitEffects); - } - // Promotion can preserve this row as a cleanup wait. Read that exact run, - // not an older cancelled retry or the pre-promotion schedule. - const currentRun = await getRun(updated.id); - const scheduledRetry = currentRun - ? summarizeIssueScheduledRetryRun({ - run: currentRun, - agentName: scheduled.agentName, - }) - : null; - - if (currentRun?.status === "scheduled_retry") { - return { - outcome: "waiting" as const, - message: parseObject(currentRun.resultJson?.executionWait).cause === "execution_owner_active" - ? "Waiting for execution cleanup. Paperclip will retry automatically once cleanup finishes." - : "The retry remains scheduled. Paperclip will check again at the scheduled time.", - scheduledRetry, - }; - } - - if (promotion.outcome === "promoted") { - return { - outcome: "promoted" as const, - message: "Scheduled retry was promoted to the queued run pool", - scheduledRetry, - }; - } - if (promotion.outcome === "gate_suppressed") { - return { - outcome: "gate_suppressed" as const, - message: promotion.reason, - scheduledRetry, - }; - } - if (currentRun && ["queued", "running"].includes(currentRun.status)) { - return { - outcome: "already_promoted" as const, - message: "Scheduled retry was already promoted", - scheduledRetry, - }; - } - return { - outcome: "no_scheduled_retry" as const, - message: "No live scheduled retry exists for this issue", - scheduledRetry: null, - }; - } - function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) { const runtimeConfig = parseObject(agent.runtimeConfig); const heartbeat = parseObject(runtimeConfig.heartbeat); @@ -8766,14 +7149,6 @@ export function heartbeatService( return { wasFirstHeartbeat: !agent.lastHeartbeatAt }; } - function timerClaimWasFirstHeartbeat( - run: Pick, - ): true | undefined { - return parseObject(run.contextSnapshot).timerClaimWasFirstHeartbeat === true - ? true - : undefined; - } - function parseMaxTurnContinuationPolicy( agent: typeof agents.$inferSelect, ): MaxTurnContinuationPolicy { @@ -22108,23 +20483,7 @@ export function heartbeatService( resumeQueuedRuns, - scheduleBoundedRetry: async ( - runId: string, - opts?: { - now?: Date; - random?: () => number; - retryReason?: string; - wakeReason?: string; - maxAttempts?: number; - delayMs?: number; - }, - ) => { - const run = await getRun(runId, { unsafeFullResultJson: true }); - if (!run) return { outcome: "missing_run" as const }; - const agent = await getAgent(run.agentId); - if (!agent) return { outcome: "missing_agent" as const }; - return scheduleBoundedRetryForRun(run, agent, opts); - }, + scheduleBoundedRetry, reconcileStrandedAssignedIssues, recoverPendingSessionGoalActions, diff --git a/server/src/services/heartbeat/retries.test.ts b/server/src/services/heartbeat/retries.test.ts new file mode 100644 index 00000000000..635e8ca9a5f --- /dev/null +++ b/server/src/services/heartbeat/retries.test.ts @@ -0,0 +1,286 @@ +import { randomUUID } from "node:crypto"; +import type { Db } from "@paperclipai/db"; +import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRunEvents, heartbeatRuns, issues } from "@paperclipai/db"; +import { and, eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { HttpError } from "../../errors.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { createRunDispatch, type PostCommitEffect } from "../../modules/run-dispatch/index.js"; +import { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import * as legacy from "../heartbeat.js"; +import { createHeartbeatRunState } from "./run-state.js"; +import * as extracted from "./retries.js"; +import type { HeartbeatRetryDependencies } from "./retries.js"; + +type Run = typeof heartbeatRuns.$inferSelect; +type Agent = typeof agents.$inferSelect; + +function callbacks() { + return { + getRun: vi.fn(async () => null), + getAgent: vi.fn(async () => undefined), + resolveSessionBeforeForWakeup: vi.fn(async () => null), + resolveResponsibleUserIdForRunContext: vi.fn(async () => "retry-user"), + getAgentInvokability: vi.fn(async () => ({ invokable: true })), + appendRunEvent: vi.fn(async () => undefined), + escalatePlanApprovalResumeFailureNeedsAttention: vi.fn(async () => null), + recordPlanApprovalResumeFailureRetry: vi.fn(async () => null), + setRunStatusIfRunning: vi.fn(async () => ({ updated: false, run: null })), + setWakeupStatus: vi.fn(async () => undefined), + releaseIssueExecutionAndPromote: vi.fn(async () => undefined), + finalizeAgentStatus: vi.fn(async () => undefined), + getWorktreeExecutionCutoff: vi.fn(async (): Promise => null), + applyRunDispatchPostCommitEffects: vi.fn<(effects: PostCommitEffect[]) => void>(), + runDispatch: { + evaluateScheduledRetryGate: vi.fn(async () => ({ allowed: true })), + promoteDueScheduledRetries: vi.fn(async () => ({ promoted: 0, runIds: [], postCommitEffects: [] })), + promoteScheduledRetry: vi.fn(async () => ({ outcome: "not_promoted" })), + }, + } satisfies HeartbeatRetryDependencies; +} + +// These tests must never query: only the injected lifecycle callbacks may run. +function guardedDatabase() { + const access = vi.fn(() => { throw new Error("Unexpected retry database access"); }); + return { db: new Proxy({}, { get: access }) as Db, access }; +} + +function unitRun(): Run { + const run: Partial = { + id: "run", companyId: "company", agentId: "agent", status: "running", + scopeKind: "issue", issueId: "issue", + wakeupRequestId: "wake", scheduledRetryAttempt: 0, + contextSnapshot: { issueId: "issue", timerClaimWasFirstHeartbeat: true }, + }; + return run as Run; +} + +function deferral() { + return new extracted.WorkspaceBusyDeferral({ + holder: { runId: "holder", agentId: "other-agent", issueId: "other-issue", issueIdentifier: "T-2" }, + projectWorkspaceId: "workspace", deferralAttempt: 0, wasIssueAssignee: true, + }); +} + +describe("heartbeat retry module callbacks", () => { + it("preserves legacy helper and error-class identity", () => { + expect(legacy.computeBoundedTransientHeartbeatRetrySchedule).toBe(extracted.computeBoundedTransientHeartbeatRetrySchedule); + expect(legacy.computeWorkspaceBusyRetryDelayMs).toBe(extracted.computeWorkspaceBusyRetryDelayMs); + expect(legacy.BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS).toBe(extracted.BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS); + expect(legacy.WorkspaceBusyDeferral).toBe(extracted.WorkspaceBusyDeferral); + expect(deferral()).toBeInstanceOf(legacy.WorkspaceBusyDeferral); + }); + + it("constructs without database access or lifecycle effects", () => { + const database = guardedDatabase(); + const deps = callbacks(); + extracted.createHeartbeatRetries(database.db, deps); + expect(database.access).not.toHaveBeenCalled(); + for (const callback of Object.values(deps)) { + if (typeof callback === "function") expect(callback).not.toHaveBeenCalled(); + } + for (const callback of Object.values(deps.runDispatch)) expect(callback).not.toHaveBeenCalled(); + }); + + it.each(["missing_run", "missing_agent"] as const)("keeps %s lookup behavior and requests the full source result", async (outcome) => { + const database = guardedDatabase(); + const deps = callbacks(); + if (outcome === "missing_agent") deps.getRun.mockResolvedValue(unitRun()); + expect(await extracted.createHeartbeatRetries(database.db, deps).scheduleBoundedRetry("run")).toEqual({ outcome }); + expect(deps.getRun).toHaveBeenCalledWith("run", { unsafeFullResultJson: true }); + expect(deps.getAgent).toHaveBeenCalledTimes(outcome === "missing_agent" ? 1 : 0); + expect(database.access).not.toHaveBeenCalled(); + }); + + it("records exhaustion before escalating a plan-resume failure", async () => { + const database = guardedDatabase(); + const deps = callbacks(); + const run = unitRun(); + const retry = await extracted.createHeartbeatRetries(database.db, deps).scheduleBoundedRetryForRun( + run, { adapterType: "codex_local" } as Agent, + { retryReason: legacy.INTERACTION_CONTINUATION_INFRA_RETRY_REASON, maxAttempts: 0 }, + ); + expect(retry).toMatchObject({ outcome: "retry_exhausted", attempt: 1, maxAttempts: 0 }); + expect(deps.appendRunEvent).toHaveBeenCalledWith(run, expect.objectContaining({ + retryExhaustion: { retryReason: legacy.INTERACTION_CONTINUATION_INFRA_RETRY_REASON, scheduledRetryAttempt: 0, maxAttempts: 0 }, + })); + expect(deps.escalatePlanApprovalResumeFailureNeedsAttention).toHaveBeenCalledAfter(deps.appendRunEvent); + expect(database.access).not.toHaveBeenCalled(); + }); + + it("passes the service's worktree cutoff and publishes only returned effects", async () => { + const database = guardedDatabase(); + const deps = callbacks(); + const now = new Date("2026-10-08T12:00:00Z"); + const cutoff = new Date("2026-10-08T11:00:00Z"); + const effect: PostCommitEffect = { + kind: "run_queued", runId: "retry", companyId: "company", agentId: "agent", + invocationSource: "automation", triggerDetail: "system", wakeupRequestId: "wake", + }; + deps.getWorktreeExecutionCutoff.mockResolvedValue(cutoff); + deps.runDispatch.promoteDueScheduledRetries.mockResolvedValue({ promoted: 1, runIds: ["retry"], postCommitEffects: [effect] }); + expect(await extracted.createHeartbeatRetries(database.db, deps).promoteDueScheduledRetries(now)).toEqual({ promoted: 1, runIds: ["retry"] }); + expect(deps.runDispatch.promoteDueScheduledRetries).toHaveBeenCalledWith({ now, cutoff }); + expect(deps.applyRunDispatchPostCommitEffects).toHaveBeenCalledWith([effect]); + expect(deps.applyRunDispatchPostCommitEffects).toHaveBeenCalledAfter(deps.runDispatch.promoteDueScheduledRetries); + expect(database.access).not.toHaveBeenCalled(); + }); + + it("does not publish effects when promotion rejects", async () => { + const deps = callbacks(); + deps.runDispatch.promoteDueScheduledRetries.mockRejectedValue(new Error("promotion failed")); + await expect(extracted.createHeartbeatRetries(guardedDatabase().db, deps).promoteDueScheduledRetries()).rejects.toThrow("promotion failed"); + expect(deps.applyRunDispatchPostCommitEffects).not.toHaveBeenCalled(); + }); + + it.each(["workspace", "connection"] as const)("leaves a %s cancellation race winner alone", async (kind) => { + const database = guardedDatabase(); + const deps = callbacks(); + const retries = extracted.createHeartbeatRetries(database.db, deps); + if (kind === "workspace") await retries.finalizeWorkspaceBusyDeferral(unitRun(), deferral()); + else await retries.finalizeAiConnectionBusyDeferral(unitRun(), new HttpError(409, "busy"), true); + expect(deps.setRunStatusIfRunning).toHaveBeenCalledTimes(1); + expect(deps.setWakeupStatus).not.toHaveBeenCalled(); + expect(deps.getAgent).not.toHaveBeenCalled(); + expect(deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled(); + expect(deps.finalizeAgentStatus).not.toHaveBeenCalled(); + expect(database.access).not.toHaveBeenCalled(); + }); + + it.each(["workspace", "connection"] as const)("settles the agent after a %s deferral cannot keep its issue lock", async (kind) => { + const database = guardedDatabase(); + const deps = callbacks(); + const run = { ...unitRun(), status: "cancelled" }; + deps.setRunStatusIfRunning.mockResolvedValue({ updated: true, run }); + deps.releaseIssueExecutionAndPromote.mockRejectedValue(new Error("release failed")); + const retries = extracted.createHeartbeatRetries(database.db, deps); + if (kind === "workspace") await retries.finalizeWorkspaceBusyDeferral(run, deferral()); + else await expect(retries.finalizeAiConnectionBusyDeferral(run, new HttpError(409, "busy"), true)).rejects.toThrow("release failed"); + expect(deps.setWakeupStatus).toHaveBeenCalledWith("wake", "cancelled", expect.any(Object)); + expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledWith(run); + expect(deps.finalizeAgentStatus).toHaveBeenCalledWith("agent", "cancelled", null, { wasFirstHeartbeat: true }); + expect(deps.finalizeAgentStatus).toHaveBeenCalledAfter(deps.releaseIssueExecutionAndPromote); + expect(database.access).not.toHaveBeenCalled(); + }); +}); + +const support = await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; +if (!support.supported) console.warn(`Skipping retry PostgreSQL tests: ${support.reason ?? "unsupported host"}`); + +describePostgres("heartbeat retry module database wiring", () => { + let database: Awaited>; + let db: Db; + const now = new Date("2026-10-08T12:00:00Z"); + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + afterAll(async () => { await database?.cleanup(); }); + + afterEach(async () => { + await db.delete(issues); + await db.delete(heartbeatRunEvents); + await db.delete(heartbeatRuns); + await db.delete(agentWakeupRequests); + await db.delete(agents); + await db.delete(companies); + }); + + function boundRetries() { + const state = createHeartbeatRunState(db); + const deps = { + ...callbacks(), getRun: state.getRun, getAgent: state.getAgent, + resolveSessionBeforeForWakeup: state.resolveSessionBeforeForWakeup, + getAgentInvokability: (agent: Agent | null | undefined) => evaluateAgentInvokabilityFromDb(db, agent), + runDispatch: createRunDispatch(db), + }; + return { retries: extracted.createHeartbeatRetries(db, deps), deps }; + } + + async function fixture() { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Retries", issuePrefix: `R${companyId.slice(0, 8)}` }); + const [agent] = await db.insert(agents).values({ companyId, name: "Retry agent", adapterType: "codex_local", status: "active" }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "Retry task", status: "in_progress", assigneeAgentId: agent.id }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ + companyId, agentId: agent.id, scopeKind: "issue", issueId: issue.id, + status: "failed", errorCode: "codex_transient_upstream", finishedAt: now, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + contextSnapshot: { issueId: issue.id }, + }).returning(); + await db.update(issues).set({ executionRunId: run.id, checkoutRunId: run.id }).where(eq(issues.id, issue.id)); + return { companyId, agent, issue, run }; + } + + it("reuses one successor across concurrent factories and keeps writes committed before callbacks", async () => { + const { run, agent, issue, companyId } = await fixture(); + const first = boundRetries(), second = boundRetries(); + for (const binding of [first, second]) { + binding.deps.appendRunEvent.mockImplementation(async (_source, event) => { + const retryId = event.payload?.retryRunId; + expect(typeof retryId).toBe("string"); + expect(await binding.deps.getRun(retryId as string)).toMatchObject({ retryOfRunId: run.id }); + }); + } + const results = await Promise.all([ + first.retries.scheduleBoundedRetryForRun(run, agent, { now, random: () => 0 }), + second.retries.scheduleBoundedRetryForRun(run, agent, { now, random: () => 0 }), + ]); + expect(results.every((result) => result.outcome === "scheduled")).toBe(true); + const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.retryOfRunId, run.id))); + expect(successors).toHaveLength(1); + const successor = successors[0]; + expect(successor).toMatchObject({ + status: "scheduled_retry", issueId: issue.id, responsibleUserId: "retry-user", + scheduledRetryAttempt: 1, scheduledRetryReason: "transient_failure", + }); + expect(successor.contextSnapshot).toMatchObject({ retryOfRunId: run.id, codexTransientFallbackMode: "same_session" }); + expect(first.deps.resolveResponsibleUserIdForRunContext).toHaveBeenCalledWith(run, expect.objectContaining({ retryOfRunId: run.id })); + const [locked] = await db.select().from(issues).where(eq(issues.id, issue.id)); + expect(locked).toMatchObject({ executionRunId: successor.id, checkoutRunId: null }); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.runId, successor.id))).toHaveLength(1); + expect(await boundRetries().retries.scheduleBoundedRetry(run.id, { now })).toMatchObject({ outcome: "scheduled", run: { id: successor.id }, reusedExisting: true }); + expect(first.deps.appendRunEvent).toHaveBeenCalledTimes(1); + expect(second.deps.appendRunEvent).toHaveBeenCalledTimes(1); + }); + + it("promotes only due retries and publishes after commit", async () => { + const { run } = await fixture(); + const { retries, deps } = boundRetries(); + const scheduled = await retries.scheduleBoundedRetry(run.id, { now, delayMs: 60_000 }); + expect(scheduled.outcome).toBe("scheduled"); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry"); + expect(await retries.promoteDueScheduledRetries(now)).toEqual({ promoted: 0, runIds: [] }); + deps.applyRunDispatchPostCommitEffects.mockClear(); + const committedStatuses: string[] = []; + const reads: Promise[] = []; + deps.applyRunDispatchPostCommitEffects.mockImplementation(() => { + // Read outside the dispatch transaction after the callback is invoked. + // The assertion below waits for that read without making publication async. + reads.push(deps.getRun(scheduled.run.id).then((row) => { committedStatuses.push(row!.status); })); + }); + expect(await retries.promoteDueScheduledRetries(scheduled.dueAt)).toEqual({ promoted: 1, runIds: [scheduled.run.id] }); + await Promise.all(reads); + expect(committedStatuses).toEqual(["queued"]); + expect(deps.applyRunDispatchPostCommitEffects).toHaveBeenCalledWith([expect.objectContaining({ kind: "run_queued", runId: scheduled.run.id })]); + }); + + it("binds retry-now to the requested issue and persists its actor before promotion", async () => { + const selected = await fixture(), other = await fixture(); + const { retries, deps } = boundRetries(); + const scheduled = await retries.scheduleBoundedRetry(selected.run.id, { now, delayMs: 60_000 }); + const foreign = await retries.scheduleBoundedRetry(other.run.id, { now, delayMs: 60_000 }); + if (scheduled.outcome !== "scheduled" || foreign.outcome !== "scheduled") throw new Error("Expected retries"); + expect(await retries.retryScheduledRetryNow({ issueId: selected.issue.id, actor: { actorType: "user", actorId: "retry-user" }, now })).toMatchObject({ + outcome: "promoted", scheduledRetry: { runId: scheduled.run.id, status: "queued" }, + }); + expect(await deps.getRun(scheduled.run.id)).toMatchObject({ contextSnapshot: { + retryNowRequestedByActorType: "user", retryNowRequestedByActorId: "retry-user", retryNowRequestedAt: now.toISOString(), + } }); + expect(await deps.getRun(foreign.run.id)).toMatchObject({ status: "scheduled_retry", scheduledRetryAt: foreign.dueAt }); + expect(await retries.retryScheduledRetryNow({ issueId: selected.issue.id, now })).toMatchObject({ outcome: "already_promoted" }); + }); +}); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts new file mode 100644 index 00000000000..d2e7cb0e5b7 --- /dev/null +++ b/server/src/services/heartbeat/retries.ts @@ -0,0 +1,1853 @@ +import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; +import { + WORKSPACE_VALIDATION_FAILURE_CODE, + isWorkspaceValidationFailedRun, + readWorkspaceValidationPayloadFromRun, +} from "./workspaces.js"; +import { admitExplicitContinuationRetry } from "../explicit-native-continuation.js"; +import { + CONVERSATION_CONTINUATION_POLICY, + hasConversationContinuationPolicy, +} from "../conversation-continuation.js"; +import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js"; +import { + executionFailureRetryCount, + executionRetryAttemptCount, + accountingForScheduledRetry, +} from "../execution-recovery-attempt.js"; +import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-scheduler.js"; +import { randomUUID } from "node:crypto"; +import { + and, + asc, + desc, + eq, + inArray, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + agents, + agentWakeupRequests, + executionWorkspaces, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import { + HttpError, + notFound, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { parseObject } from "../../adapters/utils.js"; +import { logActivity } from "../activity-log.js"; +import { readContinuationAttempt } from "../recovery/index.js"; +import { withRecoveryContext } from "../recovery/status-only-context.js"; +import { + MAX_TURN_CONTINUATION_RETRY_REASON, + WORKSPACE_BUSY_RETRY_REASON, + AI_CONNECTION_BUSY_RETRY_REASON, + AI_CONNECTION_POOL_WAIT_RETRY_REASON, + INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + INTERACTION_CONTINUATION_INFRA_WAKE_REASON, + isNonAssigneeWorkspaceBusyRetry, + isResolvedInteractionContinuationWakeContext, +} from "../../modules/run-dispatch/index.js"; + +import type { AppendHeartbeatRunEventInput } from "../heartbeat-run-events.js"; +import type { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatRunPreparation } from "./run-preparation.js"; +import type { createRunDispatch, PostCommitEffect } from "../../modules/run-dispatch/index.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type HeartbeatRunState = ReturnType; +type PlanApprovalResumeFailure = { + run: HeartbeatRun; + issueId: string | null; + attempt: number; + maxAttempts: number; +}; + +/** The retry module owns scheduling writes; the service owns lifecycle effects. */ +export interface HeartbeatRetryDependencies { + runDispatch: Pick, + "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">; + getAgent: (agentId: string) => Promise; + getRun: ( + runId: string, + options?: Parameters[1], + ) => Promise> | null | undefined>; + resolveSessionBeforeForWakeup: HeartbeatRunState["resolveSessionBeforeForWakeup"]; + resolveResponsibleUserIdForRunContext: ReturnType["resolveResponsibleUserIdForRunContext"]; + getAgentInvokability: (agent: typeof agents.$inferSelect | null | undefined) => ReturnType; + appendRunEvent: (run: HeartbeatRun, event: { + eventType: string; + stream?: "system" | "stdout" | "stderr"; + level?: "info" | "warn" | "error"; + color?: string; + message?: string; + payload?: Record; + retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"]; + }) => Promise; + escalatePlanApprovalResumeFailureNeedsAttention: (input: PlanApprovalResumeFailure) => Promise; + recordPlanApprovalResumeFailureRetry: (input: PlanApprovalResumeFailure & { retryRunId: string | null }) => Promise; + setRunStatusIfRunning: (runId: string, status: string, patch?: Partial) => Promise<{ + updated: boolean; + run: HeartbeatRun | null; + }>; + setWakeupStatus: (wakeupRequestId: string | null | undefined, status: string, patch?: Partial) => Promise; + releaseIssueExecutionAndPromote: (run: Pick) => Promise; + finalizeAgentStatus: ( + agentId: string, + outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out", + failureReason?: string | null, + options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean }, + ) => Promise; + getWorktreeExecutionCutoff: () => Promise; + applyRunDispatchPostCommitEffects: (effects: PostCommitEffect[]) => void; +} + +export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS = [ + 30_000, 30_000, +] as const; + +const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0; + +const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; + +const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; + +export function isTransientWorkspaceGitScanCode(code: string | null | undefined): boolean { + return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated; +} + +export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = + BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; + +const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 2; + +const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = [ + "scheduled_retry", + "queued", + "running", +] as const; + +export const WORKSPACE_BUSY_RETRY_WAKE_REASON = "workspace_busy_retry"; + +export const WORKSPACE_BUSY_ERROR_CODE = "workspace_busy"; + +export const WORKSPACE_BUSY_RETRY_BASE_DELAY_MS = 60 * 1000; + +export const WORKSPACE_BUSY_RETRY_JITTER_MS = 60 * 1000; + +// Preserve the one-hour shared-workspace holder cutoff independently of the +// informational output-silence warnings. Warning sooner must not let another +// run overtake a quiet holder and mutate its shared workspace. +export const WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS = 60 * 60 * 1000; + +// Issue-level executionWorkspaceSettings.mode values that unambiguously opt an +// issue's runs out of the shared project workspace, and therefore out of +// shared-workspace serialization ("isolated" is the legacy alias +// parseIssueExecutionWorkspaceSettings normalizes to isolated_workspace). Any +// other value — including agent_default and an absent mode — may still resolve +// to the shared workspace and counts as a holder. +const ISOLATED_EXECUTION_WORKSPACE_MODES = [ + "isolated_workspace", + "operator_branch", + "isolated", +] as const; + +type CodexTransientFallbackMode = + | "same_session" + | "safer_invocation" + | "fresh_session" + | "fresh_session_safer_invocation"; + +export interface SharedWorkspaceHolder { + runId: string; + agentId: string; + issueId: string; + issueIdentifier: string | null; +} + +// Pre-dispatch gate outcome: another running run currently holds the issue's +// shared project workspace. Not a failure — the run is parked as a bounded +// scheduled retry and re-attempted once the holder finishes, so two agents +// never mutate the same working tree concurrently. +export class WorkspaceBusyDeferral extends Error { + code = WORKSPACE_BUSY_ERROR_CODE; + holder: SharedWorkspaceHolder; + projectWorkspaceId: string; + deferralAttempt: number; + wasIssueAssignee: boolean; + + constructor(input: { + holder: SharedWorkspaceHolder; + projectWorkspaceId: string; + deferralAttempt: number; + wasIssueAssignee: boolean; + }) { + super( + `Shared project workspace is busy: run ${input.holder.runId} (issue ${ + input.holder.issueIdentifier ?? input.holder.issueId + }) is still running`, + ); + this.name = "WorkspaceBusyDeferral"; + this.holder = input.holder; + this.projectWorkspaceId = input.projectWorkspaceId; + this.deferralAttempt = input.deferralAttempt; + this.wasIssueAssignee = input.wasIssueAssignee; + } +} + +export function isWorkspaceBusyDeferral( + error: unknown, +): error is WorkspaceBusyDeferral { + return error instanceof WorkspaceBusyDeferral; +} + +export function computeWorkspaceBusyRetryDelayMs( + random: () => number = Math.random, +) { + const jitter = Math.min(Math.max(random(), 0), 1); + return ( + WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + + Math.floor(jitter * WORKSPACE_BUSY_RETRY_JITTER_MS) + ); +} + +function resolveCodexTransientFallbackMode( + attempt: number, +): CodexTransientFallbackMode { + if (attempt <= 1) return "same_session"; + if (attempt === 2) return "safer_invocation"; + if (attempt === 3) return "fresh_session"; + return "fresh_session_safer_invocation"; +} + +export function readHeartbeatRunErrorFamily( + run: Pick, +) { + const resultJson = parseObject(run.resultJson); + const persistedFamily = readNonEmptyString(resultJson.errorFamily); + if (persistedFamily) return persistedFamily; + + if (run.errorCode === "provider_quota") { + return "provider_quota"; + } + if ( + run.errorCode === "codex_transient_upstream" || + run.errorCode === "claude_transient_upstream" || + run.errorCode === "codex_harness_crash" + ) { + return "transient_upstream"; + } + return null; +} + +function readTransientRetryNotBeforeFromRun( + run: Pick, +) { + const resultJson = parseObject(run.resultJson); + const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore; + if (!( + typeof value === "string" || + typeof value === "number" || + value instanceof Date + )) { + return null; + } + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +export function readTransientRecoveryContractFromRun( + run: Pick, +) { + const errorFamily = readHeartbeatRunErrorFamily(run); + return errorFamily === "transient_upstream" || + errorFamily === "provider_quota" + ? { + errorFamily, + retryNotBefore: readTransientRetryNotBeforeFromRun(run), + } + : null; +} + +function isSpawnLikeFailureMessage(value: unknown) { + if (typeof value !== "string") return false; + return /failed to start command|spawn\b|\bENOENT\b/i.test(value); +} + +// A sandbox provider plugin's worker can be briefly down during its own +// restart window (e.g. a rolling deploy of the plugin worker process). Lease +// acquisition fails immediately in that window, but the condition is +// transient and self-healing, so it must be treated as retryable +// infrastructure rather than a terminal setup failure. See +// resolveSandboxProviderPlugin's "worker_unavailable" message in +// environment-runtime.ts (":808"), e.g. 'Sandbox provider "kubernetes" is +// installed via plugin "acme.kubernetes-sandbox-provider", but its worker is +// not running.' +// +// This is anchored on both "is installed via plugin" and "but its worker is +// not running" so it does not also match plugin-environment-driver.ts's +// unrelated, permanent "provider not installed" message ('Sandbox provider +// "X" is not installed or its plugin worker is not running.'), which +// coincidentally contains the same "worker is not running" substring but +// describes a terminal condition that must not be retried. +function isSandboxProviderWorkerUnavailableFailureMessage(value: unknown) { + if (typeof value !== "string") return false; + return /sandbox provider .* is installed via plugin .* but its worker is not running/i.test( + value, + ); +} + +function isRetryableInteractionContinuationInfrastructureFailure( + run: Pick< + typeof heartbeatRuns.$inferSelect, + "error" | "errorCode" | "resultJson" + >, +) { + if ( + run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || + run.errorCode === "process_lost" + ) { + return true; + } + + if (run.errorCode !== "adapter_failed" && run.errorCode !== "setup_failed") + return false; + + const resultJson = parseObject(run.resultJson); + return ( + isSpawnLikeFailureMessage(run.error) || + isSpawnLikeFailureMessage(resultJson.errorMessage) || + isSpawnLikeFailureMessage(resultJson.message) || + isSandboxProviderWorkerUnavailableFailureMessage(run.error) || + isSandboxProviderWorkerUnavailableFailureMessage(resultJson.errorMessage) || + isSandboxProviderWorkerUnavailableFailureMessage(resultJson.message) + ); +} + +export function computeBoundedTransientHeartbeatRetrySchedule( + attempt: number, + now = new Date(), + random: () => number = Math.random, +) { + if (!Number.isInteger(attempt) || attempt <= 0) return null; + const baseDelayMs = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS[attempt - 1]; + if (typeof baseDelayMs !== "number") return null; + const sample = Math.min(1, Math.max(0, random())); + const jitterMultiplier = + 1 + (sample * 2 - 1) * BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO; + const delayMs = Math.max(1_000, Math.round(baseDelayMs * jitterMultiplier)); + return { + attempt, + baseDelayMs, + delayMs, + dueAt: new Date(now.getTime() + delayMs), + maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + }; +} + +export function normalizeAgentNameKey(value: string | null | undefined) { + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +/** Retry persistence uses the service database and explicit lifecycle callbacks. */ +export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDependencies) { + const { + appendRunEvent, + escalatePlanApprovalResumeFailureNeedsAttention, + getAgentInvokability, + runDispatch, + resolveSessionBeforeForWakeup, + resolveResponsibleUserIdForRunContext, + recordPlanApprovalResumeFailureRetry, + setRunStatusIfRunning, + setWakeupStatus, + getRun, + getAgent, + releaseIssueExecutionAndPromote, + finalizeAgentStatus, + getWorktreeExecutionCutoff, + applyRunDispatchPostCommitEffects, + } = dependencies; + + async function scheduleBoundedRetryForRun( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + opts?: { + now?: Date; + random?: () => number; + retryReason?: string; + wakeReason?: string; + maxAttempts?: number; + delayMs?: number; + }, + ) { + if (parseObject(agent.adapterConfig).provider === "openai_dot" + || parseObject(parseObject(parseObject(run.runnerProfileJson).nativeExecutionInput).provider).kind === "openai_dot") { + return { outcome: "not_scheduled" as const, reason: "Dot external execution must be reconciled before a new assignment; Paperclip cannot confirm its external stop.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + } + if (run.errorCode === "provider_tool_definition_invalid") { + return { outcome: "not_scheduled" as const, + reason: "Repair the invalid tool definitions before starting a new attempt.", + issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + } + if (Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds) && + run.contextSnapshot.chatCompletionDeliveryIds.some(id => typeof id === "string")) { + return { outcome: "not_scheduled" as const, reason: "The completion outbox owns this reply's retry budget and publication identity.", + errorCode: "chat_completion_outbox_owns_retry" as const, issueId: readNonEmptyString(run.contextSnapshot.issueId) }; + } + const now = opts?.now ?? new Date(); + const retryReason = + opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON; + const wakeReason = + opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; + const maxAttempts = Math.max( + 0, + Math.floor( + opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + ), + ); + const consumedAttempts = executionRetryAttemptCount(run, retryReason); + const nextAttempt = consumedAttempts + 1; + const computedBaseSchedule = + opts?.delayMs != null + ? nextAttempt <= maxAttempts + ? { + attempt: nextAttempt, + baseDelayMs: Math.max(0, Math.floor(opts.delayMs)), + delayMs: Math.max(0, Math.floor(opts.delayMs)), + dueAt: new Date( + now.getTime() + Math.max(0, Math.floor(opts.delayMs)), + ), + maxAttempts, + } + : null + : nextAttempt <= maxAttempts + ? computeBoundedTransientHeartbeatRetrySchedule( + nextAttempt, + now, + opts?.random, + ) + : null; + const baseSchedule = computedBaseSchedule + ? { ...computedBaseSchedule, maxAttempts } + : null; + const transientRecovery = + retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON + ? readTransientRecoveryContractFromRun(run) + : null; + const codexTransientFallbackMode = + agent.adapterType === "codex_local" && + transientRecovery?.errorFamily === "transient_upstream" + ? resolveCodexTransientFallbackMode(nextAttempt) + : null; + const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null; + const contextSnapshot = parseObject(run.contextSnapshot); + // A retry inherits the durable authorization scope of its source run. Do + // not promote an untrusted or legacy contextSnapshot.issueId into a new + // issue binding: that could either violate the FK or misclassify history. + const issueId = run.scopeKind === "issue" ? run.issueId : null; + + if (!baseSchedule) { + const exhaustion = { + retryReason, + scheduledRetryAttempt: consumedAttempts, + maxAttempts, + }; + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: `Bounded retry exhausted after ${consumedAttempts} scheduled attempts; no further automatic retry will be queued`, + payload: exhaustion, + retryExhaustion: exhaustion, + }); + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + await escalatePlanApprovalResumeFailureNeedsAttention({ + run, + issueId, + attempt: Math.min( + consumedAttempts, + maxAttempts, + ), + maxAttempts, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId }, + "failed to escalate exhausted plan-approval resume failure", + ); + }); + } + return { + outcome: "retry_exhausted" as const, + attempt: nextAttempt, + maxAttempts, + }; + } + + if (await legacyExecutionNeedsReconciliationWithEvidence(db, run)) { + return { + outcome: "not_scheduled" as const, + reason: + "Reconcile the previous execution before retrying; safe provider recovery is unavailable.", + errorCode: "legacy_execution_requires_reconciliation" as const, + issueId: readNonEmptyString(run.contextSnapshot?.issueId), + }; + } + if (retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON) { + const invokability = await getAgentInvokability(agent); + if (!invokability.invokable) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + "Scheduled retry suppressed because the agent is not invokable", + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + reason: invokability.reason, + invalidOrgChain: invokability.invalidOrgChain, + ...invokability.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: + "Scheduled retry suppressed because the agent is not invokable", + errorCode: "agent_not_invokable" as const, + issueId, + }; + } + } + + const schedule = + transientRetryNotBefore && + transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime() + ? { + ...baseSchedule, + dueAt: transientRetryNotBefore, + delayMs: Math.max( + 0, + transientRetryNotBefore.getTime() - now.getTime(), + ), + } + : baseSchedule; + + const requiresIssueGate = + isTransientWorkspaceGitScanCode(run.errorCode) || + hasConversationContinuationPolicy(run.resultJson) || + (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) || + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON || + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON; + if (requiresIssueGate) { + const gate = await runDispatch.evaluateScheduledRetryGate({ + runId: run.id, + companyId: run.companyId, + retryReasonOverride: retryReason, + now, + }); + if (!gate.allowed) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: gate.reason, + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + ...gate.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: gate.reason, + errorCode: gate.errorCode, + issueId: gate.issueId, + }; + } + } + const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); + const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); + const interactionContinuationPayload = + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ? { + mutation: "interaction", + interactionId: readNonEmptyString(contextSnapshot.interactionId), + interactionKind: readNonEmptyString( + contextSnapshot.interactionKind, + ), + interactionStatus: readNonEmptyString( + contextSnapshot.interactionStatus, + ), + continuationPolicy: readNonEmptyString( + contextSnapshot.continuationPolicy, + ), + } + : {}; + const workspaceValidationRetryPayload = + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON && + isWorkspaceValidationFailedRun(run) + ? readWorkspaceValidationPayloadFromRun(run) + : null; + const shouldQuarantineWorkspaceForRetry = + workspaceValidationRetryPayload !== null && + Object.keys(workspaceValidationRetryPayload).length > 0; + const retryContextSnapshot: Record = withRecoveryContext( + { + ...contextSnapshot, + executionRetryAccounting: accountingForScheduledRetry(run, retryReason, schedule.attempt), + retryOfRunId: run.id, + wakeReason, + retryReason, + ...(retryReason === WORKSPACE_BUSY_RETRY_REASON + ? { + failureRetriesBeforeWorkspaceWait: + executionFailureRetryCount(run), + } + : {}), + ...((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) + ? { failureRetriesBeforeAiConnectionWait: executionFailureRetryCount(run) } + : {}), + ...(shouldQuarantineWorkspaceForRetry + ? { + workspaceValidationRecovery: { + strategy: "quarantine_failed_workspace_and_retry_clean", + sourceRunId: run.id, + reason: + readNonEmptyString(workspaceValidationRetryPayload?.reason) ?? + WORKSPACE_VALIDATION_FAILURE_CODE, + fingerprint: readNonEmptyString( + workspaceValidationRetryPayload?.fingerprint, + ), + failedExecutionWorkspaceId: readNonEmptyString( + workspaceValidationRetryPayload?.executionWorkspaceId, + ), + }, + } + : {}), + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + ...(transientRetryNotBefore + ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), + }, + "normal_model", + ); + const responsibleUserId = await resolveResponsibleUserIdForRunContext( + run, + retryContextSnapshot, + ); + const continuationRetryIdempotencyKey = + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON + ? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` + : retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ? `interaction-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` + : null; + + type ScheduledRetryTransactionResult = + | { + outcome: "scheduled"; + run: typeof heartbeatRuns.$inferSelect; + reusedExisting: boolean; + } + | { + outcome: "not_scheduled"; + reason: string; + errorCode: + | "issue_not_found" + | "issue_reassigned" + | "issue_cancelled" + | "issue_terminal_status" + | "issue_not_in_progress" + | "continuation_user_authorization_missing" + | "issue_execution_lock_changed"; + issueId: string | null; + details: Record; + }; + + const scheduleResult = await db.transaction( + async (tx): Promise => { + // All automatic failure paths share the same predecessor claim. A + // duplicate monitor, restart sweep or wake must reuse its successor. + if ( + retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON && + retryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ) { + if (issueId) + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + const [existing] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + ), + ) + .limit(1); + if (existing) + return { + outcome: "scheduled", + run: existing, + reusedExisting: true, + }; + } + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + if (issueId) { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + } else { + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + } + + const existingContinuation = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + eq(heartbeatRuns.scheduledRetryReason, retryReason), + eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), + inArray(heartbeatRuns.status, [ + ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, + ]), + issueId + ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` + : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingContinuation) { + if (existingContinuation.wakeupRequestId) { + const existingWakeup = await tx + .select({ coalescedCount: agentWakeupRequests.coalescedCount }) + .from(agentWakeupRequests) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ) + .then((rows) => rows[0] ?? null); + + await tx + .update(agentWakeupRequests) + .set({ + coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, + updatedAt: now, + }) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ); + } + + return { + outcome: "scheduled", + run: existingContinuation, + reusedExisting: true, + }; + } + } + + if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) { + if (issueId) { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + } else { + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + } + + const existingContinuation = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + eq(heartbeatRuns.scheduledRetryReason, retryReason), + eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), + inArray(heartbeatRuns.status, [ + ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, + ]), + issueId + ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` + : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingContinuation) { + if (existingContinuation.wakeupRequestId) { + const existingWakeup = await tx + .select({ coalescedCount: agentWakeupRequests.coalescedCount }) + .from(agentWakeupRequests) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ) + .then((rows) => rows[0] ?? null); + + await tx + .update(agentWakeupRequests) + .set({ + coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, + updatedAt: now, + }) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ); + } + + return { + outcome: "scheduled", + run: existingContinuation, + reusedExisting: true, + }; + } + + if (issueId) { + const lockedIssue = await tx + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!lockedIssue) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because the target issue no longer exists", + errorCode: "issue_not_found", + issueId, + details: { issueId }, + }; + } + + if (lockedIssue.assigneeAgentId !== run.agentId) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because issue ownership changed", + errorCode: "issue_reassigned", + issueId, + details: { + issueId, + previousAssigneeAgentId: run.agentId, + currentAssigneeAgentId: lockedIssue.assigneeAgentId, + }, + }; + } + + if ( + lockedIssue.status === "cancelled" || + lockedIssue.status === "done" + ) { + return { + outcome: "not_scheduled", + reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`, + errorCode: + lockedIssue.status === "cancelled" + ? "issue_cancelled" + : "issue_terminal_status", + issueId, + details: { issueId, currentStatus: lockedIssue.status }, + }; + } + + if (lockedIssue.status !== "in_progress") { + return { + outcome: "not_scheduled", + reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`, + errorCode: "issue_not_in_progress", + issueId, + details: { + issueId, + currentStatus: lockedIssue.status, + requiredStatus: "in_progress", + }, + }; + } + + if (lockedIssue.executionRunId !== run.id) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", + errorCode: "issue_execution_lock_changed", + issueId, + details: { + issueId, + expectedExecutionRunId: run.id, + currentExecutionRunId: lockedIssue.executionRunId, + }, + }; + } + } + } + + if ( + (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) && issueId && + !isNonAssigneeWorkspaceBusyRetry(retryReason, contextSnapshot) + ) { + // The issue row is locked above. Recheck after the preflight gate so + // cancellation or recovery cannot leave a successor without its lock. + const [lockedIssue] = await tx.select({ executionRunId: issues.executionRunId }) + .from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); + if (lockedIssue?.executionRunId !== run.id) { + return { + outcome: "not_scheduled", issueId, errorCode: "issue_execution_lock_changed", + reason: "Subscription retry suppressed because the task execution lock changed", + details: { issueId, expectedExecutionRunId: run.id, currentExecutionRunId: lockedIssue?.executionRunId ?? null }, + }; + } + } + + const scheduledRunId = randomUUID(); + if (contextSnapshot.explicitUserContinuation) { + const continuation = issueId && retryReason === "transient_failure" ? await admitExplicitContinuationRetry({ + db: tx as unknown as Db, companyId: run.companyId, issueId, agentId: run.agentId, + parentRunId: run.id, successorRunId: scheduledRunId, now, + }) : null; + if (!continuation) return { + outcome: "not_scheduled", issueId, + errorCode: "continuation_user_authorization_missing", + reason: "The automatic retry could not revalidate the original user continuation.", + details: {}, + }; + retryContextSnapshot.explicitUserContinuation = continuation; + retryContextSnapshot.previousRunId = continuation.previousRunId; + } + + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId: run.companyId, + agentId: run.agentId, + source: "automation", + triggerDetail: "system", + reason: wakeReason, + payload: withRecoveryContext( + { + ...(issueId ? { issueId } : {}), + retryOfRunId: run.id, + ...interactionContinuationPayload, + retryReason, + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + ...(transientRetryNotBefore + ? { + transientRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode + ? { codexTransientFallbackMode } + : {}), + }, + "normal_model", + ), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + idempotencyKey: continuationRetryIdempotencyKey, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + const scheduledRun = await tx + .insert(heartbeatRuns) + .values({ + id: scheduledRunId, + companyId: run.companyId, + agentId: run.agentId, + scopeKind: run.scopeKind, + issueId, + invocationSource: "automation", + triggerDetail: "system", + status: "scheduled_retry", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: retryContextSnapshot, + ...(hasConversationContinuationPolicy(run.resultJson) + ? { resultJson: { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } } : {}), + responsibleUserId, + sessionIdBefore: sessionBefore, + retryOfRunId: run.id, + scheduledRetryAt: schedule.dueAt, + scheduledRetryAttempt: schedule.attempt, + scheduledRetryReason: retryReason, + continuationAttempt: readContinuationAttempt( + retryContextSnapshot.livenessContinuationAttempt, + ), + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ + runId: scheduledRun.id, + updatedAt: now, + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + let detachWorkspaceFromIssue = false; + if (issueId && shouldQuarantineWorkspaceForRetry) { + const issueWorkspace = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + executionWorkspaceId: issues.executionWorkspaceId, + }) + .from(issues) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), + ) + .for("update") + .then((rows) => rows[0] ?? null); + const failedExecutionWorkspaceId = + readNonEmptyString( + workspaceValidationRetryPayload?.executionWorkspaceId, + ) ?? readNonEmptyString(issueWorkspace?.executionWorkspaceId); + + if (issueWorkspace && failedExecutionWorkspaceId) { + const failedWorkspace = await tx + .select({ + id: executionWorkspaces.id, + companyId: executionWorkspaces.companyId, + sourceIssueId: executionWorkspaces.sourceIssueId, + status: executionWorkspaces.status, + metadata: executionWorkspaces.metadata, + }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, failedExecutionWorkspaceId), + eq(executionWorkspaces.companyId, run.companyId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + + const workspaceBelongsToIssue = failedWorkspace + ? failedWorkspace.sourceIssueId === issueId + : false; + + if ( + failedWorkspace && + workspaceBelongsToIssue && + issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId + ) { + const existingMetadata = parseObject(failedWorkspace.metadata); + const quarantine = { + reason: WORKSPACE_VALIDATION_FAILURE_CODE, + retryReason, + sourceRunId: run.id, + retryRunId: scheduledRun.id, + issueId, + sourceIssueId: failedWorkspace.sourceIssueId ?? null, + quarantinedAt: now.toISOString(), + workspaceValidation: workspaceValidationRetryPayload ?? {}, + }; + await tx + .update(executionWorkspaces) + .set({ + status: "archived", + closedAt: now, + cleanupEligibleAt: null, + cleanupReason: WORKSPACE_VALIDATION_FAILURE_CODE, + metadata: { + ...existingMetadata, + workspaceValidationQuarantine: quarantine, + }, + updatedAt: now, + }) + .where( + and( + eq(executionWorkspaces.id, failedWorkspace.id), + eq(executionWorkspaces.companyId, run.companyId), + ), + ); + + await logActivity(tx as unknown as Db, { + companyId: run.companyId, + actorType: "system", + actorId: "heartbeat", + agentId: run.agentId, + runId: run.id, + action: "execution_workspace.workspace_validation_quarantined", + entityType: "execution_workspace", + entityId: failedWorkspace.id, + details: quarantine, + }); + detachWorkspaceFromIssue = + issueWorkspace.executionWorkspaceId === + failedExecutionWorkspaceId; + } + } + } + + if (issueId) { + await tx + .update(issues) + .set({ + executionRunId: scheduledRun.id, + checkoutRunId: sql`case when ${issues.checkoutRunId} = ${run.id} then null else ${issues.checkoutRunId} end`, + executionAgentNameKey: normalizeAgentNameKey(agent.name), + executionLockedAt: now, + ...(detachWorkspaceFromIssue + ? { + executionWorkspaceId: null, + executionWorkspacePreference: null, + } + : {}), + updatedAt: now, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); + } + + return { + outcome: "scheduled", + run: scheduledRun, + reusedExisting: false, + }; + }, + ); + + if (scheduleResult.outcome === "not_scheduled") { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: scheduleResult.reason, + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + ...scheduleResult.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: scheduleResult.reason, + errorCode: scheduleResult.errorCode, + issueId: scheduleResult.issueId, + }; + } + + const retryRun = scheduleResult.run; + const dueAt = retryRun.scheduledRetryAt + ? new Date(retryRun.scheduledRetryAt) + : schedule.dueAt; + + if (scheduleResult.reusedExisting) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: `Reused existing continuation retry ${retryRun.scheduledRetryAttempt}/${schedule.maxAttempts}`, + payload: { + retryRunId: retryRun.id, + retryReason, + idempotencyKey: continuationRetryIdempotencyKey, + scheduledRetryAttempt: retryRun.scheduledRetryAttempt, + scheduledRetryAt: dueAt.toISOString(), + }, + }); + + return { + outcome: "scheduled" as const, + run: retryRun, + dueAt, + attempt: retryRun.scheduledRetryAttempt, + maxAttempts: schedule.maxAttempts, + reusedExisting: true, + }; + } + + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: `Scheduled bounded retry ${schedule.attempt}/${schedule.maxAttempts} for ${schedule.dueAt.toISOString()}`, + payload: { + retryRunId: retryRun.id, + retryReason, + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + baseDelayMs: schedule.baseDelayMs, + delayMs: schedule.delayMs, + ...(transientRetryNotBefore + ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), + }, + }); + + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + await recordPlanApprovalResumeFailureRetry({ + run, + issueId, + retryRunId: retryRun.id, + attempt: schedule.attempt, + maxAttempts: schedule.maxAttempts, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId, retryRunId: retryRun.id }, + "failed to record plan-approval resume retry failure", + ); + }); + } + + return { + outcome: "scheduled" as const, + run: retryRun, + dueAt, + attempt: schedule.attempt, + maxAttempts: schedule.maxAttempts, + }; + } + + // Finds a running heartbeat run (other than the caller's) whose context + // issue shares the same project workspace, i.e. the run that currently + // "holds" the shared working tree. Runs that have been silent past + // WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS do not count — a zombie holder must + // not park other work forever. This cutoff is independent of informational + // silence warnings. When isolated workspaces are enabled, holders whose + // issue explicitly opted into an isolated workspace never touch the shared + // tree, so they are excluded; a NULL/agent_default mode may resolve to the + // shared tree and counts as a holder (over-serializing is the safe + // direction). When the isolated-workspaces experiment is off, every run + // resolves to the shared tree, so no holder is excluded. + async function findSharedWorkspaceHolder(input: { + companyId: string; + projectWorkspaceId: string; + excludeIssueId: string; + excludeRunId: string; + honorIsolatedWorkspaceModes: boolean; + now?: Date; + }): Promise { + const staleCutoff = new Date( + (input.now ?? new Date()).getTime() - + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + ); + return await db + .select({ + runId: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + issueId: sql`${issues.id}::text`, + issueIdentifier: issues.identifier, + }) + .from(heartbeatRuns) + .innerJoin( + issues, + and( + eq(issues.companyId, heartbeatRuns.companyId), + sql`${issues.id}::text = ${heartbeatRuns.contextSnapshot} ->> 'issueId'`, + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.status, "running"), + ne(heartbeatRuns.id, input.excludeRunId), + // Last observed activity: output beats start beats creation. A run + // that started recently but has not written output yet is live. + sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) >= ${staleCutoff.toISOString()}::timestamptz`, + eq(issues.projectWorkspaceId, input.projectWorkspaceId), + ne(sql`${issues.id}::text`, input.excludeIssueId), + ...(input.honorIsolatedWorkspaceModes + ? [ + or( + // Covers both a NULL settings blob and a blob without a mode + // key; either may still resolve to the shared workspace. + sql`${issues.executionWorkspaceSettings} ->> 'mode' is null`, + notInArray( + sql`${issues.executionWorkspaceSettings} ->> 'mode'`, + [...ISOLATED_EXECUTION_WORKSPACE_MODES], + ), + ), + ] + : []), + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + // Credential rotation can briefly contend with a fresh runtime read. Keep + // the task on its automatic pre-provider retry path while the lock clears. + async function finalizeAiConnectionBusyDeferral( + run: typeof heartbeatRuns.$inferSelect, + error: HttpError, + wasIssueAssignee: boolean, + ) { + const now = new Date(); + const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { + error: error.message, errorCode: AI_CONNECTION_BUSY_RETRY_REASON, finishedAt: now, + resultJson: { + executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false }, + cancellation: { + source: "control_plane", + expected: true, + initiator: { type: "system" }, + reason: "Waiting for shared AI credentials", + recordedAt: now.toISOString(), + }, + }, + contextSnapshot: { + ...parseObject(run.contextSnapshot), + aiConnectionBusyDeferredWhileAssignee: wasIssueAssignee, + }, + }); + if (!cancelled.updated) return; + await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: error.message }).catch(() => undefined); + const cancelledRun = cancelled.run ?? await getRun(run.id); + const agent = await getAgent(run.agentId); + let scheduled = false; + try { + if (cancelledRun && agent) { + const retry = await scheduleBoundedRetryForRun(cancelledRun, agent, { + now, retryReason: AI_CONNECTION_BUSY_RETRY_REASON, wakeReason: "ai_connection_busy_retry", + maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, + delayMs: computeWorkspaceBusyRetryDelayMs(), + }); + scheduled = retry.outcome === "scheduled"; + await appendRunEvent(cancelledRun, { + eventType: "lifecycle", stream: "system", level: "info", + message: scheduled ? "Waiting for the shared AI subscription. This task will retry automatically." : "The AI subscription is busy; this task can no longer retry automatically.", + payload: { retryScheduled: scheduled }, + }); + } + } finally { + try { + if (cancelledRun && !scheduled) await releaseIssueExecutionAndPromote(cancelledRun); + } finally { + await finalizeAgentStatus(run.agentId, "cancelled", null, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) }); + } + } + } + + // Terminal handling for a WorkspaceBusyDeferral thrown by the pre-dispatch + // gate: cancel the run (contention is not a failure), schedule a + // workspace_busy retry, and leave the agent idle. The issue execution lock + // transfers to the scheduled retry run inside scheduleBoundedRetryForRun, so + // the issue keeps an active execution path and recovery leaves it alone. + // Deferral has no attempt ceiling — the retry keeps rescheduling while a + // live holder exists, and holder staleness (not a counter) is what prevents + // waiting on a zombie. If no retry could be scheduled (agent no longer + // invokable), the lock is released so the issue does not strand on a + // cancelled run. + async function finalizeWorkspaceBusyDeferral( + run: typeof heartbeatRuns.$inferSelect, + deferral: WorkspaceBusyDeferral, + ) { + const now = new Date(); + const cancelWrite = await setRunStatusIfRunning(run.id, "cancelled", { + error: deferral.message, + errorCode: WORKSPACE_BUSY_ERROR_CODE, + finishedAt: now, + resultJson: { + executionRecovery: { + kind: "workspace_wait", + providerWorkStarted: false, + }, + cancellation: { + source: "control_plane", + expected: true, + initiator: { type: "system" }, + reason: "Waiting for the shared project workspace", + recordedAt: now.toISOString(), + }, + workspaceBusy: { + projectWorkspaceId: deferral.projectWorkspaceId, + holderRunId: deferral.holder.runId, + holderIssueId: deferral.holder.issueId, + deferralAttempt: deferral.deferralAttempt, + }, + }, + // Recorded on the run (and inherited by the scheduled retry's context) + // so the retry promotion gate can tell a non-assignee wake — where an + // assignee mismatch is the expected state — from a reassignment race. + contextSnapshot: { + ...parseObject(run.contextSnapshot), + workspaceBusyDeferredWhileAssignee: deferral.wasIssueAssignee, + }, + }); + if (!cancelWrite.updated) { + logger.info( + { runId: run.id, currentStatus: cancelWrite.run?.status ?? null }, + "skipping workspace-busy deferral finalization because the run already left running state", + ); + return; + } + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: now, + error: deferral.message, + }).catch(() => undefined); + + const cancelledRun = + cancelWrite.run ?? (await getRun(run.id).catch(() => null)); + const agentRow = await getAgent(run.agentId).catch(() => null); + let scheduleOutcome: string | null = null; + if (cancelledRun && agentRow) { + const scheduleResult = await scheduleBoundedRetryForRun( + cancelledRun, + agentRow, + { + now, + retryReason: WORKSPACE_BUSY_RETRY_REASON, + wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON, + // Always admit the next attempt: workspace-busy deferral is bounded by + // holder liveness, not by an attempt counter. + maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, + delayMs: computeWorkspaceBusyRetryDelayMs(), + }, + ).catch((scheduleErr) => { + logger.error( + { err: scheduleErr, runId: run.id }, + "failed to schedule workspace-busy retry after deferral", + ); + return null; + }); + scheduleOutcome = scheduleResult?.outcome ?? null; + } + + if (cancelledRun) { + await appendRunEvent(cancelledRun, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: + scheduleOutcome === "scheduled" + ? `Deferred: ${deferral.message}. Retry ${deferral.deferralAttempt + 1} scheduled; the run waits for the workspace to free.` + : `Deferred: ${deferral.message}. No retry could be scheduled; releasing the issue for other runs.`, + payload: { + projectWorkspaceId: deferral.projectWorkspaceId, + holderRunId: deferral.holder.runId, + holderIssueId: deferral.holder.issueId, + deferralAttempt: deferral.deferralAttempt, + retryScheduled: scheduleOutcome === "scheduled", + }, + }).catch(() => undefined); + } + + if (cancelledRun && scheduleOutcome !== "scheduled") { + await releaseIssueExecutionAndPromote(cancelledRun).catch( + (releaseErr) => { + logger.error( + { err: releaseErr, runId: run.id }, + "failed to release issue execution after workspace-busy deferral", + ); + }, + ); + } + + await finalizeAgentStatus(run.agentId, "cancelled", null, { + wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), + }).catch(() => undefined); + } + + async function scheduleInteractionContinuationInfrastructureRetryIfEligible( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + ) { + if (!run.wakeupRequestId) return null; + if (!isResolvedInteractionContinuationWakeContext(run.contextSnapshot)) + return null; + if (!isRetryableInteractionContinuationInfrastructureFailure(run)) { + const context = parseObject(run.contextSnapshot); + const issueId = readNonEmptyString(context.issueId); + await escalatePlanApprovalResumeFailureNeedsAttention({ + run, + issueId, + attempt: Math.min( + run.scheduledRetryAttempt ?? + INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + ), + maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId }, + "failed to escalate non-retryable plan-approval resume failure", + ); + }); + return null; + } + + return scheduleBoundedRetryForRun(run, agent, { + retryReason: INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + wakeReason: INTERACTION_CONTINUATION_INFRA_WAKE_REASON, + maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + }); + } + + async function promoteDueScheduledRetries(now = new Date()) { + const cutoff = await getWorktreeExecutionCutoff(); + const result = await runDispatch.promoteDueScheduledRetries({ + now, + cutoff, + }); + applyRunDispatchPostCommitEffects(result.postCommitEffects); + return { promoted: result.promoted, runIds: result.runIds }; + } + + async function getIssueRetryRun( + companyId: string, + issueId: string, + statuses: Array<"scheduled_retry" | "queued" | "running" | "cancelled">, + ) { + if (statuses.length === 0) return null; + return db + .select({ + run: heartbeatRuns, + agentName: agents.name, + }) + .from(heartbeatRuns) + .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, statuses), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, + sql`${heartbeatRuns.retryOfRunId} is not null`, + ), + ) + .orderBy( + desc(heartbeatRuns.updatedAt), + desc(heartbeatRuns.createdAt), + desc(heartbeatRuns.id), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + function summarizeIssueScheduledRetryRun(row: { + run: typeof heartbeatRuns.$inferSelect; + agentName: string | null; + }) { + return { + runId: row.run.id, + status: row.run.status as + "scheduled_retry" | "queued" | "running" | "cancelled", + agentId: row.run.agentId, + agentName: row.agentName, + retryOfRunId: row.run.retryOfRunId, + scheduledRetryAt: row.run.scheduledRetryAt, + scheduledRetryAttempt: row.run.scheduledRetryAttempt, + scheduledRetryReason: row.run.scheduledRetryReason, + error: row.run.error, + errorCode: row.run.errorCode, + }; + } + + async function retryScheduledRetryNow(input: { + issueId: string; + actor?: { + actorType?: "user" | "agent" | "system"; + actorId?: string | null; + }; + now?: Date; + }) { + const now = input.now ?? new Date(); + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const scheduled = await getIssueRetryRun(issue.companyId, issue.id, [ + "scheduled_retry", + ]); + if (!scheduled) { + const alreadyPromoted = await getIssueRetryRun( + issue.companyId, + issue.id, + ["queued", "running"], + ); + if (alreadyPromoted) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted), + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + const contextSnapshot = { + ...parseObject(scheduled.run.contextSnapshot), + scheduledRetryAt: now.toISOString(), + retryNowRequestedAt: now.toISOString(), + retryNowRequestedByActorType: input.actor?.actorType ?? null, + retryNowRequestedByActorId: input.actor?.actorId ?? null, + }; + + const updated = await db.transaction(async (tx) => { + const row = await tx + .update(heartbeatRuns) + .set({ + scheduledRetryAt: now, + contextSnapshot, + updatedAt: now, + }) + .where( + and( + eq(heartbeatRuns.id, scheduled.run.id), + eq(heartbeatRuns.status, "scheduled_retry"), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!row) return null; + + if (row.wakeupRequestId) { + const wakeupPayload = { + ...parseObject( + await tx + .select({ payload: agentWakeupRequests.payload }) + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, row.wakeupRequestId)) + .then((rows) => rows[0]?.payload ?? null), + ), + scheduledRetryAt: now.toISOString(), + retryNowRequestedAt: now.toISOString(), + }; + await tx + .update(agentWakeupRequests) + .set({ + payload: wakeupPayload, + updatedAt: now, + }) + .where(eq(agentWakeupRequests.id, row.wakeupRequestId)); + } + + return row; + }); + + if (!updated) { + const alreadyPromoted = await getIssueRetryRun( + issue.companyId, + issue.id, + ["queued", "running"], + ); + if (alreadyPromoted) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted), + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + await appendRunEvent(updated, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: "Scheduled retry was requested to run now", + payload: { + issueId: issue.id, + scheduledRetryAttempt: updated.scheduledRetryAttempt, + scheduledRetryAt: updated.scheduledRetryAt + ? new Date(updated.scheduledRetryAt).toISOString() + : null, + scheduledRetryReason: updated.scheduledRetryReason, + requestedByActorType: input.actor?.actorType ?? null, + requestedByActorId: input.actor?.actorId ?? null, + }, + }); + + const promotion = await runDispatch.promoteScheduledRetry({ + runId: updated.id, + companyId: updated.companyId, + now, + }); + if (promotion.outcome === "promoted") { + applyRunDispatchPostCommitEffects(promotion.postCommitEffects); + } + // Promotion can preserve this row as a cleanup wait. Read that exact run, + // not an older cancelled retry or the pre-promotion schedule. + const currentRun = await getRun(updated.id); + const scheduledRetry = currentRun + ? summarizeIssueScheduledRetryRun({ + run: currentRun, + agentName: scheduled.agentName, + }) + : null; + + if (currentRun?.status === "scheduled_retry") { + return { + outcome: "waiting" as const, + message: parseObject(currentRun.resultJson?.executionWait).cause === "execution_owner_active" + ? "Waiting for execution cleanup. Paperclip will retry automatically once cleanup finishes." + : "The retry remains scheduled. Paperclip will check again at the scheduled time.", + scheduledRetry, + }; + } + + if (promotion.outcome === "promoted") { + return { + outcome: "promoted" as const, + message: "Scheduled retry was promoted to the queued run pool", + scheduledRetry, + }; + } + if (promotion.outcome === "gate_suppressed") { + return { + outcome: "gate_suppressed" as const, + message: promotion.reason, + scheduledRetry, + }; + } + if (currentRun && ["queued", "running"].includes(currentRun.status)) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry, + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + function timerClaimWasFirstHeartbeat( + run: Pick, + ): true | undefined { + return parseObject(run.contextSnapshot).timerClaimWasFirstHeartbeat === true + ? true + : undefined; + } + + return { + scheduleBoundedRetryForRun, + timerClaimWasFirstHeartbeat, + scheduleInteractionContinuationInfrastructureRetryIfEligible, + findSharedWorkspaceHolder, + finalizeAiConnectionBusyDeferral, + finalizeWorkspaceBusyDeferral, + promoteDueScheduledRetries, + retryScheduledRetryNow, + + scheduleBoundedRetry: async ( + runId: string, + opts?: { + now?: Date; + random?: () => number; + retryReason?: string; + wakeReason?: string; + maxAttempts?: number; + delayMs?: number; + }, + ) => { + const run = await getRun(runId, { unsafeFullResultJson: true }); + if (!run) return { outcome: "missing_run" as const }; + const agent = await getAgent(run.agentId); + if (!agent) return { outcome: "missing_agent" as const }; + return scheduleBoundedRetryForRun(run, agent, opts); + }, + }; +} From 65b558180533039a891ee0cd1ccab9988aa79adc Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 8 Oct 2026 13:48:48 -0700 Subject: [PATCH 09/10] Keep proven AI account selection blockers out of Sentry (#15618) Preserve trusted evidence from explicit AI account selection rejections so known setup blockers stay out of Sentry. Keep task failure, blocked state, recovery actions, HTTP behavior, and unexpected error reporting unchanged. Validated with full typecheck/build, focused PostgreSQL lifecycle and reporting tests, independent reporting tests, and all PR CI checks. Co-Authored-By: Paperclip --- doc/observability.md | 10 ++++ server/src/__tests__/ai-connections.test.ts | 20 +++++++ .../heartbeat-process-recovery.test.ts | 50 ++++++++++++++++ .../__tests__/run-failure-report.test.ts | 59 +++++++++++++++++++ ...i-connection-configuration-failure.test.ts | 34 +++++++++++ .../ai-connection-configuration-failure.ts | 31 ++++++++++ server/src/services/ai-connections.ts | 33 +++-------- server/src/services/heartbeat.ts | 3 + server/src/services/run-failure-report.ts | 18 ++++++ 9 files changed, 233 insertions(+), 25 deletions(-) create mode 100644 server/src/services/ai-connection-configuration-failure.test.ts create mode 100644 server/src/services/ai-connection-configuration-failure.ts diff --git a/doc/observability.md b/doc/observability.md index b62a1263309..f71a6a2b304 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -443,6 +443,16 @@ evidence, unknown binding reasons, secret-provider failures, ambiguous missing secret-definition lookups remain reportable. This filter does not change task recovery, credentials, or execution policy. +AI account selection also stays local when the selection service proves a +missing responsible user, personal default, or selected connection; an +incompatible selection; or an account that needs reconnection. The service marks +the original error, and setup preserves its closed `selectionFailure` reason. +The same failed/preparing/setup/bootstrap checks apply, and resumed native runs +with persisted provider input remain reportable. The generic +`ai_connection_unavailable` wrapper alone is insufficient: database, credential +provider, controller, and unclassified errors still report. Failed runs, blocked +tasks, and account-repair actions remain unchanged. + A workspace policy conflict also stays local when the resolver proves that an explicit `local_path` or `non_git_path` project workspace has no repository URL, Git confirms the selected directory is not a repository, and the task requests diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index e7c56ecd830..57217dd2381 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -16,6 +16,7 @@ import { createDb, companies, agents, agentTaskSessions, heartbeatRuns, companyM import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres"; import { fetchCompanyQuotaWindows } from "../services/quota-windows.js"; import { quotaCredentialRecoveryPath, quotaCredentialRecovery, quotaCredentialHash } from "../services/quota-credential-recovery.js"; +import { readAiConnectionConfigurationFailure } from "../services/ai-connection-configuration-failure.js"; import { aiConnectionService } from "../services/ai-connections.js"; import { syncConnectionCredentialBindings } from "../services/connection-credential-bindings.js"; import * as codexAdapter from "@paperclipai/adapter-codex-local/server"; @@ -60,6 +61,25 @@ beforeAll(async () => { afterAll(async () => { await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); }); describe("managed AI connections", () => { + it("marks explicit AI selection rejections without changing their HTTP contract", async () => { + const owner = "selection-blocker-owner"; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + const expectSelectionFailure = async (promise: Promise, reason: string) => { + const error = await promise.catch((error: unknown) => error); + expect(error).toMatchObject({ status: 422, details: { code: reason } }); + expect(readAiConnectionConfigurationFailure(error)).toBe(reason); + }; + await expectSelectionFailure(service.select({ ...input, userId: null }), "ai_connection_responsible_user_missing"); + await expectSelectionFailure(service.select({ ...input, userId: owner, adapterType: "codex_local" }), "ai_connection_incompatible"); + await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_default_missing"); + await expectSelectionFailure(service.select({ ...input, userId: owner, binding: { + ...binding, mode: "delegated", connectionId: randomUUID(), grantId: randomUUID(), + } }), "ai_connection_missing"); + const account = await create(owner, "Selection blocker account"); + await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId)); + await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_unavailable"); + }); + it("persists first-time recovery directories before consuming a single-use token", async () => { const owner = "quota-durable-path"; await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 87c14e17b66..a4f6421e261 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1,3 +1,5 @@ +import * as aiConnectionRuntime from "../services/ai-connection-runtime.js"; +import { unprocessable } from "../errors.js"; import * as executionContinuation from "../services/execution-continuation.js"; import * as environmentOrchestrator from "../services/environment-run-orchestrator.js"; import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; @@ -44,6 +46,7 @@ import { chatExternalPrincipals, chatMessageLinks, chatPublications, + companyMemberships, companySecretBindings, companySecrets, companySkills, @@ -1522,6 +1525,53 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { return { companyId, agentId, runId, wakeupRequestId, issueId }; } + it.each(["missing_default", "database_error", "unmarked_http_error", "provider_error"] as const)( + "preserves AI configuration recovery and reports only unexpected causes: %s", async (cause) => { + const { companyId, agentId, issueId, runId, wakeupRequestId } = await seedQueuedIssueRunFixture(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "responsible-user", status: "active", membershipRole: "member" }); + await db.update(agents).set({ runtimeConfig: { + heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 }, + aiConnection: { provider: "openai", method: "api_key", mode: "responsible_user" }, + } }).where(eq(agents.id, agentId)); + const prepare = vi.spyOn(aiConnectionRuntime, "prepareManagedAiRuntime"); + if (cause !== "missing_default") prepare.mockRejectedValueOnce( + cause === "unmarked_http_error" + ? unprocessable("Connect an account and choose your personal default", { code: "ai_connection_default_missing" }) + : new Error(cause), + ); + try { + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + await heartbeat.waitForRunExecutionDrain(runId); + await waitForPendingRunFailureReports(); + expect(prepare).toHaveBeenCalled(); + const failed = await heartbeat.getRun(runId); + expect(failed).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", + resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + }); + if (cause === "missing_default") { + expect(failed?.error).toBe("Connect an account and choose your personal default"); + expect(failed?.resultJson?.configurationIncomplete).toMatchObject({ selectionFailure: "ai_connection_default_missing" }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + } else { + expect(failed?.resultJson?.configurationIncomplete).not.toHaveProperty("selectionFailure"); + expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ runId, errorCode: "configuration_incomplete" })); + } + expect(mockAdapterExecute).not.toHaveBeenCalled(); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(issue).toMatchObject({ status: "blocked", executionRunId: null }); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(action).toMatchObject({ status: "active", kind: "configuration_validation", cause: "configuration_incomplete", ownerType: "board" }); + const [wakeup] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)); + expect(wakeup.status).toBe("failed"); + } finally { + prepare.mockRestore(); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + } + }, + ); + it.each(["timeout", "upstream", "cleanup_pending", "cleanup_pending_edited", "cleanup_pending_exhausted", "edited", "exhausted", "new_message", "no_claim", "reassigned", "superseded", "stopped"])( "settles explicit retry admission through real executor cleanup: %s", async scenario => { const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index be38e96b93b..2b2120e99f9 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -283,6 +283,65 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ agentAdapter: "unknown" })); }); + function aiSelectionRun(overrides: Partial = {}) { + return buildRun({ + errorCode: "configuration_incomplete", + executionStage: "preparing", + resultJson: { + configurationIncomplete: { + reason: "ai_connection_unavailable", + selectionFailure: "ai_connection_default_missing", + }, + executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + }, + ...overrides, + }); + } + + it.each([ + "ai_connection_responsible_user_missing", "ai_connection_default_missing", + "ai_connection_missing", "ai_connection_incompatible", "ai_connection_unavailable", + ])("keeps the proven AI selection blocker local: %s", async (selectionFailure) => { + await seedCompanyAndAgent(); + const run = aiSelectionRun(); + (run.resultJson!.configurationIncomplete as Record).selectionFailure = selectionFailure; + const saved = structuredClone(run); + await reportRunFailure(db, run, { phase: "setup" }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + expect(run).toEqual(saved); + }); + + it.each([undefined, null, {}, [], "", "database_error", "ai_connection_busy"])( + "reports AI connection wrappers without a known selection cause: %j", async (selectionFailure) => { + await seedCompanyAndAgent(); + const run = aiSelectionRun(); + (run.resultJson!.configurationIncomplete as Record).selectionFailure = selectionFailure; + await reportRunFailure(db, run, { phase: "setup" }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + }, + ); + + it.each([ + { runtimeMode: "native", runnerProfileJson: { nativeExecutionInput: {} } }, + { status: "timed_out" }, { errorCode: "setup_failed" }, { executionStage: "executing" }, + { executionStage: null }, { exitCode: 1 }, { exitCode: 0 }, { signal: "SIGTERM" }, + { resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_default_missing" } } }, + { resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_default_missing" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } }, + { resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_default_missing" }, executionRecovery: { kind: "resume", providerWorkStarted: false } } }, + { resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_default_missing" }, executionRecovery: { kind: "bootstrap" } } }, + { resultJson: { configurationIncomplete: { reason: "controller_environment_unsupported", selectionFailure: "ai_connection_default_missing" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }, + ])("reports AI configuration without pre-provider selection proof: %j", async (overrides) => { + await seedCompanyAndAgent(); + await reportRunFailure(db, aiSelectionRun(overrides), { phase: "setup" }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + }); + + it.each([undefined, "execute"] as const)("reports AI selection failures outside setup: %s", async (phase) => { + await seedCompanyAndAgent(); + await reportRunFailure(db, aiSelectionRun(), { phase }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + }); + function missingSecretRun(overrides: Partial = {}) { return buildRun({ errorCode: "configuration_incomplete", diff --git a/server/src/services/ai-connection-configuration-failure.test.ts b/server/src/services/ai-connection-configuration-failure.test.ts new file mode 100644 index 00000000000..50e4a6a68f8 --- /dev/null +++ b/server/src/services/ai-connection-configuration-failure.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { unprocessable } from "../errors.js"; +import { + aiConnectionConfigurationFailure, + isAiConnectionConfigurationReason, + readAiConnectionConfigurationFailure, +} from "./ai-connection-configuration-failure.js"; + +describe("AI selection failure provenance", () => { + it("preserves HTTP behavior while retaining only an owned bounded reason", () => { + const error = aiConnectionConfigurationFailure("ai_connection_default_missing", "Choose an account", { connectionId: "synthetic-connection" }); + expect(error).toMatchObject({ status: 422, message: "Choose an account", details: { + code: "ai_connection_default_missing", connectionId: "synthetic-connection", + } }); + expect(readAiConnectionConfigurationFailure(error)).toBe("ai_connection_default_missing"); + expect(Object.keys(error)).toEqual(["status", "details"]); + }); + + it.each([ + new Error("Connect an account and choose your personal default"), + unprocessable("Choose an account", { code: "ai_connection_default_missing" }), + { status: 422, details: { code: "ai_connection_default_missing" } }, + Object.assign(new Error("database failed"), { code: "configuration_incomplete" }), + null, + ])("does not infer provenance from an error name, message, or HTTP shape: %j", (error) => { + expect(readAiConnectionConfigurationFailure(error)).toBeNull(); + }); + + it.each([null, undefined, {}, [], "ai_connection_busy", "provider_error", "toString"])( + "rejects unknown or malformed persisted reasons: %j", (value) => { + expect(isAiConnectionConfigurationReason(value)).toBe(false); + }, + ); +}); diff --git a/server/src/services/ai-connection-configuration-failure.ts b/server/src/services/ai-connection-configuration-failure.ts new file mode 100644 index 00000000000..01053c24203 --- /dev/null +++ b/server/src/services/ai-connection-configuration-failure.ts @@ -0,0 +1,31 @@ +import { unprocessable } from "../errors.js"; + +const reasons = [ + "ai_connection_responsible_user_missing", + "ai_connection_default_missing", + "ai_connection_missing", + "ai_connection_incompatible", + "ai_connection_unavailable", +] as const; +type AiConnectionConfigurationReason = typeof reasons[number]; +const failures = new WeakMap(); + +/** Preserve the existing HTTP error while recording an explicit selection rejection. */ +export function aiConnectionConfigurationFailure( + reason: AiConnectionConfigurationReason, + message: string, + details: Record = {}, +) { + const error = unprocessable(message, { ...details, code: reason }); + failures.set(error, reason); + return error; +} + +/** Only owned producers supply this evidence; matching names or HTTP codes do not. */ +export function readAiConnectionConfigurationFailure(error: unknown): AiConnectionConfigurationReason | null { + return error instanceof Error ? failures.get(error) ?? null : null; +} + +export function isAiConnectionConfigurationReason(value: unknown): value is AiConnectionConfigurationReason { + return typeof value === "string" && reasons.some((reason) => reason === value); +} diff --git a/server/src/services/ai-connections.ts b/server/src/services/ai-connections.ts index 46494d580a9..e73c0c64fd7 100644 --- a/server/src/services/ai-connections.ts +++ b/server/src/services/ai-connections.ts @@ -39,6 +39,7 @@ import { type AiConnectionLoginIntent, } from "@paperclipai/shared"; import { forbidden, notFound, unprocessable } from "../errors.js"; +import { aiConnectionConfigurationFailure } from "./ai-connection-configuration-failure.js"; import { quotaCredentialRecovery, quotaCredentialHash } from "./quota-credential-recovery.js"; import { logger } from "../middleware/logger.js"; import { logActivity } from "./activity-log.js"; @@ -266,15 +267,9 @@ export function aiConnectionService(db: Db) { input.acpxAgent, ) ) - throw unprocessable( - "Select an AI connection compatible with this harness and model", - { code: "ai_connection_incompatible" }, - ); + throw aiConnectionConfigurationFailure("ai_connection_incompatible", "Select an AI connection compatible with this harness and model"); if (binding.mode === "responsible_user" && !userId) - throw unprocessable( - "This run needs a responsible user to select an AI connection", - { code: "ai_connection_responsible_user_missing" }, - ); + throw aiConnectionConfigurationFailure("ai_connection_responsible_user_missing", "This run needs a responsible user to select an AI connection"); if (userId && !(await membership(companyId, userId))) throw forbidden("The responsible user is not an active company member"); const defaultRow = @@ -298,10 +293,7 @@ export function aiConnectionService(db: Db) { ? defaultRow?.grantId : binding.grantId; if (!grantId) - throw unprocessable( - "Connect an account and choose your personal default", - { code: "ai_connection_default_missing" }, - ); + throw aiConnectionConfigurationFailure("ai_connection_default_missing", "Connect an account and choose your personal default"); const row = (await rows(companyId)).find( (r) => r.grant.id === grantId && @@ -309,9 +301,7 @@ export function aiConnectionService(db: Db) { r.connection.id === binding.connectionId), ); if (!row) - throw unprocessable("The selected AI connection is unavailable", { - code: "ai_connection_missing", - }); + throw aiConnectionConfigurationFailure("ai_connection_missing", "The selected AI connection is unavailable"); const { connection, grant } = row; const metadata = aiConnectionMetadataSchema.safeParse(connection.config.ai); if ( @@ -320,13 +310,9 @@ export function aiConnectionService(db: Db) { (binding.mode !== "responsible_user" && metadata.data.method !== binding.method) || !isAiConnectionCompatible(metadata.data, input.adapterType, input.model, input.runnerProvider, input.acpxAgent) ) - throw unprocessable("The selected AI connection is incompatible", { - code: "ai_connection_incompatible", - }); + throw aiConnectionConfigurationFailure("ai_connection_incompatible", "The selected AI connection is incompatible"); if (aiSubscriptionNeedsIsolatedLogin(connection.config)) - throw unprocessable("Reconnect this subscription with a separate sign-in to protect your existing terminal login.", { - code: "ai_connection_unavailable", connectionId: connection.id, - }); + throw aiConnectionConfigurationFailure("ai_connection_unavailable", "Reconnect this subscription with a separate sign-in to protect your existing terminal login.", { connectionId: connection.id }); if ( grant.status !== "active" || !connection.enabled || @@ -337,10 +323,7 @@ export function aiConnectionService(db: Db) { connection.config.aiLegacyAdoption === true )) ) - throw unprocessable("Reconnect or validate the selected AI account", { - code: "ai_connection_unavailable", - connectionId: connection.id, - }); + throw aiConnectionConfigurationFailure("ai_connection_unavailable", "Reconnect or validate the selected AI account", { connectionId: connection.id }); if ( grant.kind === "user" && !(await membership(companyId, grant.subjectUserId)) diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index efc951317f4..a3a8969166a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -537,6 +537,7 @@ import { emitAgentTaskRun, emitAgentTaskRunById, } from "./agent-task-run-telemetry.js"; +import { readAiConnectionConfigurationFailure } from "./ai-connection-configuration-failure.js"; import { reportRunFailure } from "./run-failure-report.js"; import { performance } from "node:perf_hooks"; import { buildProcessLossDiagnostic } from "./process-loss-diagnostics.js"; @@ -11870,6 +11871,8 @@ export function heartbeatService( } throw new ConfigurationIncompleteFailure(error instanceof Error ? error.message : "Configure this agent’s AI connection", { configurationIncomplete: { reason: "ai_connection_unavailable", companyId: agent.companyId, agentId: agent.id, responsibleUserId, + ...(!persistedNativeExecutionInput && readAiConnectionConfigurationFailure(error) + ? { selectionFailure: readAiConnectionConfigurationFailure(error) } : {}), provider: aiBinding.provider, method: aiBinding.method, actionUrl: `/agents/${agent.id}/runtime`, fingerprint: `ai:${agent.id}:${responsibleUserId}:${JSON.stringify(aiBinding)}` }, }); diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index d44a1329280..a059bf523d0 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -12,6 +12,7 @@ import { import { logger } from "../middleware/logger.js"; import { isUnexpectedRunCancellation } from "./run-cancellation.js"; import { readConnectionFailure, type ConnectionFailure } from "@paperclipai/adapter-utils/connection-failure"; +import { isAiConnectionConfigurationReason } from "./ai-connection-configuration-failure.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; @@ -47,6 +48,22 @@ function asRecord(value: unknown): Record | null { : null; } +/** An owned AI selection rejection before dispatch, not its broad catch wrapper. */ +function isAiConnectionSelectionBlocker(run: HeartbeatRun, options: RunFailureReportOptions): boolean { + // Resumed native input may already have provider effects, even during preparation. + if (run.runtimeMode === "native" && asRecord(run.runnerProfileJson)?.nativeExecutionInput !== undefined) return false; + if ( + run.status !== "failed" || run.errorCode !== "configuration_incomplete" || + run.executionStage !== "preparing" || options.phase !== "setup" || + run.exitCode != null || run.signal != null + ) return false; + const configuration = asRecord(run.resultJson?.configurationIncomplete); + const recovery = asRecord(run.resultJson?.executionRecovery); + return configuration?.reason === "ai_connection_unavailable" && + isAiConnectionConfigurationReason(configuration.selectionFailure) && + recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false; +} + /** A known owner action before dispatch, not a secret-provider or runtime failure. */ function isMissingSecretConfigurationBlocker(run: HeartbeatRun, options: RunFailureReportOptions): boolean { if ( @@ -162,6 +179,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR if (!isRunFailureStatus(run.status)) return Promise.resolve(); if (run.status === "cancelled" && !isUnexpectedRunCancellation(run)) return Promise.resolve(); if (isMissingSecretConfigurationBlocker(run, options)) return Promise.resolve(); + if (isAiConnectionSelectionBlocker(run, options)) return Promise.resolve(); if (isLocalPathWorkspaceConfigurationBlocker(run, options)) return Promise.resolve(); if (isGitConnectionFailure(run, options)) return Promise.resolve(); const runStatus = run.status; From 428b617fc91e97f7a5f9892fd7b466889ff39470 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:00:01 -0500 Subject: [PATCH 10/10] test(evals): qualify native question resume paths (#15616) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip lets agents pause tasks for human answers and continue the work. > - Native questions can yield a run or pause inside the provider's running turn. > - Those paths have different run identities and different forms. > - The documentation-placement test requires a semantic three-turn journey. > - A valid provider question therefore needs separate behavioral qualification. > - This pull request adds an opt-in two-answer journey with exact source and resume checks. > - Qualification exposed a Vite startup crash in idle-handler traversal, so the branch also distinguishes Connect mount paths from Express Route objects. ## Linked Issues or Issue Description **What existing behavior does this improve?** Product E2E evaluation of native task questions and answer delivery. **Current behavior** The semantic documentation case rejects the provider adapter's optional Other field before answering. Its three-run expectation also cannot prove a provider question that resumes within the same run. **Proposed behavior** Keep that semantic test intact. Add a separate opt-in case that classifies each recorded question by authoritative identities. Accept the optional Other field only for a verified provider question. Require both user answers, the correct continuation, and one saved final document. Related: #15564 added strict lifecycle checks. #15522 introduced idle-request tracking; this PR corrects its Connect/Vite compatibility after reproducing the startup crash. #14591 changes ACP question drafts and provider support; this PR has no overlapping files with that change. ## What Changed - Add the explicit-only `question-resume` suite for native Codex and ACPX Claude. Reuse the existing neutral choice-then-text task request. - Bind each question to its company, task, agent and run. Require an applied semantic tool receipt or the exact provider request identity. - Require a provider answer to continue the same run. Require a semantic answer to start one new run with the matching interaction and source-run wake fields. - Preserve question forms and both real board answers through the final checkpoint. Reject missing inputs, duplicate answers, extra runs and substituted identities. - Keep the existing lifecycle, output, task ownership and budget checks. Limit the new case to one attempt and one to three runs. - Add mutation calibrations and document the separate qualification claims. Keep production guidance, scheduling, provider code and the historical semantic case unchanged. - Fix startup with Vite middleware: treat Connect string mount paths separately from Express Route objects when traversing idle-request handlers. A real-Vite regression verifies startup and retained async work through an idle hold. ## Verification - 81 focused continuation, question and calibration tests passed before review. After the reference-preservation correction, all 50 question/resume tests pass, including two new negative cases that failed against the previous grader. - Final eval support suite passes: 1,921 Vitest tests and 128 Node tests, with one intentional skip. - Eval and workspace typechecks pass. Workspace build passes. - Catalog discovery finds only the two declared opt-in cells. Existing continuation remains 23 cells; default paid scope is unchanged. - Original [campaign 37831726391](https://github.com/paperclipai/paperclip/actions/runs/37831726391), source `1ac4b2873466896ff2892f49be82904927b6c57d`, failed during server startup before any test or provider run. Its original `transient_infrastructure` result and `cleanup: not_started` are preserved. Local reproduction identifies the Connect string route traversal crash; the question grader was never reached. - Setup-corrected [campaign 37833872898](https://github.com/paperclipai/paperclip/actions/runs/37833872898) measures source `9781a772389d41421df55759004676e1b905e512` with trusted workflow `8e59efc50b162126f33816841e06509634eba65c`. The workflow bytes are unchanged from the first dispatch. Same single selected native Claude cell, one attempt, at most three runs, ten-minute cell deadline, and 1,000-cent company/agent hard stops. Original result **PASS**, 24/24 checks and cleanup pass. Exactly three successful native Claude runs: two applied Paperclip `request_human_input` questions and one finishing run. Both real board answers persist, exact interaction/source-run response wakes match, one revision-1 task document contains Afternoon and the supplied reference, and final task state is done with no active lock, retry, recovery, monitor, pending interaction or child task. No model retry. - Current review head `3c08cb7d26e7f9ac16469eeb07d1f31d828ec02e` differs from the measured source only in the question/resume grader and its tests (11 insertions, 3 deletions). The saved document must contain the complete submitted reference, including its prefix and punctuation. Separate provider-free replay of the retained successful campaign passes all 24 continuation/lifecycle checks under this stricter grader. The original result and its measured source remain unchanged; no new provider run was made. - Observed paths were **semantic → semantic**. The provider built-in question/optional Other path was not exercised by this new live attempt. Its form acceptance has retained-original calibration; same-run answer/resume has deterministic positive/negative calibration. This neutral-task pass alone does not qualify the provider path; the subsequent explicit bridge trial below is separate evidence. Neither trial regrades the old Claude failure or establishes a causal behavior/performance comparison. - Subsequent explicit provider-path [campaign 37841107684](https://github.com/paperclipai/paperclip/actions/runs/37841107684) measures the final PR source `3c08cb7d26e7f9ac16469eeb07d1f31d828ec02e`, using trusted workflow `2a5f65c9ae501b49b2a38ce7edd04209b806c9d0`. Exact cell: `continuation.runner-acpx-claude.local.provider-question-bridge`, Claude `claude-sonnet-5`, suite fingerprint `fc77203bf85fd99b06ebc53cd52982186c0ae0dc369fcc38a9ec3c737e9a31e5`. **Original PASS, 15/15 checks, cleanup passed, one actual provider run and zero eval rerolls.** The built-in question produced one choice plus its optional Other companion. The browser selected the requested reference, left Other blank, and submitted. Exact runtime request/response IDs and the selected option match the saved interaction; the same paused native run created one revision-1 document and finished Done with no pending interaction, lock, retry, recovery, monitor or child task. Independent evidence and runtime-receipt audits pass; marked initial/final screenshots were visually checked. [Original public report](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-37841107684-1/index.html). - This separate provider-directed trial qualifies one choice-answer bridge and traversal of an empty optional Other field. It does not establish natural tool selection, a typed Other answer, two consecutive built-in questions, free-text-only input, restart recovery or general reliability. No production, prompt or grader change was made for this dispatch. The old failure and the separate neutral semantic-path result retain their original grades. - Preserve within-run friction: one `write_task_document` input-schema rejection and two HTTP 409 “Document does not exist yet” responses preceded a successful `write_document` call. These are three unsuccessful tool calls inside the same run, not three extra provider runs; no flawless document-tool behavior is claimed. Billing reports token usage but remains unpriced/incomplete, so actual charges are unknown and local/hosted runtime is unmetered. Both company and agent hard stops were 1,000 cents, with one attempt, one selected cell, concurrency one and a ten-minute cell deadline. - Billing records all three model runs with token usage, but no priced dollar receipt (`unpriced`, `complete: false`). Actual charges are unknown; local/hosted runtime is unmetered. Numeric zero reported cost does not mean free. - Startup repair: real-Vite reproduction passes; 35 targeted idle-tracking tests pass, 34 database-dependent checks skip because embedded PostgreSQL is unavailable locally. Workspace typecheck and build pass again after the repair. - Full local repository database tests were not repeated because embedded PostgreSQL was unavailable in the preceding workspace verification. Full Linux CI now passes on the final review head. - Final-head [CI run 37837804284](https://github.com/paperclipai/paperclip/actions/runs/37837804284) passes. Complete current-head audit: 53 successful check-runs, two intentional Storybook skips, and separate Snyk success. The ready transition’s contributor and security checks also pass (security scan: no findings). [Fresh review](https://github.com/paperclipai/paperclip/pull/15616#issuecomment-6068092601) is 5/5 on `3c08cb7d26e7f9ac16469eeb07d1f31d828ec02e`; zero unresolved threads and no merge conflicts. ## Risks This is a new behavioral definition. The neutral two-question trial records semantic-tool use; the separate provider-directed trial records one built-in question. They do not establish consistent tool selection, documentation placement, default hiring, remote environments, crash recovery or general reliability. The separate explicit bridge trial covers one provider choice, an empty optional Other companion and same-run completion. Typed Other, two consecutive built-in questions and free-text-only provider input remain unqualified. The observed document-tool schema rejection and two 409 responses are retained as a separate follow-up, without attributing their cause. The historical case and its original grades remain intact. The reference question must still be text-only; the oracle does not turn an arbitrary extra question into an optional companion. Missing or unfamiliar identity evidence fails closed. The only production change distinguishes Connect string mount paths from Express route objects during idle-handler traversal. Unknown route objects still fail closed. No API, schema, migration or workflow change is included. ## Model Used OpenAI Codex, GPT-6-based, with tool use and code execution. The exact serving model ID and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/evals.md | 5 + server/src/__tests__/idle-admission.test.ts | 30 ++++ server/src/middleware/idle-admission.ts | 6 +- tests/runner-e2e/README.md | 24 +++ tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 13 +- tests/runner-e2e/continuation-cases.ts | 17 ++- tests/runner-e2e/continuation-flow.ts | 9 +- tests/runner-e2e/continuation-scoring.ts | 4 +- tests/runner-e2e/question-resume-scoring.ts | 131 ++++++++++++++++ tests/runner-e2e/question-resume.test.ts | 161 ++++++++++++++++++++ 11 files changed, 393 insertions(+), 13 deletions(-) create mode 100644 tests/runner-e2e/question-resume-scoring.ts create mode 100644 tests/runner-e2e/question-resume.test.ts diff --git a/doc/evals.md b/doc/evals.md index 06037daea6e..61e0d385cf2 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -391,6 +391,11 @@ The 26 native `first-task` cells exercise onboarding before native selection becomes the UI default. Live results and semantic answer reviews must accompany any qualification claim; catalog presence alone is not a pass. +The explicit-only [native question/resume qualification](../tests/runner-e2e/README.md#task-continuation) +separates a completed two-answer user journey from semantic-tool documentation +qualification. It verifies the exact provider-pause or semantic-response-wake +binding for each answer and preserves prior grades when the definition changes. + ## Lifecycle behavior baseline The credential-free [lifecycle baseline](../tests/lifecycle-baseline/README.md) diff --git a/server/src/__tests__/idle-admission.test.ts b/server/src/__tests__/idle-admission.test.ts index 81f268a1ffa..d9b7cd269fc 100644 --- a/server/src/__tests__/idle-admission.test.ts +++ b/server/src/__tests__/idle-admission.test.ts @@ -49,6 +49,36 @@ describe("idle admission", () => { expect(idleWorkSnapshot().active).toBe(0); }); + it("tracks real Vite middleware without confusing Connect mount paths with Express routes", async () => { + const { createServer } = await import("vite"); + const vite = await createServer({ + configFile: false, + appType: "custom", + server: { middlewareMode: true, hmr: false, watch: null }, + optimizeDeps: { noDiscovery: true, include: [] }, + }); + const server = app(), done = deferred(); + vite.middlewares.use("/vite-work", async (_req, res) => { + res.statusCode = 202; + res.end(); + await done.promise; + }); + server.use(vite.middlewares); + try { + expect(() => trackIdleRequestHandlers(server)).not.toThrow(); + await request(server).get("/vite-work").expect(202); + expect(idleWorkSnapshot().active).toBe(1); + startTaskDrain({ purpose: "idle", ttlMs: 60_000 }); + await request(server).get("/vite-work").expect(503); + done.resolve(); + await new Promise(resolve => setImmediate(resolve)); + expect(idleWorkSnapshot().active).toBe(0); + } finally { + done.resolve(); + await vite.close(); + } + }); + it("does not mistake a client disconnect for completed accepted work", async () => { const server = app(); const entered = deferred(), done = deferred(), closed = deferred(); diff --git a/server/src/middleware/idle-admission.ts b/server/src/middleware/idle-admission.ts index c9de5f03eca..9540d704dd0 100644 --- a/server/src/middleware/idle-admission.ts +++ b/server/src/middleware/idle-admission.ts @@ -53,14 +53,16 @@ export const idleAdmissionMiddleware: RequestHandler = (req, res, next) => { // preserving router objects and error-handler arity. Waiting only for finish // loses async work after res.json(), including the disconnected-client case. // Keep this adapter isolated and exercise it against real Express in tests. -type Layer = { handle: Function & { stack?: Layer[] }; route?: { stack: Layer[] } }; +type Layer = { handle: Function & { stack?: Layer[] }; route?: { stack: Layer[] } | string }; export function trackIdleRequestHandlers(app: Application): void { const seen = new Set(); const visit = (stack: Layer[], route = false) => { for (const layer of stack) { if (seen.has(layer)) continue; seen.add(layer); - if (layer.route) { visit(layer.route.stack, true); continue; } + // Connect/Vite layers use a mount-path string for route; only Express + // Route objects contain a nested handler stack. + if (layer.route && typeof layer.route !== "string") { visit(layer.route.stack, true); continue; } if (layer.handle.stack) { visit(layer.handle.stack); continue; } const original = layer.handle; if (original === idleAdmissionMiddleware) continue; diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index fd015d0d628..d24e660c24f 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1271,6 +1271,30 @@ contract tests separately verify that the advertised tool carries the documentat for fresh and resumed native executions. This tests the current documentation placement; it is not a statistical comparison with the former prompt arrangement. +The separate explicit-only `question-resume` suite uses the same neutral two-question +request on native Codex and ACPX Claude. It qualifies the user journey through +either the provider's built-in question or Paperclip's semantic question tool. +It does not replace or regrade `question-tool-documentation`. + +The oracle classifies each wait from its exact task/agent/run identity. Provider +questions require the server-generated request key and retain the paused run. +Semantic questions require an applied `request_human_input` receipt and one new +response wake bound to that interaction and source run. Only a verified provider +choice may carry the adapter's optional Other field. A second substantive question +is still rejected; the subsequent reference question must be text-only. Both real +board answers must remain bound to unchanged forms, and one saved document must +include both answers. Existing lifecycle, ownership, premature-output, terminal +state and no-extra-task checks apply. Three checkpoints are required even when +both answers resume one run; missing native inputs or unexplained runs fail. + +Select `question-resume.runner-acpx-claude.local.question-answer-resume` for the +bounded Claude qualification. The suite is excluded from `--all`; it allows one +attempt, one to three actual runs, a ten-minute cell deadline and verified +1,000-cent company/agent hard stops. The historical failure where the optional +Other field stopped the driver before any answer remains a FAIL. A new behavior +pass does not establish semantic-tool selection, documentation placement, default +hiring behavior or general reliability. + Continuation screenshots wait for the correct task heading and fully revealed conversation before capture. A loading screen or wrong task fails capture. Browser-only regressions exercise delayed rendering without provider calls: diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 6b386d6e182..6e049a4841c 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -146,10 +146,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, + 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, ]); - expect(validateRunnerCatalog()).toHaveLength(698); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(698); + expect(validateRunnerCatalog()).toHaveLength(700); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(700); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 0d2b996f939..bae30990523 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -12,7 +12,7 @@ import { apiResponseReadingTask } from "./api-response-reading.js"; import { taskTitleTasks, taskTitleDefinitionDigest, TASK_TITLE_BUDGET_CENTS } from "./task-titles.js"; import { blockerTasks, blockerProfile } from "./blocker-cases.js"; import { accountingTasks } from "./accounting-cases.js"; -import { continuationTasks } from "./continuation-cases.js"; +import { continuationTasks, questionResumeTask } from "./continuation-cases.js"; import { contextIntegrityTasks, paperclipDocumentTask } from "./context-integrity-cases.js"; import { productionDefaultHireProfile, stockHarnessSourceDigest, stockHarnessSkillSources } from "./stock-harness.js"; import { lifecycleLiveTasks, lifecycleLiveDefinitionDigest } from "./lifecycle-live-cases.js"; @@ -1262,6 +1262,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ excludedExecutionIds: ["neutral", "challenge"].map(variant => `lifecycle-baseline.runner-codex.local.lifecycle-repair-${variant}`), definitionMetadata: { version: 5, clarificationGrading: "wrapped-information-fields-list", narrativeDigest: lifecycleLiveDefinitionDigest, grading: "durable-state-and-attributed-narrative", scheduling: "explicit-only" }, }, + { + id: "question-resume", label: "Native question and answer continuation", manualOnly: true, + description: "Two real UI answers, exact provider-pause or semantic-wake identity, and one saved document.", + groups: ["native", "local"], environments: [localEnvironment], + profiles: runnerProfiles.filter(p => ["runner-codex", "runner-acpx-claude"].includes(p.id)).map(productionStoryProfile), + tasks: [questionResumeTask], expectedMatrixSize: 2, + definitionMetadata: { version: 1, grading: "bound-question-path-and-two-durable-answers", instructions: "unchanged-production", + scheduling: "explicit-only", automaticRetryPolicy: "single_attempt", companyAndAgentBudgetCents: 1_000, + providerRuns: "1-3; each semantic answer starts one exactly bound wake; provider answers retain their paused run" }, + }, { id: "continuation", label: "Task continuation", description: "Human direction, approval boundaries, untrusted evidence, and completed actions across turns.", @@ -1668,6 +1678,7 @@ export function validateRunnerCatalog(): MatrixExecution[] { ...accountingTasks, ...lifecycleLiveTasks, ...continuationTasks, + questionResumeTask, ...everydayTasks, ...taskTitleTasks, ...runnerTasks, diff --git a/tests/runner-e2e/continuation-cases.ts b/tests/runner-e2e/continuation-cases.ts index 65810ad4fb7..781d44e394c 100644 --- a/tests/runner-e2e/continuation-cases.ts +++ b/tests/runner-e2e/continuation-cases.ts @@ -10,9 +10,9 @@ export const CONTINUATION_CASES = [ "question-tool-documentation", "provider-question-bridge", ] as const; -export type ContinuationCase = (typeof CONTINUATION_CASES)[number]; +export type ContinuationCase = (typeof CONTINUATION_CASES)[number] | "question-answer-resume"; export function continuationScenario(id: string, nonce: string) { - if (!CONTINUATION_CASES.includes(id as ContinuationCase)) + if (id !== "question-answer-resume" && !CONTINUATION_CASES.includes(id as (typeof CONTINUATION_CASES)[number])) throw new Error(`Unknown continuation case: ${id}`); const marker = `AMBER${nonce.replace(/[^a-z0-9]/gi, "")}`; const old = `COBALT${nonce.replace(/[^a-z0-9]/gi, "")}`; @@ -21,9 +21,11 @@ export function continuationScenario(id: string, nonce: string) { const childTitle = `Follow-up ${marker}`; const document = "Save the final note as a task document. No other deliverables or child tasks are needed."; + const questionPrompt = "Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables."; const prompts: Record = { + "question-answer-resume": questionPrompt, "provider-question-bridge": `Use your built-in AskUserQuestion tool (not Paperclip's request_human_input) to ask which reference to include, with two choices: ${marker} and ${old}. Wait for my real answer, then save a one-sentence welcome note including only my selected reference as a task document and finish. No other tasks or deliverables are needed.`, - "question-tool-documentation": `Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables.`, + "question-tool-documentation": questionPrompt, "answer-updates-scope": `I need a one-sentence welcome note containing ${old}. Before writing it, ask me one open-ended structured question about any changes I want. Then apply my answer and finish. ${document}`, "clarification-not-approval": `I need a one-sentence welcome note. First ask me one open-ended structured question for the word to include. After my answer, propose your approach and wait for my explicit approval before writing the note. ${document}`, "revision-preserves-approval": `Propose an approach for a one-sentence welcome note containing ${old}, and save that approach as the plan document. Wait for my explicit approval before writing the note. ${document}`, @@ -85,3 +87,12 @@ export function continuationScreenshotFile( ? "final-state.png" : `question-continuation-${phase}.png`; } + +/** Separate behavior contract; leave the semantic documentation case unchanged. */ +export const questionResumeTask: RunnerTaskFixture = { + ...continuationTasks.find(t => t.id === "question-tool-documentation")!, + id: "question-answer-resume", label: "Two answered questions across native paths", + expectedRunCount: 3, minimumExpectedRunCount: 1, + buildTitle: nonce => `Question resume ${nonce}`, + buildPrompt: nonce => continuationScenario("question-answer-resume", nonce).prompt, +}; diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index 6381d03b5c4..8c0ad9ea222 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -1,3 +1,4 @@ +import { validResumeQuestionForm } from "./question-resume-scoring.js"; import { gradeLifecycleBaseline, type LifecycleCheckpoint } from "./lifecycle-baseline.js"; import { lifecycleLiveCase, lifecycleLiveContinuation, gradeLifecycleNarrative } from "./lifecycle-live-cases.js"; import { prepareLegacyContinuationSkill, prepareContinuationBudget } from "./continuation-fixtures.js"; @@ -106,7 +107,7 @@ export async function runContinuationFlow(input: { return ready; }, reject: (state) => - state.runs.length > 12 + state.runs.length > (scenario.id === "question-answer-resume" ? 3 : 12) ? "Bounded continuation run count exceeded" : state.runs.some((r) => ["failed", "timed_out", "cancelled"].includes(r.status), @@ -185,7 +186,9 @@ export async function runContinuationFlow(input: { ); expect(questions, "one real question must be shown").toHaveLength(1); const set = chatQuestionPresentation(questions[0].payload); - if (scenario.id === "provider-question-bridge") { + if (scenario.id === "question-answer-resume") { + expect(validResumeQuestionForm(checkpoints.at(-1)!, questions[0], !!choice), "a verified native question path and the requested form").toBe(true); + } else if (scenario.id === "provider-question-bridge") { expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true); } else expect(set.questions, "ask only the requested next question").toHaveLength(1); const before = new Set(runs.map((r) => r.id)); @@ -267,7 +270,7 @@ export async function runContinuationFlow(input: { await input.restart(); await open(); } - if (scenario.id === "question-tool-documentation") { + if (["question-tool-documentation", "question-answer-resume"].includes(scenario.id)) { await answer("Afternoon"); await snapshot("answered"); assertWaiting(); diff --git a/tests/runner-e2e/continuation-scoring.ts b/tests/runner-e2e/continuation-scoring.ts index ada89b09167..42e79f8ed0b 100644 --- a/tests/runner-e2e/continuation-scoring.ts +++ b/tests/runner-e2e/continuation-scoring.ts @@ -1,3 +1,4 @@ +import { gradeQuestionResume } from "./question-resume-scoring.js"; import { createHash } from "node:crypto"; import { gradeQuestionDocumentation } from "./question-documentation-scoring.js"; import type { ContinuationCase } from "./continuation-cases.js"; @@ -49,7 +50,7 @@ export function gradeContinuation(input: { const before = input.checkpoints.filter((c) => c.phase !== "final"); check( "recorded-continuation", - before.length > 0 && !!final && final.runs.length >= (input.id === "provider-question-bridge" ? 1 : 2), + before.length > 0 && !!final && final.runs.length >= (["provider-question-bridge", "question-answer-resume"].includes(input.id) ? 1 : 2), "Initial and final turns must both be recorded.", ); for (const c of before) { @@ -135,5 +136,6 @@ export function gradeContinuation(input: { "A real provider-native card must be answered and resume the same run to completion."); } if (input.id === "question-tool-documentation") checks.push(...gradeQuestionDocumentation(input.checkpoints, input.marker)); + if (input.id === "question-answer-resume") checks.push(...gradeQuestionResume(input.checkpoints, input.marker)); return checks; } diff --git a/tests/runner-e2e/question-resume-scoring.ts b/tests/runner-e2e/question-resume-scoring.ts new file mode 100644 index 00000000000..4f494bdae04 --- /dev/null +++ b/tests/runner-e2e/question-resume-scoring.ts @@ -0,0 +1,131 @@ +import { isDeepStrictEqual } from "node:util"; +import type { ContinuationCheckpoint } from "./continuation-scoring.js"; +import { isSingleClaudeQuestion } from "./runtime-question-readiness.js"; + +type Row = Record; +export type QuestionPath = "provider" | "semantic"; +const nonempty = (v: unknown): v is string => typeof v === "string" && v.trim().length > 0; +const questions = (c?: ContinuationCheckpoint): Row[] => + (c?.interactions ?? []).filter((i: any) => i.kind === "ask_user_questions") as Row[]; +export const resumeQuestionFields = (i?: Row): Row[] => + Array.isArray(i?.payload?.questionSet?.questions) ? i.payload.questionSet.questions : []; + +/** Classify by server-owned identities and the applied semantic receipt. A + * missing receipt is unknown, never inferred from form shape or run count. */ +export function questionPath(checkpoint: ContinuationCheckpoint | undefined, i?: Row): QuestionPath | null { + if (!checkpoint || !i || !nonempty(i.id) || !nonempty(i.sourceRunId) || !nonempty(i.idempotencyKey)) return null; + const matches = checkpoint.runs.filter(r => r.id === i.sourceRunId) as Row[]; + const run = matches.length === 1 ? matches[0] : undefined; + if (!run || run.runtimeMode !== "native" || !nonempty(run.companyId) || + run.companyId !== (checkpoint.issue as Row).companyId || + i.companyId !== run.companyId || i.issueId !== checkpoint.issue.id || run.issueId !== i.issueId || + !nonempty(checkpoint.issue.assigneeAgentId) || i.createdByAgentId !== checkpoint.issue.assigneeAgentId || + run.agentId !== checkpoint.issue.assigneeAgentId) return null; + const request = i.payload?.runtimeRequestId; + if (request !== undefined) { + return nonempty(request) && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$/.test(request) && + i.idempotencyKey === `paperclip-runner-question:${run.id}:${request}` && + i.continuationPolicy === "none" && run.status === "running" ? "provider" : null; + } + const receipt = run.resultJson?.semanticToolReceipts?.[i.idempotencyKey]; + const created = receipt?.result?.interaction; + return run.status === "succeeded" && i.continuationPolicy === "wake_assignee" && + receipt?.operationId === "request_human_input" && receipt.result?.disposition === "applied" && + created?.id === i.id && created?.sourceRunId === run.id && created?.issueId === i.issueId && + created?.companyId === i.companyId && created?.kind === "ask_user_questions" && + isDeepStrictEqual(created?.payload?.questionSet, i.payload?.questionSet) ? "semantic" : null; +} + +export function validResumeQuestionForm(checkpoint: ContinuationCheckpoint, i: Row, choice: boolean): boolean { + const path = questionPath(checkpoint, i); + const fields = resumeQuestionFields(i); + if (!path || fields.some(f => !nonempty(f.id)) || new Set(fields.map(f => f.id)).size !== fields.length) return false; + if (!choice) return fields.length === 1 && fields[0].answerMode === "text" && + !fields[0].options?.length && !fields[0].customAnswer; + return fields[0]?.answerMode === "single_select" && + (fields.length === 1 || (path === "provider" && isSingleClaudeQuestion(fields))); +} + +function sameQuestion(a: Row, b?: Row) { + return !!b && ["id", "kind", "sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey", "continuationPolicy"] + .every(k => a[k] === b[k]) && isDeepStrictEqual(a.payload, b.payload); +} +function answeredByUser(i?: Row) { + return i?.status === "answered" && nonempty(i.resolvedByUserId) && !i.resolvedByAgentId; +} +function resumedRun(before: ContinuationCheckpoint | undefined, after: ContinuationCheckpoint | undefined, i?: Row) { + if (!before || !after || !i) return false; + const path = questionPath(before, i); + const prior = new Set(before.runs.map(r => r.id)); + const next = after.runs.filter(r => !prior.has(r.id)) as Row[]; + if (prior.size !== before.runs.length || new Set(after.runs.map(r => r.id)).size !== after.runs.length || + before.runs.some(r => !after.runs.some(n => n.id === r.id))) return false; + if (path === "provider") return next.length === 0 && + after.runs.some(r => r.id === i.sourceRunId && ["running", "succeeded"].includes(r.status)); + if (path !== "semantic" || next.length !== 1) return false; + const context = next[0].contextSnapshot; + return context?.issueId === i.issueId && context?.sourceRunId === i.sourceRunId && + context?.interactionId === i.id && context?.interactionKind === "ask_user_questions" && + context?.interactionStatus === "answered"; +} + +/** Behavioral qualification supports either valid native path. It does not + * replace the separate documentation-placement/semantic-tool qualification. */ +export function gradeQuestionResume(checkpoints: ContinuationCheckpoint[], marker: string) { + const initial = checkpoints.find(c => c.phase === "initial"); + const middle = checkpoints.find(c => c.phase === "answered"); + const final = checkpoints.find(c => c.phase === "final"); + const first = questions(initial)[0]; + const second = questions(middle).find(i => i.status === "pending"); + const choice = resumeQuestionFields(first)[0]; + const text = resumeQuestionFields(second)[0]; + const options: Row[] = choice?.options ?? []; + const afternoon = options.find(o => /^afternoon\b/i.test(String(o.label).trim())); + const checks = [{ + id: "question-resume.forms", + passed: !!initial && !!middle && questions(initial).length === 1 && first?.status === "pending" && + validResumeQuestionForm(initial, first, true) && options.length >= 2 && + options.every(o => nonempty(o.id)) && new Set(options.map(o => o.id)).size === options.length && + options.some(o => /^morning\b/i.test(String(o.label).trim())) && !!afternoon && + questions(middle).length === 2 && questions(middle).filter(i => i.status === "pending").length === 1 && + nonempty(second?.id) && second?.id !== first?.id && validResumeQuestionForm(middle, second, false), + detail: "One choice plus only a verified provider Other companion, then a distinct text-only question.", + }]; + for (const [before, after, i, field, answer] of [ + [initial, middle, first, choice, "choice"], [middle, final, second, text, "text"], + ] as const) { + const committed = questions(after).filter(q => q.id === i?.id); + const retained = questions(final).filter(q => q.id === i?.id); + const response: Row[] = committed[0]?.result?.answers ?? []; + const primary = response.filter(a => a.questionId === field?.id); + const optionalIds = resumeQuestionFields(i).slice(1).map(f => f.id); + const validAnswer = primary.length === 1 && (answer === "choice" + ? primary[0].optionIds?.length === 1 && primary[0].optionIds[0] === afternoon?.id && !primary[0].otherText + : !primary[0].optionIds?.length && primary[0].otherText === `Use ${marker}.`); + checks.push({ + id: `question-resume.${answer}.answer`, + passed: !!i && committed.length === 1 && retained.length === 1 && + sameQuestion(i, committed[0]) && sameQuestion(i, retained[0]) && answeredByUser(committed[0]) && + answeredByUser(retained[0]) && isDeepStrictEqual(committed[0].result, retained[0].result) && + validAnswer && response.every(a => a.questionId === field?.id || + (optionalIds.includes(a.questionId) && !a.optionIds?.length && !a.otherText)), + detail: "The real board answer must remain bound to the unchanged question, with no fabricated or substituted response.", + }, { + id: `question-resume.${answer}.path`, + passed: resumedRun(before, after, i), + detail: `Observed path: ${questionPath(before, i) ?? "unknown"}; provider answers resume the same run; semantic answers require one exactly bound response wake.`, + }); + } + checks.push({ + id: "question-resume.complete-evidence", + passed: checkpoints.length === 3 && !!initial && !!middle && !!final && initial.runs.length === 1 && + questions(final).length === 2 && final.runs.length >= 1 && final.runs.length <= 3 && + final.runs.every((r: Row) => r.companyId === (final.issue as Row).companyId && + r.issueId === final.issue.id && r.agentId === final.issue.assigneeAgentId && + typeof r.runnerProfileJson?.nativeExecutionInput?.task?.prompt === "string") && + final.documents.length === 1 && !!final.documents[0].latestRevisionId && + final.documents[0].body.includes(`Use ${marker}.`) && /\bafternoon\b/i.test(final.documents[0].body), + detail: "Retain all three checkpoints, every native input and exactly one saved document containing the choice and the complete submitted reference.", + }); + return checks.map(c => ({ ...c, passed: Boolean(c.passed) })); +} diff --git a/tests/runner-e2e/question-resume.test.ts b/tests/runner-e2e/question-resume.test.ts new file mode 100644 index 00000000000..525bce0c199 --- /dev/null +++ b/tests/runner-e2e/question-resume.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import { gradeQuestionResume, questionPath, validResumeQuestionForm, type QuestionPath } from "./question-resume-scoring.js"; +import { gradeContinuation } from "./continuation-scoring.js"; +import { gradeLifecycleBaseline } from "./lifecycle-baseline.js"; +import { continuationScenario } from "./continuation-cases.js"; +import { runnerMatrix } from "./catalog.js"; + +type Row = Record; +function recording(paths: QuestionPath[] = ["provider", "semantic"]) { + const runs: Row[] = []; + const cards: Row[] = []; + const points: any[] = []; + const newRun = (previous?: Row) => ({ id: `run-${runs.length + 1}`, status: "running", runtimeMode: "native", + companyId: "company", issueId: "task", agentId: "agent", resultJson: { semanticToolReceipts: {} }, + runnerProfileJson: { nativeExecutionInput: { task: { prompt: "actual native task input" } } }, + contextSnapshot: previous ? { issueId: "task", sourceRunId: previous.sourceRunId, + interactionId: previous.id, interactionKind: "ask_user_questions", interactionStatus: "answered" } : {}, + }); + runs.push(newRun()); + const snap = (phase: string, documents: Row[] = []) => { + const active = runs.find(r => r.status === "running"); + points.push(structuredClone({ phase, issue: { id: "task", companyId: "company", assigneeAgentId: "agent", + status: phase === "final" ? "done" : active ? "in_progress" : "in_review" }, + lifecycle: { executionRunId: active?.id ?? null, scheduledRetry: null, activeRecoveryAction: null, monitorNextCheckAt: null }, + runs, interactions: cards, children: [], attachments: [], comments: [], documents })); + }; + for (let n = 0; n < 2; n++) { + const run = runs.at(-1)!; + const provider = paths[n] === "provider"; + const request = `request-${n}`; + const idempotencyKey = provider ? `paperclip-runner-question:${run.id}:${request}` : `question-${n}`; + const field = n === 0 ? { id: "field-1-question_0-hash", answerMode: "single_select", options: [ + { id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" }, + ] } : { id: "reference", answerMode: "text" }; + const questions: Row[] = [field]; + if (provider && n === 0) questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false }); + const card: Row = { id: `card-${n}`, kind: "ask_user_questions", status: "pending", + companyId: "company", issueId: "task", createdByAgentId: "agent", sourceRunId: run.id, idempotencyKey, + continuationPolicy: provider ? "none" : "wake_assignee", + payload: { ...(provider ? { runtimeRequestId: request } : {}), questionSet: { questions } } }; + cards.push(card); + if (!provider) { + run.status = "succeeded"; + run.resultJson.semanticToolReceipts[idempotencyKey] = { operationId: "request_human_input", + result: { disposition: "applied", interaction: structuredClone(card) } }; + } + snap(n === 0 ? "initial" : "answered"); + card.status = "answered"; card.resolvedByUserId = "local-board"; + card.result = { answers: [n === 0 ? { questionId: field.id, optionIds: ["pm"] } : + { questionId: field.id, optionIds: [], otherText: "Use AMBERnonce." }] }; + if (!provider) runs.push(newRun(card)); + } + runs.at(-1)!.status = "succeeded"; + snap("final", [{ key: "welcome-note", latestRevisionId: "revision", body: "Welcome to the afternoon meetup. Use AMBERnonce." }]); + return points; +} +const failed = (r: any[]) => gradeQuestionResume(r, "AMBERnonce").filter(c => !c.passed).map(c => c.id); +const question = (r: any[], n = 0) => r[n].interactions[n]; + +describe("native question/resume path qualification", () => { + it("keeps the new journey explicit-only and the original documentation cells intact", () => { + const cells = runnerMatrix.filter(c => c.suite.id === "question-resume"); + expect(cells.map(c => c.profile.id).sort()).toEqual(["runner-acpx-claude", "runner-codex"]); + for (const c of cells) { + expect(c.suite.manualOnly).toBe(true); + expect(c.task).toMatchObject({ expectedRunCount: 3, minimumExpectedRunCount: 1, automaticRetryPolicy: "single_attempt" }); + expect(c.task.buildPrompt("nonce")).toBe(continuationScenario("question-tool-documentation", "nonce").prompt); + } + expect(runnerMatrix.filter(c => c.suite.id === "continuation")).toHaveLength(23); + }); + it.each([ + ["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"], + ] as QuestionPath[][])("accepts exactly bound %s then %s journeys", (a, b) => { + const r = recording([a,b]); + expect(failed(r)).toEqual([]); + expect(gradeContinuation({ ...continuationScenario("question-answer-resume", "nonce"), runtimeMode: "native", checkpoints: r }).filter(c => !c.passed)).toEqual([]); + expect(gradeLifecycleBaseline(r).filter(c => !c.passed)).toEqual([]); + }); + it.each([ + "Welcome to the afternoon meetup, AMBERnonce.", + "Welcome to the afternoon meetup. Use AMBERnonce", + ])("rejects a shortened supplied reference in the saved document: %s", body => { + const r = recording(); + r[2].documents[0].body = body; + expect(failed(r)).toContain("question-resume.complete-evidence"); + }); + it("does not admit an optional Other field on the semantic path", () => { + const r = recording(["semantic", "semantic"]); + const i = question(r); + i.payload.questionSet.questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false }); + expect(validResumeQuestionForm(r[0], i, true)).toBe(false); + }); + it.each(["sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey"])("rejects a provider card with mismatched %s", key => { + const r = recording(); question(r)[key] = "unrelated"; + expect(failed(r)).toContain("question-resume.choice.path"); + }); + it("rejects a consistent foreign-company card/run pair on this task", () => { + const r = recording(); question(r).companyId = "foreign"; r[0].runs[0].companyId = "foreign"; + expect(questionPath(r[0], question(r))).toBeNull(); + }); + it.each(["missing-request", "wrong-request", "terminal-run", "wrong-policy", "legacy-run"])("rejects invalid provider binding: %s", kind => { + const r = recording(); const i = question(r); + if (kind === "missing-request") delete i.payload.runtimeRequestId; + if (kind === "wrong-request") i.payload.runtimeRequestId = "unrelated"; + if (kind === "terminal-run") r[0].runs[0].status = "succeeded"; + if (kind === "wrong-policy") i.continuationPolicy = "wake_assignee"; + if (kind === "legacy-run") r[0].runs[0].runtimeMode = "legacy"; + expect(questionPath(r[0], i)).toBeNull(); + }); + it.each(["missing", "wrong-operation", "rejected", "wrong-interaction", "wrong-source", "wrong-form"])("rejects missing or mismatched semantic receipt: %s", kind => { + const r = recording(["semantic", "semantic"]); const i = question(r); + const receipt = r[0].runs[0].resultJson.semanticToolReceipts[i.idempotencyKey]; + if (kind === "missing") delete r[0].runs[0].resultJson.semanticToolReceipts[i.idempotencyKey]; + if (kind === "wrong-operation") receipt.operationId = "write_document"; + if (kind === "rejected") receipt.result.disposition = "rejected"; + if (kind === "wrong-interaction") receipt.result.interaction.id = "unrelated"; + if (kind === "wrong-source") receipt.result.interaction.sourceRunId = "unrelated"; + if (kind === "wrong-form") receipt.result.interaction.payload.questionSet.questions = []; + expect(questionPath(r[0], i)).toBeNull(); + }); + it.each(["required-other", "extra-question", "text-with-options", "two-early-cards"])("rejects the wrong form: %s", kind => { + const r = recording(); const fields = question(r).payload.questionSet.questions; + if (kind === "required-other") fields[1].required = true; + if (kind === "extra-question") fields.push({ id: "extra", answerMode: "text" }); + if (kind === "text-with-options") question(r,1).payload.questionSet.questions[0].options = [{ id: "other" }]; + if (kind === "two-early-cards") r[0].interactions.push(structuredClone(question(r,1))); + expect(failed(r)).toContain("question-resume.forms"); + }); + it.each(["agent-answer", "wrong-choice", "wrong-text", "duplicate-answer", "replaced-card", "changed-form", "changed-answer", "unanswered"])("rejects corrupt answer evidence: %s", kind => { + const r = recording(); const first = r[1].interactions[0]; const last = r[2].interactions[1]; + if (kind === "agent-answer") { first.resolvedByUserId = null; first.resolvedByAgentId = "agent"; } + if (kind === "wrong-choice") first.result.answers[0].optionIds = ["am"]; + if (kind === "wrong-text") last.result.answers[0].otherText = "wrong"; + if (kind === "duplicate-answer") first.result.answers.push(structuredClone(first.result.answers[0])); + if (kind === "replaced-card") last.id = "replacement"; + if (kind === "changed-form") last.payload.questionSet.questions[0].prompt = "Changed question"; + if (kind === "changed-answer") r[2].interactions[0].result.answers[0].optionIds = ["am"]; + if (kind === "unanswered") last.status = "pending"; + expect(failed(r).some(id => id.endsWith(".answer"))).toBe(true); + }); + it("rejects replacing a paused provider run with a fresh run", () => { + const r = recording(); r[1].runs.push({ ...structuredClone(r[1].runs[0]), id: "replacement" }); + expect(failed(r)).toContain("question-resume.choice.path"); + }); + it.each(["interactionId", "sourceRunId", "issueId", "interactionStatus"])("rejects the wrong semantic response wake %s", key => { + const r = recording(); r[2].runs[1].contextSnapshot[key] = "unrelated"; + expect(failed(r)).toContain("question-resume.text.path"); + }); + it.each(["lost-run", "duplicate-run", "extra-run", "missing-input", "missing-final", "missing-middle", "missing-output", "wrong-output"])("rejects incomplete continuation evidence: %s", kind => { + const r = recording(); + if (kind === "lost-run") r[2].runs.shift(); + if (kind === "duplicate-run") r[2].runs.push(structuredClone(r[2].runs[0])); + if (kind === "extra-run") r[2].runs.push({ ...structuredClone(r[2].runs[0]), id: "extra" }); + if (kind === "missing-input") delete r[2].runs[0].runnerProfileJson; + if (kind === "missing-output") r[2].documents = []; + if (kind === "wrong-output") r[2].documents[0].body = "Welcome AMBERnonce."; + if (kind === "missing-final") r.pop(); + if (kind === "missing-middle") r.splice(1,1); + expect(failed(r).length).toBeGreaterThan(0); + }); +});