diff --git a/doc/idle-decision-effect-recovery.md b/doc/idle-decision-effect-recovery.md new file mode 100644 index 00000000000..b6724feed72 --- /dev/null +++ b/doc/idle-decision-effect-recovery.md @@ -0,0 +1,255 @@ +# Idle decision-effect recovery + +Interrupted decision effects now have a dedicated deadline worker. The main +maintenance tick no longer scans running decisions. Target invalidation remains +in that tick; TTL expiry and continuation delivery have their own workers. + +## Obligation and activation + +The existing decision row is durable truth. A row with `status = decided` and +`executionStatus = running` owes effect recovery. Its next deadline is +`updatedAt + PAPERCLIP_DECISIONS_RECOVERY_GRACE_MS`, default 60 seconds. Zero is +supported; invalid or negative values retain the existing default. Reads do not +move the timestamp. Open decisions and completed history create no recovery debt. + +The existing `writeDecisionChange` transaction boundary signals effect recovery +for running claims, effect/archive settlement, cancellation, dismissal and expiry. +The generic `notifyDecisionChange` helper signals continuation and expiry only; +open-decision creation does not owe effect recovery. Notifications are awaited +before writes. Outer commit activates work; rollback does not publish a claim. +The additional topic does not wrap provider work or agent wakes in SQL transactions +and does not introduce a new queue table. + +Committed notification bursts coalesce for up to one second after a successful +pass. This bounds sequential tenant writes to at most one notification-driven +global pass per window, rather than only coalescing overlapping scans. A saved +recovery deadline can preempt that window. New obligations discovered by a signal +can wait up to one second; foreground effects still execute immediately. The +window alone never schedules work when there is no notification or saved debt. + +The supported writer model is the application DB signal owner and its child pools. +Independent SQL clients and multiple active writers need a separate notification +contract. Startup reconstructs committed claims; it does not observe future writes +from unsupported writers. + +## Ownership and recovery + +`decision-effect-recovery.ts` owns selection, grace, bounded paging, and earliest +deadline lookup. `decisions.ts` owns execution, current membership, effect claims, +audit, archive policy, and continuation delivery. The worker adapter only binds +that domain contract to the shared coordinator. Startup registers it only when +heartbeat scheduling is enabled. The public combined sweep retains its default +recovery behavior for existing callers. + +Each pass honors the existing configured batch size (default 100, finite values +clamped to at least one integer). Its timestamp/ID cursor advances past failed +candidates within a frozen final timestamp/ID. Newer due claims wait for the next +rotation, so a continuous stream cannot postpone an earlier failure forever. +Both cursor positions retain PostgreSQL precision. Genuine execution or settlement +failures retain their retry obligation. + +A missing saved decider or chosen option is a deterministic malformed outcome. +Recovery rechecks it under a row lock, records an audited failed outcome (partial +if a completed effect receipt exists), and preserves all effect receipts. It +creates no authority and executes no effect for the malformed claim. The failure +reason and continuation obligation are saved in the same transaction as the audit. +An audit or settlement failure rolls back and leaves the claim pending. A claim +repaired before the lock is acquired remains eligible for normal recovery. + +The decision service re-reads current status before recovery. Foreground and +recovery execution share the existing per-decision flight scoped to the DB owner. +Individual effects keep their durable records and transaction advisory locks. +Completed effects are not repeated after a failed final decision status write. +Recovery derives authority from the saved decider's current active membership. +It preserves the current authorization intersection and current target checks. + +The scheduler has a 30-second minimum retry spacing for pending or failed work. +A first claim or failed retry during drain retains its wake without passive SQL. +Shutdown stops subscriptions and checks cancellation between candidates. An +already admitted effect/continuation is awaited; later candidates remain durable +for restart. No periodic pass remains after all claims settle. + +## Existing guarantees and limits + +Regular effects preserve their existing per-effect transaction boundaries. +Continuation delivery retains its separate at-least-once wake/acknowledgement +contract. This migration does not establish exactly-once external delivery. + +Archive proposals have a separate versioned manifest contract. Recovery still +verifies the saved signature/manifest and current authority. A crash after an +archive commits but before the decision settles can encounter the changed +manifest version on restart. The existing result is a failed decision with an +archive error, while the archive remains committed once. The new worker preserves +and tests that outcome; it does not claim to repair this preexisting atomicity gap. + +Cloud sleep guards remain conservative. The internal next deadline is not a +persisted external alarm and does not authorize host sleep with running decisions. +This change establishes worker quiet, not actual instance/database suspension. + +## Verification + +The new real-PostgreSQL suite covers 17 cases: empty/open/completed-history quiet, +real foreground claim and failed settlement, exact grace and zero override, +restart from a claimed effect, active foreground/recovery overlap, outer commit +and rollback, first claim and failed retry through drain, revoked authority, +continuation failure, aggregate exclusion, fair paging past missing-decider and +malformed claims, archive recovery before and after an archive commit, a claim +created during delivery, and shutdown between candidates. Quiet assertions observe +SQL methods through one hour of fake time after settling. + +The combined selection passes 212 tests: the 17 new cases, 50 existing decision +service cases, 14 TTL cases, 39 startup cases, and 92 agent-history route cases. +The first 15-case recovery run also passed. Workspace typecheck/build and the +final server typecheck pass. No failure was suppressed or timeout increased. + +The current regenerated migration `0390_salty_killraven.sql` adds one partial index concurrently, with `IF NOT EXISTS`, through +the migration owner's top-level statement path. Generation retains the newest +five schema snapshots. Fresh application, replay, safety, and numbering checks +pass. With 40,000 completed decisions and one running claim, both due selection +and earliest-deadline lookup use the partial index and read two buffers (0.012 ms +and 0.003 ms in the local probe). The probe transaction is rolled back. + +Full PR CI and review must pass separately before this slice is ready. + +A follow-up regression seeds a legacy invalid claim with a PostgreSQL-generated +microsecond timestamp ahead of a valid claim. With a one-row page, the original +Date cursor selected the invalid claim again and starved the valid effect. That +regression failed before repair. Recovery now retains the timestamp as database +text for cursor comparisons, leaving business grace deadlines unchanged. The +unresolved claim remains durable debt and is revisited after paging wraps; the +valid claim progresses without repeating its completed effect. The regression +now holds a candidate unresolved through the dispatch callback, because known +malformed claims receive a durable failed outcome as described above. +All 18 effect-recovery lifecycle cases and server typechecking pass after repair. + +### Foreground join test timing + +CI exposed a timing assumption in the foreground/recovery overlap test: it +expected exactly two lifetime recovery sweeps. A controlled PostgreSQL replay +waited for the committed claim's deadline discovery before advancing time. It +recorded two completed, zero-resumption scans before the deadline (startup and +claim notification), followed by the legitimate deadline scan. The old assertion +then failed deterministically with three calls. No recurring idle poll or duplicate +effect was found in that sequence. + +The test now waits for discovery, measures the deadline recovery relative to that +point, and proves that its promise stays pending while foreground execution holds +the effect lock. Advancing through several retry periods cannot start another +sweep or effect. After release, recovery completes, one comment and one continuation +wake exist, and the worker performs no SQL through the existing hour-long quiet +check. Counting actual lock attempts also distinguishes joining from a second +execution blocked on the same PostgreSQL advisory lock. A temporary mutation that +removed flight joining made this stronger assertion fail; production code was +restored before final verification. No runtime code, retry policy, timeout, or +business deadline changed. + +All 123 tests across the four complete effect-recovery, decision-service, expiry, +and startup suites passed, followed by server typechecking. Evidence: +`/tmp/sleep-effect-join-red.log`, `/tmp/sleep-effect-join-green.log`, +`/tmp/sleep-effect-join-mutation.log`, `/tmp/sleep-effect-join-owning-tests.log`, +and `/tmp/sleep-effect-join-root-types.log` with their status files. The initial +failure and mutation failure are expected evidence, not passing runs. Remaining +review findings and fresh stack CI remain separate acceptance requirements. + + +### Recovery review repairs (October 26) + +Two regressions failed before repair: an old failure was not revisited while newer +due claims kept each page full, and creating an open decision caused an unnecessary +effect-recovery scan. Frozen rotation boundaries now preserve both retry fairness +and forward progress. Creation still signals its continuation/expiry consumers; +the effect queue is signaled by the existing claim/settlement write boundary. + +Four malformed-outcome cases also failed before repair. Recovery now records +`effectRecoveryError` and a `decision.recovery_failed` activity atomically. Live +activity publication follows commit. Existing completed effect receipts remain +unchanged, and the normal continuation owner delivers the result. A real audit +insert failure leaves both status and audit uncommitted, then later recovery +settles successfully. A real PostgreSQL row-lock test repairs a claim while +settlement waits; the locked recheck leaves that claim running, and its valid +effect executes once on the next pass. + +The shared execution flight now loads the current claim and derives recovery +authority together. Recovery no longer performs a redundant full-row lookup +before entering the flight. Foreground overlap, current membership checks, +per-effect locks, continuation behavior, and the resumption count are preserved. +Only the two known malformed states are terminalized; other failures retain their +existing retry policy. No schema, queue, or external-wake contract is added. + +All 24 recovery lifecycle cases and server typechecking pass. Controlled failures +and final evidence are retained in `/tmp/sleep-effect-fairness-red.log`, +`/tmp/sleep-effect-invalid-red.log`, `/tmp/sleep-effect-owner-tests.log`, and +`/tmp/sleep-effect-owner-types.log`. Broader stack and remote acceptance remain +separate requirements; these results do not prove host suspension. + +The owning branch also passes all 129 cases across the four complete recovery, +decision-service, expiry, and startup suites. Database-backed suites use separate +migrated disposable databases. See `/tmp/sleep-effect-review-owner-acceptance.log`. + + +### Bounded notification review repair + +Ten sequential real cancellation transactions reproduced ten extra recovery +scans. After enabling the existing shared notification-coalescing capability, +the same transactions cause one follow-up scan and then return to SQL quiet. +A separate real-database case preserves an already loaded deadline 500 milliseconds +away. The initial deadline fixture observed the scheduler while its SQL pass was +still active; waiting for that read removed the fixture race before the baseline +was recorded. The cancellation amplification still failed before the production fix. + +All 131 tests in the four complete recovery, decision-service, expiry and startup +suites pass at the owning branch, as does server typechecking. The helper's optional +coalescing capability is promoted from the later plugin scheduling slice into this +PR; consumers that omit it keep their previous behavior. The integrated stack is +verified separately before publication. The original full-suite and external-wake +limitations remain. + + +## Current parent integration + +The expiry and continuation review repairs are integrated normally. The running +claim index is regenerated as 0390 after parent 0388/0389, with concurrent creation +registered in the existing executor. All 388 parent journal entries are unchanged; +the five newest generated snapshots remain. Schema drift, numbering/safety and +fresh migration apply/replay pass. No generated snapshot was hand-edited. + +All 141 cases in the five complete effect-recovery, decision/continuation, expiry, +startup and scoped-deadline suites pass. This includes all 26 recovery cases and +the parent's deterministic cross-pool archive contention test. Server types and +build pass. The first integrated run passed 140 cases and failed an old scheduler +assertion: the coordinator now also holds an earlier continuation-coalescing +deadline. The overlap test services that earlier registered wake, then still +requires the original effect grace deadline, one active effect flight, two total +advisory-lock attempts (test gate plus foreground), one committed effect and one +continuation. No production timing or retry was changed to pass the test. + +Existing one-second effect-notification coalescing and thirty-second failure +retry remain. Continuation notification coalescing is owned separately by the +parent. Extra upstream embedded-only tests attempted during parent integration +could not initialize locally; fresh current-head CI must cover that environment. +Full CI and both automated reviews remain merge gates. + + +After normal integration of master-target expiry parent `f7c61e9912`, the +effect-recovery owner, worker, decision execution and notification implementations +are unchanged. All 69 current recovery/startup/deadline cases and server types/ +build pass with the updated Runner vendor. The earlier full 141-case selection +remains evidence at its stated revision. Parent migration history is an exact +prefix and five generated snapshots remain. Fresh head CI and both automated +reviews are required after this parent integration. + + +## Landed expiry integration and CI interruption + +Expiry-only creation notification from `aa03d2c4c2` is integrated. All 143 cases +in the complete recovery, decision, expiry, startup and scoped-deadline suites +pass, including single/bundle creation without continuation signals. The normal +merge of landed expiry `3421366903` also preserves all 21 independently merged +master files. The same 143 cases pass after that reconciliation. + +The previous candidate `f54a707d74` had both reviews at 5/5, but CI run +40597921712 failed because server shard 6 was interrupted. Its test step was +cancelled and orphan cleanup ran; no failing test assertion or suite summary was +reported. The workflow eventually completed with that shard and its dependent +verify gate failed. It was not retried or presented as passing. The necessary +expiry parent repair creates a new candidate with fresh CI and reviews required. diff --git a/doc/plans/2026-10-24-recurring-server-sleep-stack.md b/doc/plans/2026-10-24-recurring-server-sleep-stack.md index 0ff600d0569..c4fd7944142 100644 --- a/doc/plans/2026-10-24-recurring-server-sleep-stack.md +++ b/doc/plans/2026-10-24-recurring-server-sleep-stack.md @@ -16,7 +16,8 @@ Do not mark an item done from a timer removal or a green check alone. | 02 | Remaining chat maintenance | Provider runtime discovery; voice reconciliation; milestones; wake notices; interaction continuations; terminal confirmations; GitHub recovery; Teams maintenance, separately by owner | Pending | | 03 | Run and task recovery | Orphan runs and active leases; shutdown workspace settlement; scheduled retries; queued runs; stranded assignments; resolved dependencies; task watchdogs; silent-run monitoring; stale locks | Pending | | 04 | Decision retention and notifications | Retention deadlines and notification delivery are separate reasons | Retention pending; archive delivery [#17475](https://github.com/paperclipai/paperclip/pull/17475) merged after exact-head 5/5 reviews and passing current CI | -| 05 | Decision expiry and interrupted effects | Expiry, effect recovery, continuation recovery | Continuations [#17478](https://github.com/paperclipai/paperclip/pull/17478) merged; expiry [#17479](https://github.com/paperclipai/paperclip/pull/17479), effects [#17482](https://github.com/paperclipai/paperclip/pull/17482), and targets [#17486](https://github.com/paperclipai/paperclip/pull/17486) await current-head review/CI gates | +| 05 | Decision expiry and interrupted effects | Expiry, effect recovery, continuation recovery | Continuations [#17478](https://github.com/paperclipai/paperclip/pull/17478) merged; expiry [#17479](https://github.com/paperclipai/paperclip/pull/17479) merged; effects [#17482](https://github.com/paperclipai/paperclip/pull/17482), and targets [#17486](https://github.com/paperclipai/paperclip/pull/17486) await current-head review/CI gates | + | 06 | Tool connection health | Configuration/use activation and defined proactive health deadlines | Pending | | 07 | Secret proposals, tool reviews, gateway tokens | Separate expiry/scrub/recovery owners | Slices 02–07 merged: [secret proposals](https://github.com/paperclipai/paperclip/pull/17446), [credentials](https://github.com/paperclipai/paperclip/pull/17450), [review expiry](https://github.com/paperclipai/paperclip/pull/17453), [approved recovery](https://github.com/paperclipai/paperclip/pull/17454), [interrupted claims](https://github.com/paperclipai/paperclip/pull/17458), [projection](https://github.com/paperclipai/paperclip/pull/17468) | @@ -527,3 +528,105 @@ Slice 10 creation signal follow-up: single and bundle creation now notify expiry only through the central owner. Terminal updates still signal continuation. Both real regressions failed before repair; all 117 current expiry/decision/startup/ deadline cases, server types and build pass. Fresh CI/reviews remain required. + + + + +Slice 10 is [#17479](https://github.com/paperclipai/paperclip/pull/17479), head +`63b58cc26f`, based on the repaired continuation branch. It is attached to this +chat and review was requested. Workspace typecheck/build passed on the combined +branch after parent propagation. Full CI and review are active, not complete. +Slice 09 has fresh Greptile 5/5 at `5f96892a58`, no unresolved review threads, +and CI active with no failure in the latest snapshot. The first eight published +slices remain open with their recorded passing checks. No merge is authorized. + +## Next owner audit: interrupted decision effects + +The next branch is `codex/sleep-11-decision-effect-recovery`, based on #17479. +No eleventh implementation or PR is claimed. The remaining combined decision +sweep selects `status = decided`, `executionStatus = running`, and +`updatedAt <= now - recoveryGraceMs`. The existing grace defaults to 60 seconds; +invalid values fall back and zero is permitted. The combined batch size defaults +to 100. Preserve these semantics while moving only interrupted effect recovery. +Target invalidation remains a separate pending reason after this migration. + +The existing decision update helper covers the running claim and terminal +settlements. Extend that boundary for effect-recovery intent. Creation is open +and does not itself constitute effect debt. The indexed minimum running claim +plus grace should drive the next recovery attempt; open/terminal history must +not keep it scanning. Preserve expiry and continuation as separate workers. + +Keep effect execution, signatures, recovery authority, effect claims, and +postcommit issue actions in `decisions.ts` and its domain collaborators. A +recovery selector may call a narrow decision-owned operation rather than move +that policy into the scheduler. The new per-owner execution flight already +coalesces foreground and recovery calls and re-reads the current decision before +execution. Reuse it. Do not hold a DB transaction while waiting for the whole +operation or replace durable per-effect claims with the in-memory flight. + +Regular effects use transaction-scoped advisory locks and saved effect execution +rows. Completed effects must not replay; claimed work can resume. Recovery derives +current user authority from stored `decidedByUserId` and current membership. +The existing combined sweep skips rows without a deciding user. Audit how such +invalid legacy rows are represented in readiness and sleep safety; do not allow +them to starve actionable rows or invent permission to execute them. Malformed +stored options or genuine execution failures must retain a visible obligation. + +Archive proposals have separate manifest/version checks and transaction ownership. +The parent race repair serializes archive execution through its state write in +one live owner. It is not a durable cross-process claim and does not establish +exactly-once recovery after a process dies between archive commit and decision +settlement. Preserve and explicitly test the supported failure behavior instead +of treating this limitation as solved by a new timer. + +Use bounded passes, advance past per-row failures, and propagate cancellation +between candidates. A first claim during drain, completion during recovery, and +an unresolved failed write must retain their existing notification semantics. +Reconstruct recovery deadlines on startup, keep new production SQL indexed, and +preserve the public combined sweep's default behavior while removing its effect +scan from the main tick. + +Required evidence includes a real claim with foreground interruption, exact +recovery grace, ordinary active execution coalescing, completed effect history, +claimed-effect restart, changed user authority, invalid legacy rows, provider/ +archive failure semantics, outer commit/rollback, failed retry through drain, +changes during execution, fair bounded paging, and shutdown. Run existing +archive and continuation regressions on the combined stack. The full original +inventory remains the goal after this slice. + + +## Slice 11: interrupted decision effects + +The effect-recovery owner and worker now use committed decision changes and the +minimum running claim timestamp plus the existing recovery grace. The aggregate +tick opts out of this duty while preserving target invalidation. Effect execution, +current authority, claims, audit, and archive policy remain with the decision +service. See [effect recovery](../idle-decision-effect-recovery.md). + +All 212 selected tests pass: 17 new recovery cases, 50 decision service cases, +14 TTL cases, 39 startup cases, and 92 agent-history route cases. Workspace +typecheck/build and final server typecheck pass. Migration apply/replay and a +40,000-row completed-history query probe pass; both lookups use the new partial +index with two buffers. PR creation and full CI/review are next. + +The continuation parent #17478 now has passing checks and Greptile 5/5 at +`5f96892a58`, with both review findings resolved. TTL #17479 has Greptile 5/5 at +`63b58cc26f` and no findings. Its first CI run timed out in the unchanged +agent-history route test. All 92 route tests passed locally; one failed-job retry +was requested. This evidence does not yet establish green TTL CI. The full +original inventory remains in scope; no stack PR has been merged. + +Current slice 11 parent integration passes all 141 effect/decision/expiry/startup/ +deadline cases, server types/build, generated snapshot drift and migration replay. +The recovery index is regenerated as `0390_salty_killraven.sql`; parent history +is unchanged. The initial overlap assertion failure and its earlier-queue deadline +explanation are recorded in the behavior audit. Seven slices are merged; this +PR and the remaining stack still require fresh current-head CI and both reviews. + + +Slice 10 merged after exact-head Greptile/Ponytail 5/5 and all 58 check contexts +passed or were neutral/skipped: `aa03d2c4c2` → `3421366903`. Slice 11 normally +integrates that landed parent and all 21 independent master files. All 143 cases +in its five complete suites pass. Its preceding CI shard interruption remains +recorded in the owning effect-recovery audit; it is not a passing receipt. Fresh +CI/reviews are required on the reconciled head. diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index fa192adb0d1..ff714b6919d 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -508,7 +508,7 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([ "0387_boring_professor_monster.sql", "0388_ambitious_micromax.sql", "0389_brave_hulk.sql", - + "0390_salty_killraven.sql", "0384_funny_adam_warlock.sql", diff --git a/packages/db/src/migrations/0390_salty_killraven.sql b/packages/db/src/migrations/0390_salty_killraven.sql new file mode 100644 index 00000000000..c653ae92d7f --- /dev/null +++ b/packages/db/src/migrations/0390_salty_killraven.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "decisions_running_recovery_idx" ON "decisions" USING btree ("updated_at","id") WHERE "decisions"."status" = 'decided' AND "decisions"."execution_status" = 'running'; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0385_snapshot.json b/packages/db/src/migrations/meta/0390_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0385_snapshot.json rename to packages/db/src/migrations/meta/0390_snapshot.json index 5564d696be5..346524fc38b 100644 --- a/packages/db/src/migrations/meta/0385_snapshot.json +++ b/packages/db/src/migrations/meta/0390_snapshot.json @@ -1,6 +1,6 @@ { - "id": "9965d300-e355-4f47-ba0a-951ed3b4b8a2", - "prevId": "4f96f70a-fd19-4d35-ad8a-8d5559b83cbd", + "id": "f38ffbf3-c0ce-435e-aad8-9e5afa6b9417", + "prevId": "394e85a6-7fc2-4ab1-80a3-33bb4c7003aa", "version": "7", "dialect": "postgresql", "tables": { @@ -22110,6 +22110,34 @@ "concurrently": false, "method": "btree", "with": {} + }, + "decision_archive_notification_outbox_delivery_idx": { + "name": "decision_archive_notification_outbox_delivery_idx", + "columns": [ + { + "expression": "last_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "first" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"decision_archive_notification_outbox\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -24030,6 +24058,66 @@ "method": "btree", "with": {} }, + "decisions_running_recovery_idx": { + "name": "decisions_running_recovery_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "decisions_open_expiry_idx": { + "name": "decisions_open_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"decisions\".\"status\" = 'open'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "decisions_pending_continuation_idx": { + "name": "decisions_pending_continuation_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"decisions\".\"continuation_policy\" = 'wake_origin_agent' AND \"decisions\".\"metadata\" ->> 'continuationPending' = 'true' AND (\"decisions\".\"status\" IN ('expired', 'cancelled') OR (\"decisions\".\"status\" = 'decided' AND \"decisions\".\"execution_status\" IN ('succeeded', 'partial', 'failed')))", + "concurrently": false, + "method": "btree", + "with": {} + }, "decisions_bundle_idx": { "name": "decisions_bundle_idx", "columns": [ @@ -52909,6 +52997,27 @@ "method": "btree", "with": {} }, + "tool_action_requests_company_interaction_idx": { + "name": "tool_action_requests_company_interaction_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "interaction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "tool_action_requests_issue_idx": { "name": "tool_action_requests_issue_idx", "columns": [ diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 14f1a4ad042..9db984a3918 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2717,6 +2717,13 @@ "when": 1793138457589, "tag": "0389_brave_hulk", "breakpoints": true + }, + { + "idx": 390, + "version": "7", + "when": 1793139659776, + "tag": "0390_salty_killraven", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/decisions.ts b/packages/db/src/schema/decisions.ts index 5fe03d8f090..012d2af3d58 100644 --- a/packages/db/src/schema/decisions.ts +++ b/packages/db/src/schema/decisions.ts @@ -69,6 +69,8 @@ export const decisions = pgTable( table.status, table.expiresAt, ), + runningRecoveryIdx: index("decisions_running_recovery_idx").on(table.updatedAt, table.id) + .where(sql`${table.status} = 'decided' AND ${table.executionStatus} = 'running'`), openExpiryIdx: index("decisions_open_expiry_idx").on(table.expiresAt, table.id) .where(sql`${table.status} = 'open'`), pendingContinuationIdx: index("decisions_pending_continuation_idx").on(table.id) diff --git a/server/src/__tests__/decision-effect-recovery-work.test.ts b/server/src/__tests__/decision-effect-recovery-work.test.ts new file mode 100644 index 00000000000..4358b8e3215 --- /dev/null +++ b/server/src/__tests__/decision-effect-recovery-work.test.ts @@ -0,0 +1,500 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, agents, applyPendingMigrations, authUsers, companies, companyMemberships, createDb, decisionEffectExecutions, + decisions, decisionRetention, decisionTargetIssues, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { decisionService } from "../services/decisions.js"; +import { decisionEffectRecoveryService } from "../services/decision-effect-recovery.js"; +import { registerDecisionEffectRecoveryWork } from "../services/decision-effect-recovery-work.js"; +import { registerDecisionContinuationWork } from "../services/decision-continuation-work.js"; +import { writeDecisionChange } from "../services/decision-work-notifications.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { hashAttentionArchiveManifest } from "../services/decision-retention.js"; +import type { AttentionArchiveManifestEntry } from "@paperclipai/shared"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_DECISION_EFFECTS_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("decision effect recovery worker", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, agentId: string, originId: string, targetId: string, runId: string, userId: string; + let coordinator: ReturnType | undefined; + let draining: boolean; + let recovery: ReturnType; + const errors: unknown[] = []; + const wake = vi.fn(async () => {}); + const service = () => decisionService(db, { wakeOriginAgent: wake }); + + beforeAll(async () => { + if (externalUrl) await applyPendingMigrations(externalUrl); + else temporary = await startEmbeddedPostgresTestDatabase("decision-effects-"); + db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 30_000); + beforeEach(async () => { + vi.stubEnv("PAPERCLIP_DECISION_SIGNING_SECRET", "0123456789abcdef0123456789abcdef"); + companyId = randomUUID(); agentId = randomUUID(); originId = randomUUID(); targetId = randomUUID(); runId = randomUUID(); userId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 6)}`, requireBoardApprovalForNewAgents: false }); + await db.insert(authUsers).values({ id: userId, name: "Decider", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "member" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Proposer", role: "engineer", status: "active", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} }); + await db.insert(issues).values([ + { id: originId, companyId, title: "Origin", status: "in_progress", priority: "medium", assigneeAgentId: agentId, responsibleUserId: userId }, + { id: targetId, companyId, title: "Target", status: "todo", priority: "medium", responsibleUserId: userId }, + ]); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", responsibleUserId: userId, contextSnapshot: { issueId: originId } }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + draining = false; errors.length = 0; wake.mockReset(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllEnvs(); + await db.execute(sql`drop trigger if exists decision_effect_finish_fail on decisions`); + await db.execute(sql`drop function if exists decision_effect_finish_fail()`); + await db.delete(decisionEffectExecutions); await db.delete(decisionTargetIssues); await db.delete(decisions); + await db.delete(decisionRetention); await db.delete(activityLog); await db.delete(issueComments); await db.delete(heartbeatRuns); + await db.delete(issues); await db.delete(agents); await db.delete(companyMemberships); await db.delete(authUsers); await db.delete(companies); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + const actor = () => ({ type: "agent" as const, companyId, agentId, runId, source: "agent_jwt" as const, + onBehalfOfUserId: userId, onBehalfOfMemberships: [{ companyId, membershipRole: "member", status: "active" }] }); + const board = () => ({ type: "board" as const, userId, companyIds: [companyId], source: "session" as const, + memberships: [{ companyId, membershipRole: "member", status: "active" }] }); + const create = () => service().create({ companyId, agentId, runId, actor: actor(), title: "Comment?", body: "Body", + continuationPolicy: "wake_origin_agent", options: [{ id: "yes", label: "Yes", effects: [ + { type: "comment_on_issue", targetIssueId: targetId, staleness: "lenient", bodyMarkdown: "hello" }, + ] }] }); + const decide = (id: string) => service().decide({ id, optionId: "yes", decidedByUserId: userId, userActor: board() }); + const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({ + status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId, + updatedAt, inputValues: {}, metadata: { continuationPending: true }, + }).where(eq(decisions.id, id))); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, + onError: error => errors.push(error) }); + await registerDecisionContinuationWork(coordinator, service(), () => !draining).ready; + recovery = service(); + vi.spyOn(recovery, "recoverEffects"); + await registerDecisionEffectRecoveryWork(coordinator, recovery, () => !draining).ready; + } + async function quiet() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(30_000); + expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); + }, { timeout: 5000 }); + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute"); + await vi.advanceTimersByTimeAsync(60 * 60_000); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); + reads.mockRestore(); writes.mockRestore(); executes.mockRestore(); + } + async function settled(id: string, executionStatus = "succeeded") { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(30_000); + expect(await service().get(id)).toMatchObject({ executionStatus, metadata: { continuationPending: false } }); + }, { timeout: 5000 }); + } + const comments = () => db.select().from(issueComments).where(eq(issueComments.issueId, targetId)); + async function failSettlement() { + await db.execute(sql`create function decision_effect_finish_fail() returns trigger language plpgsql as $$ begin + if OLD.execution_status = 'running' and NEW.execution_status <> 'running' then raise exception 'settlement unavailable'; end if; + return NEW; end $$`); + await db.execute(sql`create trigger decision_effect_finish_fail before update on decisions for each row execute function decision_effect_finish_fail()`); + } + + it("stays quiet with open decisions and delivered history after a real decision", async () => { + await start(); await quiet(); const row = await create(); await quiet(); + await decide(row.id); await quiet(); expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1); + }); + + it("does not inspect effect recovery when an open decision is created", async () => { + await start(); await quiet(); + const sweeps = vi.mocked(recovery.recoverEffects); sweeps.mockClear(); + await create(); await quiet(); + expect(sweeps).not.toHaveBeenCalled(); + }); + + it("coalesces sequential committed cancellations without an idle recovery poll", async () => { + const rows = []; + for (let i = 0; i < 10; i++) rows.push(await create()); + // Freeze wall time while real SQL and transaction settlement continue. + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: false }); + await start(); + const scans = vi.mocked(recovery.recoverEffects); + expect(scans).toHaveBeenCalledTimes(1); + for (const row of rows) { + await service().cancel(row.id, { actorType: "user", actorId: userId }); + await vi.advanceTimersByTimeAsync(1); + await db.execute(sql`select pg_sleep(0.01)`); // Allow each old scan to finish, rather than only testing overlap coalescing. + } + expect(scans).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1000); + await vi.waitFor(() => expect(scans).toHaveBeenCalledTimes(2)); + await quiet(); + const count = scans.mock.calls.length; + await vi.advanceTimersByTimeAsync(3_600_000); + expect(scans).toHaveBeenCalledTimes(count); + expect(wake).toHaveBeenCalledTimes(10); + }); + + it("lets an already saved recovery deadline preempt notification coalescing", async () => { + const due = await create(), cancelled = await create(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: false }); + const deadline = Date.now() + 500; + await claim(due.id, new Date(deadline - 60_000)); + await start(); + expect(coordinator!.nextWakeAt()).toBe(deadline); + await service().cancel(cancelled.id, { actorType: "user", actorId: userId }); + await vi.advanceTimersByTimeAsync(1); + await db.execute(sql`select pg_sleep(0.01)`); + expect(coordinator!.nextWakeAt()).toBe(deadline); + await vi.advanceTimersByTimeAsync(499); + await vi.waitFor(async () => expect(await comments()).toHaveLength(1)); + await settled(due.id); await quiet(); + expect(wake).toHaveBeenCalledTimes(2); + }); + + it("retries an old failure while newer due claims keep filling pages", async () => { + const failed = await create(), healthy = await create(); + const at = Date.now() - 600_000; + await claim(failed.id, new Date(at)); await claim(healthy.id, new Date(at + 1)); + // Preserve PostgreSQL precision at both ends of the first rotation. + await db.execute(sql`update decisions set updated_at = updated_at + interval '456 microseconds' + where id in (${failed.id}, ${healthy.id})`); + const failure = new Error("temporary recovery failure"); + const attempts: string[] = []; + const scan = decisionEffectRecoveryService(db, async id => { + attempts.push(id); + if (id === failed.id) throw failure; + await db.update(decisions).set({ executionStatus: "succeeded" }).where(eq(decisions.id, id)); + return true; + }); + await expect(scan.sweep(new Date(), 1)).rejects.toBe(failure); + const added: string[] = []; + for (let index = 0; index < 7; index++) { + const row = await create(); added.push(row.id); await claim(row.id, new Date(at + index + 2)); + try { await scan.sweep(new Date(), 1); } catch (error) { expect(error).toBe(failure); } + } + expect(attempts).toHaveLength(8); + expect(attempts.slice(0, 6)).toEqual([failed.id, healthy.id, failed.id, added[0], added[1], failed.id]); + expect(new Set(attempts.filter(id => id !== failed.id)).size).toBe(5); + expect((await service().get(failed.id))?.executionStatus).toBe("running"); + }); + + it("reactivates from the actual claim and recovers a failed settlement without repeating the effect", async () => { + await start(); await quiet(); const row = await create(); await failSettlement(); + await expect(decide(row.id)).rejects.toMatchObject({ cause: { message: "settlement unavailable" } }); + expect((await service().get(row.id))?.executionStatus).toBe("running"); expect(await comments()).toHaveLength(1); + await db.execute(sql`drop trigger decision_effect_finish_fail on decisions`); + await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1); + }); + + it("reconstructs the saved grace deadline at restart and recovers a claimed effect", async () => { + const row = await create(); const claimedAt = new Date(); await claim(row.id, claimedAt); + await db.insert(decisionEffectExecutions).values({ decisionId: row.id, effectIndex: 0, effectType: "comment_on_issue", targetIssueId: targetId, status: "claimed" }); + await start(); expect(coordinator!.nextWakeAt()).toBe(claimedAt.getTime() + 60_000); + await vi.advanceTimersByTimeAsync(30_000); expect(await comments()).toHaveLength(0); + await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1); + }); + + it("recovers at the exact grace boundary and preserves a zero grace override", async () => { + const row = await create(), at = new Date(); await claim(row.id, at); + expect(await service().recoverEffects(new Date(at.getTime() + 59_999))).toEqual({ resumed: 0 }); + expect(await service().recoverEffects(new Date(at.getTime() + 60_000))).toEqual({ resumed: 1 }); + vi.stubEnv("PAPERCLIP_DECISIONS_RECOVERY_GRACE_MS", "0"); + const second = await create(); await claim(second.id, at); + expect(await service().recoverEffects(at)).toEqual({ resumed: 1 }); expect(await comments()).toHaveLength(2); + }); + + it("joins active foreground effects when their recovery deadline arrives", async () => { + const row = await create(); await start(); + const effectKey = `decision-effect:${row.id}:0`; + const previousDebug = db.$client.options.debug; + let effectLockAttempts = 0; + db.$client.options.debug = (_connection, query, parameters) => { + if (query.includes("pg_advisory_xact_lock") && parameters.includes(effectKey)) effectLockAttempts++; + }; + let release!: () => void, locked!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const acquired = new Promise(resolve => { locked = resolve; }); + const lock = db.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${effectKey}, 0))`); + locked(); await gate; + }); + let deciding: ReturnType | undefined; + let recovering: ReturnType | undefined; + try { + await acquired; + deciding = decide(row.id); + await vi.waitFor(async () => expect((await service().get(row.id))?.executionStatus).toBe("running")); + const claimed = (await service().get(row.id))!; + const deadline = claimed.updatedAt.getTime() + 60_000; + await vi.waitFor(async () => { + const scheduled = coordinator!.nextWakeAt(); + if (scheduled !== null && scheduled < deadline) { + await vi.advanceTimersByTimeAsync(Math.max(1, scheduled - Date.now())); + } + expect(coordinator!.nextWakeAt()).toBe(deadline); + }); + await vi.waitFor(() => expect(effectLockAttempts).toBe(2)); // Test gate + foreground execution. + const sweeps = vi.mocked(recovery.recoverEffects); + const beforeDeadline = sweeps.mock.calls.length; + await vi.advanceTimersByTimeAsync(Math.max(0, deadline - Date.now())); + await vi.waitFor(() => expect(sweeps).toHaveBeenCalledTimes(beforeDeadline + 1)); + let recoverySettled = false; + recovering = Promise.resolve(sweeps.mock.results[beforeDeadline].value).then(result => { + recoverySettled = true; + return result; + }); + await vi.advanceTimersByTimeAsync(120_000); + expect(await comments()).toHaveLength(0); expect(wake).not.toHaveBeenCalled(); + expect(recoverySettled).toBe(false); + expect(sweeps).toHaveBeenCalledTimes(beforeDeadline + 1); + expect(effectLockAttempts).toBe(2); + release(); + await lock; await deciding; + await expect(recovering).resolves.toEqual({ resumed: 1 }); + await quiet(); + expect(effectLockAttempts).toBe(2); + expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1); + } finally { + release(); + await Promise.allSettled([lock, deciding, recovering]); + db.$client.options.debug = previousDebug; + } + }); + + it("defers an outer transaction until commit and ignores rollback", async () => { + const row = await create(); await start(); await quiet(); + await expect(db.transaction(async tx => { + await claim(row.id, new Date(Date.now() - 120_000), tx as unknown as typeof db); + await vi.advanceTimersByTimeAsync(120_000); expect(await comments()).toHaveLength(0); throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect((await service().get(row.id))?.status).toBe("open"); + await db.transaction(async tx => { + await claim(row.id, new Date(Date.now() - 120_000), tx as unknown as typeof db); + await vi.advanceTimersByTimeAsync(120_000); expect(await comments()).toHaveLength(0); + }); + await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1); + }); + + it("retains a first claim through drain without passive SQL", async () => { + const row = await create(); await start(); await quiet(); draining = true; + await claim(row.id, new Date(Date.now() - 120_000)); + const reads = vi.spyOn(db, "select"); await vi.advanceTimersByTimeAsync(120_000); + expect(reads).not.toHaveBeenCalled(); reads.mockRestore(); + draining = false; await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1); + }); + + it("retains a failed recovery through drain and does not replay the completed effect", async () => { + const row = await create(); await claim(row.id, new Date(Date.now() - 120_000)); await failSettlement(); await start(); + await vi.waitFor(() => expect(errors.length).toBeGreaterThan(0)); draining = true; + expect(await comments()).toHaveLength(1); + const reads = vi.spyOn(db, "select"); await vi.advanceTimersByTimeAsync(120_000); + expect(reads).not.toHaveBeenCalled(); reads.mockRestore(); + await db.execute(sql`drop trigger decision_effect_finish_fail on decisions`); + draining = false; await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1); + for (const error of errors) expect(error).toMatchObject({ cause: { message: "settlement unavailable" } }); errors.length = 0; + }); + + it("rechecks current membership before executing a recovered effect", async () => { + const row = await create(); await claim(row.id, new Date(Date.now() - 120_000)); + await db.update(companyMemberships).set({ status: "inactive" }).where(eq(companyMemberships.principalId, userId)); + await start(); await settled(row.id, "failed"); await quiet(); expect(await comments()).toHaveLength(0); + const result = await service().outcome(row.id); + expect(result.executions[0]?.error).toBe("deny_decision_intersection"); expect(wake).toHaveBeenCalledTimes(1); + }); + + it("leaves continuation failures to their owner after effect recovery finishes", async () => { + const row = await create(); await claim(row.id, new Date(Date.now() - 120_000)); + wake.mockRejectedValueOnce(new Error("wake unavailable")); await start(); await settled(row.id); await quiet(); + expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(2); + expect(errors.every(error => String(error).includes("wake unavailable"))).toBe(true); errors.length = 0; + }); + + it("keeps aggregate maintenance from recovering effects when the worker owns them", async () => { + const row = await create(); await claim(row.id, new Date(Date.now() - 120_000)); + expect(await service().sweepExpired(new Date(), { includeContinuations: false, includeExpiry: false, includeEffects: false })) + .toEqual({ expired: 0, resumed: 0 }); + expect(await comments()).toHaveLength(0); + expect(await service().sweepExpired()).toEqual({ expired: 0, resumed: 1 }); expect(await comments()).toHaveLength(1); + }); + + it.each(["missing_decider", "missing_option"] as const)("settles %s as durable failure and then becomes quiet", async reason => { + const row = await create(); await claim(row.id, new Date(Date.now() - 120_000)); + await db.update(decisions).set(reason === "missing_decider" ? { decidedByUserId: null } : { chosenOptionId: "missing" }) + .where(eq(decisions.id, row.id)); + expect(await service().recoverEffects()).toEqual({ resumed: 1 }); + expect(await service().get(row.id)).toMatchObject({ executionStatus: "failed", metadata: { + effectRecoveryError: reason, continuationPending: false, + } }); + expect(await comments()).toHaveLength(0); + expect(await db.select().from(decisionEffectExecutions)).toHaveLength(0); + const audit = await db.select().from(activityLog).where(eq(activityLog.action, "decision.recovery_failed")); + expect(audit).toHaveLength(1); + expect(audit[0]).toMatchObject({ companyId, actorType: "system", actorId: "decision-executor", + entityId: row.id, details: { reason } }); + expect(wake).toHaveBeenCalledTimes(1); + await start(); await quiet(); + expect(wake).toHaveBeenCalledTimes(1); + }); + + it("preserves completed effect receipts when a malformed outcome settles as partial", async () => { + const row = await create(); await claim(row.id, new Date(Date.now() - 120_000)); + await db.update(decisions).set({ chosenOptionId: "missing" }).where(eq(decisions.id, row.id)); + const [receipt] = await db.insert(decisionEffectExecutions).values({ decisionId: row.id, effectIndex: 0, + effectType: "comment_on_issue", targetIssueId: targetId, status: "executed", result: { preserved: true }, executedAt: new Date() }).returning(); + expect(await service().recoverEffects()).toEqual({ resumed: 1 }); + expect((await service().get(row.id))?.executionStatus).toBe("partial"); + expect(await db.select().from(decisionEffectExecutions)).toEqual([receipt]); + expect(await comments()).toHaveLength(0); + expect(wake).toHaveBeenCalledTimes(1); + await start(); await quiet(); + }); + + it("keeps malformed claims pending when their failure audit cannot commit", async () => { + const row = await create(); await claim(row.id, new Date(Date.now() - 120_000)); + await db.update(decisions).set({ decidedByUserId: null }).where(eq(decisions.id, row.id)); + await db.execute(sql`create function decision_recovery_audit_fail() returns trigger language plpgsql as $$ begin + if NEW.action = 'decision.recovery_failed' then raise exception 'recovery audit unavailable'; end if; + return NEW; end $$`); + await db.execute(sql`create trigger decision_recovery_audit_fail before insert on activity_log + for each row execute function decision_recovery_audit_fail()`); + try { + await expect(service().recoverEffects()).rejects.toMatchObject({ cause: { message: "recovery audit unavailable" } }); + expect((await service().get(row.id))?.executionStatus).toBe("running"); + expect(await db.select().from(activityLog).where(eq(activityLog.action, "decision.recovery_failed"))).toHaveLength(0); + expect(wake).not.toHaveBeenCalled(); + } finally { + await db.execute(sql`drop trigger decision_recovery_audit_fail on activity_log`); + await db.execute(sql`drop function decision_recovery_audit_fail()`); + } + expect(await service().recoverEffects()).toEqual({ resumed: 1 }); + expect((await service().get(row.id))?.executionStatus).toBe("failed"); + expect(wake).toHaveBeenCalledTimes(1); + await start(); await quiet(); + }); + + it("rechecks a malformed claim under lock before settling a concurrently repaired outcome", async () => { + const row = await create(); const at = new Date(Date.now() - 120_000); await claim(row.id, at); + await db.update(decisions).set({ chosenOptionId: "missing" }).where(eq(decisions.id, row.id)); + let release!: () => void, locked!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const acquired = new Promise(resolve => { locked = resolve; }); + const repair = db.transaction(async tx => { + await tx.select({ id: decisions.id }).from(decisions).where(eq(decisions.id, row.id)).for("update"); + locked(); await gate; + await tx.update(decisions).set({ chosenOptionId: "yes", updatedAt: at }).where(eq(decisions.id, row.id)); + }); + const previousDebug = db.$client.options.debug; + let attempts = 0; + let pending: ReturnType["recoverEffects"]> | undefined; + try { + await acquired; + db.$client.options.debug = (_connection, query, parameters) => { + if (query.includes("for update") && parameters.includes(row.id)) attempts++; + }; + pending = service().recoverEffects(); + await vi.waitFor(() => expect(attempts).toBe(1)); + release(); await repair; + expect(await pending).toEqual({ resumed: 0 }); + expect(await service().get(row.id)).toMatchObject({ executionStatus: "running", chosenOptionId: "yes" }); + expect(await db.select().from(activityLog).where(eq(activityLog.action, "decision.recovery_failed"))).toHaveLength(0); + expect(wake).not.toHaveBeenCalled(); + expect(await service().recoverEffects()).toEqual({ resumed: 1 }); + expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1); + } finally { + release(); await Promise.allSettled([repair, pending]); db.$client.options.debug = previousDebug; + } + await start(); await quiet(); + }); + + it("settles invalid legacy claims without inventing authority or blocking later effects", async () => { + vi.stubEnv("PAPERCLIP_DECISIONS_SWEEP_BATCH_SIZE", "1"); + const invalid = await create(), valid = await create(); + await claim(invalid.id, new Date(Date.now() - 180_000)); await claim(valid.id, new Date(Date.now() - 120_000)); + await db.update(decisions).set({ decidedByUserId: null }).where(eq(decisions.id, invalid.id)); + const recovery = service(); + expect(await recovery.recoverEffects()).toEqual({ resumed: 1 }); + expect(await comments()).toHaveLength(0); + expect(await recovery.recoverEffects()).toEqual({ resumed: 1 }); expect(await comments()).toHaveLength(1); + expect((await recovery.get(invalid.id))?.executionStatus).toBe("failed"); + expect(await recovery.nextEffectRecoveryAt()).toBeNull(); + }); + + it("pages past unresolved claims using database timestamp precision", async () => { + const pending = await create(), valid = await create(); + await claim(pending.id); await claim(valid.id, new Date(Date.now() - 120_000)); + await db.execute(sql`update decisions set updated_at = date_trunc('milliseconds', now() - interval '3 minutes') + + interval '456 microseconds' where id = ${pending.id}`); + const scan = decisionEffectRecoveryService(db, async id => { + if (id === pending.id) return false; + await decide(id); return true; + }); + expect(await scan.sweep(new Date(), 1)).toEqual({ resumed: 0 }); + expect(await scan.sweep(new Date(), 1)).toEqual({ resumed: 1 }); + expect(await comments()).toHaveLength(1); + expect((await service().get(pending.id))?.executionStatus).toBe("running"); + expect(await scan.next()).not.toBeNull(); + expect(await scan.sweep(new Date(), 1)).toEqual({ resumed: 0 }); + expect(await scan.sweep(new Date(), 1)).toEqual({ resumed: 0 }); + expect(await comments()).toHaveLength(1); + }); + + it.each([false, true])("preserves archive recovery semantics with prior committed archive: %s", async committed => { + await db.update(issues).set({ status: "in_review", assigneeUserId: userId }).where(eq(issues.id, targetId)); + const activityAt = new Date("2026-04-01T00:00:00.000Z"); + await db.insert(decisionRetention).values({ companyId, sourceKind: "review", sourceId: targetId, sourceActivityAt: activityAt }); + const manifest: AttentionArchiveManifestEntry[] = [{ companyId, sourceKind: "review", sourceId: targetId, + expectedVersion: 1, activityAt: activityAt.toISOString(), reason: "Archive" }]; + const row = await service().create({ companyId, actor: actor(), agentId, runId, title: "Archive?", body: "Exact set", + continuationPolicy: "wake_origin_agent", options: [{ id: "archive", label: "Archive", effects: [] }], + metadata: { kind: "attention_archive_proposal", manifestHash: hashAttentionArchiveManifest(manifest) }, + additionalTargetSnapshots: { [`attention:review:${targetId}`]: { status: "attention", assigneeAgentId: null, + assigneeUserId: null, updatedAt: activityAt.toISOString(), attentionArchive: manifest[0] } } }); + if (committed) { + await failSettlement(); + await expect(service().decide({ id: row.id, optionId: "archive", decidedByUserId: userId, userActor: board() })) + .rejects.toMatchObject({ cause: { message: "settlement unavailable" } }); + await db.execute(sql`drop trigger decision_effect_finish_fail on decisions`); + } else { + await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", + chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000), + metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id))); + } + await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet(); + const [retained] = await db.select().from(decisionRetention); + expect(retained.archiveVersion).toBe(1); expect(retained.archivedAt).not.toBeNull(); + expect(wake).toHaveBeenCalledTimes(1); + if (committed) expect((await service().get(row.id))?.metadata).toHaveProperty("archiveProposalError"); + }); + + it("learns another claim written during delivery", async () => { + const first = await create(), second = await create(); await claim(first.id, new Date(Date.now() - 120_000)); + wake.mockImplementationOnce(async () => { await claim(second.id); }); + await start(); await settled(first.id); await settled(second.id); await quiet(); + expect(await comments()).toHaveLength(2); expect(wake).toHaveBeenCalledTimes(2); + }); + + it("finishes the admitted recovery and leaves later claims for restart during shutdown", async () => { + const first = await create(), second = await create(); + await claim(first.id, new Date(Date.now() - 180_000)); await claim(second.id, new Date(Date.now() - 120_000)); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + wake.mockImplementationOnce(async () => gate); + const starting = start(); let stopped = false; let stopping: Promise; + try { + await vi.waitFor(() => expect(wake).toHaveBeenCalledTimes(1)); + stopping = coordinator!.stop().then(() => { stopped = true; }); + await vi.advanceTimersByTimeAsync(60_000); expect(stopped).toBe(false); + } finally { release(); } + await starting; await stopping!; coordinator = undefined; + expect(wake).toHaveBeenCalledTimes(1); expect(await comments()).toHaveLength(1); + expect((await service().get(second.id))?.executionStatus).toBe("running"); + await start(); await quiet(); expect(await comments()).toHaveLength(2); expect(wake).toHaveBeenCalledTimes(2); + }); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index c2ee1e0caa4..3c814c6dcfe 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -349,6 +349,8 @@ vi.mock("../services/index.js", () => ({ })), decisionService: vi.fn(() => ({ sweepExpired: vi.fn(async () => ({ expired: 0 })), + recoverEffects: vi.fn(async () => ({ resumed: 0 })), + nextEffectRecoveryAt: vi.fn(async () => null), sweepTtlExpiry: vi.fn(async () => ({ expired: 0 })), nextExpiryAt: vi.fn(async () => null), sweepContinuations: vi.fn(async () => ({ retry: false })), @@ -921,8 +923,10 @@ describe("startServer feedback export wiring", () => { const decisions = vi.mocked(decisionService).mock.results.at(-1)!.value; expect(decisions.sweepContinuations).toHaveBeenCalledTimes(1); expect(decisions.sweepTtlExpiry).toHaveBeenCalledTimes(1); + expect(decisions.recoverEffects).toHaveBeenCalledTimes(1); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.decisionEffectRecovery)).toBe(true); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.decisionExpiry)).toBe(true); - expect(decisions.sweepExpired).toHaveBeenCalledWith(expect.any(Date), { includeContinuations: false, includeExpiry: false }); + expect(decisions.sweepExpired).toHaveBeenCalledWith(expect.any(Date), { includeContinuations: false, includeExpiry: false, includeEffects: false }); } finally { setIntervalSpy.mockRestore(); } @@ -961,6 +965,7 @@ describe("startServer feedback export wiring", () => { expect(environmentCustomImagesServiceMock.cleanupExpiredSetupSessions).not.toHaveBeenCalled(); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.decisionContinuation)).toBe(false); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.decisionExpiry)).toBe(false); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.decisionEffectRecovery)).toBe(false); } finally { setIntervalSpy.mockRestore(); } diff --git a/server/src/index.ts b/server/src/index.ts index 883590a67a4..0d27d6f5b5e 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,4 @@ +import { registerDecisionEffectRecoveryWork } from "./services/decision-effect-recovery-work.js"; import { registerDecisionExpiryWork } from "./services/decision-expiry-work.js"; import { registerDecisionContinuationWork } from "./services/decision-continuation-work.js"; @@ -1842,7 +1843,9 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("decision_expiry_worker", () => registerDecisionExpiryWork(deliveryWork, decisionExecutor, () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); - await step("decision_expiry", () => decisionExecutor.sweepExpired(new Date(), { includeContinuations: false, includeExpiry: false })); + await step("decision_effect_recovery_worker", () => registerDecisionEffectRecoveryWork(deliveryWork, decisionExecutor, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); + await step("decision_expiry", () => decisionExecutor.sweepExpired(new Date(), { includeContinuations: false, includeExpiry: false, includeEffects: false })); const loginSchedule = loginCleanupSchedule(db, config.heartbeatSchedulerIntervalMs); await step("login_cleanup_workers", () => registerLoginCleanupWork(deliveryWork, [ { queue: DELIVERY_QUEUES.adapterLoginCleanup, run: () => adapterLoginReaper.sweep().then(logAdapterLoginReaperResult), next: loginSchedule.adapter }, @@ -1863,7 +1866,7 @@ async function startServerWithDatabaseTeardown( trackHeartbeatSchedulerWork((async () => { if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; scheduleRoutineStatusReconcile(); - trackHeartbeatSchedulerWork(decisionExecutor.sweepExpired(new Date(), { includeContinuations: false, includeExpiry: false }).catch((err: unknown) => { + trackHeartbeatSchedulerWork(decisionExecutor.sweepExpired(new Date(), { includeContinuations: false, includeExpiry: false, includeEffects: false }).catch((err: unknown) => { logger.error({ err }, "decision expiry sweep failed"); })); trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => { diff --git a/server/src/services/decision-effect-recovery-work.ts b/server/src/services/decision-effect-recovery-work.ts new file mode 100644 index 00000000000..c34ef251a7c --- /dev/null +++ b/server/src/services/decision-effect-recovery-work.ts @@ -0,0 +1,17 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import type { decisionService } from "./decisions.js"; + +export function registerDecisionEffectRecoveryWork( + coordinator: ReturnType, + service: Pick, "recoverEffects" | "nextEffectRecoveryAt">, + canRun: () => boolean, +) { + return registerDeadlineWork(coordinator, DELIVERY_QUEUES.decisionEffectRecovery, { + retryMs: 30_000, canRun, + notificationCoalesceMs: 1000, + run: signal => service.recoverEffects(new Date(), undefined, signal), + next: service.nextEffectRecoveryAt, + }); +} diff --git a/server/src/services/decision-effect-recovery.ts b/server/src/services/decision-effect-recovery.ts new file mode 100644 index 00000000000..9d987cc0568 --- /dev/null +++ b/server/src/services/decision-effect-recovery.ts @@ -0,0 +1,60 @@ +import { and, asc, desc, lte, sql } from "drizzle-orm"; +import { decisions, type Db } from "@paperclipai/db"; + +const running = sql`${decisions.status} = 'decided' AND ${decisions.executionStatus} = 'running'`; + +export function decisionSweepBatchSize() { + const value = Number(process.env.PAPERCLIP_DECISIONS_SWEEP_BATCH_SIZE ?? 100); + return Number.isFinite(value) ? Math.max(1, Math.trunc(value)) : 100; +} + +function recoveryGraceMs() { + const value = Number(process.env.PAPERCLIP_DECISIONS_RECOVERY_GRACE_MS ?? 60_000); + return Number.isFinite(value) && value >= 0 ? value : 60_000; +} + +export function decisionEffectRecoveryService(db: Db, resume: (id: string) => Promise) { + type Position = { updatedAt: string; id: string }; + let cursor: Position | undefined; + let boundary: Position | undefined; + + async function sweep(now = new Date(), limit = decisionSweepBatchSize(), signal?: AbortSignal) { + let resumed = 0; + if (signal?.aborted) return { resumed }; + const dueBefore = new Date(now.getTime() - recoveryGraceMs()); + const position = { id: decisions.id, updatedAt: sql`${decisions.updatedAt}::text` }; + if (!boundary) { + // Finish this rotation even if newer due claims arrive on every page. + [boundary] = await db.select(position).from(decisions) + .where(and(running, lte(decisions.updatedAt, dueBefore))) + .orderBy(desc(decisions.updatedAt), desc(decisions.id)).limit(1); + if (!boundary) return { resumed }; + } + const rows = await db.select(position).from(decisions) + .where(and(running, lte(decisions.updatedAt, dueBefore), + sql`(${decisions.updatedAt}, ${decisions.id}) <= (${boundary.updatedAt}::timestamptz, ${boundary.id}::uuid)`, cursor + ? sql`(${decisions.updatedAt}, ${decisions.id}) > (${cursor.updatedAt}::timestamptz, ${cursor.id}::uuid)` : undefined)) + .orderBy(asc(decisions.updatedAt), asc(decisions.id)).limit(limit); + const errors: unknown[] = []; + for (const row of rows) { + if (signal?.aborted) return { resumed }; + try { if (await resume(row.id)) resumed++; } catch (error) { errors.push(error); } + cursor = row; + } + const last = rows.at(-1); + if (rows.length < limit || (last?.id === boundary.id && last.updatedAt === boundary.updatedAt)) { + cursor = undefined; boundary = undefined; + } + if (errors.length === 1) throw errors[0]; + if (errors.length) throw new AggregateError(errors, "Decision effect recovery failed"); + return { resumed }; + } + + async function next(): Promise { + const [row] = await db.select({ at: decisions.updatedAt }).from(decisions).where(running) + .orderBy(asc(decisions.updatedAt), asc(decisions.id)).limit(1); + return row ? row.at.getTime() + recoveryGraceMs() : null; + } + + return { sweep, next }; +} diff --git a/server/src/services/decision-work-notifications.ts b/server/src/services/decision-work-notifications.ts index 06869ceff59..081edc8fc39 100644 --- a/server/src/services/decision-work-notifications.ts +++ b/server/src/services/decision-work-notifications.ts @@ -13,6 +13,7 @@ export async function notifyDecisionChange(transaction: object) { export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise): Promise { return db.transaction(async transaction => { await notifyDecisionChange(transaction); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery); return write(transaction as unknown as Db); }); } diff --git a/server/src/services/decisions.ts b/server/src/services/decisions.ts index 751e4c30406..9d915c7955d 100644 --- a/server/src/services/decisions.ts +++ b/server/src/services/decisions.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, asc, count, desc, eq, gt, gte, inArray, lte, sql } from "drizzle-orm"; +import { and, asc, count, desc, eq, gt, gte, inArray, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { databaseWorkScopeKey, companyMemberships, decisionBundles, decisionEffectExecutions, decisionRetention, decisions, decisionTargetIssues, heartbeatRuns, issueRelations, issues } from "@paperclipai/db"; import { ATTENTION_SOURCE_KINDS, decisionEffectTargetIssueIds } from "@paperclipai/shared"; @@ -15,6 +15,7 @@ import { import { decisionRetentionService, hashAttentionArchiveManifest } from "./decision-retention.js"; import { decisionContinuationService, type DecisionContinuationWake } from "./decision-continuations.js"; +import { decisionEffectRecoveryService, decisionSweepBatchSize } from "./decision-effect-recovery.js"; import { decisionExpiryService } from "./decision-expiry.js"; import { notifyDecisionCreated, writeDecisionChange } from "./decision-work-notifications.js"; @@ -24,7 +25,8 @@ type Snapshot = { status: string; assigneeAgentId: string | null; assigneeUserId export type DecisionServiceOptions = { wakeOriginAgent: DecisionContinuationWake }; const DAY = 86_400_000; type DecisionOutcome = typeof decisions.$inferSelect & { executions: Array }; -const effectFlights = new WeakMap>>(); +type EffectAttempt = { outcome: DecisionOutcome; resumed: boolean }; +const effectFlights = new WeakMap>>(); function targetIds(options: DecisionOption[]) { const result = new Set(); @@ -126,6 +128,7 @@ function boardCanActDirectly(actor: AuthorizationActor, companyId: string) { export function decisionService(db: Db, options: DecisionServiceOptions) { const continuations = decisionContinuationService(db, options.wakeOriginAgent); const expiry = decisionExpiryService(db, continuations.deliver); + const effectRecovery = decisionEffectRecoveryService(db, recoverDecisionEffects); const scope = databaseWorkScopeKey(db); let flights = effectFlights.get(scope); if (!flights) effectFlights.set(scope, flights = new Map()); @@ -557,21 +560,53 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { } } - function runEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor): Promise { - const existing = executing.get(decision.id); + function runEffects(id: string, userActor?: AuthorizationActor): Promise { + const existing = executing.get(id); if (existing) return existing; const attempt = (async () => { - const current = await get(decision.id); - if (!current) throw notFound("Decision not found"); - return current.executionStatus === "running" ? executeDecisionEffects(current, userActor) : outcome(current.id); - })().finally(() => { executing.delete(decision.id); }); - executing.set(decision.id, attempt); + const current = await get(id); + if (!current) return null; + if (current.status !== "decided" || current.executionStatus !== "running") { + return { outcome: await outcome(id), resumed: false }; + } + // Recovery derives authority from the current claim inside the shared flight. + const actor = userActor ?? (current.decidedByUserId + ? await recoveryActor(current.companyId, current.decidedByUserId) + : { type: "none" as const, source: "none" as const }); + return { outcome: await executeDecisionEffects(current, actor), resumed: true }; + })().finally(() => { executing.delete(id); }); + executing.set(id, attempt); return attempt; } + async function settleInvalidEffectOutcome(id: string) { + const publications: ActivityPublication[] = []; + await writeDecisionChange(db, async tx => { + const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update"); + if (!current || current.status !== "decided" || current.executionStatus !== "running") return; + const reason = !current.decidedByUserId ? "missing_decider" + : !current.options.some(option => option.id === current.chosenOptionId) ? "missing_option" : null; + if (!reason) return; + const [executed] = await tx.select({ id: decisionEffectExecutions.id }).from(decisionEffectExecutions) + .where(and(eq(decisionEffectExecutions.decisionId, id), eq(decisionEffectExecutions.status, "executed"))).limit(1); + const executionStatus = executed ? "partial" : "failed"; + await tx.update(decisions).set({ executionStatus, updatedAt: new Date(), metadata: { ...current.metadata, + effectRecoveryError: reason, + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}), + } }).where(eq(decisions.id, id)); + await logActivity(tx, { companyId: current.companyId, actorType: "system", actorId: "decision-executor", + agentId: current.originAgentId, runId: current.originRunId, responsibleUserIdOverride: current.decidedByUserId, + action: "decision.recovery_failed", entityType: "decision", entityId: id, + details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId }, + }, publications); + }); + for (const publication of publications) publishActivity(publication); + return outcome(id); + } + async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) { const option = decision.options.find((item) => item.id === decision.chosenOptionId); - if (!option || !decision.decidedByUserId) throw unprocessable("Stored decision outcome is invalid"); + if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null); const metadata = decision.metadata as Record; if (metadata.kind === "attention_archive_proposal") { @@ -635,7 +670,9 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { } async function resumeDecision(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) { - const result = decision.executionStatus === "running" ? await runEffects(decision, userActor) : await outcome(decision.id); + const attempt = decision.executionStatus === "running" ? await runEffects(decision.id, userActor) : undefined; + if (attempt === null) throw notFound("Decision not found"); + const result = attempt?.outcome ?? await outcome(decision.id); await continuations.deliver(result.id); return outcome(decision.id); } @@ -716,27 +753,16 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return { ...bundle, decisions: created }; }); } - async function sweepExpired(now = new Date(), options: { includeContinuations?: boolean; includeExpiry?: boolean } = {}) { - const configuredBatchSize = Number(process.env.PAPERCLIP_DECISIONS_SWEEP_BATCH_SIZE ?? 100); - const batchSize = Number.isFinite(configuredBatchSize) - ? Math.max(1, Math.trunc(configuredBatchSize)) - : 100; - const configuredRecoveryGraceMs = Number(process.env.PAPERCLIP_DECISIONS_RECOVERY_GRACE_MS ?? 60_000); - const recoveryGraceMs = Number.isFinite(configuredRecoveryGraceMs) && configuredRecoveryGraceMs >= 0 - ? configuredRecoveryGraceMs - : 60_000; - const runningRows = await db.select().from(decisions) - .where(and(eq(decisions.status, "decided"), eq(decisions.executionStatus, "running"), - lte(decisions.updatedAt, new Date(now.getTime() - recoveryGraceMs)))) - .orderBy(asc(decisions.updatedAt)).limit(batchSize); - let resumed = 0; - for (const decision of runningRows) { - if (!decision.decidedByUserId) continue; - const recovered = await runEffects(decision, await recoveryActor(decision.companyId, decision.decidedByUserId)); - if (recovered.executionStatus === "running") continue; - resumed += 1; - await continuations.deliver(recovered.id); - } + async function recoverDecisionEffects(id: string) { + const attempt = await runEffects(id); + if (!attempt?.resumed || attempt.outcome.executionStatus === "running") return false; + await continuations.deliver(attempt.outcome.id); + return true; + } + + async function sweepExpired(now = new Date(), options: { includeContinuations?: boolean; includeExpiry?: boolean; includeEffects?: boolean } = {}) { + const batchSize = decisionSweepBatchSize(); + const { resumed } = options.includeEffects === false ? { resumed: 0 } : await effectRecovery.sweep(now, batchSize); if (options.includeContinuations !== false) await continuations.sweep(batchSize); const ttlRows = options.includeExpiry === false ? [] : await expiry.due(now, batchSize); const remaining = batchSize - ttlRows.length; @@ -768,5 +794,6 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { } return { create, createBundle, get, list, stats, outcome, decide, cancel, dismiss, sweepExpired, sweepContinuations: continuations.sweep, - sweepTtlExpiry: expiry.sweep, nextExpiryAt: expiry.next }; + sweepTtlExpiry: expiry.sweep, nextExpiryAt: expiry.next, + recoverEffects: effectRecovery.sweep, nextEffectRecoveryAt: effectRecovery.next }; } diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 46e8242508c..908be12ff8d 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -9,6 +9,7 @@ export const DELIVERY_QUEUES = { executionStatus: "execution-status", executionDisposition: "execution-disposition", environmentCleanup: "environment-cleanup", + decisionEffectRecovery: "decision-effect-recovery", decisionExpiry: "decision-expiry", decisionContinuation: "decision-continuation",