From 3c15acd3fc1b200ef0e36ac904a03cd5fcba00c0 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:37:19 -0600 Subject: [PATCH] perf(recovery): look up held runs by key so the fresh-continuation deadline stops scanning run history (#21568) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - When a run is cut off, the fresh-continuation fallback continues the task in a new run. The execution-disposition worker schedules its next pass with `nextFreshContinuationCheckAt` and lists due work with `heldExecutionsAwaitingFreshContinuation` > - Both queries share two predicates (`heldExecutionWhere` and `recentInterruptionSourceSql`). These predicates matched the run as `source.id::text = ` and the task's runs as `native_issue_id::text = ::text`. A cast on an indexed column prevents an index lookup > - So for each held record, PostgreSQL read the company's whole run history (two times), and it decoded each run's wide context JSON. On staging, the deadline query took about 13.5 s of database time in each 27 s worker cycle. It was the largest statement by time > - This pull request compares the uuid columns directly. It turns the recorded run id into a uuid only when the text is the canonical lowercase form, which is exactly when `id::text = text` can be true > - The benefit is that each held record costs a few index lookups. The same records are selected, and the worker cycle no longer keeps a database session busy ## Linked Issues or Issue Description **What existing behavior does this improve?** The fresh-continuation scheduler queries: `nextFreshContinuationCheckAt` (the disposition worker's deadline), `heldExecutionsAwaitingFreshContinuation` (the backlog listing, also probed by the schedule), and the "left alone" count. They are in `server/src/services/fresh-continuation-fallback.ts`. **Subsystem affected** Server: recovery and the execution-disposition worker (`executionControlSchedule().disposition` in `execution-control-work.ts`). **Current behavior** Neon statement statistics on staging (window of 160 s on 2026-11-14): the `select min((issue_recovery_actions.evidence#>>'{freshContinuationCheck,nextCheckAt}')::bigint) …` statement ran 6 times at about 13.5 s each. That is 81 s of database time in 160 s, about half of one busy session. All other statements in the window used less than 35 s together. Staging has 31,399 runs (1.96 GB, wide `context_snapshot`), 75,285 recovery records, and 282 records with a check deadline. The local plan shows the cause. For each candidate record: - `source.id::text = coalesce(evidence->>'runId', evidence->>'sourceRunId')` cannot use the primary key. PostgreSQL filters the company's failed runs (6,767 rows removed in each loop, and 6,798 again in the outer semi-join). - In the recent-interruption check, `max(held.finished_at)` matches runs of the task with `native_issue_id::text = source_issue_id::text or issue_id::text = … or context_snapshot->>'issueId' = …`. The casts cannot use the task-link indexes, so PostgreSQL filters the company's run history and decodes each run's context JSON (29,937 rows removed in each loop). **Proposed behavior** The queries look up the source run by its primary key and the task's runs by the existing `(company_id, native_issue_id, …)`, `(company_id, issue_id, …)` and `(company_id, context_snapshot->>'issueId', …)` indexes. The results do not change. **Reason and benefit** On a staging-shaped local fixture, the deadline query goes from 2,330 ms to 27 ms (87x), and the backlog listing and count go from 3.7-5.7 s to 19-27 ms. Buffer accesses go from 2.15 M to 28 k. **Breaking changes** None. No schema change and no migration. ## What Changed - `server/src/services/fresh-continuation-fallback.ts`: - Add `sourceRunKey`. It compares the recorded run id with `heartbeat_runs.id` through the existing `canonicalExecutionUuid` guard (cast only when the text matches `^[0-9a-f]{8}-…-[0-9a-f]{12}$`). `uuid::text` always gives that lowercase form, so this matches exactly the rows `id::text = text` matched. Uppercase, braced, padded, numeric and invalid ids still match nothing, and no cast error is possible. - Use it in `heldExecutionWhere`, `interruptionEligibility` and the listing's order expression. - `runOfTask` compares `native_issue_id` and `issue_id` with the issue id as a uuid. Both callers pass a uuid: the `source_issue_id` column, or the id of a task row read in the same transaction. - Export `heldExecutionWhere` and `recentInterruptionSourceSql` so the regression test can compare them with the previous predicates row by row. - Add `server/src/services/fresh-continuation-schedule-queries.test.ts` (regression test, see Verification). - Add `scripts/benchmark-fresh-continuation-deadline.ts`. It seeds a staging-shaped fixture, captures the SQL that the baseline revision and the working tree send, compares complete positional rows, and reports timings and plans. ## Verification - Red/green: `fresh-continuation-schedule-queries.test.ts` "finds the deadline by run-key and task-link lookups, not a scan of run history per record" counts the `heartbeat_runs` rows that the deadline query's plan reads (loops x (rows + rows removed)). With the previous predicates it fails: 549,166 rows read against a bound of 9,442 (two times the run count). With this change it passes. - Equivalence: the same file seeds 900 held records with a seeded random generator. The records cover these cases: `runId`, `sourceRunId`, both keys, uppercase, braced, padded, numeric, invalid and unknown ids; a source run in another company; source statuses (failed, timed_out, interrupted, cancelled, succeeded, running) with and without `finished_at`; the assignee and other agents; newer runs linked by native id, legacy issue id, or context only; unrelated runs; cancelled-at-dispatch runs; live (queued, running, scheduled_retry) runs; earlier and later creation times; retries of the source; person Retry wakes; old and recent finish times; restart, person-owned and sandbox-resume error codes; and issue statuses and assignees. For every record it compares `heldExecutionWhere()`, `heldExecutionWhere({ includeActive: true })` and `recentInterruptionSourceSql(now)` with verbatim copies of the previous predicates. It also compares `nextFreshContinuationCheckAt` and the full ordered `heldExecutionsAwaitingFreshContinuation` rows with the previous queries. The test checks that each predicate is true and false for more than 5 records, so the comparison is meaningful. - Focused suites pass locally: `fresh-continuation-fallback.test.ts` and `fresh-continuation-schedule-queries.test.ts` (82 tests). Related recovery suites pass: `fresh-continuation-retry-delivery`, `task-retry`, `fresh-continuation-move-forward-offer`, `execution-recovery-dead-controller`, `automatic-run-brake-recovery-passes`, `execution-control-work`, `recovery-wake-herd` (with the new test: 8 files, 252 tests). `node server/scripts/typecheck-layers.mjs` passes. - `pnpm verify` exits 2 (the new benchmark script is a global CI input), so CI supplies the broad coverage. - Benchmark: local embedded PostgreSQL 18.1, `jit=on`, `shared_buffers=128MB`, macOS arm64, a shared machine with other work. Baseline `67652580ee`. Fixture: 5,669 issues, 31,351 runs with 60 KiB incompressible context JSON (1,962 MB), 75,285 recovery records (282 with a check deadline, a third overdue). One warm-up and 5 alternating timed pairs. Both versions return identical positional rows for every query. | Query | Baseline median (samples, ms) | This PR median (samples, ms) | Speedup | Shared buffers | | --- | --- | --- | --- | --- | | `nextFreshContinuationCheckAt` | 2,330 (1,962-2,421) | 27 (21-28) | 87x | 2,149,802 -> 28,227 | | `heldExecutionsAwaitingFreshContinuation` limit 5 | 3,692 (3,296-4,014) | 19 (18-24) | 194x | 2,831,539 -> 23,970 | | same, limit 1 (schedule probe) | 3,554 (3,210-4,861) | 20 (18-24) | 179x | 2,831,539 -> 23,970 | | left-alone count | 5,746 (5,090-5,992) | 27 (25-28) | 213x | 5,435,614 -> 40,953 | With 1 KiB contexts (rows stored inline) the deadline query goes from 4,096 ms to 19 ms. Staging measured 13.5 s for the deadline query. The local fixture reproduces the same plan shape, but the staging data distribution and Neon storage are not reproduced exactly, so the staging gain is not verified yet. ```sh pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-fresh-continuation-deadline.ts \ --baseline-ref=67652580ee --context-kib=60 --samples=5 --warmups=1 --print-plans ``` - Plan after the change: the source run is an `Index Scan using heartbeat_runs_pkey` (0 rows removed), and each task-run probe is a `BitmapOr` over the native-id, issue-id and context-issue indexes. The newer-run `not exists` already used that `BitmapOr` before. It does not change. ## Risks - Low risk. The SQL change is limited to how two keys are compared. The regression test compares the old and new predicates for every record of a varied fixture. - `runOfTask` now casts its issue id to uuid. A caller that passes a non-uuid text would get an error instead of no match. The two callers pass a uuid column and a task row's id. - Follow-up, not changed here: the disposition schedule calls `deadline()`, which returns "retry in 15 s" when the earliest check is already due. An active hold that the disposition pass does not pick up (for example, one that records its run only as `sourceRunId`, which `unresolvedExecutionDispositions` does not read) keeps the deadline overdue, so the worker runs again every 15 s with nothing to do. A local probe confirmed this. This explains a fixed cadence of 15 s plus the query time (about 27 s on staging). After this change each such pass costs milliseconds, not seconds. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use (shell, file editing, local PostgreSQL benchmarks and tests). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes (no behavior or command change; no doc update needed) - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../benchmark-fresh-continuation-deadline.ts | 228 +++++++++++++++ .../services/fresh-continuation-fallback.ts | 23 +- ...resh-continuation-schedule-queries.test.ts | 269 ++++++++++++++++++ 3 files changed, 512 insertions(+), 8 deletions(-) create mode 100644 scripts/benchmark-fresh-continuation-deadline.ts create mode 100644 server/src/services/fresh-continuation-schedule-queries.test.ts diff --git a/scripts/benchmark-fresh-continuation-deadline.ts b/scripts/benchmark-fresh-continuation-deadline.ts new file mode 100644 index 00000000000..9afedc8066f --- /dev/null +++ b/scripts/benchmark-fresh-continuation-deadline.ts @@ -0,0 +1,228 @@ +// Fresh-continuation scheduler queries on a staging-shaped fixture: the +// disposition worker's deadline (`nextFreshContinuationCheckAt`), its backlog +// listing (`heldExecutionsAwaitingFreshContinuation`) and the left-alone count. +// Compares a baseline revision with the working tree on identical data and +// fails on any difference in complete positional rows. +// +// pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-fresh-continuation-deadline.ts \ +// --baseline-ref= [--context-kib=60] [--samples=7] [--warmups=2] [--print-plans] +import assert from "node:assert/strict"; +import { randomBytes, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { EXECUTION_RECONCILIATION_CAUSES } from "../packages/shared/src/index.js"; +import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js"; +import * as treatment from "../server/src/services/fresh-continuation-fallback.js"; + +type Query = { sql: string; params: unknown[] }; +const arg = (name: string, fallback: string) => process.argv.find(value => value.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback; +const baselineRevision = execFileSync("git", ["rev-parse", "--verify", `${arg("baseline-ref", "origin/master")}^{commit}`], { encoding: "utf8" }).trim(); +const contextKib = Number(arg("context-kib", "60")); +const sampleCount = Number(arg("samples", "7")); +const warmupCount = Number(arg("warmups", "2")); +const keepAlive = setInterval(() => {}, 1000); +let database: Awaited> | undefined; +let baselineDir: string | undefined; +let db: Db | undefined; +try { + database = await startEmbeddedPostgresTestDatabase("fresh-continuation-deadline-benchmark-"); + db = createDb(database.connectionString); + const sqlClient = db.$client; + baselineDir = await fs.mkdtemp(path.join(fileURLToPath(new URL("../server/scripts/", import.meta.url)), ".fresh-continuation-baseline-")); + const modulePath = "server/src/services/fresh-continuation-fallback.ts"; + const original = new URL(`../${modulePath}`, import.meta.url); + const source = execFileSync("git", ["show", `${baselineRevision}:${modulePath}`], { encoding: "utf8" }); + const baselineFile = path.join(baselineDir, "fresh-continuation-fallback.ts"); + await fs.writeFile(baselineFile, source.replace(/from "(\.{1,2}\/[^"]+)"/g, + (_match, specifier: string) => `from ${JSON.stringify(fileURLToPath(new URL(specifier, original)))}`)); + const baseline = await import(pathToFileURL(baselineFile).href) as typeof treatment; + + // Staging shape (2026-11-14): 5.7k issues, 31.4k runs whose rows carry wide + // context JSON (1.96 GB), 75k recovery records of which ~280 carry a + // fresh-continuation check deadline. + const now = new Date(Date.UTC(2026, 10, 14, 5, 0, 0)); + const nowMs = now.getTime(); + const companyId = randomUUID(), decoyCompanyId = randomUUID(); + const agentIds = [randomUUID(), randomUUID(), randomUUID()]; + const fillerSeed = randomBytes(Math.ceil(contextKib * 1024 * 0.75)).toString("base64"); + const started = performance.now(); + await sqlClient.unsafe(`insert into companies (id, name, issue_prefix) values ($1, 'Synthetic', 'FCD'), ($2, 'Decoy', 'FCX')`, + [companyId, decoyCompanyId]); + await sqlClient.unsafe(`insert into agents (id, company_id, name, adapter_type) + select id, $4::uuid, 'Agent ' || n, 'codex_local' from unnest(array[$1, $2, $3]::uuid[]) with ordinality as a(id, n)`, + [...agentIds, companyId]); + const decoyAgentId = randomUUID(); + await sqlClient.unsafe(`insert into agents (id, company_id, name, adapter_type) values ($1, $2, 'Decoy', 'codex_local')`, [decoyAgentId, decoyCompanyId]); + // Issues: 5,400 in the company (60% done), 269 decoys in another company. + await sqlClient.unsafe(`create temporary table fixture_issue as + select n, gen_random_uuid() as id, case when n <= 5400 then $1::uuid else $2::uuid end as company_id, + case when n > 5400 then $3::uuid else (array[$4, $5, $6]::uuid[])[1 + n % 3] end as agent_id, + case when n % 10 < 6 then 'done' when n % 10 = 6 then 'blocked' when n % 10 = 7 then 'in_progress' else 'todo' end as status + from generate_series(1, 5669) n`, [companyId, decoyCompanyId, decoyAgentId, ...agentIds]); + await sqlClient.unsafe(`insert into issues (id, company_id, title, status, assignee_agent_id) + select id, company_id, 'Fixture task ' || n, status, agent_id from fixture_issue`); + // Runs: 31,399, about five per task, linked by native id, legacy issue id, or context only. + await sqlClient.unsafe(`insert into heartbeat_runs (company_id, agent_id, scope_kind, issue_id, native_issue_id, runtime_mode, + status, error_code, started_at, finished_at, created_at, updated_at, context_snapshot) + select i.company_id, i.agent_id, case when g.n % 5 = 4 then 'company' else 'issue' end, + case when g.n % 5 = 4 then null else i.id end, case when g.n % 5 < 2 then i.id end, + case when g.n % 5 < 2 then 'native' else 'legacy' end, + case when g.n % 7 = 0 then 'failed' when g.n % 11 = 0 then 'cancelled' else 'succeeded' end, + case when g.n % 7 = 0 then 'adapter_failed' end, + $2::timestamptz - (g.n || ' minutes')::interval, $2::timestamptz - (g.n || ' minutes')::interval + interval '30 seconds', + $2::timestamptz - (g.n || ' minutes')::interval - interval '1 second', $2::timestamptz - (g.n || ' minutes')::interval, + jsonb_build_object('issueId', i.id::text, 'wakeReason', 'issue_assigned', 'prompt', $1::text) + from generate_series(1, 31_000) g(n) join fixture_issue i on i.n = 1 + g.n % 5669`, [fillerSeed, now.toISOString()]); + + // Held records with a check deadline (282), in the shapes the predicate distinguishes. + type Held = { issueN: number; runId: string; evidence: Record; status: string; ownerUserId?: string; + cause?: string; recentRun: boolean; errorCode: string; newer?: "native" | "legacy" | "context" | "cancelled_at_dispatch" | "live" }; + const heldRows: Held[] = []; + let issueN = 7; + const nextOpenIssue = () => { do issueN += 1; while (issueN % 10 < 6 || issueN > 5400); return issueN; }; + for (let n = 0; n < 282; n += 1) { + const runId = randomUUID(); + // A third are overdue, so the backlog listing has due work too. + const check = { kind: "exact_recovery_pending", nextCheckAt: nowMs + (n % 3 === 0 ? -1 : 1) * (n + 1) * 1_000, checks: 1 }; + const kind = n % 12; + const base = { runId, automaticRecovery: { replay: "blocked" }, freshContinuationCheck: check }; + heldRows.push({ + issueN: nextOpenIssue(), runId, + evidence: kind === 1 ? { sourceRunId: runId, automaticRecovery: { replay: "blocked" }, freshContinuationCheck: check } + : kind === 2 ? { ...base, runId: runId.toUpperCase() } + : kind === 3 ? { ...base, runId: "not-a-run" } + : base, + status: kind === 4 ? "active" : "resolved", + ownerUserId: kind === 5 ? "user-1" : undefined, + recentRun: kind !== 6 && kind !== 4, + errorCode: kind === 7 ? "adapter_failed" : "process_lost", + newer: kind === 8 ? (["native", "legacy", "context"] as const)[n % 3] : kind === 9 ? "cancelled_at_dispatch" : kind === 10 ? "live" : undefined, + }); + } + const runsSql: unknown[][] = []; + for (const held of heldRows) { + const finished = new Date(nowMs - (held.recentRun ? 2 : 48) * 3_600_000); + runsSql.push([held.runId, held.issueN, held.errorCode, finished.toISOString(), new Date(finished.getTime() - 600_000).toISOString(), null]); + if (held.newer) runsSql.push([randomUUID(), held.issueN, held.newer, finished.toISOString(), + new Date(finished.getTime() + 60_000).toISOString(), held.newer]); + } + await sqlClient.unsafe(`insert into heartbeat_runs (id, company_id, agent_id, scope_kind, issue_id, native_issue_id, runtime_mode, + status, error_code, started_at, finished_at, created_at, updated_at, context_snapshot) + select r.id, i.company_id, i.agent_id, case when r.link = 'context' then 'company' else 'issue' end, + case when r.link in ('context', 'native') then null else i.id end, case when r.link = 'native' then i.id end, + case when r.link = 'native' then 'native' else 'legacy' end, + case when r.link = 'live' then 'queued' when r.link = 'cancelled_at_dispatch' then 'cancelled' + when r.link is null then 'failed' else 'succeeded' end, + case when r.link = 'cancelled_at_dispatch' then 'execution_reconciliation_required' when r.link is null then r.code end, + case when r.link is null then r.created_at end, case when r.link is null then r.finished_at end, r.created_at, r.created_at, + jsonb_build_object('issueId', i.id::text, 'wakeReason', 'issue_assigned', 'prompt', $2::text) + from jsonb_to_recordset($1::jsonb) as r(id uuid, issue_n int, code text, finished_at timestamptz, created_at timestamptz, link text) + join fixture_issue i on i.n = r.issue_n`, + [JSON.stringify(runsSql.map(([id, issue_n, code, finished_at, created_at, link]) => ({ id, issue_n, code, finished_at, created_at, link }))), fillerSeed]); + await sqlClient.unsafe(`insert into issue_recovery_actions (company_id, source_issue_id, kind, status, owner_type, owner_user_id, + cause, fingerprint, evidence, next_action) + select i.company_id, i.id, 'active_run_watchdog', h.status, 'board', h.owner_user_id, 'legacy_execution_requires_reconciliation', + 'held:' || h.ord, h.evidence, 'Fixture' + from jsonb_to_recordset($1::jsonb) as h(ord int, issue_n int, status text, owner_user_id text, evidence jsonb) + join fixture_issue i on i.n = h.issue_n`, + [JSON.stringify(heldRows.map((held, ord) => ({ ord, issue_n: held.issueN, status: held.status, owner_user_id: held.ownerUserId ?? null, + evidence: held.evidence })))]); + // Historical records (75,003): held-shaped records without a deadline on + // finished tasks, settled continuations, and records of other causes. + await sqlClient.unsafe(`insert into issue_recovery_actions (company_id, source_issue_id, kind, status, owner_type, cause, + fingerprint, evidence, next_action, resolved_at) + select i.company_id, i.id, case when g.n % 3 = 0 then 'active_run_watchdog' else 'missing_disposition' end, 'resolved', 'agent', + (case when g.n % 4 = 0 then $1::text[] else array['stranded_assigned_issue', 'missing_disposition'] end)[1 + g.n % 2], + 'history:' || g.n, + case when g.n % 15 = 0 then jsonb_build_object('runId', r.id::text, 'automaticRecovery', jsonb_build_object('replay', 'blocked')) + when g.n % 4 = 0 then jsonb_build_object('runId', r.id::text, 'executionReconciliation', jsonb_build_object('runId', r.id::text), + 'freshContinuation', jsonb_build_object('sourceRunId', r.id::text), 'note', repeat('x', 400)) + else jsonb_build_object('runId', r.id::text, 'note', repeat('y', 400)) end, + 'Fixture', $2::timestamptz + from generate_series(1, 75_003) g(n) + join fixture_issue i on i.n = 1 + (g.n * 7) % 5669 + join lateral (select id from heartbeat_runs where (issue_id = i.id or native_issue_id = i.id) limit 1) r on true`, + [[...EXECUTION_RECONCILIATION_CAUSES].slice(0, 2), now.toISOString()]); + await sqlClient.unsafe("analyze"); + const [sizes] = await sqlClient.unsafe(`select + (select count(*) from heartbeat_runs)::int as runs, pg_size_pretty(pg_total_relation_size('heartbeat_runs')) as runs_size, + (select count(*) from issue_recovery_actions)::int as actions, pg_size_pretty(pg_total_relation_size('issue_recovery_actions')) as actions_size, + (select count(*) from issue_recovery_actions where evidence #>> '{freshContinuationCheck,nextCheckAt}' is not null)::int as with_deadline, + (select count(*) from issues)::int as issues, current_setting('server_version') as server_version, current_setting('jit') as jit, + current_setting('shared_buffers') as shared_buffers`); + const fixtureSeconds = (performance.now() - started) / 1000; + + // Captures the SQL the real service sends, then stops it. + async function capture(invoke: (captureDb: Db) => Promise): Promise { + let query: Query | undefined; + const stop = new Error("captured"); + const captureDb = new Proxy(db!, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args); + selection.execute = async () => { query ??= selection.toSQL(); throw stop; }; + return selection; + }) as typeof builder.from; + return builder; + }; + }}); + try { await invoke(captureDb); } catch (error) { if (error !== stop) throw error; } + assert.ok(query, "no query captured"); + return query; + } + const queries = (module: typeof treatment) => ({ + nextCheckAt: () => capture(captureDb => module.nextFreshContinuationCheckAt(captureDb, now)), + heldListing: async () => module.heldExecutionsAwaitingFreshContinuation(db!, now, 5).toSQL(), + heldListingProbe: async () => module.heldExecutionsAwaitingFreshContinuation(db!, now, 1).toSQL(), + leftAloneCount: () => capture(captureDb => { module.resetBacklogContinuationPacingForTests(); return module.continueHeldExecutions(captureDb, now); }), + }); + async function run(query: Query) { + const start = performance.now(); + const rows = await sqlClient.unsafe(query.sql, query.params as never[]).values(); + return { ms: performance.now() - start, rows }; + } + async function plan(query: Query) { + const [row] = await sqlClient.unsafe(`explain (analyze, buffers, format json) ${query.sql}`, query.params as never[]); + const root = (row!["QUERY PLAN"] as Array<{ "Execution Time": number; Plan: Record }>)[0]!; + return { executionMs: root["Execution Time"], sharedHit: root.Plan["Shared Hit Blocks"], sharedRead: root.Plan["Shared Read Blocks"] }; + } + const before = queries(baseline), after = queries(treatment); + const results = []; + for (const name of Object.keys(before) as Array) { + const [baselineQuery, treatmentQuery] = [await before[name](), await after[name]()]; + const baselineSamples: number[] = [], treatmentSamples: number[] = []; + let baselineRows: unknown[] = [], treatmentRows: unknown[] = []; + // Warmups each, then alternating timed pairs. + for (let n = 0; n < warmupCount; n += 1) { await run(baselineQuery); await run(treatmentQuery); } + for (let n = 0; n < sampleCount; n += 1) { + const first = n % 2 === 0 ? baselineQuery : treatmentQuery, second = first === baselineQuery ? treatmentQuery : baselineQuery; + const a = await run(first), b = await run(second); + const [base, treat] = first === baselineQuery ? [a, b] : [b, a]; + baselineSamples.push(base.ms); treatmentSamples.push(treat.ms); + baselineRows = base.rows; treatmentRows = treat.rows; + } + assert.deepEqual(treatmentRows, baselineRows, `${name}: treatment rows differ from baseline`); + const median = (samples: number[]) => [...samples].sort((x, y) => x - y)[Math.floor(samples.length / 2)]!; + results.push({ name, rows: baselineRows.length, firstRow: baselineRows[0] ?? null, + baselineMedianMs: +median(baselineSamples).toFixed(1), treatmentMedianMs: +median(treatmentSamples).toFixed(1), + speedup: +(median(baselineSamples) / median(treatmentSamples)).toFixed(1), + baselineSamplesMs: baselineSamples.map(ms => +ms.toFixed(1)), treatmentSamplesMs: treatmentSamples.map(ms => +ms.toFixed(1)), + baselinePlan: await plan(baselineQuery), treatmentPlan: await plan(treatmentQuery) }); + } + if (process.argv.includes("--print-plans")) { + for (const [label, module] of [["baseline", baseline], ["treatment", treatment]] as const) { + const query = await queries(module).nextCheckAt(); + const rows = await sqlClient.unsafe(`explain (analyze, buffers) ${query.sql}`, query.params as never[]); + console.error(`--- ${label} nextFreshContinuationCheckAt plan\n${rows.map(row => row["QUERY PLAN"]).join("\n")}`); + } + } + console.log(JSON.stringify({ baselineRevision, fixture: { ...sizes, contextKib, fixtureSeconds: +fixtureSeconds.toFixed(1) }, results }, null, 2)); +} finally { + try { await db?.$client.end({ timeout: 0 }); await database?.cleanup(); + if (baselineDir) await fs.rm(baselineDir, { recursive: true, force: true }); + } finally { clearInterval(keepAlive); } +} diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index f8ad3bb99d0..4dce0254bb3 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -28,6 +28,7 @@ import { readRunCancellation } from "./run-cancellation.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js"; import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; +import { canonicalExecutionUuid } from "./execution-identity.js"; export { AUTOMATIC_CONTINUATION_WAKE_REASONS, PERSON_OWNED_FAILURE_CODES, isPersonOwnedFailureCode, isRecoveryRun } from "./continuation-cause.js"; @@ -772,7 +773,7 @@ export function heldExecutionsAwaitingFreshContinuation(db: Db, now: Date, limit .where(and(heldExecutionWhere(), freshContinuationCheckDue(now), recentInterruptionSourceSql(now))) // Newest interruption first: the most recent work is the work someone is waiting for. .orderBy(desc(sql`(select source.finished_at from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} - and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId'))`), + and source.id = ${sourceRunKey(heldRunIdSql())})`), asc(issueRecoveryActions.id)) .limit(limit); } @@ -799,13 +800,15 @@ function liveDispositionHoldSql() { const cancelledAtDispatch = (alias: "newer" | "held") => sql.raw(`coalesce(${alias}.status = 'cancelled' and ${alias}.started_at is null and ${alias}.error_code = 'execution_reconciliation_required', false)`); -const runOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id::text = ${issueId}::text - or ${sql.raw(alias)}.issue_id::text = ${issueId}::text or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; +// The issue id is a uuid (a column or a task's id), so each link is compared on +// its own indexed column; a cast of the column would scan every run's context. +const runOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id = ${issueId}::uuid + or ${sql.raw(alias)}.issue_id = ${issueId}::uuid or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQL; sourceRunId: SQL }) { const since = sql`${new Date(input.now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz`; const source = (condition: SQL) => sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${input.companyId} - and source.id::text = ${input.sourceRunId}::text and ${condition})`; + and source.id = ${sourceRunKey(input.sourceRunId)} and ${condition})`; const refusedLater = sql`held.company_id = source.company_id and held.id <> source.id and ${runOfTask("held", input.issueId)} and (held.created_at, held.id) > (source.created_at, source.id) and ${cancelledAtDispatch("held")}`; const personAsked = source(sql`(exists (select 1 from ${agentWakeupRequests} retry where retry.company_id = source.company_id @@ -822,10 +825,10 @@ function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQ } /** The backlog pass's eligibility: the one predicate its listing and its deadline query share. */ -function recentInterruptionSourceSql(now: Date) { +export function recentInterruptionSourceSql(now: Date) { const { recent, personAsked } = interruptionEligibility({ now, companyId: sql`${issueRecoveryActions.companyId}`, issueId: sql`${issueRecoveryActions.sourceIssueId}`, - sourceRunId: sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')` }); + sourceRunId: heldRunIdSql() }); return sql`(${recent} or ${personAsked})`; } @@ -847,7 +850,7 @@ let lastLeftAloneLogAt = 0; const LEFT_ALONE_LOG_INTERVAL_MS = 10 * 60_000; /** Held reconciliation records of a task's latest unfinished run (joined to its task). */ -function heldExecutionWhere(options: { includeActive?: boolean } = {}) { +export function heldExecutionWhere(options: { includeActive?: boolean } = {}) { return and( inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), sql`${issueRecoveryActions.evidence}->'executionReconciliation' is null`, @@ -868,7 +871,7 @@ function heldExecutionWhere(options: { includeActive?: boolean } = {}) { sql`${issues.assigneeAgentId} is not null`, // Only the task's latest run, unfinished and still its assignee's: other sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} - and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') + and source.id = ${sourceRunKey(heldRunIdSql())} and source.status in ('failed', 'timed_out', 'interrupted', 'cancelled') and source.finished_at is not null and source.agent_id = ${issues.assigneeAgentId} and not exists (select 1 from ${heartbeatRuns} newer where newer.company_id = source.company_id and newer.id <> source.id @@ -878,6 +881,10 @@ function heldExecutionWhere(options: { includeActive?: boolean } = {}) { )!; } +const heldRunIdSql = () => sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')`; +/** The recorded run id as the run's key: the same match as `id::text = text`, but an index lookup. */ +const sourceRunKey = (runId: SQL) => canonicalExecutionUuid(sql`(${runId})::text`); + /** The earliest time a deferred check becomes due again, if any. */ export async function nextFreshContinuationCheckAt(db: Db, now = new Date()): Promise { // The same records the held-run query and the disposition pass would select; diff --git a/server/src/services/fresh-continuation-schedule-queries.test.ts b/server/src/services/fresh-continuation-schedule-queries.test.ts new file mode 100644 index 00000000000..0b894b8c060 --- /dev/null +++ b/server/src/services/fresh-continuation-schedule-queries.test.ts @@ -0,0 +1,269 @@ +import { randomUUID } from "node:crypto"; +import { and, asc, desc, eq, inArray, isNull, or, sql, type SQL } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { EXECUTION_RECONCILIATION_CAUSES } from "@paperclipai/shared"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { BACKLOG_CONTINUATION_MAX_AGE_MS, freshContinuationCheckDue, heldExecutionWhere, heldExecutionsAwaitingFreshContinuation, + nextFreshContinuationCheckAt, recentInterruptionSourceSql } from "./fresh-continuation-fallback.js"; +import { continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, recoveryRunSql } from "./continuation-cause.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; + +// The predicates as they were before they became index lookups (2026-11-14): +// `id::text = text` on the run key and cast issue columns, one OR scan per +// newer-run probe. Kept verbatim as the oracle the rewrite must match. +const previousCancelledAtDispatch = (alias: "newer" | "held") => sql.raw(`coalesce(${alias}.status = 'cancelled' and ${alias}.started_at is null + and ${alias}.error_code = 'execution_reconciliation_required', false)`); +const previousRunOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id::text = ${issueId}::text + or ${sql.raw(alias)}.issue_id::text = ${issueId}::text or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; + +function previousRecentInterruptionSourceSql(now: Date) { + const input = { companyId: sql`${issueRecoveryActions.companyId}`, issueId: sql`${issueRecoveryActions.sourceIssueId}`, + sourceRunId: sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')` }; + const since = sql`${new Date(now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz`; + const source = (condition: SQL) => sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${input.companyId} + and source.id::text = ${input.sourceRunId}::text and ${condition})`; + const refusedLater = sql`held.company_id = source.company_id and held.id <> source.id and ${previousRunOfTask("held", input.issueId)} + and (held.created_at, held.id) > (source.created_at, source.id) and ${previousCancelledAtDispatch("held")}`; + const personAsked = source(sql`(exists (select 1 from ${agentWakeupRequests} retry where retry.company_id = source.company_id + and retry.status = 'deferred_issue_execution' and retry.reason = 'task_retry' and retry.requested_by_actor_type = 'user' + and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at) + or exists (select 1 from ${heartbeatRuns} held where ${refusedLater} + and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`); + const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")} + and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false) + and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`; + const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource}) + and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); + return sql`(${recent} or ${personAsked})`; +} + +function previousHeldExecutionWhere(options: { includeActive?: boolean } = {}) { + return and( + inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), + sql`${issueRecoveryActions.evidence}->'executionReconciliation' is null`, + sql`${issueRecoveryActions.evidence}->'freshContinuation' is null`, + sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') is not null`, + or( + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, + and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('complete', 'exhausted')`), + options.includeActive ? and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + eq(issueRecoveryActions.kind, "active_run_watchdog")) : undefined, + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence')`, + ), + isNull(issueRecoveryActions.ownerUserId), + sql`${issues.status} not in ('done', 'cancelled')`, + sql`${issues.assigneeAgentId} is not null`, + sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} + and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') + and source.status in ('failed', 'timed_out', 'interrupted', 'cancelled') and source.finished_at is not null + and source.agent_id = ${issues.assigneeAgentId} + and not exists (select 1 from ${heartbeatRuns} newer where newer.company_id = source.company_id and newer.id <> source.id + and (newer.native_issue_id = ${issues.id} or newer.issue_id = ${issues.id} or newer.context_snapshot->>'issueId' = ${issues.id}::text) + and not ${previousCancelledAtDispatch("newer")} + and (newer.status in ('queued', 'running', 'scheduled_retry') or (newer.created_at, newer.id) > (source.created_at, source.id))))`, + )!; +} + +function random(seed: number) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +type PlanNode = { "Relation Name"?: string; "Actual Loops"?: number; "Actual Rows"?: number; + "Rows Removed by Filter"?: number; "Rows Removed by Index Recheck"?: number; Plans?: PlanNode[] }; + +function heartbeatRunRowsRead(node: PlanNode): number { + const own = node["Relation Name"] === "heartbeat_runs" ? (node["Actual Loops"] ?? 0) + * ((node["Actual Rows"] ?? 0) + (node["Rows Removed by Filter"] ?? 0) + (node["Rows Removed by Index Recheck"] ?? 0)) : 0; + return own + (node.Plans ?? []).reduce((total, child) => total + heartbeatRunRowsRead(child), 0); +} + +suite("fresh-continuation schedule queries", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + const now = new Date(Date.UTC(2026, 10, 14, 5, 0, 0)); + const tenants: Array<{ companyId: string; agentIds: string[] }> = []; + let actions = 0; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-fresh-continuation-schedule-"); + db = createDb(temporary.connectionString); + for (const prefix of ["FCA", "FCB"]) { + const companyId = randomUUID(), agentIds = [randomUUID(), randomUUID()]; + await db.insert(companies).values({ id: companyId, name: prefix, issuePrefix: prefix, requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values(agentIds.map((id, index) => ({ id, companyId, name: `Coder ${index}`, role: "engineer", + status: "idle", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} }))); + tenants.push({ companyId, agentIds }); + } + actions = await seedVariedFixture(20261114, 900); + // Run history that is not linked to any held task, so a scan per candidate shows. + await db.$client.unsafe(`insert into heartbeat_runs (company_id, agent_id, status, context_snapshot, created_at) + select $1::uuid, $2::uuid, 'succeeded', jsonb_build_object('issueId', gen_random_uuid()::text, 'prompt', repeat('x', 4000)), + $3::timestamptz - (n || ' minutes')::interval from generate_series(1, 3000) n`, + [tenants[0]!.companyId, tenants[0]!.agentIds[0]!, now.toISOString()]); + await db.execute(sql`analyze`); + }, 120_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + /** Held tasks in every shape the predicates distinguish: run keys, run links, statuses, ages, retries. */ + async function seedVariedFixture(seed: number, count: number) { + const next = random(seed); + const pick = (values: readonly T[]): T => values[Math.floor(next() * values.length)]!; + const chance = (probability: number) => next() < probability; + const at = (minutesAgo: number) => new Date(now.getTime() - minutesAgo * 60_000); + const issueRows: Array = []; + const runRows: Array = []; + const wakeRows: Array = []; + const actionRows: Array = []; + const link = (issueId: string, kind: string) => kind === "native" ? { scopeKind: "issue" as const, issueId, nativeIssueId: issueId } + : kind === "issue" ? { scopeKind: "issue" as const, issueId } : kind === "context" ? { scopeKind: "company" as const } + : { scopeKind: "company" as const }; + for (let index = 0; index < count; index += 1) { + const tenant = chance(0.9) ? tenants[0]! : tenants[1]!; + const other = tenant === tenants[0] ? tenants[1]! : tenants[0]!; + const issueId = randomUUID(), runId = randomUUID(); + const assignee = chance(0.9) ? pick(tenant.agentIds) : null; + issueRows.push({ id: issueId, companyId: tenant.companyId, title: `Held ${index}`, assigneeAgentId: assignee, + status: pick(["todo", "in_progress", "blocked", "blocked", "in_review", "done", "cancelled"]) }); + const finishedMinutesAgo = pick([30, 120, 600, 1_200, 1_430, 1_450, 2_000, 4_000]); + const sourceLink = pick(["native", "issue", "context"]); + const errorCode = pick(["process_lost", "native_session_interrupted", "server_shutdown_interrupted", "controller_lease_lost", + "adapter_failed", "reusable_sandbox_resume_failed", "budget_exceeded", null]); + const sourceCreated = chance(0.92); + if (sourceCreated) { + const finished = chance(0.92) ? at(finishedMinutesAgo) : null; + runRows.push({ id: runId, companyId: chance(0.95) ? tenant.companyId : other.companyId, + agentId: chance(0.85) && assignee ? assignee : pick(tenant.agentIds), ...link(issueId, sourceLink), + status: pick(["failed", "failed", "timed_out", "interrupted", "cancelled", "succeeded", "running"]), + errorCode, error: errorCode === "reusable_sandbox_resume_failed" ? pick(["resume failed (cause=transport_timeout)", "resume failed"]) : errorCode, + startedAt: at(finishedMinutesAgo + 10), finishedAt: finished, createdAt: at(finishedMinutesAgo + 11), updatedAt: at(finishedMinutesAgo), + retryOfRunId: null, scheduledRetryReason: chance(0.1) ? "process_lost" : null, + contextSnapshot: { issueId, wakeReason: pick(["issue_assigned", "task_retry", "process_lost_retry"]), + ...(chance(0.1) ? { explicitUserContinuation: { runId } } : {}), ...(chance(0.1) ? { previousRunId: randomUUID() } : {}) } }); + } + for (let extra = Math.floor(next() * 3); extra > 0; extra -= 1) { + const extraLink = pick(["native", "issue", "context", "unrelated"]); + const cancelledAtDispatch = chance(0.35); + const createdMinutesAgo = pick([finishedMinutesAgo + 11, finishedMinutesAgo + 30, finishedMinutesAgo - 5, 5]); + runRows.push({ companyId: chance(0.95) ? tenant.companyId : other.companyId, agentId: pick(tenant.agentIds), + ...link(issueId, extraLink), + status: cancelledAtDispatch ? "cancelled" : pick(["queued", "running", "scheduled_retry", "succeeded", "failed", "cancelled"]), + errorCode: cancelledAtDispatch ? "execution_reconciliation_required" : null, + startedAt: cancelledAtDispatch && chance(0.8) ? null : at(createdMinutesAgo - 1), + finishedAt: chance(0.8) ? at(pick([createdMinutesAgo - 2, 60, 2_000])) : null, createdAt: at(createdMinutesAgo), + retryOfRunId: sourceCreated && chance(0.3) ? runId : null, + contextSnapshot: extraLink === "unrelated" ? { issueId: randomUUID() } + : { issueId, wakeReason: pick(["task_retry", "issue_assigned", "process_lost_retry"]) } }); + } + if (chance(0.25)) { + wakeRows.push({ companyId: tenant.companyId, agentId: pick(tenant.agentIds), source: "on_demand", reason: pick(["task_retry", "issue_assigned"]), + status: pick(["deferred_issue_execution", "deferred_issue_execution", "coalesced"]), + requestedByActorType: pick(["user", "user", "system"]), runId: !sourceCreated || chance(0.8) ? null : runId, + requestedAt: at(pick([finishedMinutesAgo - 5, finishedMinutesAgo + 5])), payload: { issueId } }); + } + const recordedRunId = pick(["runId", "runId", "runId", "runId", "runId", "sourceRunId", "sourceRunId", "upper", "invalid", "number", "missing", "unknown", + "braced", "padded", "both"]); + const runKey = recordedRunId === "sourceRunId" ? { sourceRunId: runId } + : recordedRunId === "upper" ? { runId: runId.toUpperCase() } : recordedRunId === "invalid" ? { runId: "not-a-run" } + : recordedRunId === "number" ? { runId: 42 } : recordedRunId === "missing" ? {} + : recordedRunId === "unknown" ? { runId: randomUUID() } : recordedRunId === "braced" ? { runId: `{${runId}}` } + : recordedRunId === "padded" ? { runId: ` ${runId}` } : recordedRunId === "both" ? { runId: "not-a-run", sourceRunId: runId } + : { runId }; + actionRows.push({ companyId: tenant.companyId, sourceIssueId: issueId, kind: pick(["active_run_watchdog", "missing_disposition"]), + status: pick(["active", "escalated", "resolved", "resolved"]), ownerType: "board", ownerUserId: chance(0.05) ? "user-1" : null, + cause: pick(["legacy_execution_requires_reconciliation", "native_continuation_requires_reconciliation", "stranded_assigned_issue"]), + fingerprint: `held:${index}`, nextAction: "Fixture", + evidence: { ...runKey, + ...(chance(0.7) ? { automaticRecovery: { replay: pick(["blocked", "blocked", "blocked", "allowed"]) } } : {}), + ...(chance(0.2) ? { systemRecovery: { state: pick(["complete", "exhausted", "scheduled", "running", "failed"]) } } : {}), + ...(chance(0.05) ? { executionReconciliation: { runId } } : {}), + ...(chance(0.05) ? { freshContinuation: { sourceRunId: runId } } : {}), + ...(chance(0.7) ? { freshContinuationCheck: { kind: "backoff", nextCheckAt: now.getTime() + Math.round((next() - 0.5) * 7_200_000) } } : {}) } }); + } + await db.insert(issues).values(issueRows); + // One statement, so a retry may name a source run listed after it. + await db.insert(heartbeatRuns).values(runRows); + if (wakeRows.length) await db.insert(agentWakeupRequests).values(wakeRows); + for (let start = 0; start < actionRows.length; start += 500) + await db.insert(issueRecoveryActions).values(actionRows.slice(start, start + 500)); + return actionRows.length; + } + + const verdicts = (where: { held: SQL; heldWithActive: SQL; eligible: SQL }) => db.select({ id: issueRecoveryActions.id, + held: sql`coalesce(${where.held}, false)`, heldWithActive: sql`coalesce(${where.heldWithActive}, false)`, + eligible: sql`coalesce(${where.eligible}, false)` }) + .from(issueRecoveryActions).innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), + eq(issues.companyId, issueRecoveryActions.companyId))).orderBy(asc(issueRecoveryActions.id)); + + it("selects exactly the records the previous predicates selected, for every run key, link and status", async () => { + const before = await verdicts({ held: previousHeldExecutionWhere(), heldWithActive: previousHeldExecutionWhere({ includeActive: true }), + eligible: previousRecentInterruptionSourceSql(now) }); + const after = await verdicts({ held: heldExecutionWhere(), heldWithActive: heldExecutionWhere({ includeActive: true }), + eligible: recentInterruptionSourceSql(now) }); + expect(after).toEqual(before); + expect(before).toHaveLength(actions); + // The fixture reaches both outcomes of each predicate, so equality is meaningful. + for (const key of ["held", "heldWithActive", "eligible"] as const) { + expect(before.filter(row => row[key]).length).toBeGreaterThan(5); + expect(before.filter(row => !row[key]).length).toBeGreaterThan(5); + } + expect(before.filter(row => row.heldWithActive && row.eligible).length).toBeGreaterThan(5); + }, 60_000); + + it("returns the deadline and the backlog listing the previous queries returned", async () => { + const liveHold = and(inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"))!; + const [previousDeadline] = await db.select({ at: sql`min((${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}')::bigint)` }) + .from(issueRecoveryActions).innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), + eq(issues.companyId, issueRecoveryActions.companyId))) + .where(and(previousHeldExecutionWhere({ includeActive: true }), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}' is not null`, + or(liveHold, previousRecentInterruptionSourceSql(now)))); + expect(previousDeadline?.at).not.toBeNull(); + expect(await nextFreshContinuationCheckAt(db, now)).toBe(Number(previousDeadline!.at)); + + const previousListing = await db.select({ id: issueRecoveryActions.id, companyId: issueRecoveryActions.companyId, + runId: sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')` }) + .from(issueRecoveryActions).innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), + eq(issues.companyId, issueRecoveryActions.companyId))) + .where(and(previousHeldExecutionWhere(), freshContinuationCheckDue(now), previousRecentInterruptionSourceSql(now))) + .orderBy(desc(sql`(select source.finished_at from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} + and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId'))`), + asc(issueRecoveryActions.id)).limit(1_000); + expect(previousListing.length).toBeGreaterThan(0); + expect(await heldExecutionsAwaitingFreshContinuation(db, now, 1_000)).toEqual(previousListing); + }, 60_000); + + it("finds the deadline by run-key and task-link lookups, not a scan of run history per record", async () => { + let captured: { sql: string; params: unknown[] } | undefined; + const capture = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args); + selection.execute = async () => { captured = selection.toSQL(); return [{ at: null }]; }; + return selection; + }) as typeof builder.from; + return builder; + }; + } }); + await nextFreshContinuationCheckAt(capture as typeof db, now); + const [row] = await db.$client.unsafe(`explain (analyze, format json) ${captured!.sql}`, captured!.params as never[]); + const plan = (row!["QUERY PLAN"] as Array<{ Plan: PlanNode }>)[0]!.Plan; + const [{ runs }] = await db.$client.unsafe(`select count(*)::int as runs from heartbeat_runs`) as unknown as Array<{ runs: number }>; + // The previous query read the whole run history once per held record + // (hundreds of times here); the lookups read a few rows per record. + expect(heartbeatRunRowsRead(plan)).toBeLessThan(2 * runs); + }, 60_000); +});