diff --git a/scripts/benchmark-fresh-continuation-deadline.ts b/scripts/benchmark-fresh-continuation-deadline.ts new file mode 100644 index 00000000000..9afedc8066f --- /dev/null +++ b/scripts/benchmark-fresh-continuation-deadline.ts @@ -0,0 +1,228 @@ +// Fresh-continuation scheduler queries on a staging-shaped fixture: the +// disposition worker's deadline (`nextFreshContinuationCheckAt`), its backlog +// listing (`heldExecutionsAwaitingFreshContinuation`) and the left-alone count. +// Compares a baseline revision with the working tree on identical data and +// fails on any difference in complete positional rows. +// +// pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-fresh-continuation-deadline.ts \ +// --baseline-ref= [--context-kib=60] [--samples=7] [--warmups=2] [--print-plans] +import assert from "node:assert/strict"; +import { randomBytes, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { EXECUTION_RECONCILIATION_CAUSES } from "../packages/shared/src/index.js"; +import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js"; +import * as treatment from "../server/src/services/fresh-continuation-fallback.js"; + +type Query = { sql: string; params: unknown[] }; +const arg = (name: string, fallback: string) => process.argv.find(value => value.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback; +const baselineRevision = execFileSync("git", ["rev-parse", "--verify", `${arg("baseline-ref", "origin/master")}^{commit}`], { encoding: "utf8" }).trim(); +const contextKib = Number(arg("context-kib", "60")); +const sampleCount = Number(arg("samples", "7")); +const warmupCount = Number(arg("warmups", "2")); +const keepAlive = setInterval(() => {}, 1000); +let database: Awaited> | undefined; +let baselineDir: string | undefined; +let db: Db | undefined; +try { + database = await startEmbeddedPostgresTestDatabase("fresh-continuation-deadline-benchmark-"); + db = createDb(database.connectionString); + const sqlClient = db.$client; + baselineDir = await fs.mkdtemp(path.join(fileURLToPath(new URL("../server/scripts/", import.meta.url)), ".fresh-continuation-baseline-")); + const modulePath = "server/src/services/fresh-continuation-fallback.ts"; + const original = new URL(`../${modulePath}`, import.meta.url); + const source = execFileSync("git", ["show", `${baselineRevision}:${modulePath}`], { encoding: "utf8" }); + const baselineFile = path.join(baselineDir, "fresh-continuation-fallback.ts"); + await fs.writeFile(baselineFile, source.replace(/from "(\.{1,2}\/[^"]+)"/g, + (_match, specifier: string) => `from ${JSON.stringify(fileURLToPath(new URL(specifier, original)))}`)); + const baseline = await import(pathToFileURL(baselineFile).href) as typeof treatment; + + // Staging shape (2026-11-14): 5.7k issues, 31.4k runs whose rows carry wide + // context JSON (1.96 GB), 75k recovery records of which ~280 carry a + // fresh-continuation check deadline. + const now = new Date(Date.UTC(2026, 10, 14, 5, 0, 0)); + const nowMs = now.getTime(); + const companyId = randomUUID(), decoyCompanyId = randomUUID(); + const agentIds = [randomUUID(), randomUUID(), randomUUID()]; + const fillerSeed = randomBytes(Math.ceil(contextKib * 1024 * 0.75)).toString("base64"); + const started = performance.now(); + await sqlClient.unsafe(`insert into companies (id, name, issue_prefix) values ($1, 'Synthetic', 'FCD'), ($2, 'Decoy', 'FCX')`, + [companyId, decoyCompanyId]); + await sqlClient.unsafe(`insert into agents (id, company_id, name, adapter_type) + select id, $4::uuid, 'Agent ' || n, 'codex_local' from unnest(array[$1, $2, $3]::uuid[]) with ordinality as a(id, n)`, + [...agentIds, companyId]); + const decoyAgentId = randomUUID(); + await sqlClient.unsafe(`insert into agents (id, company_id, name, adapter_type) values ($1, $2, 'Decoy', 'codex_local')`, [decoyAgentId, decoyCompanyId]); + // Issues: 5,400 in the company (60% done), 269 decoys in another company. + await sqlClient.unsafe(`create temporary table fixture_issue as + select n, gen_random_uuid() as id, case when n <= 5400 then $1::uuid else $2::uuid end as company_id, + case when n > 5400 then $3::uuid else (array[$4, $5, $6]::uuid[])[1 + n % 3] end as agent_id, + case when n % 10 < 6 then 'done' when n % 10 = 6 then 'blocked' when n % 10 = 7 then 'in_progress' else 'todo' end as status + from generate_series(1, 5669) n`, [companyId, decoyCompanyId, decoyAgentId, ...agentIds]); + await sqlClient.unsafe(`insert into issues (id, company_id, title, status, assignee_agent_id) + select id, company_id, 'Fixture task ' || n, status, agent_id from fixture_issue`); + // Runs: 31,399, about five per task, linked by native id, legacy issue id, or context only. + await sqlClient.unsafe(`insert into heartbeat_runs (company_id, agent_id, scope_kind, issue_id, native_issue_id, runtime_mode, + status, error_code, started_at, finished_at, created_at, updated_at, context_snapshot) + select i.company_id, i.agent_id, case when g.n % 5 = 4 then 'company' else 'issue' end, + case when g.n % 5 = 4 then null else i.id end, case when g.n % 5 < 2 then i.id end, + case when g.n % 5 < 2 then 'native' else 'legacy' end, + case when g.n % 7 = 0 then 'failed' when g.n % 11 = 0 then 'cancelled' else 'succeeded' end, + case when g.n % 7 = 0 then 'adapter_failed' end, + $2::timestamptz - (g.n || ' minutes')::interval, $2::timestamptz - (g.n || ' minutes')::interval + interval '30 seconds', + $2::timestamptz - (g.n || ' minutes')::interval - interval '1 second', $2::timestamptz - (g.n || ' minutes')::interval, + jsonb_build_object('issueId', i.id::text, 'wakeReason', 'issue_assigned', 'prompt', $1::text) + from generate_series(1, 31_000) g(n) join fixture_issue i on i.n = 1 + g.n % 5669`, [fillerSeed, now.toISOString()]); + + // Held records with a check deadline (282), in the shapes the predicate distinguishes. + type Held = { issueN: number; runId: string; evidence: Record; status: string; ownerUserId?: string; + cause?: string; recentRun: boolean; errorCode: string; newer?: "native" | "legacy" | "context" | "cancelled_at_dispatch" | "live" }; + const heldRows: Held[] = []; + let issueN = 7; + const nextOpenIssue = () => { do issueN += 1; while (issueN % 10 < 6 || issueN > 5400); return issueN; }; + for (let n = 0; n < 282; n += 1) { + const runId = randomUUID(); + // A third are overdue, so the backlog listing has due work too. + const check = { kind: "exact_recovery_pending", nextCheckAt: nowMs + (n % 3 === 0 ? -1 : 1) * (n + 1) * 1_000, checks: 1 }; + const kind = n % 12; + const base = { runId, automaticRecovery: { replay: "blocked" }, freshContinuationCheck: check }; + heldRows.push({ + issueN: nextOpenIssue(), runId, + evidence: kind === 1 ? { sourceRunId: runId, automaticRecovery: { replay: "blocked" }, freshContinuationCheck: check } + : kind === 2 ? { ...base, runId: runId.toUpperCase() } + : kind === 3 ? { ...base, runId: "not-a-run" } + : base, + status: kind === 4 ? "active" : "resolved", + ownerUserId: kind === 5 ? "user-1" : undefined, + recentRun: kind !== 6 && kind !== 4, + errorCode: kind === 7 ? "adapter_failed" : "process_lost", + newer: kind === 8 ? (["native", "legacy", "context"] as const)[n % 3] : kind === 9 ? "cancelled_at_dispatch" : kind === 10 ? "live" : undefined, + }); + } + const runsSql: unknown[][] = []; + for (const held of heldRows) { + const finished = new Date(nowMs - (held.recentRun ? 2 : 48) * 3_600_000); + runsSql.push([held.runId, held.issueN, held.errorCode, finished.toISOString(), new Date(finished.getTime() - 600_000).toISOString(), null]); + if (held.newer) runsSql.push([randomUUID(), held.issueN, held.newer, finished.toISOString(), + new Date(finished.getTime() + 60_000).toISOString(), held.newer]); + } + await sqlClient.unsafe(`insert into heartbeat_runs (id, company_id, agent_id, scope_kind, issue_id, native_issue_id, runtime_mode, + status, error_code, started_at, finished_at, created_at, updated_at, context_snapshot) + select r.id, i.company_id, i.agent_id, case when r.link = 'context' then 'company' else 'issue' end, + case when r.link in ('context', 'native') then null else i.id end, case when r.link = 'native' then i.id end, + case when r.link = 'native' then 'native' else 'legacy' end, + case when r.link = 'live' then 'queued' when r.link = 'cancelled_at_dispatch' then 'cancelled' + when r.link is null then 'failed' else 'succeeded' end, + case when r.link = 'cancelled_at_dispatch' then 'execution_reconciliation_required' when r.link is null then r.code end, + case when r.link is null then r.created_at end, case when r.link is null then r.finished_at end, r.created_at, r.created_at, + jsonb_build_object('issueId', i.id::text, 'wakeReason', 'issue_assigned', 'prompt', $2::text) + from jsonb_to_recordset($1::jsonb) as r(id uuid, issue_n int, code text, finished_at timestamptz, created_at timestamptz, link text) + join fixture_issue i on i.n = r.issue_n`, + [JSON.stringify(runsSql.map(([id, issue_n, code, finished_at, created_at, link]) => ({ id, issue_n, code, finished_at, created_at, link }))), fillerSeed]); + await sqlClient.unsafe(`insert into issue_recovery_actions (company_id, source_issue_id, kind, status, owner_type, owner_user_id, + cause, fingerprint, evidence, next_action) + select i.company_id, i.id, 'active_run_watchdog', h.status, 'board', h.owner_user_id, 'legacy_execution_requires_reconciliation', + 'held:' || h.ord, h.evidence, 'Fixture' + from jsonb_to_recordset($1::jsonb) as h(ord int, issue_n int, status text, owner_user_id text, evidence jsonb) + join fixture_issue i on i.n = h.issue_n`, + [JSON.stringify(heldRows.map((held, ord) => ({ ord, issue_n: held.issueN, status: held.status, owner_user_id: held.ownerUserId ?? null, + evidence: held.evidence })))]); + // Historical records (75,003): held-shaped records without a deadline on + // finished tasks, settled continuations, and records of other causes. + await sqlClient.unsafe(`insert into issue_recovery_actions (company_id, source_issue_id, kind, status, owner_type, cause, + fingerprint, evidence, next_action, resolved_at) + select i.company_id, i.id, case when g.n % 3 = 0 then 'active_run_watchdog' else 'missing_disposition' end, 'resolved', 'agent', + (case when g.n % 4 = 0 then $1::text[] else array['stranded_assigned_issue', 'missing_disposition'] end)[1 + g.n % 2], + 'history:' || g.n, + case when g.n % 15 = 0 then jsonb_build_object('runId', r.id::text, 'automaticRecovery', jsonb_build_object('replay', 'blocked')) + when g.n % 4 = 0 then jsonb_build_object('runId', r.id::text, 'executionReconciliation', jsonb_build_object('runId', r.id::text), + 'freshContinuation', jsonb_build_object('sourceRunId', r.id::text), 'note', repeat('x', 400)) + else jsonb_build_object('runId', r.id::text, 'note', repeat('y', 400)) end, + 'Fixture', $2::timestamptz + from generate_series(1, 75_003) g(n) + join fixture_issue i on i.n = 1 + (g.n * 7) % 5669 + join lateral (select id from heartbeat_runs where (issue_id = i.id or native_issue_id = i.id) limit 1) r on true`, + [[...EXECUTION_RECONCILIATION_CAUSES].slice(0, 2), now.toISOString()]); + await sqlClient.unsafe("analyze"); + const [sizes] = await sqlClient.unsafe(`select + (select count(*) from heartbeat_runs)::int as runs, pg_size_pretty(pg_total_relation_size('heartbeat_runs')) as runs_size, + (select count(*) from issue_recovery_actions)::int as actions, pg_size_pretty(pg_total_relation_size('issue_recovery_actions')) as actions_size, + (select count(*) from issue_recovery_actions where evidence #>> '{freshContinuationCheck,nextCheckAt}' is not null)::int as with_deadline, + (select count(*) from issues)::int as issues, current_setting('server_version') as server_version, current_setting('jit') as jit, + current_setting('shared_buffers') as shared_buffers`); + const fixtureSeconds = (performance.now() - started) / 1000; + + // Captures the SQL the real service sends, then stops it. + async function capture(invoke: (captureDb: Db) => Promise): Promise { + let query: Query | undefined; + const stop = new Error("captured"); + const captureDb = new Proxy(db!, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args); + selection.execute = async () => { query ??= selection.toSQL(); throw stop; }; + return selection; + }) as typeof builder.from; + return builder; + }; + }}); + try { await invoke(captureDb); } catch (error) { if (error !== stop) throw error; } + assert.ok(query, "no query captured"); + return query; + } + const queries = (module: typeof treatment) => ({ + nextCheckAt: () => capture(captureDb => module.nextFreshContinuationCheckAt(captureDb, now)), + heldListing: async () => module.heldExecutionsAwaitingFreshContinuation(db!, now, 5).toSQL(), + heldListingProbe: async () => module.heldExecutionsAwaitingFreshContinuation(db!, now, 1).toSQL(), + leftAloneCount: () => capture(captureDb => { module.resetBacklogContinuationPacingForTests(); return module.continueHeldExecutions(captureDb, now); }), + }); + async function run(query: Query) { + const start = performance.now(); + const rows = await sqlClient.unsafe(query.sql, query.params as never[]).values(); + return { ms: performance.now() - start, rows }; + } + async function plan(query: Query) { + const [row] = await sqlClient.unsafe(`explain (analyze, buffers, format json) ${query.sql}`, query.params as never[]); + const root = (row!["QUERY PLAN"] as Array<{ "Execution Time": number; Plan: Record }>)[0]!; + return { executionMs: root["Execution Time"], sharedHit: root.Plan["Shared Hit Blocks"], sharedRead: root.Plan["Shared Read Blocks"] }; + } + const before = queries(baseline), after = queries(treatment); + const results = []; + for (const name of Object.keys(before) as Array) { + const [baselineQuery, treatmentQuery] = [await before[name](), await after[name]()]; + const baselineSamples: number[] = [], treatmentSamples: number[] = []; + let baselineRows: unknown[] = [], treatmentRows: unknown[] = []; + // Warmups each, then alternating timed pairs. + for (let n = 0; n < warmupCount; n += 1) { await run(baselineQuery); await run(treatmentQuery); } + for (let n = 0; n < sampleCount; n += 1) { + const first = n % 2 === 0 ? baselineQuery : treatmentQuery, second = first === baselineQuery ? treatmentQuery : baselineQuery; + const a = await run(first), b = await run(second); + const [base, treat] = first === baselineQuery ? [a, b] : [b, a]; + baselineSamples.push(base.ms); treatmentSamples.push(treat.ms); + baselineRows = base.rows; treatmentRows = treat.rows; + } + assert.deepEqual(treatmentRows, baselineRows, `${name}: treatment rows differ from baseline`); + const median = (samples: number[]) => [...samples].sort((x, y) => x - y)[Math.floor(samples.length / 2)]!; + results.push({ name, rows: baselineRows.length, firstRow: baselineRows[0] ?? null, + baselineMedianMs: +median(baselineSamples).toFixed(1), treatmentMedianMs: +median(treatmentSamples).toFixed(1), + speedup: +(median(baselineSamples) / median(treatmentSamples)).toFixed(1), + baselineSamplesMs: baselineSamples.map(ms => +ms.toFixed(1)), treatmentSamplesMs: treatmentSamples.map(ms => +ms.toFixed(1)), + baselinePlan: await plan(baselineQuery), treatmentPlan: await plan(treatmentQuery) }); + } + if (process.argv.includes("--print-plans")) { + for (const [label, module] of [["baseline", baseline], ["treatment", treatment]] as const) { + const query = await queries(module).nextCheckAt(); + const rows = await sqlClient.unsafe(`explain (analyze, buffers) ${query.sql}`, query.params as never[]); + console.error(`--- ${label} nextFreshContinuationCheckAt plan\n${rows.map(row => row["QUERY PLAN"]).join("\n")}`); + } + } + console.log(JSON.stringify({ baselineRevision, fixture: { ...sizes, contextKib, fixtureSeconds: +fixtureSeconds.toFixed(1) }, results }, null, 2)); +} finally { + try { await db?.$client.end({ timeout: 0 }); await database?.cleanup(); + if (baselineDir) await fs.rm(baselineDir, { recursive: true, force: true }); + } finally { clearInterval(keepAlive); } +} diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index f8ad3bb99d0..4dce0254bb3 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -28,6 +28,7 @@ import { readRunCancellation } from "./run-cancellation.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js"; import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; +import { canonicalExecutionUuid } from "./execution-identity.js"; export { AUTOMATIC_CONTINUATION_WAKE_REASONS, PERSON_OWNED_FAILURE_CODES, isPersonOwnedFailureCode, isRecoveryRun } from "./continuation-cause.js"; @@ -772,7 +773,7 @@ export function heldExecutionsAwaitingFreshContinuation(db: Db, now: Date, limit .where(and(heldExecutionWhere(), freshContinuationCheckDue(now), recentInterruptionSourceSql(now))) // Newest interruption first: the most recent work is the work someone is waiting for. .orderBy(desc(sql`(select source.finished_at from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} - and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId'))`), + and source.id = ${sourceRunKey(heldRunIdSql())})`), asc(issueRecoveryActions.id)) .limit(limit); } @@ -799,13 +800,15 @@ function liveDispositionHoldSql() { const cancelledAtDispatch = (alias: "newer" | "held") => sql.raw(`coalesce(${alias}.status = 'cancelled' and ${alias}.started_at is null and ${alias}.error_code = 'execution_reconciliation_required', false)`); -const runOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id::text = ${issueId}::text - or ${sql.raw(alias)}.issue_id::text = ${issueId}::text or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; +// The issue id is a uuid (a column or a task's id), so each link is compared on +// its own indexed column; a cast of the column would scan every run's context. +const runOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id = ${issueId}::uuid + or ${sql.raw(alias)}.issue_id = ${issueId}::uuid or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQL; sourceRunId: SQL }) { const since = sql`${new Date(input.now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz`; const source = (condition: SQL) => sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${input.companyId} - and source.id::text = ${input.sourceRunId}::text and ${condition})`; + and source.id = ${sourceRunKey(input.sourceRunId)} and ${condition})`; const refusedLater = sql`held.company_id = source.company_id and held.id <> source.id and ${runOfTask("held", input.issueId)} and (held.created_at, held.id) > (source.created_at, source.id) and ${cancelledAtDispatch("held")}`; const personAsked = source(sql`(exists (select 1 from ${agentWakeupRequests} retry where retry.company_id = source.company_id @@ -822,10 +825,10 @@ function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQ } /** The backlog pass's eligibility: the one predicate its listing and its deadline query share. */ -function recentInterruptionSourceSql(now: Date) { +export function recentInterruptionSourceSql(now: Date) { const { recent, personAsked } = interruptionEligibility({ now, companyId: sql`${issueRecoveryActions.companyId}`, issueId: sql`${issueRecoveryActions.sourceIssueId}`, - sourceRunId: sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')` }); + sourceRunId: heldRunIdSql() }); return sql`(${recent} or ${personAsked})`; } @@ -847,7 +850,7 @@ let lastLeftAloneLogAt = 0; const LEFT_ALONE_LOG_INTERVAL_MS = 10 * 60_000; /** Held reconciliation records of a task's latest unfinished run (joined to its task). */ -function heldExecutionWhere(options: { includeActive?: boolean } = {}) { +export function heldExecutionWhere(options: { includeActive?: boolean } = {}) { return and( inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), sql`${issueRecoveryActions.evidence}->'executionReconciliation' is null`, @@ -868,7 +871,7 @@ function heldExecutionWhere(options: { includeActive?: boolean } = {}) { sql`${issues.assigneeAgentId} is not null`, // Only the task's latest run, unfinished and still its assignee's: other sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} - and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') + and source.id = ${sourceRunKey(heldRunIdSql())} and source.status in ('failed', 'timed_out', 'interrupted', 'cancelled') and source.finished_at is not null and source.agent_id = ${issues.assigneeAgentId} and not exists (select 1 from ${heartbeatRuns} newer where newer.company_id = source.company_id and newer.id <> source.id @@ -878,6 +881,10 @@ function heldExecutionWhere(options: { includeActive?: boolean } = {}) { )!; } +const heldRunIdSql = () => sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')`; +/** The recorded run id as the run's key: the same match as `id::text = text`, but an index lookup. */ +const sourceRunKey = (runId: SQL) => canonicalExecutionUuid(sql`(${runId})::text`); + /** The earliest time a deferred check becomes due again, if any. */ export async function nextFreshContinuationCheckAt(db: Db, now = new Date()): Promise { // The same records the held-run query and the disposition pass would select; diff --git a/server/src/services/fresh-continuation-schedule-queries.test.ts b/server/src/services/fresh-continuation-schedule-queries.test.ts new file mode 100644 index 00000000000..0b894b8c060 --- /dev/null +++ b/server/src/services/fresh-continuation-schedule-queries.test.ts @@ -0,0 +1,269 @@ +import { randomUUID } from "node:crypto"; +import { and, asc, desc, eq, inArray, isNull, or, sql, type SQL } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { EXECUTION_RECONCILIATION_CAUSES } from "@paperclipai/shared"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { BACKLOG_CONTINUATION_MAX_AGE_MS, freshContinuationCheckDue, heldExecutionWhere, heldExecutionsAwaitingFreshContinuation, + nextFreshContinuationCheckAt, recentInterruptionSourceSql } from "./fresh-continuation-fallback.js"; +import { continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, recoveryRunSql } from "./continuation-cause.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; + +// The predicates as they were before they became index lookups (2026-11-14): +// `id::text = text` on the run key and cast issue columns, one OR scan per +// newer-run probe. Kept verbatim as the oracle the rewrite must match. +const previousCancelledAtDispatch = (alias: "newer" | "held") => sql.raw(`coalesce(${alias}.status = 'cancelled' and ${alias}.started_at is null + and ${alias}.error_code = 'execution_reconciliation_required', false)`); +const previousRunOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id::text = ${issueId}::text + or ${sql.raw(alias)}.issue_id::text = ${issueId}::text or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; + +function previousRecentInterruptionSourceSql(now: Date) { + const input = { companyId: sql`${issueRecoveryActions.companyId}`, issueId: sql`${issueRecoveryActions.sourceIssueId}`, + sourceRunId: sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')` }; + const since = sql`${new Date(now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz`; + const source = (condition: SQL) => sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${input.companyId} + and source.id::text = ${input.sourceRunId}::text and ${condition})`; + const refusedLater = sql`held.company_id = source.company_id and held.id <> source.id and ${previousRunOfTask("held", input.issueId)} + and (held.created_at, held.id) > (source.created_at, source.id) and ${previousCancelledAtDispatch("held")}`; + const personAsked = source(sql`(exists (select 1 from ${agentWakeupRequests} retry where retry.company_id = source.company_id + and retry.status = 'deferred_issue_execution' and retry.reason = 'task_retry' and retry.requested_by_actor_type = 'user' + and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at) + or exists (select 1 from ${heartbeatRuns} held where ${refusedLater} + and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`); + const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")} + and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false) + and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`; + const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource}) + and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); + return sql`(${recent} or ${personAsked})`; +} + +function previousHeldExecutionWhere(options: { includeActive?: boolean } = {}) { + return and( + inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), + sql`${issueRecoveryActions.evidence}->'executionReconciliation' is null`, + sql`${issueRecoveryActions.evidence}->'freshContinuation' is null`, + sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') is not null`, + or( + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, + and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('complete', 'exhausted')`), + options.includeActive ? and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + eq(issueRecoveryActions.kind, "active_run_watchdog")) : undefined, + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence')`, + ), + isNull(issueRecoveryActions.ownerUserId), + sql`${issues.status} not in ('done', 'cancelled')`, + sql`${issues.assigneeAgentId} is not null`, + sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} + and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') + and source.status in ('failed', 'timed_out', 'interrupted', 'cancelled') and source.finished_at is not null + and source.agent_id = ${issues.assigneeAgentId} + and not exists (select 1 from ${heartbeatRuns} newer where newer.company_id = source.company_id and newer.id <> source.id + and (newer.native_issue_id = ${issues.id} or newer.issue_id = ${issues.id} or newer.context_snapshot->>'issueId' = ${issues.id}::text) + and not ${previousCancelledAtDispatch("newer")} + and (newer.status in ('queued', 'running', 'scheduled_retry') or (newer.created_at, newer.id) > (source.created_at, source.id))))`, + )!; +} + +function random(seed: number) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +type PlanNode = { "Relation Name"?: string; "Actual Loops"?: number; "Actual Rows"?: number; + "Rows Removed by Filter"?: number; "Rows Removed by Index Recheck"?: number; Plans?: PlanNode[] }; + +function heartbeatRunRowsRead(node: PlanNode): number { + const own = node["Relation Name"] === "heartbeat_runs" ? (node["Actual Loops"] ?? 0) + * ((node["Actual Rows"] ?? 0) + (node["Rows Removed by Filter"] ?? 0) + (node["Rows Removed by Index Recheck"] ?? 0)) : 0; + return own + (node.Plans ?? []).reduce((total, child) => total + heartbeatRunRowsRead(child), 0); +} + +suite("fresh-continuation schedule queries", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + const now = new Date(Date.UTC(2026, 10, 14, 5, 0, 0)); + const tenants: Array<{ companyId: string; agentIds: string[] }> = []; + let actions = 0; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-fresh-continuation-schedule-"); + db = createDb(temporary.connectionString); + for (const prefix of ["FCA", "FCB"]) { + const companyId = randomUUID(), agentIds = [randomUUID(), randomUUID()]; + await db.insert(companies).values({ id: companyId, name: prefix, issuePrefix: prefix, requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values(agentIds.map((id, index) => ({ id, companyId, name: `Coder ${index}`, role: "engineer", + status: "idle", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} }))); + tenants.push({ companyId, agentIds }); + } + actions = await seedVariedFixture(20261114, 900); + // Run history that is not linked to any held task, so a scan per candidate shows. + await db.$client.unsafe(`insert into heartbeat_runs (company_id, agent_id, status, context_snapshot, created_at) + select $1::uuid, $2::uuid, 'succeeded', jsonb_build_object('issueId', gen_random_uuid()::text, 'prompt', repeat('x', 4000)), + $3::timestamptz - (n || ' minutes')::interval from generate_series(1, 3000) n`, + [tenants[0]!.companyId, tenants[0]!.agentIds[0]!, now.toISOString()]); + await db.execute(sql`analyze`); + }, 120_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + /** Held tasks in every shape the predicates distinguish: run keys, run links, statuses, ages, retries. */ + async function seedVariedFixture(seed: number, count: number) { + const next = random(seed); + const pick = (values: readonly T[]): T => values[Math.floor(next() * values.length)]!; + const chance = (probability: number) => next() < probability; + const at = (minutesAgo: number) => new Date(now.getTime() - minutesAgo * 60_000); + const issueRows: Array = []; + const runRows: Array = []; + const wakeRows: Array = []; + const actionRows: Array = []; + const link = (issueId: string, kind: string) => kind === "native" ? { scopeKind: "issue" as const, issueId, nativeIssueId: issueId } + : kind === "issue" ? { scopeKind: "issue" as const, issueId } : kind === "context" ? { scopeKind: "company" as const } + : { scopeKind: "company" as const }; + for (let index = 0; index < count; index += 1) { + const tenant = chance(0.9) ? tenants[0]! : tenants[1]!; + const other = tenant === tenants[0] ? tenants[1]! : tenants[0]!; + const issueId = randomUUID(), runId = randomUUID(); + const assignee = chance(0.9) ? pick(tenant.agentIds) : null; + issueRows.push({ id: issueId, companyId: tenant.companyId, title: `Held ${index}`, assigneeAgentId: assignee, + status: pick(["todo", "in_progress", "blocked", "blocked", "in_review", "done", "cancelled"]) }); + const finishedMinutesAgo = pick([30, 120, 600, 1_200, 1_430, 1_450, 2_000, 4_000]); + const sourceLink = pick(["native", "issue", "context"]); + const errorCode = pick(["process_lost", "native_session_interrupted", "server_shutdown_interrupted", "controller_lease_lost", + "adapter_failed", "reusable_sandbox_resume_failed", "budget_exceeded", null]); + const sourceCreated = chance(0.92); + if (sourceCreated) { + const finished = chance(0.92) ? at(finishedMinutesAgo) : null; + runRows.push({ id: runId, companyId: chance(0.95) ? tenant.companyId : other.companyId, + agentId: chance(0.85) && assignee ? assignee : pick(tenant.agentIds), ...link(issueId, sourceLink), + status: pick(["failed", "failed", "timed_out", "interrupted", "cancelled", "succeeded", "running"]), + errorCode, error: errorCode === "reusable_sandbox_resume_failed" ? pick(["resume failed (cause=transport_timeout)", "resume failed"]) : errorCode, + startedAt: at(finishedMinutesAgo + 10), finishedAt: finished, createdAt: at(finishedMinutesAgo + 11), updatedAt: at(finishedMinutesAgo), + retryOfRunId: null, scheduledRetryReason: chance(0.1) ? "process_lost" : null, + contextSnapshot: { issueId, wakeReason: pick(["issue_assigned", "task_retry", "process_lost_retry"]), + ...(chance(0.1) ? { explicitUserContinuation: { runId } } : {}), ...(chance(0.1) ? { previousRunId: randomUUID() } : {}) } }); + } + for (let extra = Math.floor(next() * 3); extra > 0; extra -= 1) { + const extraLink = pick(["native", "issue", "context", "unrelated"]); + const cancelledAtDispatch = chance(0.35); + const createdMinutesAgo = pick([finishedMinutesAgo + 11, finishedMinutesAgo + 30, finishedMinutesAgo - 5, 5]); + runRows.push({ companyId: chance(0.95) ? tenant.companyId : other.companyId, agentId: pick(tenant.agentIds), + ...link(issueId, extraLink), + status: cancelledAtDispatch ? "cancelled" : pick(["queued", "running", "scheduled_retry", "succeeded", "failed", "cancelled"]), + errorCode: cancelledAtDispatch ? "execution_reconciliation_required" : null, + startedAt: cancelledAtDispatch && chance(0.8) ? null : at(createdMinutesAgo - 1), + finishedAt: chance(0.8) ? at(pick([createdMinutesAgo - 2, 60, 2_000])) : null, createdAt: at(createdMinutesAgo), + retryOfRunId: sourceCreated && chance(0.3) ? runId : null, + contextSnapshot: extraLink === "unrelated" ? { issueId: randomUUID() } + : { issueId, wakeReason: pick(["task_retry", "issue_assigned", "process_lost_retry"]) } }); + } + if (chance(0.25)) { + wakeRows.push({ companyId: tenant.companyId, agentId: pick(tenant.agentIds), source: "on_demand", reason: pick(["task_retry", "issue_assigned"]), + status: pick(["deferred_issue_execution", "deferred_issue_execution", "coalesced"]), + requestedByActorType: pick(["user", "user", "system"]), runId: !sourceCreated || chance(0.8) ? null : runId, + requestedAt: at(pick([finishedMinutesAgo - 5, finishedMinutesAgo + 5])), payload: { issueId } }); + } + const recordedRunId = pick(["runId", "runId", "runId", "runId", "runId", "sourceRunId", "sourceRunId", "upper", "invalid", "number", "missing", "unknown", + "braced", "padded", "both"]); + const runKey = recordedRunId === "sourceRunId" ? { sourceRunId: runId } + : recordedRunId === "upper" ? { runId: runId.toUpperCase() } : recordedRunId === "invalid" ? { runId: "not-a-run" } + : recordedRunId === "number" ? { runId: 42 } : recordedRunId === "missing" ? {} + : recordedRunId === "unknown" ? { runId: randomUUID() } : recordedRunId === "braced" ? { runId: `{${runId}}` } + : recordedRunId === "padded" ? { runId: ` ${runId}` } : recordedRunId === "both" ? { runId: "not-a-run", sourceRunId: runId } + : { runId }; + actionRows.push({ companyId: tenant.companyId, sourceIssueId: issueId, kind: pick(["active_run_watchdog", "missing_disposition"]), + status: pick(["active", "escalated", "resolved", "resolved"]), ownerType: "board", ownerUserId: chance(0.05) ? "user-1" : null, + cause: pick(["legacy_execution_requires_reconciliation", "native_continuation_requires_reconciliation", "stranded_assigned_issue"]), + fingerprint: `held:${index}`, nextAction: "Fixture", + evidence: { ...runKey, + ...(chance(0.7) ? { automaticRecovery: { replay: pick(["blocked", "blocked", "blocked", "allowed"]) } } : {}), + ...(chance(0.2) ? { systemRecovery: { state: pick(["complete", "exhausted", "scheduled", "running", "failed"]) } } : {}), + ...(chance(0.05) ? { executionReconciliation: { runId } } : {}), + ...(chance(0.05) ? { freshContinuation: { sourceRunId: runId } } : {}), + ...(chance(0.7) ? { freshContinuationCheck: { kind: "backoff", nextCheckAt: now.getTime() + Math.round((next() - 0.5) * 7_200_000) } } : {}) } }); + } + await db.insert(issues).values(issueRows); + // One statement, so a retry may name a source run listed after it. + await db.insert(heartbeatRuns).values(runRows); + if (wakeRows.length) await db.insert(agentWakeupRequests).values(wakeRows); + for (let start = 0; start < actionRows.length; start += 500) + await db.insert(issueRecoveryActions).values(actionRows.slice(start, start + 500)); + return actionRows.length; + } + + const verdicts = (where: { held: SQL; heldWithActive: SQL; eligible: SQL }) => db.select({ id: issueRecoveryActions.id, + held: sql`coalesce(${where.held}, false)`, heldWithActive: sql`coalesce(${where.heldWithActive}, false)`, + eligible: sql`coalesce(${where.eligible}, false)` }) + .from(issueRecoveryActions).innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), + eq(issues.companyId, issueRecoveryActions.companyId))).orderBy(asc(issueRecoveryActions.id)); + + it("selects exactly the records the previous predicates selected, for every run key, link and status", async () => { + const before = await verdicts({ held: previousHeldExecutionWhere(), heldWithActive: previousHeldExecutionWhere({ includeActive: true }), + eligible: previousRecentInterruptionSourceSql(now) }); + const after = await verdicts({ held: heldExecutionWhere(), heldWithActive: heldExecutionWhere({ includeActive: true }), + eligible: recentInterruptionSourceSql(now) }); + expect(after).toEqual(before); + expect(before).toHaveLength(actions); + // The fixture reaches both outcomes of each predicate, so equality is meaningful. + for (const key of ["held", "heldWithActive", "eligible"] as const) { + expect(before.filter(row => row[key]).length).toBeGreaterThan(5); + expect(before.filter(row => !row[key]).length).toBeGreaterThan(5); + } + expect(before.filter(row => row.heldWithActive && row.eligible).length).toBeGreaterThan(5); + }, 60_000); + + it("returns the deadline and the backlog listing the previous queries returned", async () => { + const liveHold = and(inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"))!; + const [previousDeadline] = await db.select({ at: sql`min((${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}')::bigint)` }) + .from(issueRecoveryActions).innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), + eq(issues.companyId, issueRecoveryActions.companyId))) + .where(and(previousHeldExecutionWhere({ includeActive: true }), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}' is not null`, + or(liveHold, previousRecentInterruptionSourceSql(now)))); + expect(previousDeadline?.at).not.toBeNull(); + expect(await nextFreshContinuationCheckAt(db, now)).toBe(Number(previousDeadline!.at)); + + const previousListing = await db.select({ id: issueRecoveryActions.id, companyId: issueRecoveryActions.companyId, + runId: sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')` }) + .from(issueRecoveryActions).innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), + eq(issues.companyId, issueRecoveryActions.companyId))) + .where(and(previousHeldExecutionWhere(), freshContinuationCheckDue(now), previousRecentInterruptionSourceSql(now))) + .orderBy(desc(sql`(select source.finished_at from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} + and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId'))`), + asc(issueRecoveryActions.id)).limit(1_000); + expect(previousListing.length).toBeGreaterThan(0); + expect(await heldExecutionsAwaitingFreshContinuation(db, now, 1_000)).toEqual(previousListing); + }, 60_000); + + it("finds the deadline by run-key and task-link lookups, not a scan of run history per record", async () => { + let captured: { sql: string; params: unknown[] } | undefined; + const capture = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args); + selection.execute = async () => { captured = selection.toSQL(); return [{ at: null }]; }; + return selection; + }) as typeof builder.from; + return builder; + }; + } }); + await nextFreshContinuationCheckAt(capture as typeof db, now); + const [row] = await db.$client.unsafe(`explain (analyze, format json) ${captured!.sql}`, captured!.params as never[]); + const plan = (row!["QUERY PLAN"] as Array<{ Plan: PlanNode }>)[0]!.Plan; + const [{ runs }] = await db.$client.unsafe(`select count(*)::int as runs from heartbeat_runs`) as unknown as Array<{ runs: number }>; + // The previous query read the whole run history once per held record + // (hundreds of times here); the lookups read a few rows per record. + expect(heartbeatRunRowsRead(plan)).toBeLessThan(2 * runs); + }, 60_000); +});