diff --git a/packages/db/src/migrations/0488_issue_recovery_full_passes.sql b/packages/db/src/migrations/0488_issue_recovery_full_passes.sql new file mode 100644 index 00000000000..89d3d9d86dc --- /dev/null +++ b/packages/db/src/migrations/0488_issue_recovery_full_passes.sql @@ -0,0 +1,18 @@ +CREATE TABLE "issue_recovery_deferrals" ( + "queue" text NOT NULL, + "issue_id" uuid NOT NULL, + "company_id" uuid NOT NULL, + "due_at" timestamp with time zone NOT NULL, + CONSTRAINT "issue_recovery_deferrals_pk" PRIMARY KEY("queue","issue_id") +); +--> statement-breakpoint +CREATE TABLE "issue_recovery_full_passes" ( + "company_id" uuid NOT NULL, + "queue" text NOT NULL, + "started_at" timestamp with time zone NOT NULL, + CONSTRAINT "issue_recovery_full_passes_pk" PRIMARY KEY("company_id","queue") +); +--> statement-breakpoint +ALTER TABLE "issue_recovery_deferrals" ADD CONSTRAINT "issue_recovery_deferrals_issue_id_issues_id_fk" FOREIGN KEY ("issue_id") REFERENCES "public"."issues"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "issue_recovery_deferrals" ADD CONSTRAINT "issue_recovery_deferrals_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "issue_recovery_full_passes" ADD CONSTRAINT "issue_recovery_full_passes_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0483_snapshot.json b/packages/db/src/migrations/meta/0488_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0483_snapshot.json rename to packages/db/src/migrations/meta/0488_snapshot.json index 3eb96902c08..bc96eb0f6a1 100644 --- a/packages/db/src/migrations/meta/0483_snapshot.json +++ b/packages/db/src/migrations/meta/0488_snapshot.json @@ -1,6 +1,6 @@ { - "id": "ceaa38c4-681b-430e-9578-0854650d85b3", - "prevId": "8b353a38-62df-4834-be55-7e9ebefad5c5", + "id": "8f935a52-5f1e-4afe-8d13-c1770ca33968", + "prevId": "73c8fa97-70fc-49c0-804e-e49cfdffd156", "version": "7", "dialect": "postgresql", "tables": { @@ -6582,6 +6582,111 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.automatic_run_brakes": { + "name": "automatic_run_brakes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'closed'" + }, + "reason": { + "name": "reason", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state_changed_at": { + "name": "state_changed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_probe_at": { + "name": "next_probe_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "probe_run_id": { + "name": "probe_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "probe_successes": { + "name": "probe_successes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "release_window_started_at": { + "name": "release_window_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_window_starts": { + "name": "release_window_starts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_creation_at": { + "name": "next_creation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automatic_run_brakes_company_id_companies_id_fk": { + "name": "automatic_run_brakes_company_id_companies_id_fk", + "tableFrom": "automatic_run_brakes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.board_api_keys": { "name": "board_api_keys", "schema": "", @@ -36029,6 +36134,78 @@ "method": "btree", "with": {} }, + "heartbeat_runs_company_finished_idx": { + "name": "heartbeat_runs_company_finished_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finished_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"finished_at\" is not null", + "concurrently": true, + "method": "btree", + "with": {} + }, + "heartbeat_runs_settled_process_identity_idx": { + "name": "heartbeat_runs_settled_process_identity_idx", + "columns": [ + { + "expression": "process_host_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' and \"heartbeat_runs\".\"status\" in ('failed', 'cancelled', 'timed_out', 'interrupted')\n and \"heartbeat_runs\".\"finished_at\" is not null\n and (\"heartbeat_runs\".\"process_pid\" is not null or \"heartbeat_runs\".\"process_group_id\" is not null or \"heartbeat_runs\".\"process_started_at\" is not null)", + "concurrently": true, + "method": "btree", + "with": {} + }, + "heartbeat_runs_pending_api_stop_idx": { + "name": "heartbeat_runs_pending_api_stop_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"status\" = 'cancelled' and \"heartbeat_runs\".\"driver_kind\" = 'cursor_cloud_api'\n and (\"heartbeat_runs\".\"result_json\"->'executionCancellation'->>'state') = 'requested'", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_company_created_at_desc_idx": { "name": "heartbeat_runs_company_created_at_desc_idx", "columns": [ @@ -40722,6 +40899,131 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.issue_recovery_deferrals": { + "name": "issue_recovery_deferrals", + "schema": "", + "columns": { + "queue": { + "name": "queue", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "issue_recovery_deferrals_issue_id_issues_id_fk": { + "name": "issue_recovery_deferrals_issue_id_issues_id_fk", + "tableFrom": "issue_recovery_deferrals", + "tableTo": "issues", + "columnsFrom": [ + "issue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "issue_recovery_deferrals_company_id_companies_id_fk": { + "name": "issue_recovery_deferrals_company_id_companies_id_fk", + "tableFrom": "issue_recovery_deferrals", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "issue_recovery_deferrals_pk": { + "name": "issue_recovery_deferrals_pk", + "columns": [ + "queue", + "issue_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_recovery_full_passes": { + "name": "issue_recovery_full_passes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "queue": { + "name": "queue", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "issue_recovery_full_passes_company_id_companies_id_fk": { + "name": "issue_recovery_full_passes_company_id_companies_id_fk", + "tableFrom": "issue_recovery_full_passes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "issue_recovery_full_passes_pk": { + "name": "issue_recovery_full_passes_pk", + "columns": [ + "company_id", + "queue" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.issue_reference_mentions": { "name": "issue_reference_mentions", "schema": "", @@ -59607,6 +59909,12 @@ "primaryKey": false, "notNull": false }, + "progress": { + "name": "progress", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 9790c0d8fc3..464a6addffb 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3403,6 +3403,13 @@ "when": 1794639469687, "tag": "0487_heartbeat_runs_settled_process_indexes", "breakpoints": true + }, + { + "idx": 488, + "version": "7", + "when": 1794820062730, + "tag": "0488_issue_recovery_full_passes", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index fd8c162f7ee..68878bc412f 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -58,6 +58,7 @@ export { issueAccessGrants } from "./issue_access_grants.js"; export { issueWatchdogs } from "./issue_watchdogs.js"; export { issuePlanDecompositions } from "./issue_plan_decompositions.js"; export { issueRecoveryActions } from "./issue_recovery_actions.js"; +export { issueRecoveryDeferrals, issueRecoveryFullPasses } from "./issue_recovery_full_passes.js"; export { issueReferenceMentions } from "./issue_reference_mentions.js"; export { externalObjects } from "./external_objects.js"; export { externalObjectMentions } from "./external_object_mentions.js"; diff --git a/packages/db/src/schema/issue_recovery_full_passes.ts b/packages/db/src/schema/issue_recovery_full_passes.ts new file mode 100644 index 00000000000..700c54f87fa --- /dev/null +++ b/packages/db/src/schema/issue_recovery_full_passes.ts @@ -0,0 +1,35 @@ +import { pgTable, primaryKey, text, timestamp, uuid } from "drizzle-orm/pg-core"; +import { companies } from "./companies.js"; +import { issues } from "./issues.js"; + +/** + * When a task recovery queue last finished checking every task of a company. + * Any server writes it and every server reads it, so a replaced or overlapping + * server keeps the hourly full pass on schedule instead of starting a new one + * at boot. A row missing or older than an hour means a full pass is due. + */ +export const issueRecoveryFullPasses = pgTable("issue_recovery_full_passes", { + companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), + queue: text("queue").notNull(), + /** When the pass began loading; changes committed after this are seen by the next pass. */ + startedAt: timestamp("started_at", { withTimezone: true }).notNull(), +}, table => ({ + pk: primaryKey({ name: "issue_recovery_full_passes_pk", columns: [table.companyId, table.queue] }), +})); + +/** + * A task whose recovery check deferred itself to a later deadline (a grace + * period, a back-off, a live owner's lease). The row exists while a deferral + * is pending, so the successor of a replaced or crashed server re-checks these + * tasks at boot and keeps their deadlines without re-checking every task. + * The boot check computes the deadline again from current state. + */ +export const issueRecoveryDeferrals = pgTable("issue_recovery_deferrals", { + queue: text("queue").notNull(), + issueId: uuid("issue_id").notNull().references(() => issues.id, { onDelete: "cascade" }), + companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), + /** The deadline the check deferred to, as last saved; deletes name it, so a newer one is never removed. */ + dueAt: timestamp("due_at", { withTimezone: true }).notNull(), +}, table => ({ + pk: primaryKey({ name: "issue_recovery_deferrals_pk", columns: [table.queue, table.issueId] }), +})); diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts index b46e2c74b08..a02d833c115 100644 --- a/packages/db/src/work-signals.test.ts +++ b/packages/db/src/work-signals.test.ts @@ -4,7 +4,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createDb, withDedicatedDbConnection } from "./client.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; import { workSignalDebug } from "./work-signal-debug.js"; -import { DATABASE_WORK_CHANNEL, DATABASE_WORK_NOTIFICATION_MAX_BYTES, DATABASE_WORK_TOPIC_NOTIFICATIONS_MAX_BYTES, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, publishDatabaseWorkSettled, receiveDatabaseWorkNotification, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; +import { DATABASE_WORK_CHANNEL, DATABASE_WORK_NOTIFICATION_MAX_BYTES, DATABASE_WORK_TOPIC_NOTIFICATIONS_MAX_BYTES, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, publishDatabaseWorkSettled, receiveDatabaseWorkNotification, reconcileDatabaseWork, reconcileDatabaseWorkNotifications, setDatabaseWorkReconnectHint, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; const support = await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("database work signals", () => { @@ -31,6 +31,18 @@ const support = await getEmbeddedPostgresTestSupport(); } finally { signal.mockRestore(); event.mockRestore(); } }); + it("sends a topic's declared reconnect hint, and an unscoped reload to other topics, when notifications reconnect", () => { + const received: Array<[string, string[] | undefined]> = []; + const stopHinted = subscribeDatabaseWork(db, "reconnect-hinted", (event, hints) => received.push(["hinted", hints && [...hints]])); + const stopPlain = subscribeDatabaseWork(db, "reconnect-plain", (event, hints) => received.push(["plain", hints && [...hints]])); + try { + setDatabaseWorkReconnectHint(db, "reconnect-hinted", "{\"reload\":true}"); + reconcileDatabaseWorkNotifications(db); + expect(received).toEqual(expect.arrayContaining([["hinted", ["{\"reload\":true}"]], ["plain", undefined]])); + expect(received).toHaveLength(2); + } finally { stopHinted(); stopPlain(); } + }); + it("defers nested savepoint notifications until the outer commit and preserves return values", async () => { const events: string[] = []; const unsubscribe = subscribeDatabaseWork(db, "nested", event => events.push(event)); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts index 69b3d2159a0..e42502e2dcc 100644 --- a/packages/db/src/work-signals.ts +++ b/packages/db/src/work-signals.ts @@ -7,7 +7,8 @@ import { workSignalDebug } from "./work-signal-debug.js"; // triggers, tables, timers, or query replay. Only registered work fetches an XID. type Listener = (event: "intent" | "settled", hints?: ReadonlySet) => void; type Transaction = { topics: Set; hints: Map>; xid?: Promise; uncertain: boolean }; -type Scope = { key: object; listeners: Map>; transactions: Set; root: Connection; notificationOrigin: string; notify: boolean }; +type Scope = { key: object; listeners: Map>; transactions: Set; root: Connection; notificationOrigin: string; notify: boolean; + reconnectHints?: Map }; type PendingNotification = { hints: Set; unscoped: boolean }; type Context = { scope: Scope; transaction?: Transaction; pending?: Map; settled?: Array<[string, string | undefined]>; ended?: boolean }; type Connection = { transaction: (...args: any[]) => Promise; execute: (...args: any[]) => Promise }; @@ -215,10 +216,23 @@ export function reconcileDatabaseWorkNotifications(owner: object): void { const scope = contexts.get(owner)?.scope; if (scope) for (const topic of scope.listeners.keys()) { if (workSignalDebug.enabled) workSignalDebug.event(topic, "notification_reconnect"); - publish(scope, topic); + const hint = scope.reconnectHints?.get(topic); + publish(scope, topic, "settled", hint === undefined ? undefined : new Set([hint])); } } +/** + * The hint a topic receives when the notification connection opens or + * reconnects, instead of an unscoped reload. Only a hint lost from another + * server is missing, so a queue that can find those writes from its own rows + * declares how to look. Process-local; never sent to another server. + */ +export function setDatabaseWorkReconnectHint(owner: object, topic: string, hint: string): void { + const scope = contexts.get(owner)?.scope; + if (!scope) return; + (scope.reconnectHints ??= new Map()).set(topic, hint); +} + export function databaseWorkPending(owner: object, topic: string): boolean { return [...(contexts.get(owner)?.scope.transactions ?? [])].some(state => state.topics.has(topic)); } diff --git a/server/src/__tests__/issue-recovery-company-reload-census.test.ts b/server/src/__tests__/issue-recovery-company-reload-census.test.ts new file mode 100644 index 00000000000..2137bd0fab9 --- /dev/null +++ b/server/src/__tests__/issue-recovery-company-reload-census.test.ts @@ -0,0 +1,304 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { eq } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { agents, costEvents, heartbeatRuns, issues, projects } from "@paperclipai/db"; +import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; +import { reconcileDatabaseWorkNotifications } from "@paperclipai/db/work-signals"; +import { ISSUE_RECOVERY_BACKSTOP_MS, notifyIssueGraphRecovery } from "../services/issue-recovery-notifications.js"; +import { createAgentLifecycle, createAgentLifecycleEffects } from "../services/agent-lifecycle.js"; +import { budgetService } from "../services/budgets.js"; +import { documentService } from "../services/documents.js"; +import { issueService } from "../services/issues.js"; +import { createHeartbeatAgentStatus } from "../services/heartbeat/agent-status.js"; +import { notifyExecutionRunWork } from "../services/execution-control-notifications.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; + +vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1,assigned-work-invariant:1"; }); + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const TEST_TIMEOUT_MS = 120_000; +const STEP_TIMEOUT_MS = 20_000; +const FULL_PASS_MS = 15 * MINUTE; +const TOPICS = new Set(["stranded-assignments", "assigned-work-invariant"]); +const realSetTimeout = globalThis.setTimeout; +const realClearTimeout = globalThis.clearTimeout; +type Recorded = { at: number; topic: string; name: string; fields: Record }; + +function bounded(work: Promise, waitingFor: () => string): Promise { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = realSetTimeout(() => reject(new Error(`Timed out after ${STEP_TIMEOUT_MS} ms waiting for ${waitingFor()}`)), STEP_TIMEOUT_MS); + }); + return Promise.race([work, expired]).finally(() => realClearTimeout(timer)); +} + +/** Company hints, notification reloads and server replacements during a long + * run: a staging-shaped company (settled reviews on an agent with a live + * native run, and on a second agent) driven through each writer that used to + * make the task recovery queues re-check every task of the company. No + * fixture agent can start a run: on-demand wakes are off, and the agent that + * is paused and resumed has no live run. */ +describeIssueRecovery("company hints and notification reloads (census)", () => { + const state = issueRecoveryFixture({ meter: true }); + + async function advance(ms: number, onTick?: () => Promise) { + const until = Date.now() + ms; + let nextTick = Date.now(); + while (Date.now() < until) { + const at = new Date(Date.now()).toISOString(); + if (onTick && Date.now() >= nextTick) { + await bounded(onTick(), () => `the run event at simulated ${at} to commit`); + nextTick = Date.now() + 10_000; + } + await bounded(vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())), () => `simulated time to advance past ${at}`); + await bounded(Promise.allSettled([...state.passes]), () => `${state.passes.size} delivery passes started by simulated ${at} to settle`); + await immediate(); + } + } + + function record() { + const events: Recorded[] = []; + vi.spyOn(workSignalDebug, "event").mockImplementation((topic, name, fields = {}) => { + if (TOPICS.has(topic)) events.push({ at: Date.now(), topic, name, fields }); + }); + return events; + } + + function byWake(events: Recorded[]) { + const counts: Record> = {}; + for (const event of events) if (event.name === "check") { + const topic = counts[event.topic] ??= {}; + topic[String(event.fields.wake)] = (topic[String(event.fields.wake)] ?? 0) + 1; + } + return counts; + } + + async function seed() { + const f = await state.fixture(); + const config = { adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false } }, role: "engineer" }; + const old = new Date(Date.now() - 2 * HOUR); + const idle = { ...config, companyId: f.companyId, status: "idle", createdAt: old, updatedAt: old, lastHeartbeatAt: old }; + const [big, second, spare] = await state.db.insert(agents).values([ + { ...idle, name: "Big worker" }, { ...idle, name: "Second worker" }, { ...idle, name: "Spare worker" }, + ]).returning({ id: agents.id }); + const finishedAt = new Date(Date.now() - HOUR); + const reviews = async (agentId: string, count: number) => { + const rows = await state.db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ companyId: f.companyId, + title: `Review ${index}`, status: "in_review", assigneeAgentId: agentId, createdAt: old, updatedAt: old }))) + .returning({ id: issues.id }); + await state.db.insert(heartbeatRuns).values(rows.map(row => ({ companyId: f.companyId, agentId, status: "succeeded", + runtimeMode: "native", startedAt: finishedAt, finishedAt, updatedAt: finishedAt, contextSnapshot: { issueId: row.id } }))); + return rows.map(row => row.id); + }; + const bigTasks = await reviews(big!.id, 140); + const secondTasks = await reviews(second!.id, 20); + const spareTasks = await reviews(spare!.id, 5); + const [bigLive, spec] = await state.db.insert(issues).values([ + { companyId: f.companyId, title: "Big live task", status: "in_progress", assigneeAgentId: big!.id }, + { companyId: f.companyId, title: "Spec", status: "backlog", createdAt: old, updatedAt: old }, + ]).returning({ id: issues.id }); + await documentService(state.db).upsertIssueDocument({ issueId: spec!.id, key: "spec", format: "markdown", body: "Draft" }); + await state.db.update(issues).set({ updatedAt: old }).where(eq(issues.id, spec!.id)); + const [project] = await state.db.insert(projects).values({ companyId: f.companyId, name: "Metered project", status: "in_progress" }) + .returning({ id: projects.id }); + await state.db.insert(costEvents).values({ companyId: f.companyId, agentId: second!.id, projectId: project!.id, provider: "test", + model: "test", costCents: 5, occurredAt: new Date() }); + return { ...f, bigId: big!.id, secondId: second!.id, spareId: spare!.id, bigTasks, secondTasks, spareTasks, + bigLiveTask: bigLive!.id, specIssue: spec!.id, projectId: project!.id }; + } + + async function boot(options: { fullBackstopMs?: number } = {}) { + const f = await bounded(seed(), () => "the fixture to be seeded"); + const events = record(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const bootedAt = Date.now(); + await bounded(state.start(options), () => "the recovery workers to start"); + const bigRun = await bounded((async () => { + const [run] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.bigId, status: "running", + runtimeMode: "native", startedAt: new Date(), contextSnapshot: { issueId: f.bigLiveTask }, nativeIssueId: f.bigLiveTask }) + .returning({ id: heartbeatRuns.id }); + await state.db.update(issues).set({ executionRunId: run!.id }).where(eq(issues.id, f.bigLiveTask)); + await state.db.update(agents).set({ status: "running", updatedAt: new Date() }).where(eq(agents.id, f.bigId)); + return run!.id; + })(), () => "the big agent's live run to be recorded"); + const tick = () => state.db.transaction(async tx => { + await tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, bigRun)); + await notifyIssueGraphRecovery(tx, { companyId: f.companyId, runId: bigRun }); + }); + await advance(ISSUE_RECOVERY_BACKSTOP_MS + MINUTE, tick); + events.length = 0; + /** A replacement server boots with no memory, then its notification connection opens. */ + const replaceServer = async () => { + await bounded(state.restart(options), () => "the replacement server's recovery workers to start"); + reconcileDatabaseWorkNotifications(state.db); + }; + return { ...f, events, bigRun, tick, bootedAt, replaceServer }; + } + + /** The lifecycle's resume, through its status-change effect (the lifecycle + * worker that settles a resume is not part of this fixture). */ + const resume = (companyId: string, agentId: string) => state.db.transaction(async tx => { + await createAgentLifecycleEffects().beforeStatusChange(tx as never, companyId, agentId, "paused", "idle"); + await tx.update(agents).set({ status: "idle", pauseReason: null, pausedAt: null, updatedAt: new Date() }).where(eq(agents.id, agentId)); + }); + + const writers = (f: Awaited>): Array<[string, () => Promise]> => [ + ["server replacement and notification connection open", f.replaceServer], + ["issue creation", () => issueService(state.db).create(f.companyId, { title: "New request", status: "backlog" } as never)], + ["document lock", () => documentService(state.db).lockIssueDocument({ issueId: f.specIssue, key: "spec", lockedByUserId: "recovery-user" })], + ["project budget hard stop", () => budgetService(state.db).upsertPolicy(f.companyId, + { scopeType: "project", scopeId: f.projectId, amount: 1, hardStopEnabled: true }, "recovery-user")], + ["spare agent pause", () => createAgentLifecycle(state.db).pauseAgent(f.spareId)], + ["spare agent resume", () => resume(f.companyId, f.spareId)], + ["second server replacement and notification connection open", f.replaceServer], + ]; + + const checksOf = (events: Recorded[], issueId: string, after: number) => events.filter(event => event.topic === "stranded-assignments" + && event.name === "check" && event.fields.issueId === issueId && event.at >= after); + + it("does not re-check settled tasks for company hints, reloads or server replacements", async () => { + const f = await boot(); + const before = state.meter.read(); + const steps = writers(f); + const minutesPerStep = 4; + const settled = new Set([...f.bigTasks, ...f.secondTasks]); + const settledByStep: Record = {}; + for (const [label, write] of steps) { + const from = f.events.length; + await bounded(write(), () => `the ${label} write to commit`); + await advance(minutesPerStep * MINUTE, f.tick); + settledByStep[label] = f.events.slice(from).filter(event => event.name === "check" && settled.has(String(event.fields.issueId))).length; + } + const after = state.meter.read(); + const checks = f.events.filter(event => event.name === "check"); + const result = { minutes: steps.length * minutesPerStep, + settledChecks: checks.filter(event => settled.has(String(event.fields.issueId))).length, + pausedAgentChecks: checks.filter(event => f.spareTasks.includes(String(event.fields.issueId))).length, + settledByStep, byWake: byWake(f.events), statements: after.statements - before.statements }; + console.info(`company-reload census: ${JSON.stringify(result)}`); + // Only the paused and resumed agent's own tasks are re-checked. + expect(result.settledChecks).toBe(0); + for (const topic of TOPICS) { + expect(result.byWake[topic]?.reload ?? 0).toBe(0); + expect(result.byWake[topic]?.every_task ?? 0).toBe(0); + } + }, TEST_TIMEOUT_MS); + + /** A run of the second agent (20 settled reviews plus its live task), + * through the writes and hints of a run start and a run finish: the run row, + * the agent row as execution start writes it, the execution-run hint, and + * finalizeAgentStatus. No process runs. */ + async function runCycle(f: Awaited>, issueId: string, outcome: "succeeded" | "failed") { + const run = await bounded(state.db.transaction(async tx => { + const [row] = await tx.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.secondId, status: "running", + runtimeMode: "native", startedAt: new Date(), contextSnapshot: { issueId }, nativeIssueId: issueId }).returning({ id: heartbeatRuns.id }); + await notifyExecutionRunWork(tx, { companyId: f.companyId, agentId: f.secondId, runId: row!.id }); + await tx.update(agents).set({ status: "running", updatedAt: new Date() }).where(eq(agents.id, f.secondId)); + return row!.id; + }), () => "the run start to commit"); + await advance(2 * MINUTE, f.tick); + await bounded(state.db.transaction(async tx => { + await notifyExecutionRunWork(tx, { companyId: f.companyId, runId: run, issueId }); + await tx.update(heartbeatRuns).set({ status: outcome, finishedAt: new Date(), updatedAt: new Date() }).where(eq(heartbeatRuns.id, run)); + }), () => "the run finish to commit"); + await bounded(createHeartbeatAgentStatus(state.db).finalizeAgentStatus(f.secondId, outcome), () => "the agent status to settle"); + await advance(2 * MINUTE, f.tick); + return run; + } + + it("does not re-check an agent's settled backlog when its runs start and finish", async () => { + const f = await boot(); + const before = state.meter.read(); + const [work] = await bounded(state.db.insert(issues).values({ companyId: f.companyId, title: "Second agent work", status: "in_progress", + assigneeAgentId: f.secondId }).returning({ id: issues.id }), () => "the run's task to be created"); + for (let cycle = 0; cycle < 2; cycle++) await runCycle(f, work!.id, "succeeded"); + const settled = new Set([...f.bigTasks, ...f.secondTasks]); + const checks = f.events.filter(event => event.name === "check"); + const result = { cycles: 2, settledChecks: checks.filter(event => settled.has(String(event.fields.issueId))).length, + runTaskChecks: checks.filter(event => event.fields.issueId === work!.id).length, byWake: byWake(f.events), + statements: state.meter.read().statements - before.statements }; + console.info(`run-cycle census: ${JSON.stringify(result)}`); + expect(result.settledChecks).toBe(0); + expect(result.runTaskChecks).toBeGreaterThan(0); + }, TEST_TIMEOUT_MS); + + it("still checks a task promptly when its run fails", async () => { + const f = await boot(); + const [work] = await bounded(state.db.insert(issues).values({ companyId: f.companyId, title: "Second agent work", status: "in_progress", + assigneeAgentId: f.secondId }).returning({ id: issues.id }), () => "the run's task to be created"); + const startedAt = Date.now(); + await runCycle(f, work!.id, "failed"); + const finishedAt = startedAt + 2 * MINUTE; + const [checked] = checksOf(f.events, work!.id, finishedAt); + expect(checked, "the failed run's task was not checked after the run ended").toBeDefined(); + expect(checked!.at - finishedAt).toBeLessThanOrEqual(MINUTE); + }, TEST_TIMEOUT_MS); + + it("still checks a task stranded right before each writer within its bound", async () => { + const f = await boot(); + const steps = writers(f); + // The lock and the budget stop re-check only their own task or changed + // rows; a task stranded by a write that sends no hint waits at most for + // the changed-rows backstop, as before. + const bounds: Record = { "document lock": ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE }; + const targets = [f.bigTasks[1], f.bigTasks[2], f.bigTasks[3], f.bigTasks[4], f.spareTasks[1], f.spareTasks[2], f.bigTasks[5]]; + const late: string[] = []; + for (const [index, [label, write]] of steps.entries()) { + const target = targets[index]!; + const bound = bounds[label] ?? MINUTE; + const changedAt = Date.now(); + await bounded(state.db.update(issues).set({ status: "todo", updatedAt: new Date() }).where(eq(issues.id, target)).then(() => undefined), + () => `the task write before ${label} to commit`); + await bounded(write(), () => `the ${label} write to commit`); + await advance(bound, f.tick); + const [checked] = checksOf(f.events, target, changedAt); + if (!checked || checked.at - changedAt > bound) late.push(`${label}: ${checked ? checked.at - changedAt : "never"} ms`); + } + expect(late).toEqual([]); + }, TEST_TIMEOUT_MS); + + it("checks every task a replaced server loaded but never reached in its first full pass", async () => { + const f = await bounded(seed(), () => "the fixture to be seeded"); + const events = record(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + // The first passes load every task (no saved full pass yet, two pages) but + // check one batch of 25 each, so the load completes long before the checks. + await bounded(state.start({}), () => "the first server's recovery workers to start"); + await bounded(vi.advanceTimersByTimeAsync(1_500), () => "the second page of the first full pass to load"); + await bounded(Promise.allSettled([...state.passes]), () => "the first server's passes to settle"); + const reached = new Set(events.filter(event => event.topic === "stranded-assignments" && event.name === "check") + .map(event => String(event.fields.issueId))); + const unreached = [...f.bigTasks, ...f.secondTasks].filter(id => !reached.has(id)); + expect(unreached.length, "the first pass should stop before checking every task").toBeGreaterThan(0); + const replacedAt = Date.now(); + await bounded(state.restart({}), () => "the replacement server's recovery workers to start"); + reconcileDatabaseWorkNotifications(state.db); + await advance(2 * MINUTE); + const checkedAfter = new Set(events.filter(event => event.topic === "stranded-assignments" && event.name === "check" + && event.at >= replacedAt).map(event => String(event.fields.issueId))); + expect(unreached.filter(id => !checkedAfter.has(id))).toEqual([]); + }, TEST_TIMEOUT_MS); + + it("keeps the full pass on schedule across server replacements", async () => { + const f = await boot({ fullBackstopMs: FULL_PASS_MS }); + // A change no changed-rows clause sees: only a full pass can find it. + const target = f.bigTasks[30]!; + await bounded(state.db.update(issues).set({ status: "todo" }).where(eq(issues.id, target)).then(() => undefined), + () => "the task write that keeps its old update time to commit"); + const changedAt = Date.now(); + await advance(MINUTE, f.tick); + await f.replaceServer(); + await advance(3 * MINUTE, f.tick); + await f.replaceServer(); + const replacedAt = Date.now(); + const deadline = f.bootedAt + FULL_PASS_MS + ISSUE_RECOVERY_BACKSTOP_MS + MINUTE; + await advance(deadline - Date.now(), f.tick); + const checks = checksOf(f.events, target, changedAt); + // Neither replacement checked it (no full pass at boot); the next full pass did. + expect(checks.filter(event => event.at < replacedAt + MINUTE)).toEqual([]); + expect(checks[0], `task ${target} was not checked by the next full pass`).toBeDefined(); + expect(checks[0]!.at).toBeLessThanOrEqual(deadline); + }, TEST_TIMEOUT_MS); +}); diff --git a/server/src/__tests__/issue-recovery-saved-deferrals.test.ts b/server/src/__tests__/issue-recovery-saved-deferrals.test.ts new file mode 100644 index 00000000000..15453e20fd2 --- /dev/null +++ b/server/src/__tests__/issue-recovery-saved-deferrals.test.ts @@ -0,0 +1,203 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { and, eq } from "drizzle-orm"; +import { afterEach, expect, it, vi } from "vitest"; +import { agents, issueRecoveryDeferrals, issues } from "@paperclipai/db"; +import { reconcileDatabaseWorkNotifications } from "@paperclipai/db/work-signals"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js"; +import { registerIssueRecoveryWork } from "../services/recovery/work.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const TEST_TIMEOUT_MS = 120_000; +const STEP_TIMEOUT_MS = 20_000; +const realSetTimeout = globalThis.setTimeout; +const realClearTimeout = globalThis.clearTimeout; + +function bounded(work: Promise, waitingFor: () => string): Promise { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = realSetTimeout(() => reject(new Error(`Timed out after ${STEP_TIMEOUT_MS} ms waiting for ${waitingFor()}`)), STEP_TIMEOUT_MS); + }); + return Promise.race([work, expired]).finally(() => realClearTimeout(timer)); +} + +/** The task recovery queues against the real test database, with the task + * checks replaced by a recorder, so a deferral and a server replacement can be + * placed exactly. No agent run can start: nothing but the recorder runs. */ +describeIssueRecovery("saved task recovery passes and deferrals", () => { + const state = issueRecoveryFixture(); + const passes = new Set>(); + // Every server a test starts stops after it, even when an assertion failed, + // so no worker stays subscribed while the next test reuses the database. + const running = new Set>(); + afterEach(async () => { + for (const coordinator of [...running]) await bounded(coordinator.stop(), () => "a test server to stop"); + running.clear(); passes.clear(); + }); + + async function seed(count: number) { + const f = await state.fixture(); + const old = new Date(Date.now() - 2 * HOUR); + const [agent] = await state.db.insert(agents).values({ companyId: f.companyId, name: "Reviewer", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false } }, createdAt: old, updatedAt: old, lastHeartbeatAt: old, + }).returning({ id: agents.id }); + const rows = await state.db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ companyId: f.companyId, + title: `Review ${index}`, status: "in_review", assigneeAgentId: agent!.id, createdAt: old, updatedAt: old }))) + .returning({ id: issues.id }); + return { ...f, tasks: rows.map(row => row.id) }; + } + + /** One server: a coordinator and the recovery queues, with recorded checks. */ + async function server(behavior: { deferUntil: Map; suppressed: () => boolean; checks: Array<{ id: string; at: number }> }) { + const coordinator = createDeliveryWorkCoordinator({ owner: state.db, canRun: () => true, canReconcile: () => true, + onError: error => { throw error; } }); + running.add(coordinator); + const register = coordinator.register.bind(coordinator); + coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { + const work = task.run(signal); passes.add(work); + void work.then(() => passes.delete(work), () => passes.delete(work)); + return work; + } }); + const stranded = async (scope: { issueId: string; onDeferredDeadline(at: number): void }) => { + behavior.checks.push({ id: scope.issueId, at: Date.now() }); + const until = behavior.deferUntil.get(scope.issueId); + if (until !== undefined && until > Date.now()) scope.onDeferredDeadline(until); + }; + const work = registerIssueRecoveryWork(state.db, coordinator, { + stranded, orphanBlockers: async () => {}, recoveryActions: async () => {}, + dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {}, watchdogs: async () => ({ incomplete: 0 }), + schedulingSuppressed: async () => behavior.suppressed(), + }, () => true); + await bounded(work.ready, () => "the recovery queues to start"); + return { stop: async () => { running.delete(coordinator); await coordinator.stop(); } }; + } + + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + const at = new Date(Date.now()).toISOString(); + await bounded(vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())), () => `simulated time to advance past ${at}`); + await bounded(Promise.allSettled([...passes]), () => `${passes.size} delivery passes started by simulated ${at} to settle`); + await immediate(); + } + } + + it("checks a deferred task at its deadline after the server that deferred it is replaced", async () => { + const f = await bounded(seed(30), () => "the fixture to be seeded"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const target = f.tasks[7]!; + const deadline = Date.now() + 20 * MINUTE; + const behavior = { deferUntil: new Map([[target, deadline]]), suppressed: () => false, checks: [] as Array<{ id: string; at: number }> }; + // The first server checks every task (no saved pass yet); the target defers itself 20 minutes. + const first = await server(behavior); + await advance(MINUTE); + expect(behavior.checks.filter(check => check.id === target)).toHaveLength(1); + await advance(4 * MINUTE); + // The server is replaced without shutdown work; its successor has no memory of the deadline. + await bounded(first.stop(), () => "the first server to stop"); + await server(behavior); + reconcileDatabaseWorkNotifications(state.db); + await advance(deadline - Date.now() + MINUTE); + const atDeadline = behavior.checks.filter(check => check.id === target && check.at >= deadline); + expect(atDeadline[0], "the replacement did not check the deferred task at its deadline").toBeDefined(); + expect(atDeadline[0]!.at - deadline).toBeLessThanOrEqual(MINUTE); + }, TEST_TIMEOUT_MS); + + it("keeps a deferral another server rewrote while this server clears the one it saw", async () => { + const f = await bounded(seed(10), () => "the fixture to be seeded"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const target = f.tasks[4]!; + const checks: Array<{ id: string; at: number }> = []; + const first = Date.now() + 30 * MINUTE; + const b = { deferUntil: new Map([[target, first]]), suppressed: () => false, checks }; + const a = { deferUntil: new Map([[target, first]]), suppressed: () => false, checks }; + const hint = () => bounded(state.db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId, issueId: target })), + () => "the task hint to commit"); + // Server B saves the first deadline; server A boots while it overlaps and reads that row. + const serverB = await server(b); + await advance(MINUTE); + await server(a); + reconcileDatabaseWorkNotifications(state.db); + await advance(MINUTE); + // B's check stops deferring and clears the row, then defers again to a new deadline. + b.deferUntil.delete(target); + await hint(); await advance(MINUTE); + const second = Date.now() + 15 * MINUTE; + b.deferUntil.set(target, second); + await hint(); await advance(MINUTE); + // A's check stops deferring and clears the row it read; B's new row must stay. + a.deferUntil.delete(target); + await hint(); await advance(MINUTE); + // B is replaced without shutdown work; its successor keeps B's deadline. + await bounded(serverB.stop(), () => "server B to stop"); + await server({ deferUntil: new Map([[target, second]]), suppressed: () => false, checks }); + reconcileDatabaseWorkNotifications(state.db); + const replacedAt = Date.now(); + await advance(second - Date.now() + MINUTE); + const atDeadline = checks.filter(check => check.id === target && check.at >= second && check.at > replacedAt); + expect(atDeadline[0], "the successor did not check the task at the deadline server B saved").toBeDefined(); + expect(atDeadline[0]!.at - second).toBeLessThanOrEqual(MINUTE); + }, TEST_TIMEOUT_MS); + + it("removes a saved deferral once its task has left the queue", async () => { + const f = await bounded(seed(10), () => "the fixture to be seeded"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const target = f.tasks[3]!; + const behavior = { deferUntil: new Map([[target, Date.now() + HOUR]]), suppressed: () => false, checks: [] as Array<{ id: string; at: number }> }; + const first = await server(behavior); + await advance(MINUTE); + const saved = () => state.db.select({ id: issueRecoveryDeferrals.issueId }).from(issueRecoveryDeferrals) + .where(and(eq(issueRecoveryDeferrals.queue, "stranded-assignments"), eq(issueRecoveryDeferrals.issueId, target))); + expect(await bounded(saved(), () => "the deferral to be read")).toHaveLength(1); + // The task closes while its deferral is pending; no queue loads it again. + await bounded(state.db.update(issues).set({ status: "done" }).where(eq(issues.id, target)).then(() => undefined), () => "the task to close"); + await bounded(first.stop(), () => "the first server to stop"); + await server(behavior); + await advance(MINUTE); + expect(await bounded(saved(), () => "the deferral to be read")).toHaveLength(0); + }, TEST_TIMEOUT_MS); + + it("removes a saved deferral once its open task has left the queue and its deadline is past", async () => { + const f = await bounded(seed(10), () => "the fixture to be seeded"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const target = f.tasks[5]!; + const behavior = { deferUntil: new Map([[target, Date.now() + 2 * MINUTE]]), suppressed: () => false, checks: [] as Array<{ id: string; at: number }> }; + const first = await server(behavior); + await advance(MINUTE); + const saved = () => state.db.select({ id: issueRecoveryDeferrals.issueId }).from(issueRecoveryDeferrals) + .where(and(eq(issueRecoveryDeferrals.queue, "stranded-assignments"), eq(issueRecoveryDeferrals.issueId, target))); + expect(await bounded(saved(), () => "the deferral to be read")).toHaveLength(1); + // The task stays open but leaves the queue (a user takes it over) while its deferral is pending. + await bounded(state.db.update(issues).set({ status: "in_progress", assigneeUserId: "recovery-user" }).where(eq(issues.id, target)) + .then(() => undefined), () => "the task to leave the queue"); + await bounded(first.stop(), () => "the first server to stop"); + await advance(15 * MINUTE); + await server(behavior); + await advance(MINUTE); + expect(await bounded(saved(), () => "the deferral to be read")).toHaveLength(0); + }, TEST_TIMEOUT_MS); + + it("checks every task again after scheduling suppression, even with a recent saved full pass", async () => { + const f = await bounded(seed(30), () => "the fixture to be seeded"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + let suppressed = false; + const behavior = { deferUntil: new Map(), suppressed: () => suppressed, checks: [] as Array<{ id: string; at: number }> }; + await server(behavior); + await advance(MINUTE); + expect(new Set(behavior.checks.map(check => check.id)).size).toBe(f.tasks.length); + // While suppressed, a pass drops the queue's entries. + suppressed = true; + await bounded(state.db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })), () => "the company hint to commit"); + await advance(MINUTE); + suppressed = false; + const liftedAt = Date.now(); + // A plain company hint would load only changed rows; after suppression it loads every task. + await bounded(state.db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })), () => "the company hint to commit"); + await advance(2 * MINUTE); + const checkedAfter = new Set(behavior.checks.filter(check => check.at >= liftedAt).map(check => check.id)); + expect(f.tasks.filter(id => !checkedAfter.has(id))).toEqual([]); + }, TEST_TIMEOUT_MS); +}); diff --git a/server/src/__tests__/issue-recovery.test-fixture.ts b/server/src/__tests__/issue-recovery.test-fixture.ts index 7421fc349fb..8a90d73e492 100644 --- a/server/src/__tests__/issue-recovery.test-fixture.ts +++ b/server/src/__tests__/issue-recovery.test-fixture.ts @@ -71,6 +71,12 @@ export function issueRecoveryFixture(options: { meter?: boolean } = {}) { cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining); await Promise.all([worker.ready, dispatch.ready, cancellation.ready]); } + /** A replaced server process: the old workers stop and new ones start with + * no memory of earlier passes, sharing only the database. */ + async function restart(options?: { backstopMs?: number; fullBackstopMs?: number }) { + await worker?.stop(); await dispatch?.stop(); await cancellation?.stop(); await coordinator?.stop(); + await start(options); + } async function compete() { const other = heartbeatService(db); const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); @@ -233,7 +239,7 @@ export function issueRecoveryFixture(options: { meter?: boolean } = {}) { return { ...f, settings, chat, message, replies, failures, resume }; } return { - start, compete, fixture, completingProvider, settled, quiet, finish, completed, savedConversation, + start, restart, compete, fixture, completingProvider, settled, quiet, finish, completed, savedConversation, get db() { return db; }, get meter() { return meter!; }, get heartbeat() { return heartbeat; }, diff --git a/server/src/services/agent-conversations.ts b/server/src/services/agent-conversations.ts index fd4bfe05869..887a4fb6c0e 100644 --- a/server/src/services/agent-conversations.ts +++ b/server/src/services/agent-conversations.ts @@ -604,7 +604,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId), )).for("update"); if (!isConversation(issue) || comment.conversationSessionGeneration != null) return; - await notifyIssueRecovery(tx, { companyId: issue.companyId }); + await notifyIssueRecovery(tx, { companyId: issue.companyId, everyTask: true }); const released = await tx.update(issueTreeHolds).set({ status: "released", releasedAt: new Date(), updatedAt: new Date(), releasedByActorType: "user", releasedByUserId: comment.authorUserId, diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index e38b30280d4..8ff1075e0fc 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -4,7 +4,7 @@ import { readmitAgentContinuations } from "./recovery/continuation-delivery-atte import { currentControllerProcessStamp } from "./controller-process-identity.js"; import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; -import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyAgentStatusDispatch } from "./run-dispatch-notifications.js"; import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { isAgentStatusInvokable } from "@paperclipai/shared"; @@ -62,7 +62,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li await notifyAttentionRetentionWork(tx, { companyId }); if (before !== after) { await notifyStrandedAgentStatus(tx, { companyId, agentId }); - await notifyRunDispatch(tx, { companyId }); + await notifyAgentStatusDispatch(tx, { companyId, agentId }); await notifyDeferredExecutionEligibility(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); } diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 76cda656eba..8a36d0bb329 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -434,7 +434,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { - await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyRunDispatch(db, { companyId: policy.companyId, everyTask: true }); await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await db .update(projects) @@ -451,7 +451,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; await notifyMuseContactVerification(db,{companyId:policy.companyId}); - await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyRunDispatch(db, { companyId: policy.companyId, everyTask: true }); await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); await notifyStatusCardIssueChange(db, policy.companyId); diff --git a/server/src/services/chat-run-milestone-notifications.ts b/server/src/services/chat-run-milestone-notifications.ts index f391ba9d1ca..13a8867e5d2 100644 --- a/server/src/services/chat-run-milestone-notifications.ts +++ b/server/src/services/chat-run-milestone-notifications.ts @@ -12,7 +12,8 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string }; export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) { - if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope); + // A company-wide change (a secret decision, configuration) can make waiting tasks eligible. + if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, { companyId: scope.companyId, everyTask: true }); if (scope.issueId || scope.runId) { await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.issueId ? undefined : scope.runId }); diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index d9dd948e621..75a3c8fc92e 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -347,7 +347,8 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { if(willReactivate)await notifyMuseContactVerification(tx,{companyId:id}); - await notifyRunDispatch(tx, { companyId: id }); + // A resumed company's tasks become eligible again without any task row changing. + await notifyRunDispatch(tx, { companyId: id, everyTask: companyPatch.status === "active" }); await notifyGitHubCompanyChange(tx, id); await notifyDeferredExecutionEligibility(tx, { companyId: id }); if (companyPatch.status === "active") await promoteDeferredWorkIntents(tx, { companyId: id, trigger: "pause_lifted" }); diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index c44aa8ee998..a584c71f53e 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -6557,7 +6557,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const relationCompanyId = targetCompany.id; await db.transaction(async tx => { await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId }); - await notifyIssueRecovery(tx, { companyId: relationCompanyId }); + await notifyIssueRecovery(tx, { companyId: relationCompanyId, everyTask: true }); await tx .insert(issueRelations) .values(relationRows.map((row) => ({ diff --git a/server/src/services/documents.ts b/server/src/services/documents.ts index e6a4e34589d..96c916e0993 100644 --- a/server/src/services/documents.ts +++ b/server/src/services/documents.ts @@ -670,7 +670,7 @@ export function documentService(db: Db) { }); } - await notifyIssueRecovery(tx, { companyId: existing.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: existing.companyId, issueId: existing.issueId }); const now = new Date(); const nextRevisionNumber = existing.latestRevisionNumber + 1; const [restoredRevision] = await tx @@ -754,7 +754,7 @@ export function documentService(db: Db) { }; } - await notifyIssueRecovery(tx, { companyId: existing.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: existing.companyId, issueId: existing.issueId }); const now = new Date(); await tx .update(documents) @@ -805,7 +805,7 @@ export function documentService(db: Db) { }; } - await notifyIssueRecovery(tx, { companyId: existing.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: existing.companyId, issueId: existing.issueId }); const now = new Date(); await tx .update(documents) @@ -857,7 +857,7 @@ export function documentService(db: Db) { }); } - await notifyIssueRecovery(tx, { companyId: existing.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: existing.companyId, issueId: existing.issueId }); await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id)); await notifyStatusCardIssueChange(tx, existing.companyId); await tx.delete(documents).where(eq(documents.id, existing.id)); diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index 97ca79f11cb..0c915115092 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -2,26 +2,50 @@ import { sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -/** An issue hint is reserved for local facts such as comments. Lifecycle, - * graph, run and eligibility changes retain company scope because their - * consequences can include ancestors and shared execution references. */ +/** What a task recovery queue re-checks for one hint. + * - `{ companyId }`: tasks whose own rows changed since the queue last looked + * at the company (see changed-tasks.ts). Most writes change a task, run, + * wake, comment or agent row, so this finds what they affected. + * - `everyTask`: every eligible task of the company. Only for a change that + * makes tasks eligible again without writing any of their rows, such as a + * company or project resume or a tree hold release. + * - `agentId`: the tasks one agent is assigned, reviews or holds a run lock + * on, for an agent status change. + * - `issueId` / graph: one task and its graph, for local facts. + * - `backstop`: the queue's own changed-rows deadline (see recovery/work.ts). + * - `reload`: a boot or a notification reconnect. Each company loads the rows + * changed since its saved full pass, or every task when none is recent. */ export type IssueRecoveryScope = { companyId: string; issueId?: string; graphRunId?: string; graphIssueId?: string; backstop?: true; - issueIds?: string[]; wholeCompany?: boolean }; + everyTask?: true; agentId?: string; reload?: true; issueIds?: string[]; wholeCompany?: boolean }; export const ISSUE_RECOVERY_MAX_ISSUE_IDS = 100; export const ISSUE_RECOVERY_BACKSTOP_MS = 5 * 60_000; export const ISSUE_RECOVERY_FULL_BACKSTOP_MS = 60 * 60_000; +export const ISSUE_RECOVERY_MAX_AGENT_ISSUE_IDS = 2_000; +export const ISSUE_RECOVERY_RELOAD_HINT = JSON.stringify({ companyId: "", reload: true }); const UUID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; const COMPANY_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.taskWatchdogs]; -const TASK_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.assignedWorkInvariant, - DELIVERY_QUEUES.resolvedDependencies, DELIVERY_QUEUES.staleIssueLocks]; +export const ISSUE_RECOVERY_TASK_QUEUES: readonly DeliveryQueue[] = [DELIVERY_QUEUES.strandedAssignments, + DELIVERY_QUEUES.assignedWorkInvariant, DELIVERY_QUEUES.resolvedDependencies, DELIVERY_QUEUES.staleIssueLocks]; +const TASK_QUEUES = ISSUE_RECOVERY_TASK_QUEUES; -export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) { +export async function notifyIssueRecovery(transaction: object, + scope?: { companyId: string; issueId?: string; everyTask?: boolean }) { const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; - const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*"; + const taskHint = !scope ? "*" : JSON.stringify({ companyId: scope.companyId, + ...(scope.issueId ? { issueId: scope.issueId } : scope.everyTask ? { everyTask: true } : {}) }); for (const queue of COMPANY_QUEUES) await notifyDeliveryWork(transaction, queue, companyHint); - for (const queue of TASK_QUEUES) await notifyDeliveryWork(transaction, queue, issueHint); + for (const queue of TASK_QUEUES) await notifyDeliveryWork(transaction, queue, taskHint); +} + +/** An agent status change: the company queues, and on the task queues only + * the tasks that agent is assigned, reviews or holds a run lock on. */ +export async function notifyAgentRecovery(transaction: object, scope: { companyId: string; agentId: string }) { + const companyHint = JSON.stringify({ companyId: scope.companyId }); + const agentHint = JSON.stringify({ companyId: scope.companyId, agentId: scope.agentId }); + for (const queue of COMPANY_QUEUES) await notifyDeliveryWork(transaction, queue, companyHint); + for (const queue of TASK_QUEUES) await notifyDeliveryWork(transaction, queue, agentHint); } export async function notifyCompanyRecovery(transaction: object, companyId: string) { @@ -39,12 +63,30 @@ export function decodeIssueRecoveryHint(hint: string): IssueRecoveryScope | unde export function issueRecoveryScopeMatches(entry: { id: string; companyId: string }, scope?: IssueRecoveryScope) { if (!scope) return true; - if (entry.companyId !== scope.companyId) return false; + if (scope.reload || entry.companyId !== scope.companyId) return false; if (scope.issueId) return entry.id === scope.issueId; - if (scope.graphRunId || scope.graphIssueId) return scope.wholeCompany === true || (scope.issueIds?.includes(entry.id) ?? false); + if (scope.graphRunId || scope.graphIssueId || scope.agentId) return scope.wholeCompany === true || (scope.issueIds?.includes(entry.id) ?? false); return true; } +/** The tasks an agent status change can affect: assigned to the agent, under + * its current review, or locked by one of its runs. Null when there are too + * many to list; the caller then re-checks every task of the company. */ +export async function resolveAgentIssues(db: Db, scope: { companyId: string; agentId: string }): Promise { + const rows = await db.execute<{ id: string }>(sql`select i.id::text as id from issues i + where i.company_id = ${scope.companyId}::uuid + and (i.status not in ('done', 'cancelled') or i.checkout_run_id is not null or i.execution_run_id is not null) + and (i.assignee_agent_id = ${scope.agentId}::uuid + or (i.status = 'in_review' and i.execution_state->>'status' = 'pending' + and i.execution_state->'currentParticipant'->>'type' = 'agent' + and i.execution_state->'currentParticipant'->>'agentId' = ${scope.agentId}) + or exists (select 1 from heartbeat_runs r where r.company_id = i.company_id and r.agent_id = ${scope.agentId}::uuid + and r.id in (i.checkout_run_id, i.execution_run_id))) + limit ${ISSUE_RECOVERY_MAX_AGENT_ISSUE_IDS + 1}`); + const ids = rows.map(row => row.id).sort(); + return ids.length <= ISSUE_RECOVERY_MAX_AGENT_ISSUE_IDS ? ids : null; +} + export async function notifyIssueGraphRecovery(transaction: object, scope: { companyId: string; runId?: string | null; issueId?: string | null }) { if (!scope.runId && !scope.issueId) return notifyIssueRecovery(transaction, { companyId: scope.companyId }); diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 8da95115248..5ec02c61530 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -806,7 +806,7 @@ export function issueTreeControlService(db: Db) { const releaseReason = input.reason ?? "Subtree resume applied."; const { hold: resumeHold } = await db.transaction(async (tx) => { - await notifyIssueRecovery(tx, { companyId }); + await notifyIssueRecovery(tx, { companyId, everyTask: true }); await notifyDeferredExecutionEligibility(tx, { companyId }); const [createdHold] = await tx .insert(issueTreeHolds) @@ -886,7 +886,7 @@ export function issueTreeControlService(db: Db) { } const { hold, members } = await db.transaction(async (tx) => { - await notifyIssueRecovery(tx, { companyId }); + await notifyIssueRecovery(tx, { companyId, everyTask: true }); await notifyDeferredExecutionEligibility(tx, { companyId }); const [createdHold] = await tx .insert(issueTreeHolds) diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 77d1021ac4c..0bdcf07c2af 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -7,7 +7,7 @@ import { issueDisplayStatusSql } from "./issue-display-status.js"; import { notifyStrandedAgentStatus, notifyActiveRunSilence } from "./live-run-recovery-notifications.js"; import { finishedIdleWake } from "./idle-work-proof.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; -import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; @@ -8026,7 +8026,7 @@ export function issueService(db: Db) { } await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId }); - await notifyIssueRecovery(dbOrTx, { companyId }); + await notifyIssueGraphRecovery(dbOrTx, { companyId, issueId }); await notifyDeferredExecutionEligibility(dbOrTx, { companyId }); await dbOrTx .delete(issueRelations) @@ -12221,8 +12221,9 @@ export function issueService(db: Db) { || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined || patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined || blockedByIssueIds !== undefined) { + // A moved task can unblock its old parent, whose rows do not change. await notifyRunDispatch(tx, patch.parentId !== undefined && patch.parentId !== receiptExisting.parentId - ? { companyId: receiptExisting.companyId } : { companyId: receiptExisting.companyId, issueId: id }); + ? { companyId: receiptExisting.companyId, everyTask: true } : { companyId: receiptExisting.companyId, issueId: id }); await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId }); await notifyActiveRunSilence(tx, { companyId: receiptExisting.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); @@ -12712,7 +12713,8 @@ export function issueService(db: Db) { } await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined); + // A deleted task releases its dependents and parent without changing their rows. + await notifyRunDispatch(tx, owner ? { companyId: owner.companyId, everyTask: true } : undefined); if (owner) { await notifyDeferredExecutionEligibility(tx, owner); await notifyActiveRunSilence(tx, { companyId: owner.companyId, issueId: id }); @@ -14641,7 +14643,7 @@ export function issueService(db: Db) { await assertAgentInteraction(tx, { companyId: existing.companyId, actorAgentId: options?.actorAgentId, operation: "edit", issueId: existing.issueId, lock: true }); - await notifyIssueRecovery(tx, { companyId: existing.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: existing.companyId, issueId: existing.issueId }); await tx.delete(issueAttachments).where(eq(issueAttachments.id, id)); await tx.delete(assets).where(eq(assets.id, existing.assetId)); if (options?.mcpCleanup?.id === existing.id && options.mcpCleanup.companyId === existing.companyId diff --git a/server/src/services/native-runtime/chat-attachment-reuse.ts b/server/src/services/native-runtime/chat-attachment-reuse.ts index 7878583ca01..648891c375c 100644 --- a/server/src/services/native-runtime/chat-attachment-reuse.ts +++ b/server/src/services/native-runtime/chat-attachment-reuse.ts @@ -1,4 +1,4 @@ -import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; +import { notifyIssueGraphRecovery } from "../issue-recovery-notifications.js"; import { createHash } from "node:crypto"; import { and, desc, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; @@ -1751,7 +1751,7 @@ export async function prepareReusedChatAttachment(input: { throw new Error("paperclip_runner_chat_attachment_origin_not_persisted"); } await input.db.transaction(async tx => { - await notifyIssueRecovery(tx, { companyId: input.binding.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: input.binding.companyId, issueId: input.binding.issueId, runId: input.binding.runId }); await tx .update(issueWorkProducts) .set({ diff --git a/server/src/services/native-runtime/native-runner-file-handoff.ts b/server/src/services/native-runtime/native-runner-file-handoff.ts index a55552da2a0..688564e7b31 100644 --- a/server/src/services/native-runtime/native-runner-file-handoff.ts +++ b/server/src/services/native-runtime/native-runner-file-handoff.ts @@ -1,4 +1,4 @@ -import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; +import { notifyIssueGraphRecovery } from "../issue-recovery-notifications.js"; import { createHash, randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; import { constants } from "node:fs"; @@ -1314,7 +1314,7 @@ export async function prepareNativeRunnerFileHandoff(input: { throw new Error("paperclip_runner_file_handoff_origin_not_persisted"); } await input.db.transaction(async tx => { - await notifyIssueRecovery(tx, { companyId: input.binding.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: input.binding.companyId, issueId: input.binding.issueId, runId: input.binding.runId }); await tx .update(issueWorkProducts) .set({ title: verified.title, updatedAt: new Date() }) diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 1983df05181..14a79515d93 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -20,7 +20,6 @@ import { import { logger } from "../../middleware/logger.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { evaluateAgentInvokability } from "../agent-invokability.js"; -import { changedTaskCondition } from "./changed-tasks.js"; import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; @@ -513,7 +512,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { } async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null; - changedSince?: Date }, + changed?: SQL }, cursor: string | null, limit: number) { const at = now(); const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} @@ -533,7 +532,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { scope.issueIds ? inArray(issues.id, scope.issueIds) : undefined, cursor ? sql`${issues.id} > ${cursor}` : undefined, scope.createdAfter ? gte(issues.createdAt, scope.createdAfter) : undefined, - scope.changedSince ? changedTaskCondition(scope.changedSince) : undefined, + scope.changed, inArray(issues.status, [...OPEN_STATUSES]), sql`${issues.assigneeAgentId} is not null`, isNull(issues.assigneeUserId), diff --git a/server/src/services/recovery/changed-tasks.ts b/server/src/services/recovery/changed-tasks.ts index aecbf30a0e6..df6cdbb9a66 100644 --- a/server/src/services/recovery/changed-tasks.ts +++ b/server/src/services/recovery/changed-tasks.ts @@ -1,4 +1,4 @@ -import { sql } from "drizzle-orm"; +import { sql, type SQL } from "drizzle-orm"; import { issues } from "@paperclipai/db"; /** Tasks whose own rows changed since `since`, for the backstop between full @@ -8,8 +8,8 @@ import { issues } from "@paperclipai/db"; * start flips the agent to running, which makes none of its other tasks * stranded. Every run start would otherwise re-check all of the agent's tasks * for two backstop windows. */ -export function changedTaskCondition(since: Date) { - const at = sql`${since.toISOString()}::timestamptz`; +export function changedTaskCondition(since: Date | SQL) { + const at = since instanceof Date ? sql`${since.toISOString()}::timestamptz` : since; return sql`(${issues.updatedAt} >= ${at} or exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} and (r.context_snapshot->>'issueId') = ${issues.id}::text and r.updated_at >= ${at}) diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 53511dde244..9fc274f2b70 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -3,18 +3,22 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; -import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope } - from "../issue-recovery-notifications.js"; +import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, ISSUE_RECOVERY_RELOAD_HINT, issueRecoveryBackstopHint, + issueRecoveryScopeMatches, resolveAgentIssues, resolveIssueGraph, type IssueRecoveryScope } from "../issue-recovery-notifications.js"; +import { setDatabaseWorkReconnectHint } from "@paperclipai/db/work-signals"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js"; import { changedTaskCondition } from "./changed-tasks.js"; import { createRecoveryPassReads, type RecoveryPassReads } from "./pass-reads.js"; -type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true; origin?: "reload" | "backstop" | "company" | "hint" }; +type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true; + origin?: "reload" | "reconnect" | "backstop" | "company" | "every_task" | "agent" | "hint" }; type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void; pass?: RecoveryPassReads }; const PAGE_SIZE = 100; +/** How long past its deadline a saved deferral of a task that left its queue is kept. */ +const STALE_DEFERRAL_MS = 10 * 60_000; export const resolvedDependencyCandidate = sql`exists (select 1 from ${issueRelations} r where r.company_id = ${issues.companyId} and r.related_issue_id = ${issues.id} and r.type = 'blocks' offset 0) @@ -36,7 +40,7 @@ export function registerIssueRecoveryWork( schedulingSuppressed(): Promise; /** The assigned-work invariant (assigned-work-sweep.ts). */ assignedWork?: { - listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; changedSince?: Date }, cursor: string | null, limit: number): + listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; changed?: SQL }, cursor: string | null, limit: number): Promise>; checkIssue(scope: Scope): Promise; }; @@ -46,46 +50,191 @@ export function registerIssueRecoveryWork( ) { const backstopMs = options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS; const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS; - function fullPasses() { + /** A plain company hint: changed rows only, unless no full pass is recent. */ + const isPlainCompany = (scope: IssueRecoveryScope) => !scope.issueId && !isGraph(scope) && !scope.agentId && !scope.everyTask + && !scope.backstop && !scope.reload; + /** When each company was last fully checked and last looked at for changes. + * Full passes are saved, so a replaced or overlapping server keeps the same + * hourly schedule instead of starting over with a full pass at boot. */ + function fullPasses(queue: DeliveryQueue) { const completedAt = new Map(), changedAt = new Map(); - const scans = new Map(); + type Scan = { filter: SQL | null; startedAt: number; reloadCutoff?: number; unchecked: Set; + seen: Set; savedDeferrals?: Map }; + const scans = new Map(); + // A full pass is saved only once every task it loaded has been checked, so + // a server replaced mid-pass leaves no saved pass and its replacement runs + // the full pass again instead of skipping the tasks it never reached. + const unsaved: Array<{ unchecked: Set; save: () => Promise }> = []; + // The saved deferral row (issue_recovery_deferrals) this server last read + // or wrote for a task, by its deadline. Every delete names that deadline, + // so it never removes a row another server rewrote since; a lost race + // costs one extra check, never a missed one. + const deferred = new Map(); + // Saved passes from before scheduling was suppressed are ignored: entries + // were dropped while suppressed, so the next pass must look at every task. + let distrustBefore = 0; const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0); - const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0); + const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get("*") ?? 0, changedAt.get(companyId) ?? 0); + const fullDue = (companyId: string) => Date.now() - lastFull(companyId) >= fullBackstopMs; + const at = (ms: number) => sql`${new Date(ms).toISOString()}::timestamptz`; + async function readSaved(companyId: string) { + if (!fullDue(companyId)) return; + // An unreadable record means a full pass, never a missed check. + const rows: Array<{ started_at?: string | Date | null }> = await db.execute<{ started_at: string | Date }>(sql`select started_at + from issue_recovery_full_passes where company_id = ${companyId}::uuid and queue = ${queue}`).catch(() => []); + const saved = rows[0]?.started_at ? new Date(rows[0].started_at).getTime() : 0; + if (saved > distrustBefore) completedAt.set(companyId, Math.max(completedAt.get(companyId) ?? 0, saved)); + } + async function save(startedAt: number, companies: SQL) { + // A lost record costs one extra full pass later, never a missed check, so + // it must not fail (and repeat) the pass it describes. + await db.execute(sql`insert into issue_recovery_full_passes (company_id, queue, started_at) + select c.id, ${queue}, ${at(startedAt)} from companies c where ${companies} + on conflict (company_id, queue) do update set started_at = greatest(issue_recovery_full_passes.started_at, excluded.started_at)`) + .catch(() => undefined); + } + /** Whether the company has a saved full pass newer than `cutoff`. */ + const recentFull = (companyId: SQL, cutoff: number) => sql`exists (select 1 from issue_recovery_full_passes p + where p.company_id = ${companyId} and p.queue = ${queue} and p.started_at > ${at(cutoff)})`; return { backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null, - changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null { - if (firstPage || !scans.has(key)) scans.set(key, { startedAt: Date.now(), - since: !scope?.backstop || Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null - : new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS) }); - return scans.get(key)!.since; + /** The row filter for this load: changed rows, or null to load every task. */ + async filter(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Promise { + if (firstPage || !scans.has(key)) { + const scan: Scan = { startedAt: Date.now(), filter: null, unchecked: new Set(), seen: new Set() }; + scans.set(key, scan); + if (scope?.reload) { + // Per company: every task when no saved full pass is recent, else + // the rows changed since that pass or this server's last reload. + const cutoff = scan.reloadCutoff = Math.max(Date.now() - fullBackstopMs, distrustBefore); + const since = sql`(greatest((select p.started_at from issue_recovery_full_passes p + where p.company_id = ${issues.companyId} and p.queue = ${queue}), ${at(changedAt.get("*") ?? 0)}) + - ${`${ISSUE_RECOVERY_BACKSTOP_MS} milliseconds`}::interval)`; + // A task with a pending deferral is checked again too, so its + // deadline survives the server that set it. + scan.filter = sql`(not ${recentFull(sql`${issues.companyId}`, cutoff)} or ${changedTaskCondition(since)} + or exists (select 1 from issue_recovery_deferrals d where d.queue = ${queue} and d.issue_id = ${issues.id}))`; + // Unreadable deferrals mean a full reload, never a lost deadline. + const saved = await db.execute<{ issue_id: string; due_at: string | Date }>(sql`select issue_id::text, due_at + from issue_recovery_deferrals where queue = ${queue}`).then(rows => [...rows], () => null); + if (saved) { + scan.savedDeferrals = new Map(saved.map(row => [row.issue_id, new Date(row.due_at).getTime()])); + for (const [id, dueAt] of scan.savedDeferrals) deferred.set(id, dueAt); + } else scan.filter = null; + } else if (scope && (scope.backstop || isPlainCompany(scope))) { + await readSaved(scope.companyId); + if (!fullDue(scope.companyId)) scan.filter = changedTaskCondition(new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS)); + } + } + return scans.get(key)!.filter; }, - loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) { + async loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) { if (!complete) return; const scan = scans.get(key); scans.delete(key); const startedAt = scan?.startedAt ?? Date.now(); - if (scan?.since && scope) changedAt.set(scope.companyId, Math.max(changedAt.get(scope.companyId) ?? 0, startedAt)); - else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) { + const saveOnceChecked = async (companies: SQL) => { + const pending = { unchecked: scan?.unchecked ?? new Set(), save: () => save(startedAt, companies) }; + if (pending.unchecked.size === 0) { await pending.save(); return; } + // A task that leaves the queue unchecked (deleted, or no longer + // eligible) holds its pass unsaved; drop the oldest rather than grow. + unsaved.push(pending); + if (unsaved.length > 16) unsaved.shift(); + }; + if (scope?.reload) { + changedAt.set("*", Math.max(changedAt.get("*") ?? 0, startedAt)); + // The reload loads every task with a saved deferral that this queue + // can still check. A row it did not load belongs to a task that left + // the queue, for whatever reason, and no check will clear it. It is + // deleted once its task is closed or its deadline is well past; the + // delete also requires the deadline it read, so a row another server + // wrote since stays. + const stale = [...scan?.savedDeferrals ?? []].filter(([id]) => !scan!.seen.has(id)) + .map(([id, dueAt]) => ({ id, due: new Date(dueAt).toISOString() })); + if (stale.length > 0) { + for (const { id } of stale) deferred.delete(id); + await db.execute(sql`delete from issue_recovery_deferrals d using jsonb_to_recordset(${JSON.stringify(stale)}::jsonb) + as s(id uuid, due timestamptz) + where d.queue = ${queue} and d.issue_id = s.id and d.due_at = s.due + and (d.due_at < ${at(Date.now() - STALE_DEFERRAL_MS)} or exists (select 1 from issues i + where i.id = d.issue_id and i.status in ('done', 'cancelled')))`).catch(() => undefined); + } + // The companies this reload loaded in full now have a recent full pass. + await saveOnceChecked(sql`not ${recentFull(sql`c.id`, scan?.reloadCutoff ?? startedAt)}`); + } else if (scan?.filter && scope) changedAt.set(scope.companyId, Math.max(changedAt.get(scope.companyId) ?? 0, startedAt)); + else if (!scope?.issueId && !scope?.agentId && (!isGraph(scope) || scope?.wholeCompany)) { const companyKey = scope?.companyId ?? "*"; completedAt.set(companyKey, Math.max(completedAt.get(companyKey) ?? 0, startedAt)); + await saveOnceChecked(scope ? sql`c.id = ${scope.companyId}::uuid` : sql`true`); } }, + /** Rows a load returned; each must be checked before its pass is saved. */ + seen(key: string, ids: readonly string[]) { + const scan = scans.get(key); + if (scan) for (const id of ids) { scan.unchecked.add(id); scan.seen.add(id); } + }, + /** Saves a checked task's deferral deadline, or clears the row this + * server knows of when the check no longer defers. One statement when + * the deadline changed or a known row is cleared; none otherwise. False + * when a deadline could not be saved: the check then does not count + * toward a saved full pass, so a successor's full pass rebuilds it. */ + async deferral(id: string, companyId: string, dueAt: number | null): Promise { + const known = deferred.get(id); + if (dueAt !== null) { + if (known === dueAt) return true; + deferred.delete(id); + const saved = await db.execute(sql`insert into issue_recovery_deferrals (queue, issue_id, company_id, due_at) + values (${queue}, ${id}::uuid, ${companyId}::uuid, ${at(dueAt)}) + on conflict (queue, issue_id) do update set due_at = excluded.due_at`).then(() => true, () => false); + if (saved) deferred.set(id, dueAt); + return saved; + } + if (known !== undefined) { + deferred.delete(id); + // Only the row this server read or wrote; a newer deadline from + // another server stays. A row left behind costs one extra check. + await db.execute(sql`delete from issue_recovery_deferrals + where queue = ${queue} and issue_id = ${id}::uuid and due_at = ${at(known)}`).catch(() => undefined); + } + return true; + }, + /** A task was checked; save each full pass that has no unchecked task left. */ + async checked(id: string) { + for (const scan of scans.values()) scan.unchecked.delete(id); + for (let index = unsaved.length - 1; index >= 0; index--) { + const pending = unsaved[index]!; + pending.unchecked.delete(id); + if (pending.unchecked.size > 0) continue; + unsaved.splice(index, 1); + await pending.save(); + } + }, + suppressed() { + completedAt.clear(); changedAt.clear(); scans.clear(); unsaved.length = 0; + distrustBefore = Date.now(); + }, }; } - const reconstructionIntervalMs = (scope: IssueRecoveryScope | undefined) => scope?.backstop || scope?.graphRunId || scope?.graphIssueId ? 0 : 30_000; - const scopeKey = (scope: IssueRecoveryScope | undefined) => scope - ? JSON.stringify([scope.companyId, scope.issueId, scope.graphRunId, scope.graphIssueId, scope.backstop]) : "*"; + const reconstructionIntervalMs = (scope: IssueRecoveryScope | undefined) => + scope?.backstop || scope?.reload || scope?.agentId || scope?.graphRunId || scope?.graphIssueId ? 0 : 30_000; + const scopeKey = (scope: IssueRecoveryScope | undefined) => { + if (!scope) return "*"; + const key: unknown[] = [scope.companyId, scope.issueId, scope.graphRunId, scope.graphIssueId, scope.backstop]; + return JSON.stringify(scope.everyTask || scope.agentId || scope.reload ? [...key, scope.everyTask, scope.agentId, scope.reload] : key); + }; const isGraph = (scope: IssueRecoveryScope | undefined) => Boolean(scope?.graphRunId || scope?.graphIssueId); const loadOrigin = (scope: IssueRecoveryScope | undefined) => !workSignalDebug.enabled ? undefined : !scope ? "reload" as const - : scope.backstop ? "backstop" as const : scope.issueId || isGraph(scope) ? "hint" as const : "company" as const; + : scope.reload ? "reconnect" as const : scope.backstop ? "backstop" as const : scope.everyTask ? "every_task" as const + : scope.agentId ? "agent" as const : scope.issueId || isGraph(scope) ? "hint" as const : "company" as const; const graphs = new Map }>(); async function resolveGraph(scope: IssueRecoveryScope | undefined) { - if (!scope || !isGraph(scope)) return; + if (!scope || (!isGraph(scope) && !scope.agentId)) return; const key = scopeKey(scope); for (const [other, cached] of graphs) if (cached.at < Date.now() - 2_000) graphs.delete(other); let cached = graphs.get(key); if (!cached) { - cached = { at: Date.now(), ids: resolveIssueGraph(db, { companyId: scope.companyId, runId: scope.graphRunId, issueId: scope.graphIssueId }) }; + cached = { at: Date.now(), ids: scope.agentId ? resolveAgentIssues(db, { companyId: scope.companyId, agentId: scope.agentId }) + : resolveIssueGraph(db, { companyId: scope.companyId, runId: scope.graphRunId, issueId: scope.graphIssueId }) }; graphs.set(key, cached); } const ids = await cached.ids.catch(error => { graphs.delete(key); throw error; }); @@ -96,17 +245,18 @@ export function registerIssueRecoveryWork( if (scope && isGraph(scope)) worker.invalidate(issueRecoveryBackstopHint(scope.companyId)); await resolveGraph(scope); if (!scope?.wholeCompany) return false; - worker.invalidate(JSON.stringify({ companyId: scope.companyId })); + worker.invalidate(JSON.stringify({ companyId: scope.companyId, everyTask: true })); return true; } function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise, eligible?: SQL) { const cursors = new Map(); - const passes = fullPasses(); + const passes = fullPasses(queue); let enabled = false; let pass = createRecoveryPassReads(); const worker: ReturnType> = registerScopedDeadlineWork(coordinator, { queue, reconstructionIntervalMs, + initialHint: ISSUE_RECOVERY_RELOAD_HINT, backstop: passes.backstop, beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, beforeChecks: entries => { pass = createRecoveryPassReads(entries.map(entry => entry.id)); }, @@ -115,14 +265,14 @@ export function registerIssueRecoveryWork( load: async scope => { const key = scopeKey(scope); if (workSignalDebug.enabled) workSignalDebug.event(queue, "load", { scope: key }); - if (!enabled) { cursors.delete(key); return []; } + if (!enabled) { cursors.delete(key); passes.suppressed(); return []; } if (await toCompany(scope, worker)) return { rows: null, complete: true }; if (scope?.issueIds?.length === 0) return { rows: [], complete: true }; const cursor = cursors.get(key); - const changedSince = passes.changedSince(scope, key, !cursor); + const changed = await passes.filter(scope, key, !cursor); const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( - scope ? eq(issues.companyId, scope.companyId) : undefined, - changedSince ? changedTaskCondition(changedSince) : undefined, + scope && !scope.reload ? eq(issues.companyId, scope.companyId) : undefined, + changed ?? undefined, scope?.issueId ? eq(issues.id, scope.issueId) : undefined, scope?.issueIds ? inArray(issues.id, scope.issueIds) : undefined, cursor ? gt(issues.id, cursor) : undefined, @@ -134,8 +284,9 @@ export function registerIssueRecoveryWork( )).orderBy(asc(issues.id)).limit(PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); - passes.loaded(scope, key, complete); - const keepUnseen = changedSince ? { keepUnseen: true as const } : {}; + passes.seen(key, rows.map(row => row.id)); + await passes.loaded(scope, key, complete); + const keepUnseen = changed ? { keepUnseen: true as const } : {}; const origin = loadOrigin(scope); return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete, ...keepUnseen }; }, @@ -146,6 +297,7 @@ export function registerIssueRecoveryWork( let next: number | null = null; await check({ companyId: entry.companyId, issueId: entry.id, pass, onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); + if (await passes.deferral(entry.id, entry.companyId, next)) await passes.checked(entry.id); return next === null ? null : { ...entry, dueAt: next, ...(workSignalDebug.enabled ? { fromDeadline: true as const } : {}) }; }, }, canRun); @@ -157,11 +309,12 @@ export function registerIssueRecoveryWork( function registerAssignedWork(sweep: NonNullable) { type Suspect = Entry & { updatedAt: number; unchanged?: true; fromBackstop?: true }; const cursors = new Map(); - const passes = fullPasses(); + const passes = fullPasses(DELIVERY_QUEUES.assignedWorkInvariant); let enabled = false; const worker: ReturnType> = registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.assignedWorkInvariant, reconstructionIntervalMs, + initialHint: ISSUE_RECOVERY_RELOAD_HINT, backstop: passes.backstop, batchSize: 25, beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, @@ -172,19 +325,20 @@ export function registerIssueRecoveryWork( : { ...loaded, dueAt: Math.max(pending.dueAt, loaded.dueAt), ...(pending.unchanged ? { unchanged: true as const } : {}) }, load: async scope => { const key = scopeKey(scope); - if (!enabled) { cursors.delete(key); return []; } + if (!enabled) { cursors.delete(key); passes.suppressed(); return []; } if (await toCompany(scope, worker)) return { rows: null, complete: true }; if (scope?.issueIds?.length === 0) return { rows: [], complete: true }; - const changedSince = passes.changedSince(scope, key, !cursors.has(key)); - const rows = await sweep.listSuspects(scope ? { companyId: scope.companyId, issueId: scope.issueId, - issueIds: scope.issueIds, ...(changedSince ? { changedSince } : {}) } : {}, cursors.get(key) ?? null, PAGE_SIZE); + const changed = await passes.filter(scope, key, !cursors.has(key)); + const rows = await sweep.listSuspects(scope && !scope.reload ? { companyId: scope.companyId, issueId: scope.issueId, + issueIds: scope.issueIds, ...(changed ? { changed } : {}) } : changed ? { changed } : {}, cursors.get(key) ?? null, PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); - passes.loaded(scope, key, complete); + passes.seen(key, rows.map(row => row.id)); + await passes.loaded(scope, key, complete); const origin = loadOrigin(scope); return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(), ...(scope?.backstop ? { fromBackstop: true as const } : {}), ...(origin ? { origin } : {}) })), complete, - ...(changedSince ? { keepUnseen: true as const } : {}) }; + ...(changed ? { keepUnseen: true as const } : {}) }; }, check: async entry => { if (!enabled) return null; @@ -193,6 +347,7 @@ export function registerIssueRecoveryWork( let next: number | null = null, unchanged = false; await sweep.checkIssue({ companyId: entry.companyId, issueId: entry.id, onDeferredDeadline: (at, detail) => { next = next === null ? at : Math.min(next, at); unchanged ||= detail?.unchanged === true; } }); + if (await passes.deferral(entry.id, entry.companyId, next)) await passes.checked(entry.id); const { unchanged: _previous, fromBackstop: _source, ...rest } = entry; return next === null ? null : { ...rest, dueAt: next, ...(unchanged ? { unchanged: true as const } : {}), ...(workSignalDebug.enabled ? { fromDeadline: true as const } : {}) }; @@ -258,6 +413,10 @@ export function registerIssueRecoveryWork( }, resolvedDependencyCandidate); const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks); const assignedWork = owner.assignedWork ? registerAssignedWork(owner.assignedWork) : null; + // A reconnect can lose only another server's hints, whose writes the changed + // rows find, so it rebuilds company by company instead of re-checking all. + for (const queue of [DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies, DELIVERY_QUEUES.staleIssueLocks, + ...(assignedWork ? [DELIVERY_QUEUES.assignedWorkInvariant] : [])]) setDatabaseWorkReconnectHint(db, queue, ISSUE_RECOVERY_RELOAD_HINT); const watchdogs = registerTaskWatchdogWork(db, coordinator, { reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()), }, canRun); diff --git a/server/src/services/run-dispatch-notifications.ts b/server/src/services/run-dispatch-notifications.ts index 5f79c3e65c4..52151fa90cd 100644 --- a/server/src/services/run-dispatch-notifications.ts +++ b/server/src/services/run-dispatch-notifications.ts @@ -1,7 +1,8 @@ -import { notifyCompanyRecovery, notifyIssueRecovery, notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; +import { notifyAgentRecovery, notifyCompanyRecovery, notifyIssueRecovery, notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export type RunDispatchScope = { companyId: string; issueId?: string | null; runId?: string | null }; +/** `everyTask` re-checks every task of the company; see IssueRecoveryScope. */ +export type RunDispatchScope = { companyId: string; issueId?: string | null; runId?: string | null; everyTask?: boolean }; export async function notifyQueuedRunDispatch(transaction: object, scope?: RunDispatchScope) { await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, @@ -10,7 +11,7 @@ export async function notifyQueuedRunDispatch(transaction: object, scope?: RunDi export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) { if (scope?.issueId || scope?.runId) await notifyIssueGraphRecovery(transaction, scope); - else await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined); + else await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId, everyTask: scope.everyTask } : undefined); await notifyDispatchQueues(transaction, scope); } @@ -19,6 +20,12 @@ export async function notifyAgentConfigurationDispatch(transaction: object, scop await notifyDispatchQueues(transaction, scope); } +/** An agent status change re-checks that agent's tasks, not the whole company. */ +export async function notifyAgentStatusDispatch(transaction: object, scope: { companyId: string; agentId: string }) { + await notifyAgentRecovery(transaction, scope); + await notifyDispatchQueues(transaction, scope); +} + async function notifyDispatchQueues(transaction: object, scope?: RunDispatchScope) { const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; await notifyDeliveryWork(transaction, DELIVERY_QUEUES.nativeProcessSessions, hint); diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index 647f75d9c57..c33cf38f467 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -26,6 +26,8 @@ export function registerScopedDeadlineWork( admitDueWhileActive?: boolean; }; reconstructionIntervalMs?: number | ((scope: Scope | undefined) => number); + /** What the first pass after registration reconstructs; an unscoped reload by default. */ + initialHint?: string; backstop?(scope: Scope | undefined): number | null; beforePass?(): Promise; beforeChecks?(entries: readonly Entry[]): void; @@ -51,7 +53,7 @@ export function registerScopedDeadlineWork( } const ownerRevision = randomUUID(); let revision = 0, running = false, stopped = false, lastRunFailed = false; - const dirty = new Set(["*"]); + const dirty = new Set([options.initialHint ?? "*"]); const versions = new Map(); const scans = new Map }>(); const entries = new Map(); diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.ts index 7de3fa17126..bbcf0c1eab0 100644 --- a/server/src/services/task-watchdogs.ts +++ b/server/src/services/task-watchdogs.ts @@ -851,7 +851,7 @@ async function upsertIssueWatchdogInTransaction( .where(and(eq(issueWatchdogs.companyId, companyId), eq(issueWatchdogs.issueId, issueId))) .then((rows: IssueWatchdogRow[]) => rows[0] ?? null); - await notifyIssueRecovery(dbOrTx, { companyId }); + await notifyIssueRecovery(dbOrTx, { companyId, issueId }); if (existing) { const updated = await updateIssueWatchdogRow(dbOrTx, existing, input, now); return { watchdog: toIssueWatchdog(updated), created: false }; @@ -1838,7 +1838,7 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) .where(and(eq(issueWatchdogs.companyId, companyId), eq(issueWatchdogs.issueId, issueId))) .then((rows) => rows[0] ?? null); if (!existing || existing.status === "disabled") return null; - await notifyIssueRecovery(tx, { companyId }); + await notifyIssueRecovery(tx, { companyId, issueId }); const [updated] = await tx .update(issueWatchdogs) .set({ diff --git a/server/src/services/work-intent-outbox.ts b/server/src/services/work-intent-outbox.ts index 7ec9e6dee10..a1eb2edbc71 100644 --- a/server/src/services/work-intent-outbox.ts +++ b/server/src/services/work-intent-outbox.ts @@ -74,9 +74,11 @@ export async function promoteDeferredWorkIntents(tx: object, input: { : input.trigger === "lock_released" && input.runId ? { companyId: input.companyId, runId: input.runId } : input.trigger === "accounting_settled" && input.runId && !hardStop ? { companyId: input.companyId, runId: input.runId } : null; // Register the work signals before the write they announce. - await notifyRunDispatch(tx, scope ?? { companyId: input.companyId }); + // A lifted pause or tree hold makes tasks eligible without changing their rows. + const liftsHold = input.trigger === "pause_lifted" || input.trigger === "tree_hold_released"; + await notifyRunDispatch(tx, scope ?? { companyId: input.companyId, everyTask: liftsHold }); await notifyDeferredExecutionEligibility(tx, { companyId: input.companyId }); - if (!scope) await notifyIssueRecovery(tx, { companyId: input.companyId }); + if (!scope) await notifyIssueRecovery(tx, { companyId: input.companyId, everyTask: liftsHold }); if (input.issueId) await notifyDeferredWakesForIssue(tx, { companyId: input.companyId, issueId: input.issueId }); const db = tx as Pick; await db.update(agentWakeupRequests).set({