diff --git a/server/src/__tests__/automatic-run-brake-import.test.ts b/server/src/__tests__/automatic-run-brake-import.test.ts index a46adb46a18..a64c70cbf68 100644 --- a/server/src/__tests__/automatic-run-brake-import.test.ts +++ b/server/src/__tests__/automatic-run-brake-import.test.ts @@ -6,6 +6,6 @@ describe("automatic-run brake module graph", () => { const brake = await import("../services/recovery/automatic-run-brake.js"); const rules = await brake.automaticRunOutcomeRules(); expect(rules.isTransientSetupCode("environment_provider_unavailable")).toBe(true); - expect(rules.isPersonOwnedFailureCode("budget_hard_stop")).toBe(true); + expect(rules.hasNativePreDispatchFailureRecord({ nativePreDispatchFailure: { version: 1, providerWorkStarted: false } })).toBe(true); }, 600_000); }); diff --git a/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts b/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts index a8830f57ea7..e52119ec75d 100644 --- a/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts +++ b/server/src/__tests__/automatic-run-brake-recovery-passes.test.ts @@ -13,6 +13,7 @@ import { taskAutomaticAttempts } from "../services/recovery/task-automatic-attem import { CONTINUATION_CAP } from "../services/recovery/assigned-work-invariant.js"; import { deliverReconciledExecutions } from "../services/execution-recovery-resolution.js"; import { continueHeldExecutions, resetBacklogContinuationPacingForTests } from "../services/fresh-continuation-fallback.js"; +import { createSystemRecovery } from "../services/system-recovery-state.js"; const support = await getEmbeddedPostgresTestSupport(); const suite = support.supported ? describe : describe.skip; @@ -370,6 +371,44 @@ suite("automatic-run brake in the recovery passes", () => { expect(wake).toHaveBeenCalledTimes(1); }, 120_000); + it("the sweep's continuation behind a run refused at dispatch only records, and its delivery creates the run only after the pause", async () => { + await seedCompany(); + const issueId = randomUUID(); + const blockedAt = ago(20 * MINUTE); + await db.insert(issues).values({ id: issueId, companyId, title: "Finish the report", status: "blocked", assigneeAgentId: agentId, + statusVersion: 3, blockedTransitionAt: blockedAt, updatedAt: ago(2 * HOUR), createdAt: ago(5 * HOUR) }); + const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", status: "failed", + runtimeMode: "legacy", errorCode: "adapter_failed", error: "native_instruction_materialization_handoff_unavailable", + startedAt: new Date(blockedAt.getTime() - MINUTE), finishedAt: new Date(blockedAt.getTime() - 500), + createdAt: new Date(blockedAt.getTime() - 2 * MINUTE), livenessState: "failed", + runnerProfileJson: { adapterDispatch: { adapterType: "codex_local" } }, resultJson: { workspaceRestoreFailure: "restore_failed" }, + contextSnapshot: { issueId, taskId: issueId, wakeReason: "issue_children_completed" } }).returning(); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", status: "active", + ownerType: "board", returnOwnerAgentId: agentId, cause: "legacy_execution_requires_reconciliation", + fingerprint: `legacy-execution:${source!.id}`, nextAction: "Held.", createdAt: blockedAt, + evidence: { runId: source!.id, originalFailureCode: "adapter_failed", workspaceRestoreFailure: "restore_failed", + systemRecovery: { ...createSystemRecovery(source!.id), state: "complete" } } }); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", status: "cancelled", + errorCode: "execution_reconciliation_required", startedAt: null, retryOfRunId: source!.id, runtimeMode: "legacy", + finishedAt: new Date(blockedAt.getTime() + 1_000), createdAt: new Date(blockedAt.getTime() - 100), resultJson: {}, + contextSnapshot: { issueId, taskId: issueId, wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed" } }); + const enqueueWakeup = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup, scheduleRecoveryRetry: async () => null } as never); + const runsBefore = (await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).length; + expect(await recovery.assignedWork.checkIssue({ companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect((await continuationRecords(source!.id))[0]?.evidence).toMatchObject({ continuationDelivery: "pending" }); + expect(enqueueWakeup).not.toHaveBeenCalled(); + expect((await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).length).toBe(runsBefore); + const failures = await openBrake(); + const wake = runWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + await recover(failures); + await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running"))); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + }, 120_000); + it("one held task spends one creation slot while open", async () => { await seedCompany(); await heldBacklog(); diff --git a/server/src/__tests__/automatic-run-brake.test.ts b/server/src/__tests__/automatic-run-brake.test.ts index 48afc8a7c2a..31db15dafb5 100644 --- a/server/src/__tests__/automatic-run-brake.test.ts +++ b/server/src/__tests__/automatic-run-brake.test.ts @@ -436,6 +436,7 @@ suite("automatic-run brake", () => { for (let index = 0; index < AUTOMATIC_RUN_BRAKE.maxRuns + 200; index += 1) { await brake.holdAutomaticRunStart(db, { id: randomUUID(), companyId, contextSnapshot: { wakeReason: "issue_continuation_needed" } }); } - expect(brake.cacheSizes()).toEqual({ companies: AUTOMATIC_RUN_BRAKE.maxCompanies, runs: AUTOMATIC_RUN_BRAKE.maxRuns }); + expect(brake.cacheSizes()).toMatchObject({ companies: AUTOMATIC_RUN_BRAKE.maxCompanies, runs: AUTOMATIC_RUN_BRAKE.maxRuns }); + expect(brake.cacheSizes().wakes).toBeLessThanOrEqual(AUTOMATIC_RUN_BRAKE.maxCompanies); }, 180_000); }); diff --git a/server/src/__tests__/automatic-run-start-gate.test.ts b/server/src/__tests__/automatic-run-start-gate.test.ts new file mode 100644 index 00000000000..72a96dfc9fd --- /dev/null +++ b/server/src/__tests__/automatic-run-start-gate.test.ts @@ -0,0 +1,311 @@ +import { randomUUID } from "node:crypto"; +import { eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agentWakeupRequests, agents, authUsers, automaticRunBrakes, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { runningProcesses } from "../adapters/index.ts"; +import { AUTOMATIC_RUN_BRAKE, automaticRunBrake, takeDueAutomaticRunWakes } from "../services/recovery/automatic-run-brake.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerRunDispatchWork } from "../services/heartbeat/dispatch-work.js"; + +const gate = vi.hoisted(() => ({ open: false, releases: [] as Array<() => void> })); + +const mockAdapterExecute = vi.hoisted(() => vi.fn(async () => { + if (!gate.open) await new Promise(resolve => gate.releases.push(resolve)); + return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "Start gate test run.", provider: "test", model: "test-model" }; +})); + +vi.mock("../adapters/index.ts", async () => { + const actual = await vi.importActual("../adapters/index.ts"); + return { ...actual, getServerAdapter: vi.fn(() => ({ supportsLocalAgentJwt: false, execute: mockAdapterExecute })) }; +}); + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const MINUTE = 60_000; + +suite("automatic-run start gate in the dispatch pass", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let heartbeat!: ReturnType; + const userId = randomUUID(); + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-automatic-run-start-gate-"); + db = createDb(temporary.connectionString); + heartbeat = heartbeatService(db); + await db.insert(authUsers).values({ id: userId, name: "Board", email: `${userId}@example.test`, emailVerified: true, + createdAt: new Date(), updatedAt: new Date() }); + }, 60_000); + + afterEach(async () => { + vi.restoreAllMocks(); + gate.open = true; + for (const release of gate.releases.splice(0)) release(); + await heartbeat.drainActiveRunExecutions(); + gate.open = false; + mockAdapterExecute.mockClear(); + runningProcesses.clear(); + automaticRunBrake.reset(); + await db.execute(sql`truncate table ${companies} cascade`); + }, 180_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }, 120_000); + + async function seedCompany(agentCount: number, maxConcurrentRuns: number) { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Start gate", status: "active", + issuePrefix: `G${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); + const agentIds: string[] = []; + for (let index = 0; index < agentCount; index += 1) { + const agentId = randomUUID(); + await db.insert(agents).values({ id: agentId, companyId, name: `Gate ${index}`, role: "engineer", status: "idle", + adapterType: "codex_local", adapterConfig: {}, permissions: {}, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 60, wakeOnDemand: true, maxConcurrentRuns } } }); + agentIds.push(agentId); + } + return { companyId, agentIds }; + } + + async function queued(companyId: string, agentId: string, count: number, context: (index: number) => Record, + ageMs = 30 * MINUTE) { + if (count === 0) return []; + const tasks = await db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ companyId, title: `Task ${index}`, + status: "todo", assigneeAgentId: agentId }))).returning({ id: issues.id }); + return db.insert(heartbeatRuns).values(tasks.map((task, index) => ({ companyId, agentId, issueId: task.id, scopeKind: "issue" as const, + invocationSource: "automation", triggerDetail: "system", status: "queued", createdAt: new Date(Date.now() - ageMs + index), + contextSnapshot: { ...context(index), issueId: task.id, taskId: task.id } }))).returning(); + } + + const automatic = () => ({ wakeReason: "issue_continuation_needed", previousRunId: randomUUID() }); + + async function openBrake(companyId: string, agentId: string, options: { probeAllowed: boolean }) { + await db.insert(heartbeatRuns).values(Array.from({ length: 20 }, (_, index) => ({ companyId, agentId, invocationSource: "automation", + triggerDetail: "system", runtimeMode: "legacy", status: "failed", errorCode: "environment_provider_unavailable", + executionStage: "preparing", createdAt: new Date(Date.now() - 4 * MINUTE), startedAt: new Date(Date.now() - 3 * MINUTE), + finishedAt: new Date(Date.now() - MINUTE + index * 100), resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + contextSnapshot: { wakeReason: "transient_failure_retry" } }))); + expect((await automaticRunBrake.refresh(db, companyId)).state).toBe("open"); + if (!options.probeAllowed) await db.update(automaticRunBrakes).set({ nextProbeAt: sql`now() + interval '10 minutes'` }) + .where(eq(automaticRunBrakes.companyId, companyId)); + automaticRunBrake.reset(); + } + + const statuses = async (ids: string[]) => db.select({ id: heartbeatRuns.id, status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode }) + .from(heartbeatRuns).where(inArray(heartbeatRuns.id, ids)); + const started = async (ids: string[]) => (await statuses(ids)).filter(run => run.status !== "queued").length; + + function countQueries() { + let count = 0; + for (const method of ["select", "insert", "update", "delete", "execute", "transaction"] as const) { + const original = db[method].bind(db) as (...args: unknown[]) => unknown; + vi.spyOn(db, method).mockImplementation(((...args: unknown[]) => { count += 1; return original(...args); }) as never); + } + return () => count; + } + + async function passCost(heldRuns: number, passes: number) { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 1); + await openBrake(companyId, agentId!, { probeAllowed: false }); + const runs = await queued(companyId, agentId!, heldRuns, automatic); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + const read = countQueries(); + for (let pass = 0; pass < passes; pass += 1) await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + const total = read(); + vi.restoreAllMocks(); + expect(await started(runs.map(run => run.id))).toBe(0); + expect((await statuses(runs.map(run => run.id))).every(run => run.errorCode === null)).toBe(true); + return total; + } + + it("walks hundreds of held runs without a claim attempt or a row read, and still starts other work in the same pass", async () => { + const small = await passCost(10, 100); + const large = await passCost(250, 100); + expect(large).toBeLessThanOrEqual(small + 100); + const beyondCache = await passCost(AUTOMATIC_RUN_BRAKE.maxRuns + 476, 3); + expect(beyondCache).toBeLessThanOrEqual(await passCost(10, 3) + 100); + + const { companyId, agentIds: [agentId] } = await seedCompany(1, 1); + await openBrake(companyId, agentId!, { probeAllowed: false }); + const held = await queued(companyId, agentId!, 250, automatic); + const [work] = await queued(companyId, agentId!, 1, () => ({ wakeReason: "issue_assigned" }), 0); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(await started([work!.id])).toBe(1); + expect(await started(held.map(run => run.id))).toBe(0); + }, 600_000); + + it("returns the probe when its run's claim is declined, so another automatic run starts in the same pass", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 5); + await openBrake(companyId, agentId!, { probeAllowed: true }); + const [declined, next] = await queued(companyId, agentId!, 2, () => ({ wakeReason: "transient_failure_retry" })); + await db.update(heartbeatRuns).set({ runnerProfileJson: { chatControlRecoveryAdmission: { phase: "unknown" } } }) + .where(eq(heartbeatRuns.id, declined!.id)); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(await started([declined!.id])).toBe(0); + expect(await started([next!.id])).toBe(1); + const [brake] = await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId)); + expect(brake?.probeRunId).toBe(next!.id); + automaticRunBrake.reset(); + await heartbeatService(db).resumeQueuedRunsAfterDrain({ companyId }); + expect(await started([declined!.id])).toBe(0); + expect((await db.select().from(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId)))[0]?.probeRunId).toBe(next!.id); + }, 300_000); + + it("returns an unused half-open release start to its window", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 1); + const runs = await queued(companyId, agentId!, 4, automatic); + await db.insert(automaticRunBrakes).values({ companyId, state: "half_open", stateChangedAt: sql`date_trunc('milliseconds', now())`, + openedAt: sql`now() - interval '5 minutes'` }); + const reserve = (index: number) => automaticRunBrake.reserveStart(db, runs[index]!, { personOrigin: false }); + const first = await reserve(0); + expect(first.held).toBe(false); + expect((await reserve(1)).held).toBe(false); + automaticRunBrake.reset(); + expect((await reserve(2)).held).toBe(true); + if (!first.held) await first.release(); + expect((await reserve(2)).held).toBe(false); + automaticRunBrake.reset(); + expect((await reserve(3)).held).toBe(true); + }, 300_000); + + it("never holds a person's Retry", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 1); + await openBrake(companyId, agentId!, { probeAllowed: false }); + const held = await queued(companyId, agentId!, 20, automatic); + const [retry] = await queued(companyId, agentId!, 1, () => ({ wakeReason: "task_retry", previousRunId: randomUUID() }), 0); + const [request] = await db.insert(agentWakeupRequests).values({ companyId, agentId: agentId!, source: "on_demand", triggerDetail: "manual", + reason: "task_retry", status: "queued", requestedByActorType: "user", requestedByActorId: userId, runId: retry!.id }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: request!.id }).where(eq(heartbeatRuns.id, retry!.id)); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(await started([retry!.id])).toBe(1); + expect(await started(held.map(run => run.id))).toBe(0); + }, 300_000); + + it("never holds the only queued run when it continues a person's Retry", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 1); + await openBrake(companyId, agentId!, { probeAllowed: false }); + const [retry] = await db.insert(heartbeatRuns).values({ companyId, agentId: agentId!, invocationSource: "on_demand", + triggerDetail: "manual", status: "failed", errorCode: "process_lost", startedAt: new Date(Date.now() - 2 * MINUTE), + finishedAt: new Date(Date.now() - MINUTE), contextSnapshot: { wakeReason: "task_retry" } }).returning(); + const [continuation] = await queued(companyId, agentId!, 1, + () => ({ wakeReason: "issue_continuation_needed", previousRunId: retry!.id }), 0); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(await started([continuation!.id])).toBe(1); + }, 300_000); + + it("starts exactly one probe across two server instances", async () => { + const { companyId, agentIds } = await seedCompany(2, 5); + await openBrake(companyId, agentIds[0]!, { probeAllowed: true }); + const runs = [...await queued(companyId, agentIds[0]!, 10, automatic), ...await queued(companyId, agentIds[1]!, 10, automatic)]; + const other = heartbeatService(db); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + automaticRunBrake.reset(); + await other.resumeQueuedRunsAfterDrain({ companyId }); + automaticRunBrake.reset(); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(await started(runs.map(run => run.id))).toBe(1); + for (const release of gate.releases.splice(0)) release(); + await other.drainActiveRunExecutions(); + }, 300_000); + + it("releases held runs at a doubling rate per window", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 100); + const runs = await queued(companyId, agentId!, 40, automatic); + await db.insert(automaticRunBrakes).values({ companyId, state: "half_open", stateChangedAt: sql`date_trunc('milliseconds', now())`, + openedAt: sql`now() - interval '5 minutes'` }); + const counts: number[] = []; + for (let step = 0; step < 3; step += 1) { + automaticRunBrake.reset(); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + counts.push(await started(runs.map(run => run.id))); + await db.update(automaticRunBrakes).set({ + stateChangedAt: sql`state_changed_at - ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval`, + releaseWindowStartedAt: sql`release_window_started_at - ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval`, + }).where(eq(automaticRunBrakes.companyId, companyId)); + } + expect(counts).toEqual([2, 6, 14]); + }, 300_000); + + it("keeps held runs out of the declined-claim backoff, so they start on the first pass after the pause", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 10); + await openBrake(companyId, agentId!, { probeAllowed: false }); + const runs = await queued(companyId, agentId!, 5, automatic); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(await started(runs.map(run => run.id))).toBe(0); + await db.delete(automaticRunBrakes).where(eq(automaticRunBrakes.companyId, companyId)); + automaticRunBrake.reset(); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(await started(runs.map(run => run.id))).toBeGreaterThanOrEqual(1); + }, 300_000); + + it("wakes the company's dispatch at the next try without polling", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 1); + await openBrake(companyId, agentId!, { probeAllowed: false }); + await db.update(automaticRunBrakes).set({ nextProbeAt: sql`now() + interval '2 seconds'` }).where(eq(automaticRunBrakes.companyId, companyId)); + automaticRunBrake.reset(); + await queued(companyId, agentId!, 3, automatic); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(takeDueAutomaticRunWakes().companyIds).toEqual([]); + const resumed: Array = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: () => undefined }); + try { + await registerRunDispatchWork(coordinator, { + promoteDueScheduledRetries: async () => undefined, + nextScheduledRetryAt: async () => null, + schedulingSuppressed: async () => false, + resumeQueuedRunsAfterDrain: async scope => { resumed.push(scope?.companyId); }, + listUndispatchedQueuedRuns: async () => [], + }, () => true).ready; + await expect.poll(() => resumed, { timeout: 30_000, interval: 250 }).toContain(companyId); + } finally { + await coordinator.stop(); + } + }, 300_000); + + it("still wakes every held company at its next try when more companies are held than the wake list keeps", async () => { + const companyIds: string[] = []; + for (let index = 0; index < AUTOMATIC_RUN_BRAKE.maxCompanies + 20; index += 1) { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 1); + await openBrake(companyId, agentId!, { probeAllowed: false }); + companyIds.push(companyId); + } + for (const companyId of companyIds) { + expect(await automaticRunBrake.holdAutomaticRunStart(db, { id: randomUUID(), companyId, contextSnapshot: automatic() }, + { personOrigin: false })).toBe(true); + } + expect(automaticRunBrake.cacheSizes().wakes).toBeLessThanOrEqual(AUTOMATIC_RUN_BRAKE.maxCompanies); + const nextAt = automaticRunBrake.takeDueWakes(Number.NEGATIVE_INFINITY).nextAt; + expect(nextAt).not.toBeNull(); + expect(nextAt!).toBeLessThanOrEqual(Date.now() + 11 * MINUTE); + const due = takeDueAutomaticRunWakes(Date.now() + 20 * MINUTE); + expect(due.all || companyIds.every(companyId => due.companyIds.includes(companyId))).toBe(true); + expect(due.all).toBe(true); + }, 600_000); + + it("keeps a held automatic run queued while paused and starts it on its own at the next try", async () => { + const { companyId, agentIds: [agentId] } = await seedCompany(1, 1); + await openBrake(companyId, agentId!, { probeAllowed: false }); + await db.update(automaticRunBrakes).set({ nextProbeAt: sql`now() + interval '3 seconds'` }).where(eq(automaticRunBrakes.companyId, companyId)); + automaticRunBrake.reset(); + const [run] = await queued(companyId, agentId!, 1, () => ({ wakeReason: "transient_failure_retry" })); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: agentId!, source: "automation", triggerDetail: "system", + reason: "transient_failure_retry", status: "queued", requestedByActorType: "system", runId: run!.id }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: wake!.id }).where(eq(heartbeatRuns.id, run!.id)); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: () => undefined }); + try { + await heartbeat.registerRunDispatchWork(coordinator, () => true).ready; + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + expect(await statuses([run!.id])).toEqual([{ id: run!.id, status: "queued", errorCode: null }]); + expect(mockAdapterExecute).not.toHaveBeenCalled(); + await expect.poll(() => mockAdapterExecute.mock.calls.length, { timeout: 60_000, interval: 250 }).toBe(1); + expect((await statuses([run!.id]))[0]?.status).toBe("running"); + for (const release of gate.releases.splice(0)) release(); + await expect.poll(async () => (await statuses([run!.id]))[0]?.status, { timeout: 60_000, interval: 250 }).toBe("succeeded"); + } finally { + await coordinator.stop(); + } + }, 300_000); +}); diff --git a/server/src/services/heartbeat/dispatch-work.ts b/server/src/services/heartbeat/dispatch-work.ts index f3ce5834046..6e757b94fdd 100644 --- a/server/src/services/heartbeat/dispatch-work.ts +++ b/server/src/services/heartbeat/dispatch-work.ts @@ -4,6 +4,7 @@ import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator import { registerDeadlineWork } from "../deadline-work.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import { createQueuedRunRedriveSchedule, type QueuedRunRedrivePlan, type UndispatchedQueuedRun } from "../queued-run-dispatch-liveness.js"; +import { nextAutomaticRunWakeAt, takeDueAutomaticRunWakes } from "../recovery/automatic-run-brake.js"; export function registerRunDispatchWork( coordinator: ReturnType, @@ -39,6 +40,9 @@ export function registerRunDispatchWork( onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); }, onStop: () => { unsubscribeRefresh?.(); unsubscribeRefresh = undefined; }, run: async signal => { + const dueWakes = takeDueAutomaticRunWakes(); + if (dueWakes.all) scopes.add("*"); + for (const companyId of dueWakes.companyIds) scopes.add(JSON.stringify({ companyId })); const requested = scopes.has("*") ? ["*"] : [...scopes]; if (requested[0] === "*") scopes.clear(); else for (const hint of requested) scopes.delete(hint); @@ -78,7 +82,8 @@ export function registerRunDispatchWork( throw error; } for (const companyId of plan.companyIds) scopes.add(JSON.stringify({ companyId })); - return plan.nextAt; + const wakeAt = nextAutomaticRunWakeAt(); + return plan.nextAt === null ? wakeAt : wakeAt === null ? plan.nextAt : Math.min(plan.nextAt, wakeAt); }, }); const refresh = () => { diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index f382ea499ba..18606daa274 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -13,6 +13,7 @@ import { deferredWorkIntentWait } from "../work-intent-outbox.js"; import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHeartbeatTimerPolicy } from "./timer-policy.js"; import { QUEUED_RUN_AGING_MS, RECOVERY_WAKE_REASONS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js"; +import { reserveAutomaticRunStart } from "../recovery/automatic-run-brake.js"; import { queuedRunDispatchFacts } from "./queued-run-line.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; @@ -2478,7 +2479,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const now = new Date(); pruneDeclinedAgedClaims(now); const runById = new Map(queuedRuns.map(run => [run.id, run])); - const { facts, dispatchInput } = await queuedRunDispatchFacts(db, { + const { facts, dispatchInput, personOriginChecked } = await queuedRunDispatchFacts(db, { agent, queuedRuns, dependencyReadiness, issueById, now, shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now, queuedRunTaskStamp(runById.get(runId)!, issueById)) }); const dispatcher = createQueuedRunDispatcher(facts, dispatchInput); @@ -2490,7 +2491,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const entry = byRun.get(run); return Boolean(entry && queuedRunDispatchClass(entry, now) === "person"); }, - personOrigin: (run: QueuedRunCandidate) => byRun.get(run)?.requestedByUser ?? false, + personOrigin: (run: QueuedRunCandidate) => personOriginChecked ? byRun.get(run)?.requestedByUser ?? false : undefined, }; } @@ -2599,15 +2600,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende continuesChain: true, triedRunIds: new Set([...triedRunIds, ...triedThisPass]) }); break; } + const start = await reserveAutomaticRunStart(db, candidate, { personOrigin: dispatcher.personOrigin(candidate) }); + if (start.held) continue; claimAttempts += 1; triedThisPass.push(candidate.id); const [queuedRun] = await db.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.status, "queued"))); - if (!queuedRun) continue; + if (!queuedRun) { + await start.release(); + continue; + } let claimed: typeof heartbeatRuns.$inferSelect | null; try { claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects, deferredCancellations, slots); } catch (err) { + await start.release(); if (isPermanentClaimRejection(err)) { rejectedClaims.push({ run: queuedRun, err }); continue; @@ -2622,6 +2629,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende continue; } if (!claimed) { + await start.release(); if (slots.full) break; noteDeclinedAgedClaim(candidate, "claim_declined"); backOffClaim(candidate, dispatcher.personRun(candidate), taskStamp); diff --git a/server/src/services/heartbeat/queued-run-line.ts b/server/src/services/heartbeat/queued-run-line.ts index 9fb3047faba..573b7e3e348 100644 --- a/server/src/services/heartbeat/queued-run-line.ts +++ b/server/src/services/heartbeat/queued-run-line.ts @@ -106,17 +106,18 @@ export async function queuedRunDispatchFacts( shareDeferred?: (runId: string) => boolean; now: Date; }, -): Promise<{ facts: Array; dispatchInput: QueuedRunDispatchInput }> { +): Promise<{ facts: Array; dispatchInput: QueuedRunDispatchInput; personOriginChecked: boolean }> { const { agent, queuedRuns, dependencyReadiness, issueById, now } = input; const wakeReasonOf = (run: R) => readNonEmptyString(parseObject(run.contextSnapshot).wakeReason); + const personOriginChecked = queuedRuns.length > 1; const requestIds = [...new Set(queuedRuns.flatMap(run => run.wakeupRequestId ? [run.wakeupRequestId] : []))]; - const userRequests = queuedRuns.length > 1 && requestIds.length + const userRequests = personOriginChecked && requestIds.length ? await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, agent.companyId), inArray(agentWakeupRequests.id, requestIds), eq(agentWakeupRequests.requestedByActorType, "user"))) : []; const userRequestIds = new Set(userRequests.map(row => row.id)); - const joinedUserRequests = queuedRuns.length > 1 + const joinedUserRequests = personOriginChecked ? await db.select({ runId: agentWakeupRequests.runId }).from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, agent.companyId), inArray(agentWakeupRequests.runId, queuedRuns.map(run => run.id)), eq(agentWakeupRequests.agentId, agent.id), eq(agentWakeupRequests.requestedByActorType, "user"))) @@ -126,7 +127,7 @@ export async function queuedRunDispatchFacts( userRunIds.has(run.id) || Boolean(run.wakeupRequestId && userRequestIds.has(run.wakeupRequestId)); const sourceRunIdOf = (run: R) => readNonEmptyString(parseObject(run.contextSnapshot).previousRunId) ?? run.retryOfRunId ?? null; - const sourceRunIds = queuedRuns.length > 1 + const sourceRunIds = personOriginChecked ? [...new Set(queuedRuns.flatMap(run => { const sourceRunId = joinedByUser(run) ? null : sourceRunIdOf(run); return sourceRunId && RUN_ID_PATTERN.test(sourceRunId) ? [sourceRunId] : []; @@ -184,7 +185,7 @@ export async function queuedRunDispatchFacts( const recentStarts = startRows.map(row => ({ ...row, personOrigin: personStarts.has(row.id), personWaiting: personStarts.has(row.id) || Boolean(row.issueId && personTaskIds.has(row.issueId)) })); const { startsSinceRecovery, startsSinceOldest } = dispatchCountersFromStarts(facts, recentStarts, now); - return { facts, dispatchInput: { now, startsSinceRecovery, startsSinceOldest } }; + return { facts, dispatchInput: { now, startsSinceRecovery, startsSinceOldest }, personOriginChecked }; } /** At most this many agents' lines are computed for one projection read. */ diff --git a/server/src/services/recovery/automatic-run-brake.ts b/server/src/services/recovery/automatic-run-brake.ts index fa345482221..9964f23cbf2 100644 --- a/server/src/services/recovery/automatic-run-brake.ts +++ b/server/src/services/recovery/automatic-run-brake.ts @@ -1,7 +1,8 @@ import { eq, sql } from "drizzle-orm"; import { automaticRunBrakes, type Db } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; -import { PERSON_WAKE_REASONS, RECOVERY_WAKE_REASONS } from "../heartbeat/queued-run-dispatch-order.js"; +import { PERSON_WAKE_REASONS } from "../heartbeat/queued-run-dispatch-order.js"; +import { isPersonOwnedFailureCode, isRecoveryRun, isRestartClassFailureCode } from "../continuation-cause.js"; import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js"; export const AUTOMATIC_RUN_BRAKE = { @@ -37,15 +38,12 @@ export type AutomaticRunFacts = { contextSnapshot?: Record | null; }; -const EXTRA_AUTOMATIC_WAKE_REASONS: ReadonlySet = new Set(["issue_assignment_recovery", "transient_failure_retry"]); - export function isAutomaticRun(run: AutomaticRunFacts): boolean { const context = run.contextSnapshot ?? {}; const reason = typeof context.wakeReason === "string" ? context.wakeReason : null; - if (context.explicitUserContinuation || context.taskRetryRequestId || (reason && PERSON_WAKE_REASONS.has(reason))) return false; - return Boolean(run.retryOfRunId || run.scheduledRetryReason || context.previousRunId || context.retryReason - || context.freshContinuationIsolation - || (reason && (RECOVERY_WAKE_REASONS.has(reason) || EXTRA_AUTOMATIC_WAKE_REASONS.has(reason)))); + if (context.taskRetryRequestId || (reason && PERSON_WAKE_REASONS.has(reason))) return false; + return isRecoveryRun({ retryOfRunId: run.retryOfRunId ?? null, scheduledRetryReason: run.scheduledRetryReason ?? null, + contextSnapshot: context }); } export type AutomaticRunOutcome = "failed_before_starting" | "healthy" | "neutral"; @@ -62,8 +60,6 @@ export type AutomaticRunOutcomeFacts = { }; export type AutomaticRunOutcomeRules = { - isPersonOwnedFailureCode: (code: string | null) => boolean; - restartCodes: readonly string[]; isTransientSetupCode: (code: string | null) => boolean; hasNativePreDispatchFailureRecord: (resultJson: unknown) => boolean; listPersonOriginSourceRunIds: (db: Db, companyId: string, runIds: string[]) => Promise; @@ -72,11 +68,9 @@ export type AutomaticRunOutcomeRules = { let outcomeRules: Promise | null = null; export function automaticRunOutcomeRules(): Promise { - outcomeRules ??= Promise.all([import("../fresh-continuation-fallback.js"), import("../heartbeat/retries.js"), + outcomeRules ??= Promise.all([import("../heartbeat/retries.js"), import("../heartbeat/run-failure-cause.js"), import("../heartbeat/queued-run-line.js")]) - .then(([fallback, retries, failureCause, queuedRunLine]) => ({ - isPersonOwnedFailureCode: fallback.isPersonOwnedFailureCode, - restartCodes: fallback.BACKLOG_CONTINUATION_SOURCE_CODES, + .then(([retries, failureCause, queuedRunLine]) => ({ isTransientSetupCode: retries.isTransientWorkspacePreparationCode, hasNativePreDispatchFailureRecord: failureCause.hasNativePreDispatchFailureRecord, listPersonOriginSourceRunIds: queuedRunLine.listPersonOriginSourceRunIds, @@ -88,8 +82,8 @@ export function automaticRunOutcomeRules(): Promise { export function classifyAutomaticRunOutcome(run: AutomaticRunOutcomeFacts, rules: AutomaticRunOutcomeRules): AutomaticRunOutcome { if (run.status === "running") return run.executionStage === "dispatching" || run.hasOutput ? "healthy" : "neutral"; if (!run.startedAt || run.status === "cancelled" || !["succeeded", "failed", "timed_out", "interrupted"].includes(run.status)) return "neutral"; - if (run.status === "succeeded" || rules.isPersonOwnedFailureCode(run.errorCode)) return "healthy"; - if (run.errorCode && rules.restartCodes.includes(run.errorCode)) return "neutral"; + if (run.status === "succeeded" || isPersonOwnedFailureCode(run.errorCode)) return "healthy"; + if (isRestartClassFailureCode(run.errorCode)) return "neutral"; const startPhase = run.providerWorkStarted === "false" || rules.isTransientSetupCode(run.errorCode) || rules.hasNativePreDispatchFailureRecord({ nativePreDispatchFailure: run.nativePreDispatchFailure }) || (run.runtimeMode === "legacy" && run.executionStage === "preparing"); @@ -156,6 +150,7 @@ function boundedMap(max: number) { while (map.size > max) map.delete(map.keys().next().value as K); }, delete(key: K) { map.delete(key); }, + entries() { return [...map.entries()]; }, clear() { map.clear(); }, get size() { return map.size; }, }; @@ -212,6 +207,10 @@ export type AutomaticWork = { issueId: string; sourceRunId: string | null }; export type AutomaticRunCreationSlot = { heldUntil: Date } | { heldUntil: null; release: () => Promise }; +export type AutomaticRunStartSlot = { held: true } | { held: false; release: () => Promise }; + +const NOT_HELD: AutomaticRunStartSlot = { held: false, release: async () => undefined }; + const NO_RESERVATION = async () => undefined; type CacheEntry = { snapshot: AutomaticRunBrakeSnapshot; readAt: number; skewMs: number; heldUntil: number }; @@ -220,6 +219,8 @@ export function createAutomaticRunBrake(options: { loadRules?: () => Promise(AUTOMATIC_RUN_BRAKE.maxCompanies); const personOrigin = boundedMap(AUTOMATIC_RUN_BRAKE.maxRuns); + const wakes = boundedMap(AUTOMATIC_RUN_BRAKE.maxCompanies); + let evictedWakeAt: number | null = null; let rulesFailureLogged = false; async function rulesOrNull() { @@ -476,15 +477,22 @@ export function createAutomaticRunBrake(options: { loadRules?: () => Promise { + await db.update(automaticRunBrakes).set({ probeRunId: null, nextProbeAt: null, updatedAt: sql`now()` }) + .where(sql`${automaticRunBrakes.companyId} = ${run.companyId} and ${automaticRunBrakes.state} = 'open' + and ${automaticRunBrakes.probeRunId} = ${run.id}::uuid + and not exists (select 1 from heartbeat_runs p where p.company_id = ${run.companyId} and p.id = ${run.id}::uuid + and p.started_at is not null)`); + companies.delete(run.companyId); + }; } const nextProbeAt = entry.snapshot.nextProbeAt ? entry.snapshot.nextProbeAt.getTime() - entry.skewMs : 0; entry.heldUntil = Math.max(now + AUTOMATIC_RUN_BRAKE.cacheTtlMs, nextProbeAt); - return false; + return null; } async function grantRelease(db: Db, entry: CacheEntry, companyId: string) { - const [granted] = [...await db.execute<{ starts: number; window_ends_at: string | Date }>(sql` + const [granted] = [...await db.execute<{ starts: number; window_start: string | Date; window_ends_at: string | Date }>(sql` with ramp as ( select b.company_id, b.state_changed_at + floor(extract(epoch from now() - b.state_changed_at) * 1000 / ${AUTOMATIC_RUN_BRAKE.releaseIntervalMs}) * ${`${AUTOMATIC_RUN_BRAKE.releaseIntervalMs} milliseconds`}::interval as window_start, @@ -498,25 +506,71 @@ export function createAutomaticRunBrake(options: { loadRules?: () => Promise { + await db.update(automaticRunBrakes).set({ releaseWindowStarts: sql`${automaticRunBrakes.releaseWindowStarts} - 1`, updatedAt: sql`now()` }) + .where(sql`${automaticRunBrakes.companyId} = ${companyId} and ${automaticRunBrakes.state} = 'half_open' + and ${automaticRunBrakes.releaseWindowStartedAt} = ${windowStart}::timestamptz and ${automaticRunBrakes.releaseWindowStarts} > 0`); + companies.delete(companyId); + }; + } const step = rampStep(entry.snapshot.stateChangedAt ?? entry.snapshot.dbNow, new Date(Date.now() + entry.skewMs)); const windowEnds = (entry.snapshot.stateChangedAt?.getTime() ?? 0) + (step + 1) * AUTOMATIC_RUN_BRAKE.releaseIntervalMs; entry.heldUntil = Math.max(Date.now() + 1_000, windowEnds - entry.skewMs); - return false; + return null; } - async function holdAutomaticRunStart(db: Db, run: AutomaticRunFacts, options: { personOrigin?: boolean } = {}): Promise { - if (!isAutomaticRun(run)) return false; + async function reserveStart(db: Db, run: AutomaticRunFacts, options: { personOrigin?: boolean } = {}): Promise { + if (!isAutomaticRun(run)) return NOT_HELD; const cached = companies.get(run.companyId); - if (cached && cached.snapshot.state === "closed" && Date.now() - cached.readAt < AUTOMATIC_RUN_BRAKE.cacheTtlMs) return false; - if (options.personOrigin ?? await isPersonOrigin(db, run)) return false; - if (cached && cached.heldUntil > Date.now()) return true; + if (cached && cached.snapshot.state === "closed" && Date.now() - cached.readAt < AUTOMATIC_RUN_BRAKE.cacheTtlMs) return NOT_HELD; + if (options.personOrigin ?? await isPersonOrigin(db, run)) return NOT_HELD; + if (cached && cached.heldUntil > Date.now()) return held(cached); const entry = await current(db, run.companyId); - if (entry.snapshot.state === "closed") return false; - if (entry.heldUntil > Date.now()) return true; - const allowed = entry.snapshot.state === "open" ? await grantProbe(db, entry, run) : await grantRelease(db, entry, run.companyId); - return !allowed; + if (entry.snapshot.state === "closed") return NOT_HELD; + if (entry.heldUntil > Date.now()) return held(entry); + const release = entry.snapshot.state === "open" ? await grantProbe(db, entry, run) : await grantRelease(db, entry, run.companyId); + if (!release) return held(entry); + return { held: false, release: () => release().catch(error => { + logger.warn({ err: error, companyId: run.companyId, runId: run.id }, "Automatic work pause could not return an unused start"); + }) }; + } + + async function holdAutomaticRunStart(db: Db, run: AutomaticRunFacts, options: { personOrigin?: boolean } = {}) { + return (await reserveStart(db, run, options)).held; + } + + function held(entry: CacheEntry): AutomaticRunStartSlot { + const now = Date.now(); + const nextProbeAt = entry.snapshot.nextProbeAt ? entry.snapshot.nextProbeAt.getTime() - entry.skewMs : 0; + const due = entry.snapshot.state === "open" ? Math.max(entry.heldUntil, nextProbeAt) : entry.heldUntil; + const at = due - now < 1_000 ? now + AUTOMATIC_RUN_BRAKE.probeIntervalMs : due; + const known = wakes.get(entry.snapshot.companyId); + if (known === undefined && wakes.size >= AUTOMATIC_RUN_BRAKE.maxCompanies) { + const [oldest] = wakes.entries(); + if (oldest) { + evictedWakeAt = evictedWakeAt === null ? oldest[1] : Math.min(evictedWakeAt, oldest[1]); + wakes.delete(oldest[0]); + } + } + if (known === undefined || known <= now || at < known) wakes.set(entry.snapshot.companyId, at); + return { held: true }; + } + + function takeDueWakes(now = Date.now()) { + const due: string[] = []; + let nextAt: number | null = null; + for (const [companyId, at] of wakes.entries()) { + if (at <= now) due.push(companyId); + else nextAt = nextAt === null ? at : Math.min(nextAt, at); + } + for (const companyId of due) wakes.delete(companyId); + const all = evictedWakeAt !== null && evictedWakeAt <= now; + if (all) evictedWakeAt = null; + else if (evictedWakeAt !== null) nextAt = nextAt === null ? evictedWakeAt : Math.min(nextAt, evictedWakeAt); + return { companyIds: due, all, nextAt }; } return { @@ -525,13 +579,27 @@ export function createAutomaticRunBrake(options: { loadRules?: () => Promise ({ companies: companies.size, runs: personOrigin.size }), - reset: () => { companies.clear(); personOrigin.clear(); }, + reserveStart, + takeDueWakes, + cacheSizes: () => ({ companies: companies.size, runs: personOrigin.size, wakes: wakes.size }), + reset: () => { companies.clear(); personOrigin.clear(); wakes.clear(); evictedWakeAt = null; }, }; } export const automaticRunBrake = createAutomaticRunBrake(); +export function reserveAutomaticRunStart(db: Db, run: AutomaticRunFacts, options: { personOrigin?: boolean } = {}) { + return automaticRunBrake.reserveStart(db, run, options); +} + +export function takeDueAutomaticRunWakes(now = Date.now()) { + return automaticRunBrake.takeDueWakes(now); +} + +export function nextAutomaticRunWakeAt() { + return automaticRunBrake.takeDueWakes(Number.NEGATIVE_INFINITY).nextAt; +} + export function automaticRunCreationHeldUntil(db: Db, companyId: string, work: AutomaticWork) { return automaticRunBrake.creationHeldUntil(db, companyId, work); }