diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index 18b449ee20f..1fc494596d0 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -726,6 +726,26 @@ describePostgres("assigned-work invariant sweep", () => { expect(wake.every(row => !("retryOfRunId" in (row.payload ?? {})))).toBe(true); }, 120_000); + it("counts the scheduler's transient retries of the same task against the one continuation cap", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + for (let index = 0; index < CONTINUATION_CAP - 2; index++) { + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: ASSIGNED_WORK_CONTINUED_ACTION, + entityType: "issue", entityId: issueId, details: {}, createdAt: new Date(Date.now() - (50 - index * 10) * 60_000) }); + } + const catalogFailure = (minutesAgo: number, attempt: number) => f.run(issueId, { status: "failed", errorCode: "repository_catalog_unavailable", + livenessState: null, scheduledRetryReason: "transient_failure", scheduledRetryAttempt: attempt, + resultJson: { errorFamily: "transient_upstream", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + createdAt: new Date(Date.now() - minutesAgo * 60_000), startedAt: new Date(Date.now() - minutesAgo * 60_000), + finishedAt: new Date(Date.now() - (minutesAgo - 1) * 60_000) }); + await catalogFailure(9, 1); + await catalogFailure(8, 2); + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "deferred_cap" }); + expect(queue.calls).toHaveLength(0); + }, 60_000); + it("evaluates old-shaped rows: native awaiting reattach, a running run saved without its controller process, and stale blocks", async () => { const f = await seedCompany(); const reattach = await f.issue(); diff --git a/server/src/__tests__/assigned-work-sweep-round-trips.test.ts b/server/src/__tests__/assigned-work-sweep-round-trips.test.ts new file mode 100644 index 00000000000..c103fd60749 --- /dev/null +++ b/server/src/__tests__/assigned-work-sweep-round-trips.test.ts @@ -0,0 +1,146 @@ +import { randomUUID } from "node:crypto"; +import { sql as sqlTag } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issues, issueTreeHolds } from "@paperclipai/db"; +import { assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; + +const gate = vi.hoisted(() => ({ perIssueCalls: 0 })); +vi.mock("../services/issue-tree-control.js", async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + issueTreeControlService: (db: Parameters[0]) => { + const service = actual.issueTreeControlService(db); + return { ...service, getActivePauseHoldGate: async (...args: Parameters) => { + gate.perIssueCalls += 1; + return service.getActivePauseHoldGate(...args); + } }; + }, + }; +}); + +const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip; +if (!recoveryPostgresSupport.supported) console.warn(`Skipping assigned-work round-trip tests: ${recoveryPostgresSupport.reason}`); + +const HOUR = 60 * 60_000; + +describePostgres("assigned-work sweep database round trips", () => { + let temporary: Awaited>; + let db: ReturnType; + let statements = 0; + + beforeAll(async () => { + temporary = await startRecoveryPostgres("assigned-work-round-trips-"); + db = createDb(temporary.connectionString); + (db.$client as unknown as { options: { debug?: unknown } }).options.debug = () => { statements += 1; }; + }, 240_000); + + afterAll(async () => { + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + }); + + it("reads pause holds and assignees once per batch on 1,500 open tasks with hundreds of holds and wakes", async () => { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Round trips", issuePrefix: `R${companyId.slice(0, 7)}` }); + const agentIds = Array.from({ length: 20 }, () => randomUUID()); + await db.insert(agents).values(agentIds.map((id, index) => ({ id, companyId, name: `Agent ${index}`, adapterType: "codex_local", + status: "idle", adapterConfig: {} }))); + const rows: Array = []; + const roots: string[] = []; + for (let index = 0; index < 1_500; index += 1) { + const id = randomUUID(); + const isRoot = index < 300; + const isChild = index >= 300 && index < 600; + if (isRoot) roots.push(id); + rows.push({ id, companyId, title: `Task ${index}`, status: index % 5 === 0 ? "todo" : "in_progress", + assigneeAgentId: agentIds[index % agentIds.length], issueNumber: index + 1, identifier: `R${companyId.slice(0, 5)}-${index + 1}`, + parentId: isChild ? roots[index - 300] : null, updatedAt: new Date(Date.now() - (index + 1) * 60_000), + createdAt: new Date(Date.now() - 5 * HOUR) }); + } + for (let index = 0; index < rows.length; index += 250) await db.insert(issues).values(rows.slice(index, index + 250)); + await db.insert(issueTreeHolds).values(roots.map(rootIssueId => ({ companyId, rootIssueId, mode: "pause", status: "active", + reason: "fixture" }))); + const runs = rows.map((row, index) => ({ companyId, agentId: row.assigneeAgentId!, invocationSource: "automation", + status: "succeeded", startedAt: new Date(Date.now() - 4 * HOUR - index), finishedAt: new Date(Date.now() - 3 * HOUR - index), + createdAt: new Date(Date.now() - 4 * HOUR - index), contextSnapshot: { issueId: row.id, taskId: row.id } })); + for (let index = 0; index < runs.length; index += 500) await db.insert(heartbeatRuns).values(runs.slice(index, index + 500)); + await db.insert(agentWakeupRequests).values(rows.slice(600, 1_000).map(row => ({ companyId, agentId: row.assigneeAgentId!, + source: "automation", triggerDetail: "system", reason: "issue_assigned", status: "queued", payload: { issueId: row.id } }))); + await db.execute(sqlTag`analyze`); + + const enqueueWakeup = vi.fn(async () => null); + const sweep = assignedWorkSweep(db, { enqueueWakeup }); + const measure = async (work: () => Promise) => { + gate.perIssueCalls = 0; statements = 0; + const started = performance.now(); + const value = await work(); + return { value, ms: Math.round(performance.now() - started), gateCalls: gate.perIssueCalls, statements }; + }; + + const report = await measure(() => sweep.report(companyId)); + + const pass = await measure(async () => { + const suspects = await sweep.listSuspects({ companyId }, null, 100); + const verdicts: Record = {}; + for (const suspect of suspects) { + const check = await sweep.checkIssue({ companyId, issueId: suspect.id }); + const key = check?.verdict.kind === "ok" && check.verdict.clause === "waiting" ? check.verdict.reason : check?.verdict.kind ?? "none"; + verdicts[key] = (verdicts[key] ?? 0) + 1; + } + return { suspects: suspects.length, verdicts }; + }); + const heldChild = rows[300]!.id!; + const held = await measure(() => sweep.checkIssue({ companyId, issueId: heldChild })); + expect(held.value).toMatchObject({ outcome: "ok", verdict: { clause: "waiting", reason: "agent_paused" } }); + + console.info(JSON.stringify({ assignedWorkRoundTrips: { + report: { ms: report.ms, statements: report.statements, perIssueGateCalls: report.gateCalls }, + sweepPass: { ms: pass.ms, statements: pass.statements, perIssueGateCalls: pass.gateCalls, ...pass.value }, + oneHeldCheck: { ms: held.ms, statements: held.statements, perIssueGateCalls: held.gateCalls }, + } })); + expect(report.value).toMatchObject({ checked: 1_500, violationCount: 500 }); + expect(report.value.ok).toEqual({ "waiting:agent_paused": 300, "waiting:dependency": 300, "pending:queued_wake": 400 }); + expect(report.gateCalls).toBe(0); + expect(pass.gateCalls).toBe(0); + expect(held.gateCalls).toBe(0); + expect(report.statements).toBeLessThan(8 * 40); + }, 300_000); + + it("the batched pause read gives the per-issue gate's answer: deep chains, cycles, released holds, other companies", async () => { + const { issueTreeControlService } = await import("../services/issue-tree-control.js"); + const companyId = randomUUID(), otherCompanyId = randomUUID(); + await db.insert(companies).values([{ id: companyId, name: "Gate", issuePrefix: `G${companyId.slice(0, 7)}` }, + { id: otherCompanyId, name: "Other", issuePrefix: `O${otherCompanyId.slice(0, 7)}` }]); + let number = 0; + const issue = async (values: Partial = {}) => { + const id = randomUUID(); number += 1; + await db.insert(issues).values({ id, companyId, title: `Gate ${number}`, status: "todo", issueNumber: number, ...values }); + return id; + }; + const root = await issue(); + const child = await issue({ parentId: root }); + const grandchild = await issue({ parentId: child }); + const releasedRoot = await issue(); + const underReleased = await issue({ parentId: releasedRoot }); + const loopA = await issue(); + const loopB = await issue({ parentId: loopA }); + await db.update(issues).set({ parentId: loopB }).where(sqlTag`${issues.id} = ${loopA}`); + const lone = await issue(); + const otherRoot = randomUUID(); + await db.insert(issues).values({ id: otherRoot, companyId: otherCompanyId, title: "Other root", status: "todo", issueNumber: 1 }); + await db.insert(issueTreeHolds).values([ + { companyId, rootIssueId: root, mode: "pause", status: "active" }, + { companyId, rootIssueId: releasedRoot, mode: "pause", status: "released" }, + { companyId: otherCompanyId, rootIssueId: otherRoot, mode: "pause", status: "active" }, + ]); + const service = issueTreeControlService(db); + const ids = [root, child, grandchild, releasedRoot, underReleased, loopA, loopB, lone, otherRoot]; + const batched = await service.listActivePauseHeldIssueIds(companyId, ids); + const perIssue = new Set(); + for (const id of ids) if (await service.getActivePauseHoldGate(companyId, id)) perIssue.add(id); + expect([...batched].sort()).toEqual([...perIssue].sort()); + expect([...batched].sort()).toEqual([root, child, grandchild].sort()); + }, 60_000); +}); diff --git a/server/src/__tests__/native-session-resumption.test.ts b/server/src/__tests__/native-session-resumption.test.ts index db322b28ef5..9a581d27596 100644 --- a/server/src/__tests__/native-session-resumption.test.ts +++ b/server/src/__tests__/native-session-resumption.test.ts @@ -384,6 +384,17 @@ describe("P6-25 pre-result native session recovery", () => { ]); }); + it("judges another owner's lease by the database clock, not a server clock that runs fast", async () => { + await db.update(nativeRunFinalizations).set({ + leaseOwner: "other-server:resume:live", leaseExpiresAt: new Date(Date.now() + 2 * 60_000), nextAttemptAt: null, + }).where(eq(nativeRunFinalizations.runId, runId)); + await expect(claimNativeSessionResumptions({ db, runnerInstanceId: "fast-clock", runIds: [runId], + now: new Date(Date.now() + 10 * 60_000) })).resolves.toEqual([]); + await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toEqual([ + expect.objectContaining({ leaseOwner: "other-server:resume:live", nextAttemptAt: null }), + ]); + }); + it("dispatches the persisted run id and lease to the live same-run resume consumer", async () => { await db.update(nativeRunFinalizations).set({ phase: "retryable_failure", diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 00a84fe64f9..b164039d5bc 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -77,6 +77,7 @@ export const FRESH_CONTINUATION_PENDING_BOUND_MS = 10 * 60_000; export const FRESH_CONTINUATION_PROGRESS_BOUND_MS = 15 * 60_000; /** Saved-work recovery that is running and advancing keeps its own budget. */ export const FRESH_CONTINUATION_SAVED_WORK_BOUND_MS = 30 * 60_000; +export const NATIVE_INTERRUPTION_NO_PROGRESS_DEADLINE_MS = FRESH_CONTINUATION_PROGRESS_BOUND_MS; const EXACT_RECOVERY_RECHECK_MS = 60_000; /** diff --git a/server/src/services/heartbeat/retries.test.ts b/server/src/services/heartbeat/retries.test.ts index b12e2f0374d..f5798b7555a 100644 --- a/server/src/services/heartbeat/retries.test.ts +++ b/server/src/services/heartbeat/retries.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { Db } from "@paperclipai/db"; -import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRunEvents, heartbeatRuns, issues, projects, projectWorkspaces } from "@paperclipai/db"; +import { activityLog, agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRunEvents, heartbeatRuns, issues, projects, projectWorkspaces } from "@paperclipai/db"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { writePaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/server-utils"; @@ -260,6 +260,7 @@ describePostgres("heartbeat retry module database wiring", () => { afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); afterEach(async () => { + await db.delete(activityLog); await db.delete(issues); await db.delete(heartbeatRunEvents); await db.delete(heartbeatRuns); @@ -387,6 +388,50 @@ describePostgres("heartbeat retry module database wiring", () => { expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(3); }); + it("a lasting outage gets at most the task's hourly cap of automatic attempts across the sweep and the transient lane", async () => { + const { run, agent, issue, companyId } = await fixture(); + const minutesAgo = (minutes: number) => new Date(now.getTime() - minutes * 60_000); + const failed = { ...run, errorCode: "repository_catalog_unavailable", + resultJson: { errorFamily: "transient_upstream", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }; + const priorTransient = (minutes: number) => db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, scopeKind: "issue", + issueId: issue.id, status: "failed", errorCode: "repository_catalog_unavailable", scheduledRetryReason: "transient_failure", + scheduledRetryAttempt: 1, contextSnapshot: { issueId: issue.id }, createdAt: minutesAgo(minutes), finishedAt: minutesAgo(minutes) }); + for (const minutes of [50, 40, 30, 20, 10]) await priorTransient(minutes); + expect(await boundRetries().retries.scheduleBoundedRetryForRun(failed, agent, { now, random: () => 0 })) + .toMatchObject({ outcome: "scheduled" }); + await db.delete(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.retryOfRunId, run.id))); + await db.insert(activityLog).values({ companyId, actorType: "system", actorId: "system", action: "issue.continued_automatically", + entityType: "issue", entityId: issue.id, details: {}, createdAt: minutesAgo(55) }); + expect(await boundRetries().retries.scheduleBoundedRetryForRun(failed, agent, { now, random: () => 0 })) + .toMatchObject({ outcome: "retry_exhausted" }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run.id))).toHaveLength(0); + }); + + it("re-checks the task's hourly cap under the task lock when the sweep records an attempt while the retry is being prepared", async () => { + const { run, agent, issue, companyId } = await fixture(); + const minutesAgo = (minutes: number) => new Date(now.getTime() - minutes * 60_000); + const failed = { ...run, errorCode: "repository_catalog_unavailable", + resultJson: { errorFamily: "transient_upstream", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }; + for (const minutes of [50, 40, 30, 20]) await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, scopeKind: "issue", + issueId: issue.id, status: "failed", errorCode: "repository_catalog_unavailable", scheduledRetryReason: "transient_failure", + scheduledRetryAttempt: 1, contextSnapshot: { issueId: issue.id }, createdAt: minutesAgo(minutes), finishedAt: minutesAgo(minutes) }); + const sweepContinuation = (createdAt: Date) => db.insert(activityLog).values({ companyId, actorType: "system", actorId: "system", + action: "issue.continued_automatically", entityType: "issue", entityId: issue.id, details: {}, createdAt }); + await sweepContinuation(minutesAgo(55)); + const { deps } = boundRetries(); + const racing = extracted.createHeartbeatRetries(db, { ...deps, getAgentInvokability: async (target: Agent | null | undefined) => { + await db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, issue.id)).for("update"); + await tx.insert(activityLog).values({ companyId, actorType: "system", actorId: "system", + action: "issue.continued_automatically", entityType: "issue", entityId: issue.id, details: {}, createdAt: minutesAgo(1) }); + }); + return deps.getAgentInvokability(target); + } }); + expect(await racing.scheduleBoundedRetryForRun(failed, agent, { now, random: () => 0 })) + .toMatchObject({ outcome: "retry_exhausted" }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run.id))).toHaveLength(0); + }); + it("keeps the accepted interaction owner and its catalog provider deadline", async () => { const { run, agent } = await fixture(); const retryAt = new Date(now.getTime() + 3600_000); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 0f19daa43cb..ec839348aee 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -70,6 +70,7 @@ import { logger } from "../../middleware/logger.js"; import { parseObject } from "../../adapters/utils.js"; import { logActivity } from "../activity-log.js"; import { readContinuationAttempt } from "../recovery/index.js"; +import { transientRetryWithinTaskCap } from "../recovery/task-automatic-attempts.js"; import { withRecoveryContext } from "../recovery/status-only-context.js"; import { MAX_TURN_CONTINUATION_RETRY_REASON, @@ -602,7 +603,11 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen // a chain whose real failures already spent the failure budget either. const failureBudgetSpent = interruptedPreparationLane && opts?.maxAttempts == null && executionFailureRetryCount(run) >= BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS; - const computedBaseSchedule = failureBudgetSpent ? null + const transientIssueId = retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON && opts?.maxAttempts == null + ? readNonEmptyString(run.contextSnapshot?.issueId) : null; + const taskCapSpent = transientIssueId !== null + && !(await transientRetryWithinTaskCap(db, run.companyId, transientIssueId, now)); + const computedBaseSchedule = failureBudgetSpent || taskCapSpent ? null : opts?.delayMs != null ? nextAttempt <= maxAttempts ? { @@ -646,7 +651,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen // issue binding: that could either violate the FK or misclassify history. const issueId = run.scopeKind === "issue" ? run.issueId : null; - if (!baseSchedule) { + const reportRetryExhausted = async () => { const exhaustion = { retryReason, scheduledRetryAttempt: consumedAttempts, @@ -682,7 +687,8 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen attempt: nextAttempt, maxAttempts, }; - } + }; + if (!baseSchedule) return reportRetryExhausted(); if (await legacyExecutionNeedsReconciliationWithEvidence(db, run)) { return { @@ -874,6 +880,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen run: typeof heartbeatRuns.$inferSelect; reusedExisting: boolean; } + | { outcome: "task_cap_spent" } | { outcome: "not_scheduled"; reason: string; @@ -922,6 +929,14 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen run: existing, reusedExisting: true, }; + if (transientIssueId !== null) { + if (transientIssueId !== issueId) + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${transientIssueId} for update`, + ); + if (!(await transientRetryWithinTaskCap(tx as unknown as Db, run.companyId, transientIssueId, now))) + return { outcome: "task_cap_spent" }; + } } if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { if (issueId) { @@ -1410,6 +1425,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }, ); + if (scheduleResult.outcome === "task_cap_spent") return reportRetryExhausted(); if (scheduleResult.outcome === "not_scheduled") { await appendRunEvent(run, { eventType: "lifecycle", diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 350c7684613..8da95115248 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -672,6 +672,29 @@ export function issueTreeControlService(db: Db) { return null; } + async function listActivePauseHeldIssueIds(companyId: string, issueIds: string[]): Promise> { + if (issueIds.length === 0) return new Set(); + const [anyHold] = await db.select({ id: issueTreeHolds.id }).from(issueTreeHolds).where(and( + eq(issueTreeHolds.companyId, companyId), eq(issueTreeHolds.status, "active"), eq(issueTreeHolds.mode, "pause"))).limit(1); + if (!anyHold) return new Set(); + const result = await db.execute<{ issueId: string }>(sql` + with recursive chain(start_id, id, depth, path) as ( + select requested.id, requested.id, 1, array[requested.id] + from unnest(${sql.param(issueIds)}::uuid[]) as requested(id) + union all + select chain.start_id, parent.parent_id, chain.depth + 1, chain.path || parent.parent_id + from chain + join issues parent on parent.id = chain.id and parent.company_id = ${companyId} + where parent.parent_id is not null and chain.depth < ${MAX_PAUSE_HOLD_ANCESTOR_DEPTH} + and not parent.parent_id = any(chain.path) + ) + select distinct chain.start_id::text as "issueId" from chain + where exists (select 1 from issue_tree_holds hold where hold.company_id = ${companyId} + and hold.status = 'active' and hold.mode = 'pause' and hold.root_issue_id = chain.id)`); + const rows = Array.isArray(result) ? result : (result as { rows?: Array<{ issueId: string }> }).rows ?? []; + return new Set(rows.map(row => row.issueId)); + } + async function preview( companyId: string, rootIssueId: string, @@ -1348,6 +1371,7 @@ export function issueTreeControlService(db: Db) { getHold, listHolds, getActivePauseHoldGate, + listActivePauseHeldIssueIds, releaseHold, findOwedPauseResume, cancelUnclaimedWakeupsForTree, diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index 0b638f5522e..0f9d07e25b0 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -158,6 +158,7 @@ export async function claimNativeSessionResumptions(input: { let terminalRunToEmit: typeof heartbeatRuns.$inferSelect | null = null; let claimFailed = false; let claimedAttempt = 0; + let leaseLiveByDatabaseClock = false; const claimed = await input.db.transaction(async (tx) => { await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`); const [issue] = await tx.select().from(issues).where(and( @@ -331,8 +332,13 @@ export async function claimNativeSessionResumptions(input: { }).where(and( eq(nativeRunFinalizations.runId, row.run.id), eq(nativeRunFinalizations.phase, row.coordinator.phase), + or(isNull(nativeRunFinalizations.leaseOwner), isNull(nativeRunFinalizations.leaseExpiresAt), + sql`${nativeRunFinalizations.leaseExpiresAt} <= clock_timestamp()`), )).returning({ runId: nativeRunFinalizations.runId }).then((rows) => rows[0] ?? null); - if (!updated) return false; + if (!updated) { + leaseLiveByDatabaseClock = true; + return false; + } claimedAttempt = row.coordinator.attempt; await tx.update(heartbeatRuns).set({ status: "running", @@ -360,7 +366,7 @@ export async function claimNativeSessionResumptions(input: { void reportRunFailure(input.db, terminalRunToEmit); } if (claimed) claims.push({ runId: candidate.runId, leaseOwner, attempt: claimedAttempt }); - else if (claimFailed || input.deferIneligible !== false) await input.db.transaction(async tx => { + else if (!leaseLiveByDatabaseClock && (claimFailed || input.deferIneligible !== false)) await input.db.transaction(async tx => { await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`); await notifyLiveRunRecovery(tx, { companyId: candidate.companyId, runId: candidate.runId }); await tx.update(nativeRunFinalizations).set({ diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index cd63032c87a..d2ae6c18e23 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -55,6 +55,7 @@ import { nativeControllerAuthorityDigest, isNativeRunnerOwnershipHeld, readNativ import { nativeRestartContinuationInFlight, nativeRestartInterruptedTurnId, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { retireSupersededNativeRetry } from "./native-superseded-retry.js"; import { legacyWorkspaceRecoveryNotMigrated } from "../legacy-workspace-recovery-migration.js"; +import { NATIVE_INTERRUPTION_NO_PROGRESS_DEADLINE_MS } from "../fresh-continuation-fallback.js"; import { requestNativeStartupCleanup, readNativeStartupContext, promoteNativeStartupCleanup } from "./native-startup-cleanup.js"; import { createSystemRecovery, failSystemRecovery, readSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure, type SystemRecoveryState } from "../system-recovery-state.js"; @@ -202,6 +203,16 @@ type RecoveryClaimProgressEntry = { disposition?: unknown; progressWatermarkSeq?: unknown; progressAttempt?: unknown; noProgressRestarts?: unknown; }; +function previousInterruptionStartedAt(recoveryHistory: ReadonlyArray): number | null { + for (let index = recoveryHistory.length - 1; index >= 0; index -= 1) { + const entry = recoveryHistory[index]; + if (!RECOVERY_CLAIM_DISPOSITIONS.has(entry?.disposition)) continue; + const at = entry?.interruptionStartedAt; + return typeof at === "number" && Number.isFinite(at) && at > 0 ? at : null; + } + return null; +} + function previousNoProgressRestarts(recoveryHistory: ReadonlyArray): number { for (let index = recoveryHistory.length - 1; index >= 0; index -= 1) { const entry = recoveryHistory[index]; @@ -1784,10 +1795,16 @@ export async function claimNativeRestartRecoveries(input: { // Read before this claim appends its own history entry. const cleanDrainSuspension = suspendedByCleanDrain(row.coordinator); + const progressCredit = await restartResumeKeepsAttempt(tx, { runId: row.run.id, coordinator: row.coordinator }); const restartCredit = cleanDrainSuspension ? { keepsAttempt: true, noProgressRestarts: previousNoProgressRestarts(row.coordinator.recoveryHistory) } - : await restartResumeKeepsAttempt(tx, { runId: row.run.id, coordinator: row.coordinator }); + : progressCredit; const resumeKeepsAttempt = restartCredit.keepsAttempt; + const interruptionProgressed = progressCredit.noProgressRestarts === 0; + const priorInterruptionStart = previousInterruptionStartedAt(row.coordinator.recoveryHistory); + const interruptionStartedAt = !interruptionProgressed && priorInterruptionStart !== null + ? priorInterruptionStart : now.getTime(); + const interruptionExpired = now.getTime() - interruptionStartedAt >= NATIVE_INTERRUPTION_NO_PROGRESS_DEADLINE_MS; const classification = classifyNativeRunnerRecoveryEvidence({ remoteSandbox: remoteSandbox || Boolean(pluginTask && durableAuthority) || portablePreparation, unclaimedBootstrap: unclaimedBootstrap || portablePreparation && row.coordinator.attempt === 0, @@ -1834,10 +1851,11 @@ export async function claimNativeRestartRecoveries(input: { const hold = admission.exhausted; const preserveExecutionBinding = hold !== null; - const claimKind = hold || ownershipChanged || exhaustedRemoteRetry ? null : classification.claimKind; + const claimKind = hold || ownershipChanged || exhaustedRemoteRetry || interruptionExpired ? null : classification.claimKind; const reason = hold?.reason ?? (ownershipChanged ? "task_ownership_or_status_changed" - : exhaustedRemoteRetry ? "execution_recovery_budget_exhausted" : classification.reason); + : exhaustedRemoteRetry ? "execution_recovery_budget_exhausted" + : interruptionExpired ? "native_interruption_deadline_exceeded" : classification.reason); // No person is asked to inspect or reconcile: unless the cause is the // person's, the fresh-continuation fallback continues the task. // Shown to people: plain words only. A person's Stop or pause keeps the @@ -2028,6 +2046,7 @@ export async function claimNativeRestartRecoveries(input: { recoveryRequestId: input.recoveryRequestId ?? null, recoveryHistory: appendBoundedRecoveryHistory({ ...event, progressWatermarkSeq, progressAttempt, noProgressRestarts: restartCredit.noProgressRestarts, + interruptionStartedAt, ...(attemptKeptForProgress ? { attemptKeptForProgress } : {}), }), failureCode: null, diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index cbbe0422a00..12f75658633 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -4070,6 +4070,93 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); { eventType: "turn.accepted", turnId: `past-limit-${round}` }, { eventType: "tool.execution.completed", turnId: `past-limit-${round}` }]; + describe("one deadline per interruption across server generations", () => { + const MINUTE = 60_000; + async function claimAt(fixture: Awaited>, at: Date) { + const [disposition] = await claimNativeRestartRecoveries({ + db: fixture.db, controller: successor, restartKind: "hard", runIds: [fixture.runId], now: at, + }); + if (disposition?.kind === "resume_dead_runner") { + await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: randomUUID(), controllerPid: 2_000_000_001 }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + } + return disposition?.kind === "resume_dead_runner" ? disposition.kind + : (disposition as { reason?: string } | undefined)?.reason ?? disposition?.kind; + } + + it("ends a run whose generations show no progress at the deadline from the first interruption, not per generation", async () => { + const fixture = await seedInterruptedCheckpoint("DEADLINE-NO-PROGRESS"); + const start = Date.now(); + const kinds = []; + for (const minutes of [0, 7, 14]) { + await appendRunEvents(fixture, providerNeverStarted); + kinds.push(await claimAt(fixture, new Date(start + minutes * MINUTE))); + } + await appendRunEvents(fixture, providerNeverStarted); + kinds.push(await claimAt(fixture, new Date(start + 16 * MINUTE))); + expect(kinds).toEqual(["resume_dead_runner", "resume_dead_runner", "resume_dead_runner", "native_interruption_deadline_exceeded"]); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_restart_recovery_blocked", leaseOwner: null }); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run).toMatchObject({ status: "failed", errorCode: "native_restart_recovery_blocked" }); + }); + + it("restarts the clock when a generation's resume made provider progress", async () => { + const fixture = await seedInterruptedCheckpoint("DEADLINE-PROGRESS-RESETS"); + const start = Date.now(); + expect(await claimAt(fixture, new Date(start))).toBe("resume_dead_runner"); + await appendRunEvents(fixture, providerNeverStarted); + expect(await claimAt(fixture, new Date(start + 10 * MINUTE))).toBe("resume_dead_runner"); + await appendRunEvents(fixture, providerProgressed(1)); + expect(await claimAt(fixture, new Date(start + 20 * MINUTE))).toBe("resume_dead_runner"); + await appendRunEvents(fixture, providerNeverStarted); + expect(await claimAt(fixture, new Date(start + 30 * MINUTE))).toBe("resume_dead_runner"); + await appendRunEvents(fixture, providerNeverStarted); + expect(await claimAt(fixture, new Date(start + 36 * MINUTE))).toBe("native_interruption_deadline_exceeded"); + }); + + it("starts the clock at the first claim for a coordinator saved before the deadline existed", async () => { + const fixture = await seedInterruptedCheckpoint("DEADLINE-OLD-ROW"); + const start = Date.now(); + expect(await claimAt(fixture, new Date(start))).toBe("resume_dead_runner"); + await fixture.db.update(nativeRunFinalizations).set({ + recoveryHistory: sql`jsonb_set(${nativeRunFinalizations.recoveryHistory}, + array[(jsonb_array_length(${nativeRunFinalizations.recoveryHistory}) - 1)::text], + (${nativeRunFinalizations.recoveryHistory} -> -1) - 'interruptionStartedAt')`, + }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + await appendRunEvents(fixture, providerNeverStarted); + expect(await claimAt(fixture, new Date(start + 60 * MINUTE))).toBe("resume_dead_runner"); + expect(await lastClaimEntry(fixture)).toMatchObject({ interruptionStartedAt: start + 60 * MINUTE }); + }); + + async function suspendByGracefulDrain(fixture: Awaited>, at: Date) { + await fixture.db.update(nativeRunFinalizations).set({ + recoveryHistory: sql`${nativeRunFinalizations.recoveryHistory} || ${JSON.stringify([{ disposition: "restart_suspended", at: at.toISOString() }])}::jsonb`, + }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + } + + it("resumes a run that progressed for 20 minutes before a graceful drain suspended it", async () => { + const fixture = await seedInterruptedCheckpoint("DEADLINE-DRAIN-PROGRESS"); + const start = Date.now(); + expect(await claimAt(fixture, new Date(start))).toBe("resume_dead_runner"); + await appendRunEvents(fixture, providerProgressed(1)); + await suspendByGracefulDrain(fixture, new Date(start + 20 * MINUTE)); + expect(await claimAt(fixture, new Date(start + 21 * MINUTE))).toBe("resume_dead_runner"); + expect(await lastClaimEntry(fixture)).toMatchObject({ interruptionStartedAt: start + 21 * MINUTE }); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run!.errorCode).not.toBe("native_restart_recovery_blocked"); + }); + + it("still ends a run with no provider progress at the deadline when a graceful drain suspended it", async () => { + const fixture = await seedInterruptedCheckpoint("DEADLINE-DRAIN-NO-PROGRESS"); + const start = Date.now(); + expect(await claimAt(fixture, new Date(start))).toBe("resume_dead_runner"); + await appendRunEvents(fixture, providerNeverStarted); + await suspendByGracefulDrain(fixture, new Date(start + 15 * MINUTE)); + expect(await claimAt(fixture, new Date(start + 16 * MINUTE))).toBe("native_interruption_deadline_exceeded"); + }); + }); + it("does not charge a restart past the generation limit whose resume never started the provider", async () => { const fixture = await seedPastGenerationLimit("PAST-LIMIT-UNSTARTED"); expect(await resumeThenCrash(fixture)).toMatchObject({ diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 703348cb5f7..c362330a984 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -18,7 +18,7 @@ import { } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; -import { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import { evaluateAgentInvokability } from "../agent-invokability.js"; import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; @@ -31,6 +31,7 @@ import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js"; import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION, taskAutomaticAttempts } from "./task-automatic-attempts.js"; import { STRANDED_CONTINUATION_MAX_AGE_MS, STRANDED_KIND, strandedHoldContinuesAutomatically, type StrandedHoldRun } from "./stranded-hold-continuation.js"; import { continuationActivityMessage, @@ -39,7 +40,6 @@ import { CONTINUATION_REFUSED_MESSAGE, CONTINUATION_REVIEW_MESSAGE, evaluateAssignedWork, - CONTINUATION_CAP_WINDOW_MS, STALE_BLOCK_MS, NATIVE_STALL_MS, type AssignedWorkFacts, @@ -50,7 +50,7 @@ import { /** Activity action recorded for every automatic continuation. It is also the * durable claim two servers serialize on, and the per-task rate-limit ledger. */ -export const ASSIGNED_WORK_CONTINUED_ACTION = "issue.continued_automatically"; +export { ASSIGNED_WORK_CONTINUED_ACTION }; export const ASSIGNED_WORK_CONTINUATION_SOURCE = "issue.assigned_work_invariant"; export const STRANDED_TRANSIENT_CONTINUED_OUTCOME = "stranded_transient_continued"; /** Repairs per process per minute, so a deploy with many violations does not @@ -286,11 +286,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, strandedRunIds))) : []) .map(({ id, ...run }) => [id, run])); const agentsById = new Map(); + const companyAgents = agentRows.length ? await database.select({ id: agents.id, companyId: agents.companyId, name: agents.name, + reportsTo: agents.reportsTo, status: agents.status }).from(agents).where(eq(agents.companyId, companyId)) : []; for (const agent of agentRows as Array) { // An agent still being set up, or held by a lifecycle hold, takes no work // yet; its setup or release is the next move, not a continuation. const ready = !isAgentAwaitingSetup(agent) && (agent.lifecycleHolds ?? []).length === 0; - agentsById.set(agent.id, { ...agent, invokable: ready && (await evaluateAgentInvokabilityFromDb(database, agent)).invokable }); + agentsById.set(agent.id, { ...agent, invokable: ready && evaluateAgentInvokability(agent, companyAgents).invokable }); } const company = companyRows[0]; const companyBudget = company?.pauseReason === "budget" || company?.pauseReason === "budget_exceeded"; @@ -329,13 +331,21 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const blockActivityByIssue = new Map(rowsOf<{ issueId: string; actorType: string; actorId: string; agentId: string | null; details: unknown }>(blockActivity) .map(row => [row.issueId, row])); + const ownedRunsByIssue = new Map(); for (const issue of issueRows) { - const agent = issue.assigneeAgentId ? agentsById.get(issue.assigneeAgentId) ?? null : null; - const runs = activeRuns.filter(run => run.issueId === issue.id) as RunRow[]; - const ownedRuns = []; - for (const run of runs) { + const ownedRuns: AssignedWorkFacts["activeRuns"] = []; + for (const run of activeRuns.filter(row => row.issueId === issue.id) as RunRow[]) { ownedRuns.push({ id: run.id, status: run.status, scheduledRetryAt: run.scheduledRetryAt, owner: stalledRunIds.has(run.id) && run.status === "running" ? "stalled" : await runOwner(run, at, proveOwners) }); } + ownedRunsByIssue.set(issue.id, ownedRuns); + } + const pauseCheckIds = issueRows.filter(issue => !ownedRunsByIssue.get(issue.id)!.some(run => run.owner !== "gone") + && !(wakeCounts.get(issue.id) ?? 0)).map(issue => issue.id); + const pauseHeldIds = await treeControl.listActivePauseHeldIssueIds(companyId, pauseCheckIds); + + for (const issue of issueRows) { + const agent = issue.assigneeAgentId ? agentsById.get(issue.assigneeAgentId) ?? null : null; + const ownedRuns = ownedRunsByIssue.get(issue.id)!; const latest = latestByIssue.get(issue.id) ?? null; const latestFinishedAt = latest?.finishedAt ? new Date(latest.finishedAt) : null; const leaseUntil = latest?.controllerLeaseExpiresAt ? new Date(latest.controllerLeaseExpiresAt) : null; @@ -344,10 +354,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const interruption = interruptions.get(issue.id) ?? null; const executionState = json(issue.executionState); const blockedEntry = blockActivityByIssue.get(issue.id); - // The tree pause gate walks ancestors; read it only when nothing else - // already gives the task a path. - const needsPauseCheck = !ownedRuns.some(run => run.owner !== "gone") && !(wakeCounts.get(issue.id) ?? 0); - const pauseHold = needsPauseCheck && Boolean(await treeControl.getActivePauseHoldGate(companyId, issue.id)); + const pauseHold = pauseHeldIds.has(issue.id); result.set(issue.id, { issue: { status: issue.status, assigneeAgentId: issue.assigneeAgentId, assigneeUserId: issue.assigneeUserId, @@ -582,11 +589,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const verdict = evaluateAssignedWork(fresh, at); if (verdict.kind !== "violation" || verdict.stale || verdict.reason === "run_owner_gone" || verdict.reason === "native_run_stalled" || (current.status === "blocked" && fresh.activeRecoveryAction && !fresh.activeRecoveryAction.continuedBySweep)) return { kind: "changed" as const }; - const recent = await tx.select({ createdAt: activityLog.createdAt }).from(activityLog).where(and( - eq(activityLog.companyId, issue.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), - eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), - gte(activityLog.createdAt, new Date(at.getTime() - CONTINUATION_CAP_WINDOW_MS)))).orderBy(desc(activityLog.createdAt)).limit(64); - const allowance = continuationAllowance(recent.map(row => row.createdAt), at); + const attempts = await taskAutomaticAttempts(tx as unknown as Db, issue.companyId, issue.id, at); + const recent = attempts.continuations; + const allowance = continuationAllowance(attempts.all, at); if (!allowance.allowed) return { kind: allowance.capped ? "capped" as const : "spaced" as const, retryAt: allowance.retryAt }; let restoredTo: Claimed["restoredTo"] = null; let restoredStatusVersion: number | null = null; @@ -617,7 +622,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { ...(restoredTo ? { status: restoredTo, previousStatus: "blocked" } : {}), attempt: recent.length + 1 }, }, publications); - const next = continuationAllowance([at, ...recent.map(row => row.createdAt)], at); + const next = continuationAllowance([at, ...attempts.all], at); return { kind: "claimed" as const, activityId: activity.id, publications, verdict, reason: verdict.reason, restoredTo, restoredStatusVersion, blockedTransitionAt: current.blockedTransitionAt, unblockDescriptor: current.unblockDescriptor, blockedOwnerNotifiedAt: current.blockedOwnerNotifiedAt, diff --git a/server/src/services/recovery/task-automatic-attempts.ts b/server/src/services/recovery/task-automatic-attempts.ts new file mode 100644 index 00000000000..f84b53bdb5a --- /dev/null +++ b/server/src/services/recovery/task-automatic-attempts.ts @@ -0,0 +1,27 @@ +import { and, desc, eq, gte, inArray, sql } from "drizzle-orm"; +import { activityLog, heartbeatRuns, type Db } from "@paperclipai/db"; +import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js"; + +export const ASSIGNED_WORK_CONTINUED_ACTION = "issue.continued_automatically"; +export const SCHEDULER_TRANSIENT_RETRY_REASON = "transient_failure"; + +export async function taskAutomaticAttempts(db: Db, companyId: string, issueId: string, at: Date) { + const since = new Date(at.getTime() - CONTINUATION_CAP_WINDOW_MS); + const [continuations, transientRetries] = await Promise.all([ + db.select({ createdAt: activityLog.createdAt }).from(activityLog).where(and( + eq(activityLog.companyId, companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since))) + .orderBy(desc(activityLog.createdAt)).limit(64), + db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + inArray(heartbeatRuns.scheduledRetryReason, [SCHEDULER_TRANSIENT_RETRY_REASON]), gte(heartbeatRuns.createdAt, since))) + .orderBy(desc(heartbeatRuns.createdAt)).limit(64), + ]); + const all = [...continuations, ...transientRetries].map(row => row.createdAt); + return { continuations: continuations.map(row => row.createdAt), all }; +} + +export async function transientRetryWithinTaskCap(db: Db, companyId: string, issueId: string, at: Date) { + const attempts = await taskAutomaticAttempts(db, companyId, issueId, at); + return attempts.continuations.length === 0 || attempts.all.length < CONTINUATION_CAP; +}