diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 8cf3d65df14..b7c82ae21d5 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -10,6 +10,7 @@ import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./ex import { getExecutionBlocker } from "./execution-blocker.js"; import { FRESH_CONTINUATIONS_PER_HOUR, FRESH_CONTINUATION_SCHEMA, PERSON_OWNED_FAILURE_CODES, continueHeldExecutions, ensureFreshContinuation, + resetBacklogContinuationPacingForTests, freshContinuationNote, heldExecutionsAwaitingFreshContinuation, nextFreshContinuationCheckAt, recordFreshContinuationCheck, FRESH_CONTINUATION_PENDING_BOUND_MS, FRESH_CONTINUATION_PROGRESS_BOUND_MS, FRESH_CONTINUATION_SAVED_WORK_BOUND_MS, } from "./fresh-continuation-fallback.js"; import { heartbeatService } from "./heartbeat.js"; @@ -55,6 +56,7 @@ suite("fresh-continuation fallback", () => { }, 60_000); afterEach(async () => { + resetBacklogContinuationPacingForTests(db); await db.execute(sql`truncate table issue_recovery_actions, native_run_finalizations, tool_invocations, issue_approvals, approvals, issue_thread_interactions, activity_log, agent_wakeup_requests, environment_leases, execution_workspaces, heartbeat_runs, issues cascade`); }); @@ -240,6 +242,55 @@ suite("fresh-continuation fallback", () => { await expectOneFreshSuccessor(f); }, 60_000); + /** A held record from an older release (settled blocked), its source run ended `ageMs` ago with `errorCode`. */ + async function heldBacklog(errorCode: string, ageMs: number) { + const f = await unfinishedRun({ runtime: "native", errorCode, hold: "settled_blocked" }); + await db.update(heartbeatRuns).set({ finishedAt: new Date(Date.now() - ageMs) }).where(eq(heartbeatRuns.id, f.runId)); + return f; + } + const continuedFrom = async (runId: string) => db.select().from(issueRecoveryActions) + .where(sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${runId}`); + const evidenceOf = async (actionId: string) => + (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, actionId)))[0]!; + + it("continues only recent interruptions from the held backlog, newest first, and leaves the rest untouched (staging herd)", async () => { + // paperclip.staging, 22:53Z: the first release with the fallback continued 176 held tasks in 30 minutes; + // 74 had a source run that ended more than 7 days ago, many with configuration or provider failures. + const HOUR = 60 * 60_000; + const recentOld = await heldBacklog("native_restart_recovery_blocked", 6 * HOUR); + const recentNew = await heldBacklog("native_continuation_requires_reconciliation", 1 * HOUR); + const weekOld = await heldBacklog("native_restart_recovery_blocked", 3 * 24 * HOUR); + const monthOld = await heldBacklog("native_session_interrupted", 28 * 24 * HOUR); + const providerFailure = await heldBacklog("runner_remote_provider_artifact_incompatible", 2 * HOUR); + const adapterFailure = await heldBacklog("adapter_failed", 2 * HOUR); + const untouched = [weekOld, monthOld, providerFailure, adapterFailure]; + const before = await Promise.all(untouched.map(f => evidenceOf(f.actionId!))); + + expect(await continueHeldExecutions(db, new Date(), 1)).toMatchObject({ continued: 1 }); + expect(await continuedFrom(recentNew.runId)).toHaveLength(1); + expect(await continuedFrom(recentOld.runId)).toHaveLength(0); + for (let pass = 0; pass < 3; pass += 1) await continueHeldExecutions(db, new Date()); + expect(await continuedFrom(recentOld.runId)).toHaveLength(1); + for (const [index, f] of untouched.entries()) { + expect(await continuedFrom(f.runId)).toHaveLength(0); + const after = await evidenceOf(f.actionId!); + expect({ status: after.status, evidence: after.evidence, updatedAt: after.updatedAt }) + .toEqual({ status: before[index]!.status, evidence: before[index]!.evidence, updatedAt: before[index]!.updatedAt }); + } + }, 120_000); + + it("paces the held backlog to ten continuations a minute per server", async () => { + const records = []; + for (let index = 0; index < 12; index += 1) records.push(await heldBacklog("native_restart_recovery_blocked", (index + 1) * 60_000)); + const now = new Date(); + for (let pass = 0; pass < 4; pass += 1) await continueHeldExecutions(db, now); + const started = (await Promise.all(records.map(f => continuedFrom(f.runId)))).filter(rows => rows.length > 0).length; + expect(started).toBe(10); + // The next minute has room again. + await continueHeldExecutions(db, new Date(now.getTime() + 61_000)); + expect((await Promise.all(records.map(f => continuedFrom(f.runId)))).filter(rows => rows.length > 0).length).toBe(12); + }, 180_000); + const sweepWake = (issueId: string) => heartbeatService(db).wakeup(agentId, { source: "automation", triggerDetail: "system", reason: "issue_continuation_needed", payload: { issueId }, requestedByActorType: "system", requestedByActorId: "idle-task-sweep", contextSnapshot: { issueId, taskId: issueId, wakeReason: "issue_continuation_needed" } }); @@ -713,6 +764,19 @@ suite("fresh-continuation fallback", () => { expect(await executionControlSchedule(db).disposition()).toBeNull(); }, 60_000); + it("schedules no disposition scan for a held record the backlog pass leaves alone, and still does for an eligible one", async () => { + const overdueCheck = (actionId: string) => db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + freshContinuationCheck: { kind: "backoff", checkedAt: Date.now() - 7_200_000, nextCheckAt: Date.now() - 3_600_000 } })}::jsonb` }) + .where(eq(issueRecoveryActions.id, actionId)); + const old = await heldBacklog("native_restart_recovery_blocked", 3 * 24 * 60 * 60_000); + await overdueCheck(old.actionId!); + expect(await nextFreshContinuationCheckAt(db)).toBeNull(); + expect(await executionControlSchedule(db).disposition()).toBeNull(); + const recent = await heldBacklog("native_restart_recovery_blocked", 60 * 60_000); + await overdueCheck(recent.actionId!); + expect(await nextFreshContinuationCheckAt(db)).not.toBeNull(); + }, 60_000); + it("tells the agent the previous attempt was cut off and which actions are unverified", async () => { const f = await unfinishedRun({ runtime: "native", errorCode: "native_session_interrupted" }); await db.insert(toolInvocations).values({ companyId, agentId, issueId: f.issueId, runId: f.runId, toolName: "github.create_pull_request", diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 27b9b3709c9..e5e5ab91c6c 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -665,11 +665,64 @@ export function heldExecutionsAwaitingFreshContinuation(db: Db, now: Date, limit runId: sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')` }) .from(issueRecoveryActions).innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), eq(issues.companyId, issueRecoveryActions.companyId))) - .where(and(heldExecutionWhere(), freshContinuationCheckDue(now))) - .orderBy(asc(issueRecoveryActions.updatedAt), asc(issueRecoveryActions.id)) + .where(and(heldExecutionWhere(), freshContinuationCheckDue(now), recentInterruptionSourceSql(now))) + // Newest interruption first: the most recent work is the work someone is waiting for. + .orderBy(desc(sql`(select source.finished_at from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} + and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId'))`), + asc(issueRecoveryActions.id)) .limit(limit); } +/** + * The backlog pass continues a held record only when it is a recent + * interruption: its source run ended within BACKLOG_CONTINUATION_MAX_AGE_MS + * and ended with a restart, deploy, lease, process-loss or recovery-refusal + * code. A configuration or provider failure, or anything older, stays exactly + * as it is for a person's Retry: re-running last week's failures is not + * "an interruption is invisible". Fresh interruptions reach the fallback + * through restart recovery and the disposition pass, not through this list. + */ +export const BACKLOG_CONTINUATION_MAX_AGE_MS = 24 * 60 * 60_000; +export const BACKLOG_CONTINUATION_SOURCE_CODES = [ + "native_restart_recovery_blocked", "native_continuation_requires_reconciliation", "native_session_interrupted", + "native_runner_process_exited", "server_shutdown_interrupted", "process_lost", "controller_lease_lost", + "execution_finalization_deadline_exceeded", "local_runtime_api_listener_unavailable", + "runner_remote_process_identity_unavailable", "continuation_user_authorization_missing", +] as const; +/** At most this many backlog continuations per server per rolling minute, and per pass. */ +export const BACKLOG_CONTINUATIONS_PER_MINUTE = 10; +const BACKLOG_CONTINUATIONS_PER_PASS = 5; + +/** Holds the disposition pass (unresolvedExecutionDispositions) handles itself, whatever their age. */ +function liveDispositionHoldSql() { + return and(inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"))!; +} + +/** The backlog pass's eligibility: the one predicate its listing and its deadline query share. */ +function recentInterruptionSourceSql(now: Date) { + return sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} + and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') + and source.finished_at > ${new Date(now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz + and source.error_code in (${sql.join(BACKLOG_CONTINUATION_SOURCE_CODES.map(code => sql`${code}`), sql`, `)}))`; +} + +/** Held records the backlog pass leaves for a person (older, or not an interruption). */ +async function countHeldExecutionsLeftAlone(db: Db, now: Date): Promise { + const [row] = await db.select({ count: sql`count(*)::int` }).from(issueRecoveryActions) + .innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), eq(issues.companyId, issueRecoveryActions.companyId))) + .where(and(heldExecutionWhere(), sql`not ${recentInterruptionSourceSql(now)}`)); + return Number(row?.count ?? 0); +} + +/** Per database client: one per server process in production, one per test database in tests. */ +const backlogContinuationTimes = new WeakMap(); +export function resetBacklogContinuationPacingForTests(db?: Db) { + if (db) backlogContinuationTimes.delete(db); + lastLeftAloneLogAt = 0; +} +let lastLeftAloneLogAt = 0; +const LEFT_ALONE_LOG_INTERVAL_MS = 10 * 60_000; + /** Held reconciliation records of a task's latest unfinished run (joined to its task). */ function heldExecutionWhere(options: { includeActive?: boolean } = {}) { return and( @@ -703,7 +756,7 @@ function heldExecutionWhere(options: { includeActive?: boolean } = {}) { } /** The earliest time a deferred check becomes due again, if any. */ -export async function nextFreshContinuationCheckAt(db: Db): Promise { +export async function nextFreshContinuationCheckAt(db: Db, now = new Date()): Promise { // The same records the held-run query and the disposition pass would select; // a deadline on a record nothing would act on never schedules a scan. // That includes active holds that backed off: the disposition pass rechecks @@ -712,7 +765,10 @@ export async function nextFreshContinuationCheckAt(db: Db): Promise>'{freshContinuationCheck,nextCheckAt}' is not null`)); + sql`${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}' is not null`, + // A record the backlog pass leaves alone schedules nothing; active holds + // stay with the disposition pass, which rechecks them when due. + or(liveDispositionHoldSql(), recentInterruptionSourceSql(now)))); return held?.at ? Number(held.at) : null; } @@ -721,13 +777,26 @@ export async function nextFreshContinuationCheckAt(db: Db): Promise= LEFT_ALONE_LOG_INTERVAL_MS) { + lastLeftAloneLogAt = now.getTime(); + const leftAlone = await countHeldExecutionsLeftAlone(db, now).catch(() => null); + if (leftAlone) logger.info({ leftAlone, maxAgeHours: BACKLOG_CONTINUATION_MAX_AGE_MS / 3_600_000 }, + "Held tasks left for a person: older than the backlog window or not an interruption"); + } + // Per-server pacing: a backlog saved by older releases must not start a herd of runs. + let times = backlogContinuationTimes.get(db); + if (!times) backlogContinuationTimes.set(db, times = []); + while (times.length && times[0]! <= now.getTime() - 60_000) times.shift(); + const room = Math.min(limit, BACKLOG_CONTINUATIONS_PER_MINUTE - times.length); + if (room <= 0) return { checked: 0, continued: 0 }; + const candidates = await heldExecutionsAwaitingFreshContinuation(db, now, room); let continued = 0; const seen = new Set(); for (const candidate of candidates) { if (!candidate.runId || seen.has(candidate.runId)) continue; seen.add(candidate.runId); + times.push(now.getTime()); try { const result = await ensureFreshContinuation(db, { companyId: candidate.companyId, runId: candidate.runId, trigger: "held_recovery", now });