diff --git a/server/src/services/continuation-cause.ts b/server/src/services/continuation-cause.ts index f73093de6f8..f9ffd23c748 100644 --- a/server/src/services/continuation-cause.ts +++ b/server/src/services/continuation-cause.ts @@ -64,6 +64,8 @@ const TRANSIENT_SETUP_FAILURE_CODES = [ "workspace_git_scan_timeout", "workspace_git_scan_saturated", "repository_catalog_unavailable", ] as const; +export const SETUP_FAILURE_CODES = ["setup_failed", ...TRANSIENT_SETUP_FAILURE_CODES] as const; + const object = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; diff --git a/server/src/services/fresh-continuation-eligibility.ts b/server/src/services/fresh-continuation-eligibility.ts new file mode 100644 index 00000000000..3ab1ca1582f --- /dev/null +++ b/server/src/services/fresh-continuation-eligibility.ts @@ -0,0 +1,172 @@ +import { and, eq, inArray, isNull, or, sql, type SQL } from "drizzle-orm"; +import { agentWakeupRequests, approvals, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, + type Db } from "@paperclipai/db"; +import { EXECUTION_RECONCILIATION_CAUSES } from "@paperclipai/shared"; +import { PERMANENT_SOURCE_RETRY_CLASSES, UNVERIFIABLE_SOURCE_CODES } from "./system-recovery-state.js"; +import { RESTART_CLASS_FAILURE_CODES, SETUP_FAILURE_CODES, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, + recoveryRunSql } from "./continuation-cause.js"; +import { canonicalExecutionUuid } from "./execution-identity.js"; +import { holdOwnsBlockedStatus } from "./recovery-hold-blocked-status.js"; + +const object = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +/** + * The backlog pass continues a held record only when it is a recent + * interruption: its source run ended within BACKLOG_CONTINUATION_MAX_AGE_MS + * and ended with a restart, deploy, lease, process-loss or recovery-refusal + * code. A configuration or provider failure, or anything older, stays exactly + * as it is for a person's Retry: re-running last week's failures is not + * "an interruption is invisible". Fresh interruptions reach the fallback + * through restart recovery and the disposition pass, not through this list. + */ +export const BACKLOG_CONTINUATION_MAX_AGE_MS = 24 * 60 * 60_000; +export const BACKLOG_CONTINUATION_SOURCE_CODES = RESTART_CLASS_FAILURE_CODES; +type BlockFacts = { holds: Array; + issue: Pick; + run: Pick }; + +export function failureOwnsBlock(holds: BlockFacts["holds"], issue: BlockFacts["issue"], run: BlockFacts["run"]) { + const receipts = holds.filter(hold => hold.evidence.nativeFailureBlock !== undefined || hold.evidence.legacyCancellationBlock !== undefined + || hold.evidence.nativeBootstrapFailureBlock !== undefined); + return (receipts.length ? receipts : holds).some(hold => holdOwnsBlockedStatus(hold, issue, run)) + || receipts.some(hold => [hold.evidence.legacyCancellationBlock, hold.evidence.nativeBootstrapFailureBlock] + .some(block => object(block).runId === run.id && object(block).statusVersion === Number(issue.statusVersion))) + || (!holds.length && issue.lastStatusDecisionId === null && issue.blockedTransitionAt !== null && run.finishedAt !== null + && issue.blockedTransitionAt.getTime() >= run.finishedAt.getTime() - 1_000); +} + +/** Holds the disposition pass (unresolvedExecutionDispositions) handles itself, whatever their age. */ +export function liveDispositionHoldSql() { + return and(inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"))!; +} + +const cancelledAtDispatch = (alias: "newer" | "held") => sql.raw(`coalesce(${alias}.status = 'cancelled' and ${alias}.started_at is null + and ${alias}.error_code = 'execution_reconciliation_required', false)`); +// The issue id is a uuid (a column or a task's id), so each link is compared on +// its own indexed column; a cast of the column would scan every run's context. +const runOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id = ${issueId}::uuid + or ${sql.raw(alias)}.issue_id = ${issueId}::uuid or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; + +const sqlList = (values: readonly string[]) => sql.join(values.map(value => sql`${value}`), sql`, `); + +export function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQL; sourceRunId: SQL }) { + const since = sql`${new Date(input.now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz`; + const source = (condition: SQL) => sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${input.companyId} + and source.id = ${sourceRunKey(input.sourceRunId)} and ${condition})`; + const refusedLater = sql`held.company_id = source.company_id and held.id <> source.id and ${runOfTask("held", input.issueId)} + and (held.created_at, held.id) > (source.created_at, source.id) and ${cancelledAtDispatch("held")}`; + const personAsked = source(sql`(exists (select 1 from ${agentWakeupRequests} retry where retry.company_id = source.company_id + and retry.status = 'deferred_issue_execution' and retry.reason = 'task_retry' and retry.requested_by_actor_type = 'user' + and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at) + or exists (select 1 from ${heartbeatRuns} held where ${refusedLater} + and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`); + const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")} + and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false) + and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`; + // Saved-work recovery proved the run's unsaved files are permanently gone + // (`source_unavailable`, use another strategy). The only way on is a fresh + // run in a new workspace, which is exactly what Retry would do: a person + // adds nothing, so the system takes that step itself (needed-retry is 0%). + // Uncast keys, so the (company, task, evidence runId) index serves the lookup. + const savedFilesLost = sql`exists (select 1 from ${issueRecoveryActions} lost where lost.company_id = source.company_id + and lost.source_issue_id = ${input.issueId} + and (lost.evidence->>'runId' = source.id::text or (lost.evidence->>'runId' is null and lost.evidence->>'sourceRunId' = source.id::text)) + and lost.evidence#>>'{systemRecovery,state}' = 'exhausted' + and lost.evidence#>>'{systemRecovery,failure,code}' in (${sqlList(UNVERIFIABLE_SOURCE_CODES)}) + and lost.evidence#>>'{systemRecovery,failure,retryClass}' in (${sqlList(PERMANENT_SOURCE_RETRY_CLASSES)}))`; + const setupFailedAfterWork = sql`(coalesce(source.result_json#>>'{executionRecovery,providerWorkStarted}', '') <> 'false' + and source.error_code in (${sqlList(SETUP_FAILURE_CODES)}) + and exists (select 1 from ${issueRecoveryActions} saved where saved.company_id = source.company_id + and saved.source_issue_id = ${input.issueId} + and (saved.evidence->>'runId' = source.id::text or (saved.evidence->>'runId' is null and saved.evidence->>'sourceRunId' = source.id::text)) + and saved.evidence ?| array['workspaceRestoreRecovery', 'workspaceRestoreFailure', 'systemRecovery'] + and coalesce(saved.evidence#>>'{systemRecovery,state}', '') <> 'exhausted'))`; + const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource} + or (${continuationCauseIsSystemOwnedSql("source")} and (${savedFilesLost} or ${setupFailedAfterWork}))) + and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); + return { recent, personAsked }; +} + +/** The backlog pass's eligibility: the one predicate its listing and its deadline query share. */ +export function recentInterruptionSourceSql(now: Date) { + const { recent, personAsked } = interruptionEligibility({ now, companyId: sql`${issueRecoveryActions.companyId}`, + issueId: sql`${issueRecoveryActions.sourceIssueId}`, + sourceRunId: heldRunIdSql() }); + return sql`(${recent} or ${personAsked})`; +} + +export async function runsAwaitingSystemContinuation(db: Db, companyId: string, issueIds: string[], runIds: string[], + now = new Date()): Promise> { + if (!issueIds.length || !runIds.length) return new Set(); + const rows = await db.select({ issueId: issueRecoveryActions.sourceIssueId, runId: sql`${heldRunIdSql()}` }) + .from(issueRecoveryActions).innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), + eq(issues.companyId, issueRecoveryActions.companyId))) + .where(and(eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, issueIds), + inArray(heldRunIdSql(), runIds), heldExecutionWhere(), interruptionEligibility({ now, + companyId: sql`${issueRecoveryActions.companyId}`, issueId: sql`${issueRecoveryActions.sourceIssueId}`, + sourceRunId: heldRunIdSql() }).recent, + sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,kind}', 'exact_recovery_pending') = 'exact_recovery_pending'`, + sql`not exists (select 1 from ${issueThreadInteractions} question where question.company_id = ${issueRecoveryActions.companyId} + and question.issue_id = ${issueRecoveryActions.sourceIssueId} and question.status = 'pending')`, + sql`not exists (select 1 from ${issueApprovals} link join ${approvals} approval on approval.id = link.approval_id + and approval.company_id = link.company_id where link.company_id = ${issueRecoveryActions.companyId} + and link.issue_id = ${issueRecoveryActions.sourceIssueId} and approval.status in ('pending', 'revision_requested'))`)); + if (!rows.length) return new Set(); + const candidateIssueIds = [...new Set(rows.map(row => row.issueId))]; + const candidateRunIds = [...new Set(rows.map(row => row.runId))]; + const runKeys = candidateRunIds.filter(id => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id)); + if (!runKeys.length) return new Set(); + const [issueRows, runRows, holdRows] = await Promise.all([ + db.select().from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, candidateIssueIds))), + db.select({ id: heartbeatRuns.id, finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runKeys))), + db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.sourceIssueId, candidateIssueIds), inArray(heldRunIdSql(), candidateRunIds))), + ]); + return new Set(rows.filter(row => { + const issue = issueRows.find(candidate => candidate.id === row.issueId); + const run = runRows.find(candidate => candidate.id === row.runId); + if (!issue || !run) return false; + if (issue.status !== "blocked") return true; + const holds = holdRows.filter(hold => hold.sourceIssueId === row.issueId + && (hold.evidence.runId === row.runId || hold.evidence.sourceRunId === row.runId)); + return failureOwnsBlock(holds, issue, run); + }).map(row => `${row.issueId}:${row.runId}`)); +} + +/** Held reconciliation records of a task's latest unfinished run (joined to its task). */ +export function heldExecutionWhere(options: { includeActive?: boolean } = {}) { + return and( + inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), + sql`${issueRecoveryActions.evidence}->'executionReconciliation' is null`, + sql`${issueRecoveryActions.evidence}->'freshContinuation' is null`, + sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') is not null`, + or( + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, + and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('complete', 'exhausted')`), + options.includeActive ? and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + eq(issueRecoveryActions.kind, "active_run_watchdog")) : undefined, + // Saved-work recovery still under way: checked so its bound is reached + // (exact recovery keeps its first chance inside the fallback). + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence')`, + ), + isNull(issueRecoveryActions.ownerUserId), + sql`${issues.status} not in ('done', 'cancelled')`, + sql`${issues.assigneeAgentId} is not null`, + // Only the task's latest run, unfinished and still its assignee's: other + sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} + and source.id = ${sourceRunKey(heldRunIdSql())} + and source.status in ('failed', 'timed_out', 'interrupted', 'cancelled') and source.finished_at is not null + and source.agent_id = ${issues.assigneeAgentId} + and not exists (select 1 from ${heartbeatRuns} newer where newer.company_id = source.company_id and newer.id <> source.id + and (newer.native_issue_id = ${issues.id} or newer.issue_id = ${issues.id} or newer.context_snapshot->>'issueId' = ${issues.id}::text) + and not ${cancelledAtDispatch("newer")} + and (newer.status in ('queued', 'running', 'scheduled_retry') or (newer.created_at, newer.id) > (source.created_at, source.id))))`, + )!; +} + +export const heldRunIdSql = () => sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')`; +/** The recorded run id as the run's key: the same match as `id::text = text`, but an index lookup. */ +export const sourceRunKey = (runId: SQL) => canonicalExecutionUuid(sql`(${runId})::text`); diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 6deeed02562..9ef35c330c1 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -12,7 +12,7 @@ import { getExecutionBlocker } from "./execution-blocker.js"; import { FRESH_CONTINUATIONS_PER_HOUR, FRESH_CONTINUATION_SCHEMA, PERSON_OWNED_FAILURE_CODES, continueHeldExecutions, ensureFreshContinuation, resetBacklogContinuationPacingForTests, - freshContinuationNote, heldExecutionsAwaitingFreshContinuation, nextFreshContinuationCheckAt, recordFreshContinuationCheck, FRESH_CONTINUATION_PENDING_BOUND_MS, FRESH_CONTINUATION_PROGRESS_BOUND_MS, FRESH_CONTINUATION_SAVED_WORK_BOUND_MS, + continueFreshInTransaction, expediteFreshContinuationChecks, freshContinuationNote, heldExecutionsAwaitingFreshContinuation, runsAwaitingSystemContinuation, nextFreshContinuationCheckAt, recordFreshContinuationCheck, FRESH_CONTINUATION_PENDING_BOUND_MS, FRESH_CONTINUATION_PROGRESS_BOUND_MS, FRESH_CONTINUATION_SAVED_WORK_BOUND_MS, } from "./fresh-continuation-fallback.js"; import { heartbeatService } from "./heartbeat.js"; import { buildExecutionContinuation } from "./execution-continuation.js"; @@ -837,6 +837,78 @@ suite("fresh-continuation fallback", () => { freshContinuation: { isolation: "fresh_isolated_workspace" } }); }, 60_000); + async function setupFailedAfterWork(state: "scheduled" | "complete" | "exhausted") { + const f = await unfinishedRun({ runtime: "native", errorCode: "setup_failed", hold: "settled_blocked" }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!)); + const recovery = { ...createSystemRecovery(f.runId), state, phase: state === "complete" ? "reconcile_effects" : "verify_execution", + ...(state === "exhausted" ? { failure: { code: "export_interrupted", retryClass: "retry_transient", evidenceFingerprint: "x" } } : {}) }; + await db.update(issueRecoveryActions).set({ ownerType: state === "exhausted" ? "board" : "system", evidence: { ...hold!.evidence, + workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [] }, + systemRecovery: recovery } }).where(eq(issueRecoveryActions.id, f.actionId!)); + return f; + } + + it("continues a setup failure of a run that already did work once its saved work is recovered", async () => { + const f = await setupFailedAfterWork("complete"); + expect(await heldExecutionsAwaitingFreshContinuation(db, new Date(), 10)).toEqual([expect.objectContaining({ id: f.actionId })]); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 }); + expect((await evidenceOf(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId, failureCode: "setup_failed" } }); + }, 60_000); + + it("keeps a setup failure whose saved-work recovery is still under way with that recovery, then continues it on completion", async () => { + const f = await setupFailedAfterWork("scheduled"); + expect(await continueHeldExecutions(db)).toMatchObject({ checked: 1, continued: 0 }); + const waiting = (await evidenceOf(f.actionId!)).evidence.freshContinuationCheck as { kind: string; nextCheckAt: number }; + expect(waiting).toMatchObject({ kind: "exact_recovery_pending" }); + expect(waiting.nextCheckAt).toBeGreaterThan(Date.now()); + expect(await runsAwaitingSystemContinuation(db, companyId, [f.issueId], [f.runId])).toEqual(new Set([`${f.issueId}:${f.runId}`])); + await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{systemRecovery,state}', '"complete"')` }) + .where(eq(issueRecoveryActions.id, f.actionId!)); + await db.transaction(tx => expediteFreshContinuationChecks(tx as unknown as typeof db, { companyId, issueId: f.issueId, runId: f.runId })); + const due = (await evidenceOf(f.actionId!)).evidence.freshContinuationCheck as { nextCheckAt: number; firstPendingAt: number }; + expect(due.nextCheckAt).toBeLessThanOrEqual(Date.now()); + expect(due.firstPendingAt).toBe((waiting as unknown as { firstPendingAt: number }).firstPendingAt); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 }); + }, 60_000); + + it("makes a recovery that completes while the backlog pass decides due at once, not at the backed-off check", async () => { + const f = await setupFailedAfterWork("scheduled"); + let completion: Promise | undefined; + await db.transaction(async tx => { + const decided = await continueFreshInTransaction(tx as unknown as typeof db, { companyId, runId: f.runId, trigger: "held_recovery" }); + expect(decided).toMatchObject({ kind: "exact_recovery_pending" }); + completion = db.transaction(async other => { + await other.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{systemRecovery,state}', '"complete"')` }) + .where(eq(issueRecoveryActions.id, f.actionId!)); + await expediteFreshContinuationChecks(other as unknown as typeof db, { companyId, issueId: f.issueId, runId: f.runId }); + }); + await new Promise(resolve => setTimeout(resolve, 200)); + await recordFreshContinuationCheck(tx as unknown as typeof db, { companyId, actionId: f.actionId! }, decided); + }); + await completion; + const check = (await evidenceOf(f.actionId!)).evidence.freshContinuationCheck as { nextCheckAt: number }; + expect(check.nextCheckAt).toBeLessThanOrEqual(Date.now()); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 }); + }, 60_000); + + it("shows a task at the hourly continuation cap as scheduled, not as continuing", async () => { + const f = await setupFailedAfterWork("complete"); + const nextCheckAt = Date.now() + 30 * 60_000; + await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + freshContinuationCheck: { kind: "backoff", checks: 1, checkedAt: Date.now(), nextCheckAt } })}::jsonb` }) + .where(eq(issueRecoveryActions.id, f.actionId!)); + expect(await runsAwaitingSystemContinuation(db, companyId, [f.issueId], [f.runId])).toEqual(new Set()); + expect(await taskInterruptionPresentation(db, await issue(f.issueId))).toMatchObject({ state: "retry_scheduled", canRetry: false, + nextRetryAt: new Date(nextCheckAt).toISOString() }); + }, 60_000); + + it("leaves a setup failure for a person once its saved-work recovery is exhausted", async () => { + const f = await setupFailedAfterWork("exhausted"); + expect(await heldExecutionsAwaitingFreshContinuation(db, new Date(), 10)).toEqual([]); + expect(await runsAwaitingSystemContinuation(db, companyId, [f.issueId], [f.runId])).toEqual(new Set()); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 0 }); + }, 60_000); + it("lets a running saved-work recovery that keeps advancing use its own budget", async () => { const f = await unfinishedRun({ runtime: "legacy", errorCode: "server_shutdown_interrupted", status: "interrupted" }); await savedWorkRecovery(f, "running"); diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index a59dada1c1f..16886a1a560 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -1,4 +1,4 @@ -import { and, asc, desc, eq, gt, inArray, isNull, or, sql, type SQL } from "drizzle-orm"; +import { and, asc, desc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; import { agentWakeupRequests, agents, approvals, companies, documentRevisions, documents, executionWorkspaces, heartbeatRuns, issueApprovals, issueComments, issueRecoveryActions, issueThreadInteractions, issues, nativeRunFinalizations, toolInvocations, type Db, @@ -25,11 +25,14 @@ import { abandonUnverifiedRemoteWorkspaceEpoch } from "./remote-workspace-epoch- import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { readRunCancellation } from "./run-cancellation.js"; -import { PERMANENT_SOURCE_RETRY_CLASSES, UNVERIFIABLE_SOURCE_CODES, readSystemRecovery } from "./system-recovery-state.js"; -import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, - isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js"; +import { readSystemRecovery } from "./system-recovery-state.js"; +import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, isPersonOwnedFailureCode } from "./continuation-cause.js"; import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; -import { canonicalExecutionUuid } from "./execution-identity.js"; +import { BACKLOG_CONTINUATION_MAX_AGE_MS, failureOwnsBlock, heldExecutionWhere, heldRunIdSql, interruptionEligibility, + liveDispositionHoldSql, recentInterruptionSourceSql, sourceRunKey } from "./fresh-continuation-eligibility.js"; + +export { BACKLOG_CONTINUATION_MAX_AGE_MS, BACKLOG_CONTINUATION_SOURCE_CODES, heldExecutionWhere, recentInterruptionSourceSql, + runsAwaitingSystemContinuation } from "./fresh-continuation-eligibility.js"; export { AUTOMATIC_CONTINUATION_WAKE_REASONS, PERSON_OWNED_FAILURE_CODES, isPersonOwnedFailureCode, isRecoveryRun } from "./continuation-cause.js"; @@ -540,14 +543,7 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< if (taskWait) return { kind: "person_owned", reason: taskWait }; let ownsBlock = false; if (issue.status === "blocked") { - const receipts = holds.filter(hold => hold.evidence.nativeFailureBlock !== undefined || hold.evidence.legacyCancellationBlock !== undefined - || hold.evidence.nativeBootstrapFailureBlock !== undefined); - ownsBlock = (receipts.length ? receipts : holds).some(hold => holdOwnsBlockedStatus(hold, issue, run)) - || receipts.some(hold => [hold.evidence.legacyCancellationBlock, hold.evidence.nativeBootstrapFailureBlock] - .some(block => object(block).runId === run.id && object(block).statusVersion === Number(issue.statusVersion))) - // A run that left no hold: its terminal failure set the block and nothing changed it since. - || (!holds.length && issue.lastStatusDecisionId === null && issue.blockedTransitionAt !== null - && issue.blockedTransitionAt.getTime() >= run.finishedAt.getTime() - 1_000); + ownsBlock = failureOwnsBlock(holds, issue, run); if (!ownsBlock) return { kind: "person_owned", reason: "blocked_by_another_decision" }; const readiness = await issueService(tx).listDependencyReadiness(companyId, [issue.id], tx); if (readiness.get(issue.id)?.isDependencyReady === false) return { kind: "dependency_wait" }; @@ -757,6 +753,17 @@ export async function recordFreshContinuationCheck(db: Db, input: { companyId: s eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, input.actionId))); } +export async function expediteFreshContinuationChecks(tx: Db, input: { companyId: string; issueId: string; runId: string }, + now = new Date()) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); + await tx.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{freshContinuationCheck,nextCheckAt}', to_jsonb(${now.getTime()}::bigint))` }).where(and( + eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, input.issueId), + or(sql`${issueRecoveryActions.evidence}->>'runId' = ${input.runId}`, sql`${issueRecoveryActions.evidence}->>'sourceRunId' = ${input.runId}`), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,kind}' = 'exact_recovery_pending'`, + sql`(${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}')::bigint > ${now.getTime()}`)); +} + /** Not yet checked, or due for another check. */ export function freshContinuationCheckDue(now: Date) { return sql`(${issueRecoveryActions.evidence}->${FRESH_CONTINUATION_CHECK_KEY} is null @@ -782,74 +789,10 @@ export function heldExecutionsAwaitingFreshContinuation(db: Db, now: Date, limit .limit(limit); } -/** - * The backlog pass continues a held record only when it is a recent - * interruption: its source run ended within BACKLOG_CONTINUATION_MAX_AGE_MS - * and ended with a restart, deploy, lease, process-loss or recovery-refusal - * code. A configuration or provider failure, or anything older, stays exactly - * as it is for a person's Retry: re-running last week's failures is not - * "an interruption is invisible". Fresh interruptions reach the fallback - * through restart recovery and the disposition pass, not through this list. - */ -export const BACKLOG_CONTINUATION_MAX_AGE_MS = 24 * 60 * 60_000; -export const BACKLOG_CONTINUATION_SOURCE_CODES = RESTART_CLASS_FAILURE_CODES; /** At most this many backlog continuations per server per rolling minute, and per pass. */ export const BACKLOG_CONTINUATIONS_PER_MINUTE = 10; const BACKLOG_CONTINUATIONS_PER_PASS = 5; -/** Holds the disposition pass (unresolvedExecutionDispositions) handles itself, whatever their age. */ -function liveDispositionHoldSql() { - return and(inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"))!; -} - -const cancelledAtDispatch = (alias: "newer" | "held") => sql.raw(`coalesce(${alias}.status = 'cancelled' and ${alias}.started_at is null - and ${alias}.error_code = 'execution_reconciliation_required', false)`); -// The issue id is a uuid (a column or a task's id), so each link is compared on -// its own indexed column; a cast of the column would scan every run's context. -const runOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id = ${issueId}::uuid - or ${sql.raw(alias)}.issue_id = ${issueId}::uuid or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; - -const sqlList = (values: readonly string[]) => sql.join(values.map(value => sql`${value}`), sql`, `); - -function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQL; sourceRunId: SQL }) { - const since = sql`${new Date(input.now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz`; - const source = (condition: SQL) => sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${input.companyId} - and source.id = ${sourceRunKey(input.sourceRunId)} and ${condition})`; - const refusedLater = sql`held.company_id = source.company_id and held.id <> source.id and ${runOfTask("held", input.issueId)} - and (held.created_at, held.id) > (source.created_at, source.id) and ${cancelledAtDispatch("held")}`; - const personAsked = source(sql`(exists (select 1 from ${agentWakeupRequests} retry where retry.company_id = source.company_id - and retry.status = 'deferred_issue_execution' and retry.reason = 'task_retry' and retry.requested_by_actor_type = 'user' - and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at) - or exists (select 1 from ${heartbeatRuns} held where ${refusedLater} - and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`); - const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")} - and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false) - and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`; - // Saved-work recovery proved the run's unsaved files are permanently gone - // (`source_unavailable`, use another strategy). The only way on is a fresh - // run in a new workspace, which is exactly what Retry would do: a person - // adds nothing, so the system takes that step itself (needed-retry is 0%). - // Uncast keys, so the (company, task, evidence runId) index serves the lookup. - const savedFilesLost = sql`exists (select 1 from ${issueRecoveryActions} lost where lost.company_id = source.company_id - and lost.source_issue_id = ${input.issueId} - and (lost.evidence->>'runId' = source.id::text or (lost.evidence->>'runId' is null and lost.evidence->>'sourceRunId' = source.id::text)) - and lost.evidence#>>'{systemRecovery,state}' = 'exhausted' - and lost.evidence#>>'{systemRecovery,failure,code}' in (${sqlList(UNVERIFIABLE_SOURCE_CODES)}) - and lost.evidence#>>'{systemRecovery,failure,retryClass}' in (${sqlList(PERMANENT_SOURCE_RETRY_CLASSES)}))`; - const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource} - or (${continuationCauseIsSystemOwnedSql("source")} and ${savedFilesLost})) - and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); - return { recent, personAsked }; -} - -/** The backlog pass's eligibility: the one predicate its listing and its deadline query share. */ -export function recentInterruptionSourceSql(now: Date) { - const { recent, personAsked } = interruptionEligibility({ now, companyId: sql`${issueRecoveryActions.companyId}`, - issueId: sql`${issueRecoveryActions.sourceIssueId}`, - sourceRunId: heldRunIdSql() }); - return sql`(${recent} or ${personAsked})`; -} - /** Held records the backlog pass leaves for a person (older, or not an interruption). */ async function countHeldExecutionsLeftAlone(db: Db, now: Date): Promise { const [row] = await db.select({ count: sql`count(*)::int` }).from(issueRecoveryActions) @@ -867,42 +810,6 @@ export function resetBacklogContinuationPacingForTests(db?: Db) { let lastLeftAloneLogAt = 0; const LEFT_ALONE_LOG_INTERVAL_MS = 10 * 60_000; -/** Held reconciliation records of a task's latest unfinished run (joined to its task). */ -export function heldExecutionWhere(options: { includeActive?: boolean } = {}) { - return and( - inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), - sql`${issueRecoveryActions.evidence}->'executionReconciliation' is null`, - sql`${issueRecoveryActions.evidence}->'freshContinuation' is null`, - sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') is not null`, - or( - sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, - and(inArray(issueRecoveryActions.status, ["active", "escalated"]), - sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('complete', 'exhausted')`), - options.includeActive ? and(inArray(issueRecoveryActions.status, ["active", "escalated"]), - eq(issueRecoveryActions.kind, "active_run_watchdog")) : undefined, - // Saved-work recovery still under way: checked so its bound is reached - // (exact recovery keeps its first chance inside the fallback). - sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence')`, - ), - isNull(issueRecoveryActions.ownerUserId), - sql`${issues.status} not in ('done', 'cancelled')`, - sql`${issues.assigneeAgentId} is not null`, - // Only the task's latest run, unfinished and still its assignee's: other - sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} - and source.id = ${sourceRunKey(heldRunIdSql())} - and source.status in ('failed', 'timed_out', 'interrupted', 'cancelled') and source.finished_at is not null - and source.agent_id = ${issues.assigneeAgentId} - and not exists (select 1 from ${heartbeatRuns} newer where newer.company_id = source.company_id and newer.id <> source.id - and (newer.native_issue_id = ${issues.id} or newer.issue_id = ${issues.id} or newer.context_snapshot->>'issueId' = ${issues.id}::text) - and not ${cancelledAtDispatch("newer")} - and (newer.status in ('queued', 'running', 'scheduled_retry') or (newer.created_at, newer.id) > (source.created_at, source.id))))`, - )!; -} - -const heldRunIdSql = () => sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')`; -/** The recorded run id as the run's key: the same match as `id::text = text`, but an index lookup. */ -const sourceRunKey = (runId: SQL) => canonicalExecutionUuid(sql`(${runId})::text`); - /** The earliest time a deferred check becomes due again, if any. */ export async function nextFreshContinuationCheckAt(db: Db, now = new Date()): Promise { // The same records the held-run query and the disposition pass would select; @@ -946,10 +853,13 @@ export async function continueHeldExecutions(db: Db, now = new Date(), limit = B seen.add(candidate.runId); times.push(now.getTime()); try { - const result = await ensureFreshContinuation(db, { companyId: candidate.companyId, runId: candidate.runId, - trigger: "held_recovery", now }); + const result = await db.transaction(async tx => { + const decided = await continueFreshInTransaction(tx as unknown as Db, { companyId: candidate.companyId, runId: candidate.runId, + trigger: "held_recovery", now }); + await recordFreshContinuationCheck(tx as unknown as Db, { companyId: candidate.companyId, actionId: candidate.id }, decided, now); + return decided; + }); if (result.kind === "continued") continued += 1; - else await recordFreshContinuationCheck(db, { companyId: candidate.companyId, actionId: candidate.id }, result, now); } catch (error) { logger.warn({ err: error, recoveryActionId: candidate.id }, "Fresh continuation of a held task deferred"); await recordFreshContinuationCheck(db, { companyId: candidate.companyId, actionId: candidate.id }, diff --git a/server/src/services/fresh-continuation-schedule-queries.test.ts b/server/src/services/fresh-continuation-schedule-queries.test.ts index 7171025de75..e2e533e9367 100644 --- a/server/src/services/fresh-continuation-schedule-queries.test.ts +++ b/server/src/services/fresh-continuation-schedule-queries.test.ts @@ -6,7 +6,7 @@ import { EXECUTION_RECONCILIATION_CAUSES } from "@paperclipai/shared"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { BACKLOG_CONTINUATION_MAX_AGE_MS, freshContinuationCheckDue, heldExecutionWhere, heldExecutionsAwaitingFreshContinuation, nextFreshContinuationCheckAt, recentInterruptionSourceSql } from "./fresh-continuation-fallback.js"; -import { continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, recoveryRunSql } from "./continuation-cause.js"; +import { SETUP_FAILURE_CODES, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, recoveryRunSql } from "./continuation-cause.js"; const support = await getEmbeddedPostgresTestSupport(); const suite = support.supported ? describe : describe.skip; @@ -35,7 +35,15 @@ function previousRecentInterruptionSourceSql(now: Date) { const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")} and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false) and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`; - const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource}) + const setupFailedAfterWork = sql`(coalesce(source.result_json#>>'{executionRecovery,providerWorkStarted}', '') <> 'false' + and source.error_code in (${sql.join(SETUP_FAILURE_CODES.map(code => sql`${code}`), sql`, `)}) + and exists (select 1 from ${issueRecoveryActions} saved where saved.company_id = source.company_id + and saved.source_issue_id = ${input.issueId} + and (saved.evidence->>'runId' = source.id::text or (saved.evidence->>'runId' is null and saved.evidence->>'sourceRunId' = source.id::text)) + and saved.evidence ?| array['workspaceRestoreRecovery', 'workspaceRestoreFailure', 'systemRecovery'] + and coalesce(saved.evidence#>>'{systemRecovery,state}', '') <> 'exhausted'))`; + const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource} + or (${continuationCauseIsSystemOwnedSql("source")} and ${setupFailedAfterWork})) and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); return sql`(${recent} or ${personAsked})`; } @@ -140,7 +148,7 @@ suite("fresh-continuation schedule queries", () => { const finishedMinutesAgo = pick([30, 120, 600, 1_200, 1_430, 1_450, 2_000, 4_000]); const sourceLink = pick(["native", "issue", "context"]); const errorCode = pick(["process_lost", "native_session_interrupted", "server_shutdown_interrupted", "controller_lease_lost", - "adapter_failed", "reusable_sandbox_resume_failed", "budget_exceeded", null]); + "adapter_failed", "reusable_sandbox_resume_failed", "setup_failed", "budget_exceeded", null]); const sourceCreated = chance(0.92); if (sourceCreated) { const finished = chance(0.92) ? at(finishedMinutesAgo) : null; diff --git a/server/src/services/setup-failure-after-work-continuation.test.ts b/server/src/services/setup-failure-after-work-continuation.test.ts new file mode 100644 index 00000000000..a6492d70a80 --- /dev/null +++ b/server/src/services/setup-failure-after-work-continuation.test.ts @@ -0,0 +1,101 @@ +import { and, eq, ne } from "drizzle-orm"; +import { describe, expect, it, vi } from "vitest"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; +import { heartbeatService } from "./heartbeat.js"; +import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./execution-recovery-resolution.js"; +import { resetBacklogContinuationPacingForTests } from "./fresh-continuation-fallback.js"; +import { recordRequiredWorkspaceRestoreFailure } from "./legacy-execution-recovery.js"; +import { readSystemRecovery } from "./system-recovery-state.js"; +import { taskInterruptionPresentation } from "./task-interruption-presentation.js"; +import { reconcileRetainedWorkspaces } from "./workspace-recovery.js"; +import { useWorkspaceRecoveryFixture, db, run, seed } from "./__tests__/workspace-recovery-fixture.js"; + +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +describe("a native run that already did work and then failed setup", () => { + useWorkspaceRecoveryFixture(targets); + + async function failedAfterWork(errorCode = "setup_failed", providerWorkStarted = true) { + const f = await seed("native"); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)); + const finishedAt = new Date(); + const source = await run(f.runId); + await db.update(heartbeatRuns).set({ errorCode, error: "Bad Gateway", nativePhase: "terminal_failure", + startedAt: new Date(finishedAt.getTime() - 13 * 60_000), finishedAt, + contextSnapshot: { issueId: f.issueId, wakeReason: "issue_assigned" }, + ...(providerWorkStarted ? {} : { resultJson: { ...source.resultJson, + executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }) }).where(eq(heartbeatRuns.id, f.runId)); + for (const eventType of ["turn.started", "tool.execution.started", "tool.execution.completed"]) + await appendHeartbeatRunEvent(db, { companyId: f.companyId, runId: f.runId, agentId: source.agentId, + eventType, stream: "system", level: "info", message: eventType, payload: {} }); + await db.update(issues).set({ status: "blocked", blockedTransitionAt: finishedAt, lastStatusDecisionId: null }) + .where(eq(issues.id, f.issueId)); + await recordRequiredWorkspaceRestoreFailure(db, await run(f.runId), (await run(f.runId)).resultJson!); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); + expect(hold).toMatchObject({ status: "resolved", outcome: "blocked", + evidence: expect.objectContaining({ automaticRecovery: expect.objectContaining({ replay: "blocked" }) }) }); + return { ...f, holdId: hold!.id, agentId: source.agentId, service: heartbeatService(db, { environmentRuntime: f.environmentRuntime }) }; + } + + const shown = async (issueId: string) => { + const [task] = await db.select().from(issues).where(eq(issues.id, issueId)); + return taskInterruptionPresentation(db, task!); + }; + const successors = (companyId: string, runId: string) => db.select().from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), ne(heartbeatRuns.id, runId), eq(heartbeatRuns.retryOfRunId, runId))); + const disposition = async () => { + resetBacklogContinuationPacingForTests(db); + await settleUnrecoverableExecutions(db); + }; + + it("continues in a fresh run as soon as its saved work is recovered, and never offers Retry", async () => { + const f = await failedAfterWork(); + const noRetry = { responsibility: "system", state: "working", canRetry: false }; + + expect(await shown(f.issueId)).toMatchObject(noRetry); + await disposition(); + expect(await successors(f.companyId, f.runId)).toEqual([]); + expect(await shown(f.issueId)).toMatchObject(noRetry); + + await vi.waitFor(async () => { + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, current => f.service.resumeRemoteStopComments(current)) ?? []); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.holdId)); + expect(readSystemRecovery(hold!.evidence)?.state).toBe("complete"); + }, { timeout: 10_000, interval: 10 }); + const [recovered] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.holdId)); + const check = recovered!.evidence.freshContinuationCheck as { nextCheckAt: number } | undefined; + if (check) expect(check.nextCheckAt).toBeLessThanOrEqual(Date.now()); + expect(await shown(f.issueId)).toMatchObject(noRetry); + + const decidedAt = Date.now(); + await disposition(); + await deliverReconciledExecutions(db, f.service.wakeup); + const next = await successors(f.companyId, f.runId); + expect(next).toHaveLength(1); + expect(next[0]!.createdAt.getTime() - decidedAt).toBeLessThan(5_000); + expect(next[0]!.contextSnapshot).toMatchObject({ previousRunId: f.runId, forceFreshSession: true, + wakeReason: "issue_recovery_action_restored" }); + expect(await shown(f.issueId)).toMatchObject({ state: "working", canRetry: false }); + }, 60_000); + + it("leaves a setup failure before any provider work on its Retry path", async () => { + const f = await failedAfterWork("setup_failed", false); + await disposition(); + expect(await successors(f.companyId, f.runId)).toEqual([]); + expect(await shown(f.issueId)).not.toMatchObject({ state: "working" }); + }, 60_000); + + it("leaves a person-owned setup failure for the person", async () => { + const f = await failedAfterWork("configuration_incomplete"); + await disposition(); + expect(await successors(f.companyId, f.runId)).toEqual([]); + expect(await shown(f.issueId)).not.toMatchObject({ state: "working" }); + }, 60_000); +}); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 052174fadae..e2552a978a0 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -653,6 +653,12 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i sql`(${issueRecoveryActions.evidence} ? 'freshContinuationSettlement' or ${issueRecoveryActions.evidence} ? 'freshContinuationCheck')`)) : []; const pendingSchema = freshRows.some(row => row.delivery === "pending") ? (await import("./fresh-continuation-fallback.js")).FRESH_CONTINUATION_SCHEMA : null; + const unfinished = runs.filter(run => ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) + && selected.some(issue => issue.id === run.issueId && !["done", "cancelled"].includes(issue.status))); + const awaitingSystemContinuation = unfinished.length + ? await (await import("./fresh-continuation-fallback.js")).runsAwaitingSystemContinuation(db, companyId, + unfinished.map(run => run.issueId), unfinished.map(run => run.id)) + : new Set(); const freshContinuationFor = (issueId: string, runId: string | undefined): ProjectionState["freshContinuation"] => { const rows = freshRows.filter(row => row.issueId === issueId && row.runId === runId); // Only the primary record (it carries `freshContinuation`) says whether a @@ -664,6 +670,8 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i } const scheduled = rows.filter(row => ["exact_recovery_pending", "backoff"].includes(row.checkKind ?? "") && row.nextCheckAt) .map(row => Number(row.nextCheckAt)).sort((a, b) => a - b)[0]; + if (awaitingSystemContinuation.has(`${issueId}:${runId}`) && retryWaitMap.get(issueId)?.sourceRunId !== runId) + return { state: "continuing" }; return scheduled ? { state: "scheduled", nextCheckAt: scheduled } : undefined; }; const dispositionRows = runIds.length ? await db.select({ issueId: issueRecoveryActions.sourceIssueId, diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index d48a9668b8f..f7dac753918 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -41,6 +41,7 @@ import { capturePolicyRearmDueCondition, retainedSourceFallbackDueCondition, sto import { rearmExhaustedCapturePolicy } from "./workspace-recovery-capture-policy.js"; import { rearmExhaustedStopProof } from "./workspace-recovery-stop-proof.js"; import { settleRecoveredWorkspaceRestoreHolds } from "./workspace-restore-hold-settlement.js"; +import { expediteFreshContinuationChecks } from "./fresh-continuation-fallback.js"; import { reusableSandboxNotKeptAfterReplacementSql } from "./reusable-sandbox-kept-marker.js"; const exportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"]; @@ -473,6 +474,7 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ // A settled restore hold returns its warm sandbox to reuse. A hold that // fails any guard keeps its protection; the saved files still count. await settleRecoveredWorkspaceRestoreHolds(tx as unknown as Db, input); + await expediteFreshContinuationChecks(tx as unknown as Db, input); return persistActivity(tx as unknown as Db, { companyId: input.companyId, actorType: "system", actorId: "workspace-recovery", action: "issue.workspace_recovered", entityType: "issue", entityId: input.issueId, runId: input.runId, details: { recoveryActionId: input.actionId, summary: "Saved work recovered." } });