diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index d7cf5b54fe0..2b1a7098c17 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -534,7 +534,7 @@ export async function deliverReconciledExecutions( .where(pendingDecision); await settleCoalescedFreshContinuationHolds(db, { companyId: action.companyId, primaryId: action.id, delivery: "delivered", continuationRunId: newer.id }); - if (newer.contextSnapshot?.freshContinuationIsolation === action.id) + if (newer.contextSnapshot?.recoveryActionId === action.id) await db.transaction(async tx => { await tx.execute(sql`select id from issues where company_id = ${action.companyId} and id = ${task.id} for update`); await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: action.companyId, issueId: task.id, runId: newer.id, @@ -600,7 +600,7 @@ export async function deliverReconciledExecutions( .where(pendingDecision); await settleCoalescedFreshContinuationHolds(tx as unknown as Db, { companyId: action.companyId, primaryId: action.id, delivery: "delivered", continuationRunId: run.id }); - if ((action.evidence.freshContinuation as { isolation?: unknown } | undefined)?.isolation === "fresh_isolated_workspace") + if (fresh) await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: action.companyId, issueId: task.id, runId: run.id, sourceRunId: decision.runId }); }); diff --git a/server/src/services/fresh-continuation-retry-delivery.test.ts b/server/src/services/fresh-continuation-retry-delivery.test.ts new file mode 100644 index 00000000000..af078c0c878 --- /dev/null +++ b/server/src/services/fresh-continuation-retry-delivery.test.ts @@ -0,0 +1,203 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + activityLog, agentWakeupRequests, agents, authUsers, companies, createDb, heartbeatRuns, issueRecoveryActions, issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { deliverReconciledExecutions } from "./execution-recovery-resolution.js"; +import { FRESH_CONTINUATIONS_PER_HOUR, FRESH_CONTINUATION_SCHEMA, ensureFreshContinuation, + resetBacklogContinuationPacingForTests } from "./fresh-continuation-fallback.js"; +import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js"; +import { heartbeatService } from "./heartbeat.js"; +import { queuedRunDispatchClass } from "./heartbeat/queued-run-dispatch-order.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION } from "./recovery/task-automatic-attempts.js"; +import { CONTINUATION_CAP } from "./recovery/assigned-work-invariant.js"; +import { requestTaskRetry } from "./task-retry.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; + +const KINDS = ["fresh_isolated_workspace", "writer_ended", "native"] as const; +type Kind = (typeof KINDS)[number]; +const RECORDED_ISOLATION: Record = { + fresh_isolated_workspace: "fresh_isolated_workspace", writer_ended: "writer_ended", native: "remote_epoch" }; + +suite("a person's Retry while a continuation waits for delivery", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let companyId = "", agentId = "", userId = ""; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-retry-delivery-"); + db = createDb(temporary.connectionString); + companyId = randomUUID(); agentId = randomUUID(); userId = randomUUID(); + await db.insert(authUsers).values({ id: userId, name: "Board", email: `${userId}@example.test`, + emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix: "FRD", + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: userId }); + await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "idle", + adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, permissions: {} }); + }, 60_000); + + afterEach(async () => { + resetBacklogContinuationPacingForTests(db); + await db.execute(sql`truncate table issue_recovery_actions, native_run_finalizations, activity_log, + agent_wakeup_requests, heartbeat_runs, issues cascade`); + }); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + async function pendingContinuation(kind: Kind, endedAgoMs = 60_000) { + const issueId = randomUUID(), runId = randomUUID(); + const finishedAt = new Date(Date.now() - endedAgoMs); + const native = kind === "native"; + await db.insert(issues).values({ id: issueId, companyId, title: "Ship the release notes", status: "blocked", + assigneeAgentId: agentId, statusVersion: 4, blockedTransitionAt: finishedAt }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "assignment", issueId, + nativeIssueId: native ? issueId : null, runtimeMode: native ? "native" : "legacy", + status: "failed", errorCode: native ? "native_session_interrupted" : kind === "writer_ended" ? "process_lost" : "controller_lease_lost", + error: "cut off", startedAt: new Date(finishedAt.getTime() - 10 * 60_000), finishedAt, + createdAt: new Date(finishedAt.getTime() - 11 * 60_000), responsibleUserId: userId, + contextSnapshot: { issueId, wakeReason: "issue_assigned" }, + ...(native ? { nativePhase: "terminal_failure", runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", + workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: "sandbox-1", remoteCwd: "/home/daytona/work", epoch: 3, + state: "prepared", snapshotId: null } } } + : { runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } } }), + ...(kind === "fresh_isolated_workspace" + ? { processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000) } : {}) }); + if (native) { + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "terminal_failure", + recoveryState: "blocked", failureCode: "native_continuation_requires_reconciliation", leaseOwner: null }); + } + const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "active", ownerType: "board", returnOwnerAgentId: agentId, + cause: native ? "native_continuation_requires_reconciliation" : "legacy_execution_requires_reconciliation", + fingerprint: `${native ? "native-continuation" : "legacy-execution"}:${runId}`, nextAction: "Held.", maxAttempts: 3, + createdAt: finishedAt, evidence: { runId, originalFailureCode: "cut_off", nativeFailureBlock: { runId, statusVersion: 4 } } }) + .returning(); + await db.update(issues).set({ projectId: null, projectWorkspaceId: null, executionWorkspaceId: null }).where(eq(issues.id, issueId)); + expect(await ensureFreshContinuation(db, { companyId, runId, trigger: "recovery_refused" })) + .toMatchObject({ kind: "continued", delivery: "recorded", isolation: RECORDED_ISOLATION[kind] }); + return { issueId, runId, actionId: action!.id, isolation: RECORDED_ISOLATION[kind] }; + } + + async function occupySlot() { + const [other] = await db.insert(issues).values({ companyId, title: "Other work", status: "in_progress", assigneeAgentId: agentId }).returning(); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: other!.id, status: "running", contextSnapshot: { issueId: other!.id } }); + } + const taskRuns = (f: { issueId: string; runId: string }) => db.select().from(heartbeatRuns).where(and( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`)); + const record = async (actionId: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, actionId)))[0]!; + const wake = async (id: string) => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, id)))[0]!; + const deliveryPass = () => deliverReconciledExecutions(db, heartbeatService(db).wakeup); + async function deliver() { await deliveryPass(); await deliveryPass(); } + async function requestRetry(issueId: string, observedRunId: string) { + const receipt = await requestTaskRetry({ db, companyId, issueId, actorId: userId, observedRunId, + idempotencyKey: randomUUID(), actorSource: "local_implicit" }); + expect(receipt).toMatchObject({ accepted: true }); + return receipt.requestId!; + } + async function pressRetry(f: { issueId: string; runId: string }) { + const requestId = await requestRetry(f.issueId, f.runId); + await heartbeatService(db).resumeTaskRetry(companyId, requestId); + return requestId; + } + + async function expectSingleContinuationRun(f: Awaited>, requestId: string) { + const runs = await taskRuns(f); + expect(runs).toHaveLength(1); + const run = runs[0]!; + expect(run.contextSnapshot).toMatchObject({ recoveryActionId: f.actionId, previousRunId: f.runId, forceFreshSession: true }); + expect(run.retryOfRunId).toBe(f.runId); + if (f.isolation === "fresh_isolated_workspace") { + expect(run.contextSnapshot).toMatchObject({ freshContinuationIsolation: f.actionId }); + expect(await readFreshWorkspaceReadiness(db, run)).toMatchObject({ mode: "fresh_isolated_workspace", retryIntentId: f.actionId }); + } else expect(run.contextSnapshot?.freshContinuationIsolation).toBeUndefined(); + expect(await wake(requestId)).toMatchObject({ status: "coalesced", runId: run.id }); + expect((await record(f.actionId)).evidence).toMatchObject({ continuationDelivery: "delivered", continuationRunId: run.id }); + return run; + } + + it.each(KINDS)("runs a Retry pressed before delivery as the recorded continuation itself, ranked as person work (%s)", async kind => { + const f = await pendingContinuation(kind); + await occupySlot(); + const requestId = await pressRetry(f); + const run = await expectSingleContinuationRun(f, requestId); + expect(run.contextSnapshot).toMatchObject({ wakeReason: "task_retry", taskRetryRequestId: requestId }); + const [runWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, run.wakeupRequestId!)); + expect(runWake!.requestedByActorType).toBe("user"); + expect(queuedRunDispatchClass({ id: run.id, createdAt: run.createdAt, wakeReason: "task_retry", requestedByUser: true, + sourceEndedAt: null, readinessRank: 1, priorityRank: 0 }, new Date())).toBe("person"); + await deliver(); + expect((await taskRuns(f)).map(row => row.id)).toEqual([run.id]); + }, 60_000); + + it.each(KINDS)("answers a Retry saved before delivery with the delivered continuation, never a second run (%s)", async kind => { + const f = await pendingContinuation(kind); + await occupySlot(); + const requestId = await requestRetry(f.issueId, f.runId); + await deliver(); + await heartbeatService(db).resumeTaskRetry(companyId, requestId); + const runs = await taskRuns(f); + expect(runs).toHaveLength(1); + expect(runs[0]!.contextSnapshot).toMatchObject({ recoveryActionId: f.actionId, wakeReason: "issue_recovery_action_restored" }); + expect(await wake(requestId)).toMatchObject({ status: "coalesced", runId: runs[0]!.id }); + const joined = await requestRetry(f.issueId, runs[0]!.id); + expect(await wake(joined)).toMatchObject({ status: "coalesced", runId: runs[0]!.id }); + expect(await taskRuns(f)).toHaveLength(1); + }, 60_000); + + it.each(KINDS)("starts exactly one run when a Retry and the delivery pass race (%s)", async kind => { + const f = await pendingContinuation(kind); + await occupySlot(); + const requestId = await requestRetry(f.issueId, f.runId); + await Promise.all([heartbeatService(db).resumeTaskRetry(companyId, requestId), deliveryPass(), deliveryPass()]); + await deliver(); + await heartbeatService(db).resumeTaskRetry(companyId, requestId); + const runs = await taskRuns(f); + expect(runs).toHaveLength(1); + expect(runs[0]!.contextSnapshot).toMatchObject({ recoveryActionId: f.actionId }); + if (f.isolation === "fresh_isolated_workspace") expect(runs[0]!.contextSnapshot).toMatchObject({ freshContinuationIsolation: f.actionId }); + expect(await wake(requestId)).toMatchObject({ status: "coalesced", runId: runs[0]!.id }); + expect((await record(f.actionId)).evidence).toMatchObject({ continuationDelivery: "delivered", continuationRunId: runs[0]!.id }); + }, 60_000); + + it.each(["fresh_isolated_workspace", "writer_ended"] as const)("gives a Retry of the continuation's own failure its own run (%s)", async kind => { + const f = await pendingContinuation(kind); + await occupySlot(); + await deliver(); + const [continuation] = await taskRuns(f); + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "adapter_failed", startedAt: new Date(), finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, continuation!.id)); + const requestId = await pressRetry({ issueId: f.issueId, runId: continuation!.id }); + const retryWake = await wake(requestId); + expect(retryWake.runId, JSON.stringify(retryWake)).toBeTruthy(); + expect(retryWake.runId).not.toBe(continuation!.id); + const [own] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retryWake.runId!)); + expect(own!.contextSnapshot).toMatchObject({ wakeReason: "task_retry", previousRunId: continuation!.id }); + expect(own!.contextSnapshot?.recoveryActionId).toBeUndefined(); + expect((await record(f.actionId)).evidence).toMatchObject({ continuationRunId: continuation!.id }); + }, 60_000); + + it("starts the person's run while the delivery is held by pacing and the task's automatic attempts are spent", async () => { + const f = await pendingContinuation("fresh_isolated_workspace", 45 * 60_000); + const [other] = await db.insert(issues).values({ companyId, title: "Queued work", status: "todo", assigneeAgentId: agentId }).returning(); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: other!.id, status: "queued", contextSnapshot: { issueId: other!.id } }); + const now = new Date(); + for (let index = 0; index < FRESH_CONTINUATIONS_PER_HOUR; index += 1) + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog", status: "resolved", + ownerType: "system", cause: "fresh_continuation", fingerprint: `spent:${index}:${f.issueId}`, nextAction: "Continued.", + evidence: { freshContinuation: { schema: FRESH_CONTINUATION_SCHEMA, recordedAt: new Date(now.getTime() - 60_000).toISOString() }, + continuationDelivery: "delivered" } }); + for (let index = 0; index < CONTINUATION_CAP; index += 1) + await db.insert(activityLog).values({ companyId, actorType: "system", actorId: "assigned-work", action: ASSIGNED_WORK_CONTINUED_ACTION, + entityType: "issue", entityId: f.issueId }); + await deliver(); + expect(await taskRuns(f)).toHaveLength(0); + expect((await record(f.actionId)).evidence.continuationDelivery).toBe("pending"); + const requestId = await pressRetry(f); + await expectSingleContinuationRun(f, requestId); + }, 60_000); +}); diff --git a/server/src/services/fresh-continuation-retry-fold.ts b/server/src/services/fresh-continuation-retry-fold.ts index f09509b581e..3544c9736de 100644 --- a/server/src/services/fresh-continuation-retry-fold.ts +++ b/server/src/services/fresh-continuation-retry-fold.ts @@ -1,6 +1,7 @@ -import { and, eq, inArray, isNull, sql } from "drizzle-orm"; -import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; +import { and, desc, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { agentWakeupRequests, heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; import { queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js"; +import { FRESH_CONTINUATION_SCHEMA, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; const LIVE_CONTINUATION_STATUSES = ["queued", "running"]; @@ -29,3 +30,60 @@ export async function foldWaitingRetriesIntoContinuation(tx: Db, input: { .where(and(eq(agentWakeupRequests.companyId, input.companyId), inArray(agentWakeupRequests.id, retries), eq(agentWakeupRequests.status, "deferred_issue_execution"))); } + +export async function foldWaitingRetriesIntoRecordedContinuation(tx: Db, input: { + companyId: string; issueId: string; runId: string; recoveryActionId: string | null; sourceRunId: string | null; +}) { + if (!input.recoveryActionId || !input.sourceRunId) return; + const [record] = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, input.issueId), + sql`${issueRecoveryActions.id}::text = ${input.recoveryActionId}`, + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' = ${FRESH_CONTINUATION_SCHEMA}`, + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${input.sourceRunId}`)).limit(1); + if (record) await foldWaitingRetriesIntoContinuation(tx, input); +} + +export type UndeliveredContinuation = { recoveryActionId: string; sourceRunId: string; isolation: string | null }; + +/** Caller holds the task row lock. The task's recorded continuation that no run took yet: a run admitted now is that continuation. */ +export async function undeliveredContinuation(tx: Db, input: { companyId: string; issueId: string }): Promise { + const [record] = await tx.select({ id: issueRecoveryActions.id, evidence: issueRecoveryActions.evidence }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, input.issueId), + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' = ${FRESH_CONTINUATION_SCHEMA}`)) + .orderBy(desc(sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'`)).limit(1).for("update"); + if (!record) return null; + const fresh = record.evidence.freshContinuation as { recordedAt?: unknown; isolation?: unknown } | undefined; + const sourceRunId = (record.evidence.executionReconciliation as { runId?: unknown } | undefined)?.runId; + if (typeof sourceRunId !== "string" || typeof fresh?.recordedAt !== "string") return null; + const [intervening] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), sql`${heartbeatRuns.id}::text <> ${sourceRunId}`, + or(eq(heartbeatRuns.nativeIssueId, input.issueId), eq(heartbeatRuns.issueId, input.issueId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${input.issueId}::text`), + sql`${heartbeatRuns.createdAt} >= ${fresh.recordedAt}::timestamptz`)).limit(1); + if (intervening) return null; + return { recoveryActionId: record.id, sourceRunId, isolation: typeof fresh.isolation === "string" ? fresh.isolation : null }; +} + +export function undeliveredContinuationContext(continuation: UndeliveredContinuation, context: Record) { + return { recoveryActionId: continuation.recoveryActionId, retryOfRunId: continuation.sourceRunId, forceFreshSession: true, + ...(typeof context.previousRunId === "string" ? {} : { previousRunId: continuation.sourceRunId }), + ...(continuation.isolation === "fresh_isolated_workspace" ? { freshContinuationIsolation: continuation.recoveryActionId } : {}) }; +} + +/** In the admitting transaction: the new run is the recorded continuation, so the delivery pass finds nothing left to start. */ +export async function deliverContinuationAsRun(tx: Db, input: { companyId: string; issueId: string; runId: string; + continuation: UndeliveredContinuation }) { + const { continuation } = input; + await tx.update(heartbeatRuns).set({ retryOfRunId: continuation.sourceRunId }) + .where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId))); + await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + continuationDelivery: "delivered", continuationRunId: input.runId, continuationDeliveredByPersonRun: true })}::jsonb` }) + .where(and(eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, continuation.recoveryActionId), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); + await settleCoalescedFreshContinuationHolds(tx, { companyId: input.companyId, primaryId: continuation.recoveryActionId, + delivery: "delivered", continuationRunId: input.runId }); + await foldWaitingRetriesIntoContinuation(tx, { companyId: input.companyId, issueId: input.issueId, runId: input.runId, + sourceRunId: continuation.sourceRunId }); +} diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 3e25cddcb02..40d2af26f65 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -4,7 +4,8 @@ import { verifyBlockerStallWake } from "../recovery/blocker-stall-wakeups.js"; import { CONTEXT_COMPACTION_WAKE_REASON, ContextCompactionError, isContextCompactionRun, readContextCompactionRequest } from "../context-compaction.js"; import { isProcessRunAdmissionClosed } from "./run-admission.js"; import { continueFreshInTransaction, hasPendingFreshContinuation, isFreshIsolatedContinuationWake, settleCoalescedFreshContinuationHolds } from "../fresh-continuation-fallback.js"; -import { foldWaitingRetriesIntoContinuation } from "../fresh-continuation-retry-fold.js"; +import { deliverContinuationAsRun, foldWaitingRetriesIntoContinuation, foldWaitingRetriesIntoRecordedContinuation, undeliveredContinuation, + undeliveredContinuationContext } from "../fresh-continuation-retry-fold.js"; import { refreshRunStartYieldIfStale } from "../run-start-handover.js"; import { deliveredQueuedCommentIds } from "../queued-comment-delivery.js"; import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, recheckQueuedRunInteractions, withInteractionDeliveryOrigin } from "../agent-interaction-delivery.js"; @@ -4028,6 +4029,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }); return { kind: "skipped" as const }; } + const continuationToDeliver = !systemWake && !executionReconciliationWake && !executionBlocker + ? await undeliveredContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id }) : null; + if (continuationToDeliver) + Object.assign(enrichedContextSnapshot, undeliveredContinuationContext(continuationToDeliver, enrichedContextSnapshot)); if ((executionBlocker || opts.queuedCommentInterruptId) && !ownsReconstructionContinuation && !(await admitExplicitNativeContinuation({ db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, @@ -5083,9 +5088,12 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); } - if (readNonEmptyString(enrichedContextSnapshot.freshContinuationIsolation)) - await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, runId: newRun.id, - sourceRunId: readNonEmptyString(enrichedContextSnapshot.previousRunId) }); + if (continuationToDeliver) + await deliverContinuationAsRun(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, runId: newRun.id, + continuation: continuationToDeliver }); + else if (reconciledSourceRunId) + await foldWaitingRetriesIntoRecordedContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, + runId: newRun.id, recoveryActionId: readNonEmptyString(enrichedContextSnapshot.recoveryActionId), sourceRunId: reconciledSourceRunId }); await tx .update(agentWakeupRequests)