diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index b4adc245c4e..8d38ffc4245 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -340,9 +340,12 @@ company and issue lock, the admission transaction, and the handoff after commit for both fresh task runs and wakes arriving behind an active execution. Wake-queue checks conversation eligibility before locking the task. The locked task supplies the attribution fields used to select the responsible user for a fresh run. -Heartbeat uses the transaction for its specialized eligibility gates and -continuation writes. Wake-queue selects eligible saved comment receipts in request order and -binds them to the new run. Chat authorization and its admission callback run in +Wake-queue reads recovery state inside the admission transaction. It checks +continuation eligibility before allowing a wake to join active work, then +settles any explicit continuation before allowing a fresh run. Heartbeat +uses the same transaction for its other eligibility gates and specialized +receipt handling. Wake-queue selects eligible saved comment receipts in request +order and binds them to the new run. Chat authorization and its admission callback run in the same transaction. Authorization contention defers the wake and rolls back without publishing buffered effects. `createHeartbeatQueue` binds these operations without querying or starting work. diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 41890c69acf..407a80abe04 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -257,6 +257,31 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { } }); + it("checks conversation reset eligibility using the row refreshed under the admission lock", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); + const userId = randomUUID(); + const [comment] = await db.insert(issueComments).values({ + companyId, issueId, authorType: "user", authorUserId: userId, body: "/new", + }).returning(); + await db.insert(issueRecoveryActions).values({ + companyId, sourceIssueId: issueId, kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", + fingerprint: randomUUID(), nextAction: "Inspect the stopped run.", + }); + + await createPostgresIssueWakeAdmission(db)({ companyId, issueId }, async (tx, { lockedIssue, lockIssue, recovery }) => { + const input = { commentId: comment.id, requestedByActorType: "user", requestedByActorId: userId }; + expect(lockedIssue?.conversationAgentId).toBeNull(); + await expect(recovery.readBlocker(input)).resolves.toMatchObject({ verifiedReset: false, blocker: { cause: "legacy_execution_requires_reconciliation" } }); + await tx.update(issues).set({ conversationAgentId: agentId, conversationUserId: userId, conversationState: "active" }) + .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))); + await expect(lockIssue()).resolves.toMatchObject({ conversationAgentId: agentId }); + await expect(recovery.readBlocker(input)).resolves.toEqual({ verifiedReset: true, blocker: null }); + return { kind: "skipped" }; + }); + }); + it("locks an issue added after the initial admission lock missed", async () => { const companyId = await seedCompany(); const issueId = randomUUID(); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index efc3ac254de..dad4943d701 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -13,8 +13,10 @@ import { readRunCancellation } from "../../../services/run-cancellation.js"; import { withRunRedactionHistory } from "../../../services/run-secret-redaction.js"; import { CHAT_COMPLETION_WAKE_REASON, isCompletedOnboardingHandoffWake, settleReportedChatCompletions } from "../../../services/chat-completion-delivery.js"; import { earlierDeclinedOutcomeReply } from "../../../services/chat-interaction-outcome-report.js"; -import { currentConversationCommentCondition } from "../../../services/agent-conversations.js"; +import { currentConversationCommentCondition, isConversation, isConversationReset } from "../../../services/agent-conversations.js"; import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; +import { preserveTaskRemoteWorkspaceWriter } from "../../../services/native-runtime/native-failed-workspace-recovery.js"; +import type { WakeRecoveryAdmissionStore } from "../application/recovery-admission.js"; import { nativeManualAdmissionHoldsExecution } from "../../../services/native-runtime/native-runner-ownership.js"; import { and, asc, eq, inArray, isNull, ne, notInArray, or, sql } from "drizzle-orm"; import { readReleaseRun, type ReleaseRun } from "./release-run-read.js"; @@ -56,7 +58,7 @@ import { executionRetryAccounting } from "../../../services/execution-recovery-a import { issueService } from "../../../services/issues.js"; import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js"; import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js"; -import { undeliveredLegacyUserCommentIds } from "../../../services/explicit-native-continuation.js"; +import { admitExplicitNativeContinuation, undeliveredLegacyUserCommentIds } from "../../../services/explicit-native-continuation.js"; import { withRecoveryContext } from "../../../services/recovery/status-only-context.js"; import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js"; import { @@ -1320,6 +1322,8 @@ const issueAdmissionFields = { }; export type IssueAdmissionRow = Pick; +export type IssueWakeContinuationInput = Omit[0], "db" | "companyId" | "issueId" | "dryRun" | "onBlocked">; +export type IssueWakeContinuationResult = NonNullable>>; export function createPostgresIssueWakeAdmission(db: Db) { return async function withIssueAdmission( @@ -1327,6 +1331,7 @@ export function createPostgresIssueWakeAdmission(db: Db) { evaluate: (tx: IssueAdmissionDbTransaction, stores: { lockedIssue: IssueAdmissionRow | null; lockIssue: () => Promise; + recovery: WakeRecoveryAdmissionStore; reader: WakeAdmissionReader; writer: WakeAdmissionWriter; fresh: FreshIssueRunStore; @@ -1335,15 +1340,41 @@ export function createPostgresIssueWakeAdmission(db: Db) { try { return await db.transaction(withDurableChatAdmission(durableChatRequest, async (tx) => { // Mentions may target another agent, but every admission must share the issue lock. + let lockedIssue: IssueAdmissionRow | null = null; const lockIssue = async () => { const [issue] = await tx.select(issueAdmissionFields).from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))).for("update"); - return issue ?? null; + lockedIssue = issue ?? null; + return lockedIssue; }; - const lockedIssue = await lockIssue(); + await lockIssue(); return evaluate(tx, { lockedIssue, lockIssue, + recovery: { + async readBlocker({ commentId, requestedByActorType, requestedByActorId }) { + const issue = lockedIssue; + const userComment = requestedByActorType === "user" ? commentId : null; + const [resetComment] = isConversation(issue) && userComment && requestedByActorId + ? await tx.select({ body: issueComments.body }).from(issueComments).where(and( + eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), + eq(issueComments.id, userComment), eq(issueComments.authorUserId, requestedByActorId), + eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt), isNull(issueComments.createdByRunId), + )).limit(1) + : []; + const verifiedReset = Boolean(resetComment && isConversationReset(resetComment.body)); + const preserveWorkspace = userComment && issue?.executionWorkspaceId; + if (preserveWorkspace) await preserveTaskRemoteWorkspaceWriter(tx as unknown as Db, companyId, issueId); + const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId, { + conversationResetCommentId: verifiedReset ? userComment : null, + lockedIssue: preserveWorkspace ? undefined : issue ?? undefined, + }); + return { blocker, verifiedReset }; + }, + admitExplicit(input, dryRun, onBlocked) { + return admitExplicitNativeContinuation({ ...input, db: tx as unknown as Db, companyId, issueId, dryRun, onBlocked }); + }, + }, reader: createWakeAdmissionReader(tx, companyId), writer: createWakeAdmissionWriter(tx, companyId), fresh: createFreshIssueRunStore(tx, companyId), diff --git a/server/src/modules/wake-queue/application/recovery-admission.test.ts b/server/src/modules/wake-queue/application/recovery-admission.test.ts new file mode 100644 index 00000000000..d959105a969 --- /dev/null +++ b/server/src/modules/wake-queue/application/recovery-admission.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, vi } from "vitest"; +import { createWakeRecoveryAdmission } from "./recovery-admission.js"; + +const blocker = { + cause: "legacy_execution_requires_reconciliation" as const, + recoveryActionId: "recovery", + runId: "stopped", + nextAction: "Inspect the stopped run.", +}; +const wake = { commentId: "message" }; + +describe("wake recovery admission", () => { + it("withholds both ordinary admission capabilities when recovery is required", async () => { + const admitExplicit = vi.fn().mockResolvedValue(null); + const prepare = createWakeRecoveryAdmission({ + readBlocker: async () => ({ blocker, verifiedReset: false }), + admitExplicit, + }, vi.fn(), vi.fn()); + + const recovery = await prepare(wake); + expect(recovery).not.toHaveProperty("admitBusy"); + expect(recovery).not.toHaveProperty("admitFresh"); + const result = await recovery.checkBusyAdmission(); + expect(result.kind).toBe("blocked"); + expect(result).not.toHaveProperty("admitBusy"); + expect(result).not.toHaveProperty("checkFreshAdmission"); + expect(admitExplicit).toHaveBeenCalledExactlyOnceWith(wake, true, expect.any(Function)); + }); + + it("requires actual fresh continuation after the successful busy dry run", async () => { + const admitExplicit = vi.fn().mockResolvedValueOnce({ previousRunId: "stopped" }).mockResolvedValueOnce(null); + const admitBusy = vi.fn(), admitFresh = vi.fn(); + const prepare = createWakeRecoveryAdmission({ + readBlocker: async () => ({ blocker, verifiedReset: false }), + admitExplicit, + }, admitBusy, admitFresh); + + const busy = await (await prepare(wake)).checkBusyAdmission(); + expect(busy.kind).toBe("allowed"); + if (busy.kind !== "allowed") throw new Error("Expected busy admission"); + expect(busy.admitBusy).toBe(admitBusy); + expect(admitExplicit).toHaveBeenCalledExactlyOnceWith(wake, true, expect.any(Function)); + expect(busy).not.toHaveProperty("admitFresh"); + const fresh = await busy.checkFreshAdmission(); + expect(fresh.kind).toBe("blocked"); + expect(fresh).not.toHaveProperty("admitFresh"); + expect(admitExplicit).toHaveBeenLastCalledWith(wake, false, expect.any(Function)); + }); + + it("exposes normal fresh admission after the final continuation check", async () => { + const admitExplicit = vi.fn().mockResolvedValue(null); + const admitFresh = vi.fn(); + const prepare = createWakeRecoveryAdmission({ + readBlocker: async () => ({ blocker: null, verifiedReset: false }), + admitExplicit, + }, vi.fn(), admitFresh); + const busy = await (await prepare(wake)).checkBusyAdmission(); + expect(admitExplicit).not.toHaveBeenCalled(); + if (busy.kind !== "allowed") throw new Error("Expected busy admission"); + const fresh = await busy.checkFreshAdmission(); + expect(fresh).toMatchObject({ kind: "allowed", admitFresh, explicitContinuation: null }); + expect(admitExplicit).toHaveBeenCalledExactlyOnceWith(wake, false, expect.any(Function)); + }); +}); diff --git a/server/src/modules/wake-queue/application/recovery-admission.ts b/server/src/modules/wake-queue/application/recovery-admission.ts new file mode 100644 index 00000000000..f92a4c3171e --- /dev/null +++ b/server/src/modules/wake-queue/application/recovery-admission.ts @@ -0,0 +1,60 @@ +import type { ExecutionBlocker } from "@paperclipai/shared"; + +type ContinuationWait = { reason: string; message: string }; +type AdmissionBlocker = Pick; + +type WakeRecoveryReadInput = { + commentId: string | null; + requestedByActorType?: string | null; + requestedByActorId?: string | null; +}; +type ExplicitContinuationIntent = { commentId: string | null; queuedCommentInterruptId?: string }; + +export type WakeRecoveryAdmissionStore = { + readBlocker(input: WakeRecoveryReadInput): Promise<{ + blocker: AdmissionBlocker | null; + verifiedReset: boolean; + }>; + admitExplicit(input: ExplicitInput, dryRun: boolean, onBlocked: (reason: string, message: string) => void): + Promise; +}; + +export function createWakeRecoveryAdmission( + store: WakeRecoveryAdmissionStore, + admitBusy: Busy, + admitFresh: Fresh, +) { + return async function prepareRecovery( + input: ExplicitInput & WakeRecoveryReadInput & { + reconciliation?: { sourceRunId: string; actionId: string }; + }, + ) { + const { blocker, verifiedReset } = await store.readBlocker(input); + const ownsReconstruction = Boolean(input.reconciliation && blocker && + blocker.runId === input.reconciliation.sourceRunId && blocker.recoveryActionId === input.reconciliation.actionId); + + return { + blocker, + ownsReconstruction, + async checkBusyAdmission() { + let wait: ContinuationWait | undefined; + const onBlocked = (reason: string, message: string) => { wait = { reason, message }; }; + const blocked = () => Boolean(blocker || (wait && !verifiedReset)); + if ((blocker || input.queuedCommentInterruptId) && !ownsReconstruction && + !(await store.admitExplicit(input, true, onBlocked)) && blocked()) { + return { kind: "blocked" as const, wait }; + } + return { + kind: "allowed" as const, + admitBusy, + wait, + async checkFreshAdmission() { + const explicitContinuation = await store.admitExplicit(input, false, onBlocked); + if (!explicitContinuation && blocked() && !ownsReconstruction) return { kind: "blocked" as const, wait }; + return { kind: "allowed" as const, explicitContinuation, admitFresh, wait }; + }, + }; + }, + }; + }; +} diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts index 405d9d82e7f..04f23066ba2 100644 --- a/server/src/modules/wake-queue/index.ts +++ b/server/src/modules/wake-queue/index.ts @@ -7,11 +7,14 @@ import { type IssueAdmissionDbTransaction, type IssueAdmissionRow, type IssueAdmissionTarget, + type IssueWakeContinuationInput, + type IssueWakeContinuationResult, } from "./adapters/postgres.js"; import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js"; import { createQueuedCommentSteeringAdapter, lockBoardQueuedComments, readBoardQueuedComments } from "./adapters/queued-comment-board-postgres.js"; import type { QueuedCommentQueuePostgresAdapterDeps } from "./adapters/queued-comment-postgres.js"; import { createFreshIssueRunAdmission } from "./application/fresh-issue-run.js"; +import { createWakeRecoveryAdmission } from "./application/recovery-admission.js"; import { admitIssueWake, createIssueWakePreflight, type IssueWakeAdmissionEffects, type IssueWakeAdmissionOutcome } from "./application/issue-wake-admission.js"; import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "./application/use-cases.js"; import { @@ -88,8 +91,9 @@ type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type LockedIssueAdmission = { lockedIssue: IssueAdmissionRow | null; lockIssue: () => Promise; - admitWakeBehindIssueExecution: ReturnType; - admitFreshIssueRun: ReturnType>; + prepareRecovery: ReturnType, ReturnType> + >>; }; /** Composes the issue admission and release workflows with their Postgres locks. */ @@ -117,24 +121,23 @@ export function createWakeQueue(db: Db, deps: WakeQueueDeps) { evaluateHeartbeatPolicy: (tx: IssueAdmissionDbTransaction, admission: LockedIssueAdmission) => Promise>, effects: IssueWakeAdmissionEffects, ) { - return admitIssueWake(() => withIssueAdmission({ companyId, issueId, durableChatRequest }, (tx, { lockedIssue, lockIssue, reader, writer, fresh }) => { + return admitIssueWake(() => withIssueAdmission({ companyId, issueId, durableChatRequest }, (tx, { lockedIssue, lockIssue, recovery, reader, writer, fresh }) => { const admitBusy = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: deps.wakeAdmissionHelpers }); const admitFresh = createFreshIssueRunAdmission(fresh); return evaluateHeartbeatPolicy(tx, { lockedIssue, lockIssue, - async admitWakeBehindIssueExecution(input) { + prepareRecovery: createWakeRecoveryAdmission(recovery, async (input: Parameters[0]) => { if (input.companyId !== companyId || input.issueId !== issueId || input.contextSnapshot.issueId !== issueId) { throw new Error("wake-queue: admission does not match the locked company and issue"); } return admitBusy(input); - }, - async admitFreshIssueRun(input) { + }, async (input: Parameters[0]) => { if (input.issue.id !== issueId || input.contextSnapshot.issueId !== issueId) { throw new Error("wake-queue: admission does not match the locked issue"); } return admitFresh(input); - }, + }), }); }), effects); }, diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index e03f3ba14e4..ecc292533aa 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -284,10 +284,30 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { deps.wakeQueue.admitIssueWake.mockImplementation((target, evaluate, effects) => admit(target, (tx, admission) => evaluate(tx, otherIssue ? { ...admission, - admitFreshIssueRun: (input) => admission.admitFreshIssueRun({ ...input, - issue: { ...input.issue, id: otherIssue.id }, contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }), - admitWakeBehindIssueExecution: (input) => admission.admitWakeBehindIssueExecution({ ...input, - issueId: otherIssue.id, contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }), + async prepareRecovery(input) { + const recovery = await admission.prepareRecovery(input); + return { + ...recovery, + async checkBusyAdmission() { + const busy = await recovery.checkBusyAdmission(); + if (busy.kind !== "allowed") return busy; + return { + ...busy, + admitBusy: (input) => busy.admitBusy({ ...input, issueId: otherIssue.id, + contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }), + async checkFreshAdmission() { + const fresh = await busy.checkFreshAdmission(); + if (fresh.kind !== "allowed") return fresh; + return { + ...fresh, + admitFresh: (input) => fresh.admitFresh({ ...input, + issue: { ...input.issue, id: otherIssue.id }, contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }), + }; + }, + }; + }, + }; + }, } : admission), { ...effects, publishIssueActivity })); const wake = queue.enqueueWakeup(agent.id, { @@ -1013,9 +1033,16 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { }); const busyAdmission = vi.fn(); deps.wakeQueue.admitIssueWake.mockImplementation((target, evaluate, effects) => admission.admitIssueWake(target, - (tx, stores) => evaluate(tx, { ...stores, admitWakeBehindIssueExecution(input) { - busyAdmission(input.activeExecutionRun.id, input.issueExecutionAgentNameKey); - return stores.admitWakeBehindIssueExecution(input); + (tx, stores) => evaluate(tx, { ...stores, async prepareRecovery(input) { + const recovery = await stores.prepareRecovery(input); + return { ...recovery, async checkBusyAdmission() { + const busy = await recovery.checkBusyAdmission(); + if (busy.kind !== "allowed") return busy; + return { ...busy, admitBusy(input) { + busyAdmission(input.activeExecutionRun.id, input.issueExecutionAgentNameKey); + return busy.admitBusy(input); + } }; + } }; } }), effects)); const queue = createHeartbeatQueue(db, deps); if (scenario === "running") { diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 4d63a52832d..96130f81e02 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -56,14 +56,12 @@ import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { isVerifiedDependencyInteractionResponse, readQueuedInteractionResponse } from "../queued-interaction-response.js"; import { isConversation, - isConversationReset, isWaitingConversation, } from "../agent-conversations.js"; import { onboardingGreetingNeeded } from "../onboarding-chat.js"; import { ONBOARDING_CHAT_GREETING_REASON } from "../onboarding-chat-prompt.js"; import { legacyControllerClaim } from "../legacy-controller-lease.js"; import { - admitExplicitNativeContinuation, undeliveredLegacyUserCommentIds, } from "../explicit-native-continuation.js"; import { canRetryStoppedRun } from "../cancelled-native-startup.js"; @@ -80,7 +78,6 @@ import { import { claimQueuedNativeReviewRun } from "../native-runtime/native-review-dispatch.js"; import { nativeCleanupHoldsAdmission } from "../native-runtime/native-cleanup-admission-hold.js"; import { adapterExecutionControls } from "../adapter-execution-control.js"; -import { preserveTaskRemoteWorkspaceWriter } from "../native-runtime/native-failed-workspace-recovery.js"; import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; import { slackMentionAllowsRunStart, @@ -3520,7 +3517,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return wakeQueue.admitIssueWake( { companyId: agent.companyId, issueId, durableChatRequest: durableRequest }, - async (tx, { lockedIssue, lockIssue, admitWakeBehindIssueExecution, admitFreshIssueRun }) => { + async (tx, { lockedIssue, lockIssue, prepareRecovery }) => { if (executionWaitRequestId) { const [pending] = await tx.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.id, executionWaitRequestId), eq(agentWakeupRequests.companyId, agent.companyId), @@ -3870,6 +3867,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } let reconciledSourceRunId: string | null = null; + let reconciledRecoveryActionId: string | null = null; let reconciledRestoreRetryCount: number | null = null; if (executionReconciliationWake) { const actionId = readNonEmptyString( @@ -4012,10 +4010,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende reconciledRestoreRetryCount = executionFailureRetryCount(reconciledRun!); } reconciledSourceRunId = sourceRunId; + if (enrichedContextSnapshot.recoveryActionId === action.id) reconciledRecoveryActionId = action.id; } let continuationWait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; - let continuationBlocked = false; const deferBlockedExecution = async ( executionBlocker: Pick>>, "recoveryActionId" | "cause" | "nextAction">, ) => { @@ -4066,32 +4064,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return { kind: "deferred" as const, admitted: true }; }; const explicitContinuationRunId = randomUUID(); - const [resetComment] = isConversation(issue) && opts.requestedByActorType === "user" && opts.requestedByActorId && wakeCommentId - ? await tx.select({ body: issueComments.body }).from(issueComments).where(and( - eq(issueComments.companyId, issue.companyId), eq(issueComments.issueId, issue.id), - eq(issueComments.id, wakeCommentId), eq(issueComments.authorUserId, opts.requestedByActorId), - eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt), isNull(issueComments.createdByRunId), - )).limit(1) - : []; - const verifiedConversationReset = Boolean(resetComment && isConversationReset(resetComment.body)); - const preserveWorkspace = wakeCommentId && opts.requestedByActorType === "user" - && issue.executionWorkspaceId !== null; - if (preserveWorkspace) - await preserveTaskRemoteWorkspaceWriter(tx as unknown as Db, issue.companyId, issue.id); - const executionBlocker = await getExecutionBlocker( - tx as unknown as Db, issue.companyId, issue.id, - { - conversationResetCommentId: verifiedConversationReset ? wakeCommentId : null, - lockedIssue: preserveWorkspace ? undefined : issue, - }, - ); - // Prove eligibility without retiring the hold. Later gates can still - // decline this wake; hold retirement and successor creation stay atomic. - // A bound chat request has already rechecked its current principal - // above. Treat its new user message like a board comment, but keep - // failed-run retry actions on their separate exact-request path. - const ownsReconstructionContinuation = reconciledSourceRunId && executionBlocker?.runId === reconciledSourceRunId - && executionBlocker.recoveryActionId === enrichedContextSnapshot.recoveryActionId; + const recovery = await prepareRecovery({ + agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, + requestedByActorType: opts.requestedByActorType, requestedByActorId: opts.requestedByActorId, + reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, + commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, + queuedCommentInterruptId: opts.queuedCommentInterruptId, + queuedCommentRequestId: opts.queuedCommentRequestId, + taskRetryRequestId: opts.taskRetryRequestId, + ...(durableRequest?.resumeDeferred && !failedChatRetry ? { durableChatRequest: durableRequest } : {}), + interactionPayload: payload, + reconciliation: reconciledSourceRunId && reconciledRecoveryActionId ? { + sourceRunId: reconciledSourceRunId, actionId: reconciledRecoveryActionId, + } : undefined, + }); + const { blocker: executionBlocker, ownsReconstruction: ownsReconstructionContinuation } = recovery; const continuationHold = () => executionBlocker ?? { recoveryActionId: null, cause: "native_continuation_requires_reconciliation" as const, nextAction: continuationWait.message, }; @@ -4118,19 +4105,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende ? await undeliveredContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id }) : null; if (continuationToDeliver) Object.assign(enrichedContextSnapshot, undeliveredContinuationContext(continuationToDeliver, enrichedContextSnapshot)); - if ((executionBlocker || opts.queuedCommentInterruptId) && !ownsReconstructionContinuation && !(await admitExplicitNativeContinuation({ - db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, - agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, - reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, - commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, - queuedCommentInterruptId: opts.queuedCommentInterruptId, - queuedCommentRequestId: opts.queuedCommentRequestId, - taskRetryRequestId: opts.taskRetryRequestId, - ...(durableRequest?.resumeDeferred && !failedChatRetry ? { durableChatRequest: durableRequest } : {}), - interactionPayload: payload, - dryRun: true, - onBlocked: (reason, message) => { continuationBlocked = true; continuationWait = { reason, message }; }, - })) && (executionBlocker || (continuationBlocked && !verifiedConversationReset))) { + const busyRecovery = await recovery.checkBusyAdmission(); + if (busyRecovery.wait) continuationWait = busyRecovery.wait; + if (busyRecovery.kind === "blocked") { // A hold left by an unfinished run never refuses an ordinary // continuation: unless its cause is the person's, the fallback // settles it in this transaction and this wake becomes the fresh @@ -4764,7 +4741,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return { kind: "coalesced" as const, run: activeExecutionRun }; } - const admission = await admitWakeBehindIssueExecution({ + const admission = await busyRecovery.admitBusy({ companyId: agent.companyId, issueId: issue.id, agentId, @@ -4999,20 +4976,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return { kind: "skipped" as const }; } - const explicitContinuation = await admitExplicitNativeContinuation({ - db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, - agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, - reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, - commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, - queuedCommentInterruptId: opts.queuedCommentInterruptId, - queuedCommentRequestId: opts.queuedCommentRequestId, - taskRetryRequestId: opts.taskRetryRequestId, - ...(durableRequest?.resumeDeferred && !failedChatRetry ? { durableChatRequest: durableRequest } : {}), - interactionPayload: payload, - onBlocked: (reason, message) => { continuationBlocked = true; continuationWait = { reason, message }; }, - }); - if (!explicitContinuation && (executionBlocker || (continuationBlocked && !verifiedConversationReset)) && !ownsReconstructionContinuation) - return deferBlockedExecution(continuationHold()); + const freshRecovery = await busyRecovery.checkFreshAdmission(); + if (freshRecovery.wait) continuationWait = freshRecovery.wait; + if (freshRecovery.kind === "blocked") return deferBlockedExecution(continuationHold()); + const { explicitContinuation } = freshRecovery; if (explicitContinuation) { enrichedContextSnapshot.forceFreshSession = true; enrichedContextSnapshot.previousRunId = explicitContinuation.previousRunId; @@ -5030,7 +4997,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }, issueActivityPublications); } - const newRun = await admitFreshIssueRun({ + const newRun = await freshRecovery.admitFresh({ receipt: { agentId, source, diff --git a/server/src/services/workspace-recovery-admission.test.ts b/server/src/services/workspace-recovery-admission.test.ts index 56534a0f4dd..76bd35a91f9 100644 --- a/server/src/services/workspace-recovery-admission.test.ts +++ b/server/src/services/workspace-recovery-admission.test.ts @@ -1457,9 +1457,16 @@ describe("automatic retained workspace recovery", () => { queueDeps.wakeQueue.admitIssueWake.mockImplementation((target, evaluate, effects) => admitIssueWake(target, (tx, admission) => evaluate(tx, { ...admission, - admitWakeBehindIssueExecution(input) { - expect(input.durableReceipt).toBeUndefined(); - return admission.admitWakeBehindIssueExecution(input); + async prepareRecovery(input) { + const recovery = await admission.prepareRecovery(input); + return { ...recovery, async checkBusyAdmission() { + const busy = await recovery.checkBusyAdmission(); + if (busy.kind !== "allowed") return busy; + return { ...busy, admitBusy(input) { + expect(input.durableReceipt).toBeUndefined(); + return busy.admitBusy(input); + } }; + } }; }, }), effects)); queueDeps.liveRunExecutions.has.mockImplementation(id => t.deps.activeRunExecutions.has(id));