diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index a235ab12885..d846c688622 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -225,6 +225,17 @@ provider has stopped or change native cancellation acknowledgement. A known execution hold is a waiting condition, not a new execution attempt. Every issue wake must read the current effective reconciliation hold under the issue admission lock before creating a run. Resolved recovery bookkeeping can still carry a no-replay hold; only clearing the effective hold makes admission eligible again. The final dispatch gate remains required for changes after admission. +A queued legacy run denied by that final gate is still a wait. Its matching +dispatch-gate receipt, task identity, and absence of runtime selection, execution +ownership, environment leases, coordinator, or execution events prove that it +never started. Recovery must not create another unknown-outcome incident for +that admission denial. Historical incidents with the same verified evidence are +nonblocking without deleting their audit records. The original hold and any +workspace repair obligation remain effective until their own recovery completes. +Once all current gates clear, startup and periodic queue recovery reconsider the +saved input through normal admission. The cancelled admission is not replayed, +and the user need not certify infrastructure state in a reconciliation form. + Repeated automatic signals for an unchanged gate share one durable skipped-wake diagnostic, scoped to company, agent, issue, gate code, and condition identity. The diagnostic retains the first request and counts later observations. This applies to execution reconciliation, dependencies, pause holds, company and agent availability, budget blocks, and disabled heartbeats. These diagnostics do not consume provider attempts and are never proof that a future wake was delivered. All current gates are checked again on the next wake, including the periodic dependency reconciliation sweep. Clearing one gate does not bypass another. New comments received during an execution hold retain their individual deferred receipts and ordered comment ids. Release cannot drain those receipts while replay remains blocked; the next eligible wake can adopt them. Authorized external-chat requests also remain deferred with their exact durable receipt. They must use normal promotion and current authorization; a generic wake cannot adopt only their comment ids and discard their actor or session contract. A wait does not authorize replay, reset an incident retry budget, or bypass an interaction's delivery rules. diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index 9bb221e6ca2..bd6a174ab93 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -46,6 +46,16 @@ cannot authorize replacement or erase the original ownership marker. This is internal supervision; it grants no new agent recovery action. Existing task, company, process-identity and workspace-recovery checks remain required. +An execution-hold denial before legacy runtime selection is not a provider +attempt. The server verifies the dispatch receipt and task identity against +persisted execution, lease and coordinator evidence. The same proof excludes +historical false incidents from effective holds while retaining audit history. +This internal classification grants no new agent API or resolution authority. +The original hold, file recovery, company scope and normal admission gates stay +effective. Local [database regressions](../../server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts) +cover automatic saved-message admission and denied cases; live provider recovery +remains a separate verification step. + Evidence: `source-only` — [implementation](../../server/src/routes/issues.ts#L9478) at the reviewed commit above. diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 8f3d23b61c6..df950319767 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -65,6 +65,16 @@ After a stop, send a new instruction on the same task or inspect one already saved while cleanup was pending. Observe the waiting message, continuation, and eventual result rather than manually changing the task status to look complete. +A continuation denied before startup by an existing execution hold remains a +wait. Once the original hold clears, the server verifies the dispatch receipt +and absence of execution evidence, ignores any historical false incident for +that unused admission, and reconsiders saved input through normal admission. +No infrastructure-certification form is needed. The +[database regressions](../../server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts) +cover incident prevention, retained audit history, concurrent restart admission, +ordered messages, contradictory execution evidence, and unchanged pause and +task gates. These tests do not qualify a live provider or prove task completion. + ### `browser-reconnect` While viewing an active or recovering task, interrupt the browser connection diff --git a/server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts b/server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts new file mode 100644 index 00000000000..40ad08c67f4 --- /dev/null +++ b/server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts @@ -0,0 +1,182 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + agentWakeupRequests, agents, companies, createDb, environmentLeases, + heartbeatRunEvents, heartbeatRuns, issueComments, issueRecoveryActions, issues, issueTreeHoldMembers, issueTreeHolds, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { getExecutionBlocker } from "../services/execution-blocker.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("undispatched execution waits", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("undispatched-execution-wait-"); + db = createDb(database.connectionString); + }, 90_000); + afterAll(async () => { await database?.cleanup(); }); + + async function seed(withHistoricalHold = false) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + const runId = randomUUID(), actionId = randomUUID(), originalActionId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Admission wait", defaultResponsibleUserId: "board", issuePrefix: "W" + companyId.replaceAll("-", "") }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", adapterType: "paperclip_runner", status: "idle", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue saved work", status: "todo", assigneeAgentId: agentId }); + const [run] = await db.insert(heartbeatRuns).values({ + id: runId, companyId, agentId, runtimeMode: "legacy", status: "cancelled", + errorCode: "execution_reconciliation_required", finishedAt: new Date(Date.now() - 60_000), + contextSnapshot: { issueId, taskId: issueId }, + resultJson: { stopReason: "execution_reconciliation_required", timeoutSource: "stale_queued_run_gate", + timeoutFired: false, timeoutConfigured: false, effectiveTimeoutSec: 0, + executionWait: { issueId, recoveryActionId: originalActionId } }, + }).returning(); + if (withHistoricalHold) await db.insert(issueRecoveryActions).values({ + id: actionId, companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + cause: "legacy_execution_requires_reconciliation", fingerprint: "legacy-execution:" + runId, + status: "resolved", outcome: "blocked", ownerType: "board", nextAction: "Inspect the stopped work.", + evidence: { runId, automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } }, + }); + return { companyId, agentId, issueId, runId, actionId, originalActionId, run }; + } + + async function saveMessages(f: Awaited>) { + // Occupy the agent so admission cannot launch a provider in this test. + const [busy] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running" }).returning(); + const commentIds = [randomUUID(), randomUUID()]; + await db.insert(issueComments).values(commentIds.map((id, index) => ({ id, companyId: f.companyId, + issueId: f.issueId, authorType: "user", authorUserId: index ? "second-user" : "board", + body: index ? "Preserve the recovered files." : "Continue the original work.", + createdAt: new Date(Date.now() - 20_000 + index) }))); + const queueId = randomUUID(); + await db.insert(agentWakeupRequests).values({ id: queueId, companyId: f.companyId, agentId: f.agentId, + source: "automation", reason: "issue_commented", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: "board", payload: { issueId: f.issueId, + commentId: commentIds[1], _paperclipWakeContext: { issueId: f.issueId, wakeReason: "issue_commented", + wakeCommentId: commentIds[1], wakeCommentIds: commentIds } } }); + return { commentIds, queueId, busy }; + } + + it("does not create another provider recovery incident for an admission denial", async () => { + const f = await seed(); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, f.run)).toBe(false); + await terminalizeLegacyExecution({ db, run: f.run, status: "cancelled", reconcileIfNeeded: true }); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId))).toHaveLength(0); + }); + + it("recognizes a historical false hold without deleting its audit history", async () => { + const f = await seed(true); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId)))[0]) + .toMatchObject({ status: "resolved", outcome: "blocked", evidence: { automaticRecovery: { replay: "blocked" } } }); + }); + + it("resumes all saved messages once through normal admission after restart", async () => { + const f = await seed(true); + const { commentIds, queueId } = await saveMessages(f); + await Promise.all([heartbeatService(db).resumeQueuedRuns(), heartbeatService(db).resumeQueuedRuns()]); + const successors = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued"))); + expect(successors).toHaveLength(1); + expect(successors[0].contextSnapshot).toMatchObject({ issueId: f.issueId, wakeCommentIds: commentIds }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, queueId)))[0]) + .toMatchObject({ status: "queued", runId: successors[0].id }); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + }); + + it.each(["pause", "daily_limit", "reassigned", "agent_paused", "company_inactive"])( + "still honors %s during automatic admission", async gate => { + const f = await seed(true); + const { busy } = await saveMessages(f); + if (gate === "pause") { + const holdId = randomUUID(); + await db.insert(issueTreeHolds).values({ id: holdId, companyId: f.companyId, rootIssueId: f.issueId, mode: "pause", status: "active" }); + await db.insert(issueTreeHoldMembers).values({ companyId: f.companyId, holdId, issueId: f.issueId, depth: 0, issueTitle: "Continue saved work", issueStatus: "todo" }); + } else if (gate === "daily_limit") { + await db.update(agents).set({ runtimeConfig: { heartbeat: { maxConcurrentRuns: 1, maxDailyRuns: 0 } } }).where(eq(agents.id, f.agentId)); + } else if (gate === "company_inactive") { + await db.update(companies).set({ status: "paused" }).where(eq(companies.id, f.companyId)); + } else if (gate === "agent_paused") { + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId)); + } else { + await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId)); + } + await heartbeatService(db).resumeQueuedRuns(); + if (gate === "daily_limit") { + // This cap is rechecked at dispatch, after ordinary deferred promotion. + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, busy.id)); + await heartbeatService(db).resumeQueuedRunsForAgent(f.companyId, f.agentId); + const attempts = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + expect(attempts.filter(run => run.id !== f.runId && run.id !== busy.id)) + .toEqual([expect.objectContaining({ status: "cancelled", startedAt: null, errorCode: "heartbeat.daily_run_limit" })]); + } + expect(await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")))).toHaveLength(0); + }, + ); + + it.each(["foreign_run", "wrong_issue", "malformed_run", "workspace"])( + "retains historical actions with %s evidence", async kind => { + const f = await seed(true); + let issueId = f.issueId; + if (kind === "wrong_issue") { + issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId: f.companyId, title: "Different source", status: "todo", assigneeAgentId: f.agentId }); + await db.update(issueRecoveryActions).set({ sourceIssueId: issueId }).where(eq(issueRecoveryActions.id, f.actionId)); + } else { + const evidence = { runId: kind === "foreign_run" ? (await seed()).runId : kind === "malformed_run" ? "invalid-uuid" : f.runId, + automaticRecovery: { replay: "blocked" }, ...(kind === "workspace" ? { workspaceRestoreFailure: "restore_unsafe_archive" } : {}) }; + await db.update(issueRecoveryActions).set({ evidence }).where(eq(issueRecoveryActions.id, f.actionId)); + } + expect(await getExecutionBlocker(db, f.companyId, issueId)).toMatchObject({ recoveryActionId: f.actionId }); + }, + ); + + it("keeps the real upstream workspace obligation effective", async () => { + const f = await seed(true), sourceId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: sourceId, companyId: f.companyId, agentId: f.agentId, + runtimeMode: "native", nativeIssueId: f.issueId, status: "failed", startedAt: new Date(), finishedAt: new Date(), + contextSnapshot: { issueId: f.issueId }, resultJson: { workspaceRestoreFailure: "restore_unsafe_archive" } }); + await db.insert(issueRecoveryActions).values({ id: f.originalActionId, companyId: f.companyId, + sourceIssueId: f.issueId, kind: "active_run_watchdog", cause: "native_continuation_requires_reconciliation", + fingerprint: sourceId, status: "resolved", outcome: "blocked", ownerType: "board", + evidence: { runId: sourceId, workspaceRestoreFailure: "restore_unsafe_archive", automaticRecovery: { replay: "blocked" } }, + nextAction: "Recover the saved files before continuing." }); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ + recoveryActionId: f.originalActionId, workspaceRepairRequired: true, + }); + }); + + it.each(["started", "runtime_selected", "process", "session", "controller", "stage", "receipt", "receipt_issue", "receipt_action", "lease", "coordinator", "launch", "provider", "workspace"])( + "retains a hold with conflicting or incomplete %s evidence", async kind => { + const f = await seed(true); + const patch = kind === "started" ? { startedAt: new Date() } + : kind === "runtime_selected" ? { runtimeModeResolvedAt: new Date() } + : kind === "process" ? { processPid: 999999999 } + : kind === "session" ? { nativeSessionId: randomUUID() } + : kind === "controller" ? { controllerBootId: randomUUID() } + : kind === "coordinator" ? { nativeIssueId: f.issueId } + : kind === "stage" ? { executionStage: "dispatching" } + : kind === "receipt" ? { resultJson: { stopReason: "execution_reconciliation_required" } } + : kind === "receipt_issue" ? { resultJson: { ...f.run.resultJson, executionWait: { issueId: randomUUID(), recoveryActionId: f.originalActionId } } } + : kind === "receipt_action" ? { resultJson: { ...f.run.resultJson, executionWait: { issueId: f.issueId, recoveryActionId: "invalid-uuid" } } } + : kind === "workspace" ? { resultJson: { ...f.run.resultJson, workspaceRestoreFailure: "restore_unsafe_archive" } } : {}; + const run = Object.keys(patch).length + ? (await db.update(heartbeatRuns).set(patch).where(eq(heartbeatRuns.id, f.runId)).returning())[0] + : f.run; + if (kind === "lease") await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: f.runId, + provider: "local", status: "released", releasedAt: new Date(), cleanupStatus: "succeeded" }); + if (kind === "coordinator") await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, + issueId: f.issueId, runId: f.runId, phase: "observed" }); + if (kind === "launch" || kind === "provider") await db.insert(heartbeatRunEvents).values({ + companyId: f.companyId, agentId: f.agentId, runId: f.runId, seq: 1, + eventType: kind === "launch" ? "native.process_start_requested" : "provider.event", stream: "system" }); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(true); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ recoveryActionId: f.actionId }); + }, + ); +}); diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index f2d9b5b4357..cce2bc3e372 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -1,4 +1,5 @@ import { isPreDispatchReviewWait } from "./pre-dispatch-review-wait.js"; +import { preDispatchExecutionWaitRecoveryActionPredicate } from "./pre-dispatch-execution-wait.js"; import { and, desc, eq, gt, inArray, not, or, sql } from "drizzle-orm"; import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; import { z } from "zod"; @@ -43,6 +44,7 @@ export async function hasPendingRunCleanup(db: Db, companyId: string, runId: str export function executionBlockerPredicate() { return and( not(conversationRecoveryActionPredicate()!), + not(preDispatchExecutionWaitRecoveryActionPredicate()), or(unreconciledNativeExportPredicate(), and( inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), or(inArray(issueRecoveryActions.status, ["active", "escalated"]), diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index b3b3609f31b..8c08c103647 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -1,6 +1,7 @@ import { hasRequiredWorkspaceRecovery, preserveWorkspaceRestoreRecoveryMetadata, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources, type LegacyWorkspaceRestoreSource } from "./legacy-workspace-restore-recovery.js"; import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js"; +import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wait.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js"; import { claimedAdapterType, hasConversationContinuationPolicy } from "./conversation-continuation.js"; @@ -56,11 +57,12 @@ export function legacyExecutionNeedsReconciliation( ); } -/** Review-wait receipts are only exempt after retained execution evidence agrees. +/** Admission-wait receipts are only exempt after retained execution evidence agrees. * The synchronous classifier stays conservative for callers without a DB proof. */ export async function legacyExecutionNeedsReconciliationWithEvidence(db: Db, run: Run): Promise { return await hasUnrestoredRemoteWorkspace(db, run) || - (legacyExecutionNeedsReconciliation(run) && !(await isPreDispatchReviewWaitVerified(db, run))); + (legacyExecutionNeedsReconciliation(run) && !(await isPreDispatchReviewWaitVerified(db, run)) + && !(await isPreDispatchExecutionWaitVerified(db, run))); } /** Persist the failed legacy run, owned lock release and operator decision together. */ diff --git a/server/src/services/pre-dispatch-execution-wait.ts b/server/src/services/pre-dispatch-execution-wait.ts new file mode 100644 index 00000000000..228d1a12d74 --- /dev/null +++ b/server/src/services/pre-dispatch-execution-wait.ts @@ -0,0 +1,63 @@ +import { and, eq, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { canonicalExecutionUuid } from "./execution-identity.js"; +import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; +import { LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; + +/** Admission denied by an existing hold is not a provider attempt. Require the + * dispatcher's receipt and absence of execution authority, resources and events. */ +function preDispatchExecutionWaitPredicate() { + return sql`coalesce( + ${heartbeatRuns.runtimeMode} = 'legacy' AND ${heartbeatRuns.status} = 'cancelled' + AND ${heartbeatRuns.finishedAt} IS NOT NULL + AND ${heartbeatRuns.errorCode} = 'execution_reconciliation_required' + AND ${heartbeatRuns.resultJson}->>'stopReason' = 'execution_reconciliation_required' + AND ${heartbeatRuns.resultJson}->>'timeoutSource' = 'stale_queued_run_gate' + AND ${canonicalExecutionUuid(sql`${heartbeatRuns.resultJson}->'executionWait'->>'issueId'`)} + = ${canonicalExecutionUuid(sql`${heartbeatRuns.contextSnapshot}->>'issueId'`)} + AND (${heartbeatRuns.issueId} IS NULL OR ${heartbeatRuns.issueId} + = ${canonicalExecutionUuid(sql`${heartbeatRuns.contextSnapshot}->>'issueId'`)}) + AND ${canonicalExecutionUuid(sql`${heartbeatRuns.resultJson}->'executionWait'->>'recoveryActionId'`)} IS NOT NULL + AND ${heartbeatRuns.startedAt} IS NULL AND ${heartbeatRuns.runtimeModeResolvedAt} IS NULL + AND ${heartbeatRuns.processPid} IS NULL AND ${heartbeatRuns.processGroupId} IS NULL + AND ${heartbeatRuns.processStartedAt} IS NULL AND ${heartbeatRuns.nativeIssueId} IS NULL + AND ${heartbeatRuns.nativeSessionId} IS NULL AND ${heartbeatRuns.sessionIdAfter} IS NULL + AND ${heartbeatRuns.controllerBootId} IS NULL AND ${heartbeatRuns.controllerLeaseExpiresAt} IS NULL + AND ${heartbeatRuns.executionStage} IS NULL + AND coalesce(${heartbeatRuns.resultJson}->>'workspaceRestoreFailure', '') <> 'restore_unsafe_archive' + AND coalesce(${heartbeatRuns.resultJson}->'workspaceRestoreRecovery'->>'schema', '') <> ${LEGACY_WORKSPACE_RECOVERY_SCHEMA} + AND NOT EXISTS (SELECT 1 FROM ${environmentLeases} + WHERE ${environmentLeases.companyId} = ${heartbeatRuns.companyId} + AND ${environmentLeases.heartbeatRunId} = ${heartbeatRuns.id}) + AND NOT EXISTS (SELECT 1 FROM ${nativeRunFinalizations} + WHERE ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId} + AND ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}) + AND NOT EXISTS (SELECT 1 FROM ${heartbeatRunEvents} + WHERE ${heartbeatRunEvents.companyId} = ${heartbeatRuns.companyId} + AND ${heartbeatRunEvents.runId} = ${heartbeatRuns.id} + AND (${heartbeatRunEvents.sourceEventId} IS NOT NULL OR ${heartbeatRunEvents.eventType} IN + ('adapter.invoke', ${PROCESS_START_REQUESTED}, ${PROCESS_IDENTITY_RECORDED}, 'harness.ready', + 'session.started', 'session.resumed', 'session.updated', 'turn.started', + 'provider.event', 'provider.rpc_result', 'tool.execution.started'))), false)`; +} + +export async function isPreDispatchExecutionWaitVerified(db: Db, run: typeof heartbeatRuns.$inferSelect): Promise { + if (run.errorCode !== "execution_reconciliation_required" || run.runtimeMode !== "legacy") return false; + const [verified] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id), preDispatchExecutionWaitPredicate(), + )).limit(1); + return Boolean(verified); +} + +/** Re-evaluate old false incidents with the same proof; retain their audit rows + * and leave the original execution/workspace hold to its own recovery policy. */ +export function preDispatchExecutionWaitRecoveryActionPredicate() { + return sql`(${issueRecoveryActions.cause} = 'legacy_execution_requires_reconciliation' + AND coalesce(${issueRecoveryActions.evidence}->>'workspaceRestoreFailure', '') <> 'restore_unsafe_archive' + AND coalesce(${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema', '') <> ${LEGACY_WORKSPACE_RECOVERY_SCHEMA} + AND EXISTS (SELECT 1 FROM ${heartbeatRuns} + WHERE ${heartbeatRuns.companyId} = ${issueRecoveryActions.companyId} + AND ${heartbeatRuns.id} = ${canonicalExecutionUuid(sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')`)} + AND ${issueRecoveryActions.sourceIssueId} = ${canonicalExecutionUuid(sql`${heartbeatRuns.contextSnapshot}->>'issueId'`)} + AND ${preDispatchExecutionWaitPredicate()}))`; +}