diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0101065c965..62622d23111 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3561,6 +3561,8 @@ fails closed as before. Provider execution and control-plane finalization have different clocks. A healthy provider can think or execute a long tool without output. Once execution settles, recovery and finalization control steps have a 60-second deadline, checked on startup and every 15 seconds. With a healthy database and scheduler, an abandoned transition must be repaired or surfaced within 90 seconds. Terminal persistence must not wait on provider cleanup or publication; a late finalizer cannot change a reassigned or closed task or release another run's locks. In particular, a done report from a run whose task now belongs to another agent, a user or nobody is kept as an accepted result but records `completion_owner_superseded` and preserves the task status for the current owner. Historical ambiguous runs are never automatically replayed after an upgrade. +A control deadline bounds the settlement of one attempt, never a later attempt of the same run. While a native run's finalization owner holds a live lease (for example, a controller that reclaimed the run after a server restart) or has a same-run resume scheduled, that owner is responsible for the run. The reconciler clears a run deadline left by an earlier attempt instead of failing the resumed work. When the live owner's own settlement deadline is due, the reconciler waits until that lease lapses before it surfaces the deadline. A run whose deadline genuinely expires fails with `execution_finalization_deadline_exceeded` and shows plain wording with Retry: its saved work is kept, and actions with unconfirmed outcomes are not repeated. Runs saved by earlier releases with the previous operator-facing wording show the same plain wording when the task is read; no migration is needed. + When that deadline fails a native run with a prepared workspace, the same transaction records the exact source's file-preservation obligation. Existing stop-and-retain recovery contains the provider and saves the files before diff --git a/server/src/services/execution-blocker-read.test.ts b/server/src/services/execution-blocker-read.test.ts index f23a8d23a55..bc9b56e9a0c 100644 --- a/server/src/services/execution-blocker-read.test.ts +++ b/server/src/services/execution-blocker-read.test.ts @@ -4,6 +4,7 @@ import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { EXECUTION_BLOCKER_PERSON_RETRY_ACTION, getExecutionBlocker } from "./execution-blocker.js"; +import { EXECUTION_FINALIZATION_DEADLINE_MESSAGE } from "./execution-control-deadline.js"; const support = await getEmbeddedPostgresTestSupport(); describe.skipIf(!support.supported)("execution blocker PostgreSQL read projection", () => { @@ -84,6 +85,20 @@ describe.skipIf(!support.supported)("execution blocker PostgreSQL read projectio expect(EXECUTION_BLOCKER_PERSON_RETRY_ACTION).toContain("Agents cannot retry it for them."); }); + it("shows plain wording and Retry for a finalization deadline saved by the previous release", async () => { + const previousReleaseText = "Inspect the failed run and verify its provider has stopped. Reconcile any uncertain external action before explicitly continuing this task."; + const f = await fixture({ runtimeMode: "native", errorCode: "execution_finalization_deadline_exceeded", + error: previousReleaseText, nextAction: previousReleaseText }); + await db.update(issueRecoveryActions).set({ cause: "execution_finalization_deadline_exceeded", ownerType: "board", + nextAction: previousReleaseText }).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); + + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + + expect(blocker).toMatchObject({ canRetry: true, runError: EXECUTION_FINALIZATION_DEADLINE_MESSAGE, + nextAction: `${EXECUTION_FINALIZATION_DEADLINE_MESSAGE} ${EXECUTION_BLOCKER_PERSON_RETRY_ACTION}` }); + expect(JSON.stringify(blocker)).not.toMatch(/Inspect the failed run|verify its provider|Reconcile any uncertain/); + }); + it("retains the claimed adapter and startup receipts for a cancelled pre-provider run", async () => { const f = await fixture({ status: "cancelled", startedAt: null, errorCode: "operator_interrupted", resultJson: { startupCancellation: { beforeNativeSelection: true }, startupPreparationSettledAt: "2020-01-01T00:01:00Z" } }); diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index 7bfaf9b054f..35229f1f2a3 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -18,6 +18,7 @@ import { completedWorkspaceRecoveryCondition, hasVerifiedWorkspaceRepair, worksp import { nativeManualAdmissionHoldsExecution } from "./native-runtime/native-runner-ownership.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED } from "./native-runtime/completed-turn-cleanup.js"; +import { EXECUTION_FINALIZATION_DEADLINE_EXCEEDED, EXECUTION_FINALIZATION_DEADLINE_MESSAGE } from "./execution-control-deadline.js"; const blockerContextKeys = ["issueId", "taskId", "source", "connectionIntentResolution", "interactionKind", "interactionStatus", "mutation", "interactionId"] as const; @@ -410,6 +411,7 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st // Earlier releases saved operator-facing text for the cleanup gate. Show // the plain wording from the code instead of the stored message. : run?.errorCode === NATIVE_SESSION_CLEANUP_QUARANTINED ? NATIVE_CLEANUP_SETTLING_MESSAGE + : run?.errorCode === EXECUTION_FINALIZATION_DEADLINE_EXCEEDED ? EXECUTION_FINALIZATION_DEADLINE_MESSAGE : cancellation?.reason ?? (run?.status === "cancelled" ? "Execution was cancelled; its source was not recorded." // Rows written before SQL text was kept out of run errors still carry it. @@ -424,7 +426,8 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st )).where(and( eq(chatConversations.companyId, companyId), eq(chatConversations.issueId, issueId), )).limit(1) : []; - let nextAction = action.nextAction; + let nextAction = action.status === "active" && action.cause === EXECUTION_FINALIZATION_DEADLINE_EXCEEDED + ? EXECUTION_FINALIZATION_DEADLINE_MESSAGE : action.nextAction; // Retry is a person's control on this task. Say so, so an agent reading this // record points the person to it instead of re-running the work itself. if (canRetry && !eligibleContinuation) nextAction += ` ${EXECUTION_BLOCKER_PERSON_RETRY_ACTION}`; diff --git a/server/src/services/execution-control-deadline.ts b/server/src/services/execution-control-deadline.ts index 244a9adbdb2..2ac86971c19 100644 --- a/server/src/services/execution-control-deadline.ts +++ b/server/src/services/execution-control-deadline.ts @@ -20,3 +20,6 @@ export async function boundedExecutionCleanup( if (timer) clearTimeout(timer); } } +export const EXECUTION_FINALIZATION_DEADLINE_EXCEEDED = "execution_finalization_deadline_exceeded"; +export const EXECUTION_FINALIZATION_DEADLINE_MESSAGE = + "The run stopped before it finished saving its work. Saved work is kept, and actions with unconfirmed outcomes will not be repeated."; diff --git a/server/src/services/execution-control-reconciliation.test.ts b/server/src/services/execution-control-reconciliation.test.ts index d53a2870965..6f19a577991 100644 --- a/server/src/services/execution-control-reconciliation.test.ts +++ b/server/src/services/execution-control-reconciliation.test.ts @@ -19,6 +19,7 @@ vi.mock("../sentry.js", async () => { }); import { reconcileAbandonedExecutionControl } from "./execution-control-reconciliation.js"; +import { EXECUTION_FINALIZATION_DEADLINE_MESSAGE } from "./execution-control-deadline.js"; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_EXECUTION_CONTROL_TEST_DATABASE_URL"); const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); @@ -224,6 +225,70 @@ describeEmbeddedPostgres("reconcileAbandonedExecutionControl reports a genuine f }, { timeout: 5_000 }); }); + async function seedResumedNativeRun(coordinator: Partial) { + const f = await seedAbandonedRunFixture(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.insert(nativeRunFinalizations).values({ runId: f.runId, companyId: f.companyId, issueId: f.issueId, + phase: "observed", controlDeadlineAt: null, ...coordinator }); + return f; + } + + it.each([ + ["reclaimed after a server restart", { phase: "observed", leaseOwner: "restart-claim", controllerGeneration: 2, + recoveryState: "awaiting_runner_reattach", leaseExpiresAt: new Date(Date.now() + 60_000) }], + ["waiting for its scheduled same-run resume", { phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, + recoveryState: "resuming_session", nextAttemptAt: new Date(Date.now() + 120_000) }], + ] as const)("keeps a native run %s running past the previous attempt's run deadline", async (_shape, coordinator) => { + const f = await seedResumedNativeRun(coordinator); + const captureCallsBefore = mockCaptureRunFailure.mock.calls.length; + + const result = await reconcileAbandonedExecutionControl(db); + + expect(result.surfaced).toBe(0); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run).toMatchObject({ status: "running", errorCode: null, error: null, executionControlDeadlineAt: null }); + const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect(owner).toMatchObject({ phase: coordinator.phase, leaseOwner: coordinator.leaseOwner, failureCode: null }); + const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + expect(task).toMatchObject({ status: "in_progress", executionRunId: f.runId, checkoutRunId: f.runId }); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toHaveLength(0); + expect(mockCaptureRunFailure.mock.calls.slice(captureCallsBefore)).toHaveLength(0); + }); + + it("waits for a live native owner to finish its own failure settlement before surfacing the deadline", async () => { + const leaseExpiresAt = new Date(Date.now() + 30_000); + const f = await seedResumedNativeRun({ phase: "observed", leaseOwner: "settling-controller", leaseExpiresAt, + controlDeadlineAt: new Date(Date.now() - 1_000) }); + + expect((await reconcileAbandonedExecutionControl(db)).surfaced).toBe(0); + const [waiting] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect(waiting).toMatchObject({ status: "running", errorCode: null }); + expect(owner.controlDeadlineAt?.getTime()).toBe(leaseExpiresAt.getTime()); + + const lapsed = new Date(Date.now() - 1_000); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: lapsed, controlDeadlineAt: lapsed }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + expect((await reconcileAbandonedExecutionControl(db)).surfaced).toBe(1); + const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(failed).toMatchObject({ status: "failed", errorCode: "execution_finalization_deadline_exceeded" }); + }); + + it("explains a genuine finalization deadline in plain words without asking a person to repair the run", async () => { + const f = await seedAbandonedRunFixture(); + + await reconcileAbandonedExecutionControl(db); + + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); + expect(run.status).toBe("failed"); + for (const text of [run.error, run.nextAction, action?.nextAction]) { + expect(text).toBe(EXECUTION_FINALIZATION_DEADLINE_MESSAGE); + expect(text).not.toMatch(/inspect|verify|reconcile|provider/i); + } + }); + it("reports zero events for a repeated sweep over the same already-failed run", async () => { const { runId } = await seedAbandonedRunFixture(); await reconcileAbandonedExecutionControl(db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index dffbf539735..8b8e6d41617 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -22,7 +22,8 @@ import { hasLiveLegacyController } from "./legacy-controller-lease.js"; import { reportRunFailure } from "./run-failure-report.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js"; import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js"; -import { EXECUTION_RECONCILIATION_INTERVAL_MS } from "./execution-control-deadline.js"; +import { EXECUTION_FINALIZATION_DEADLINE_EXCEEDED, EXECUTION_FINALIZATION_DEADLINE_MESSAGE, + EXECUTION_RECONCILIATION_INTERVAL_MS } from "./execution-control-deadline.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { NATIVE_STARTUP_CLEANUP_PENDING, readNativeStartupContext, promoteNativeStartupCleanup } from "./native-runtime/native-startup-cleanup.js"; import { releaseAbandonedNativeOwners } from "./native-runtime/native-failed-run-owner-release.js"; @@ -176,6 +177,24 @@ export async function reconcileAbandonedExecutionControl( } return false; } + if (coordinator && !coordinator.resultId && run.status === "running") { + const nativeOwnerLeaseAt = coordinator.leaseOwner && coordinator.leaseExpiresAt && coordinator.leaseExpiresAt > now + ? coordinator.leaseExpiresAt : null; + const sameRunResumeScheduled = coordinator.phase === "retryable_failure" && !coordinator.leaseOwner; + const nativeSettlementDue = Boolean(coordinator.controlDeadlineAt && coordinator.controlDeadlineAt <= now); + if (!nativeSettlementDue && (nativeOwnerLeaseAt || sameRunResumeScheduled)) { + await tx.update(heartbeatRuns).set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)); + return false; + } + if (nativeOwnerLeaseAt) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization); + await tx.update(nativeRunFinalizations).set({ controlDeadlineAt: nativeOwnerLeaseAt }) + .where(eq(nativeRunFinalizations.runId, run.id)); + await tx.update(heartbeatRuns).set({ executionControlDeadlineAt: run.executionControlDeadlineAt ? nativeOwnerLeaseAt : null }) + .where(eq(heartbeatRuns.id, run.id)); + return false; + } + } await tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) @@ -217,9 +236,8 @@ export async function reconcileAbandonedExecutionControl( .where(eq(heartbeatRuns.id, run.id)); return false; } - const cause = "execution_finalization_deadline_exceeded"; - const nextAction = - "Inspect the failed run and verify its provider has stopped. Reconcile any uncertain external action before explicitly continuing this task."; + const cause = EXECUTION_FINALIZATION_DEADLINE_EXCEEDED; + const nextAction = EXECUTION_FINALIZATION_DEADLINE_MESSAGE; // The deadline replaces the run's error code. Keep the earlier // diagnosis as its root cause, as the native finalizer does. const priorFailureCode = coordinator?.failureCode ?? run.errorCode;