diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 326da73f0bd..72a0cb98324 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1156,10 +1156,12 @@ controller, lease, or result). It also checks for contradictory launch/process evidence and verifies local cleanup or exact remote termination receipts. The preparer must have finished or its startup lease must have expired. A missing PID alone does not establish this proof. -For older interrupted preparation rows without a cancellation receipt, the +For older interrupted preparation rows without a cancellation receipt, or with +a retained startup fence but no preparation-settled marker, the immutable Paperclip Runner adapter claim, unresolved runtime, and preparing stage must agree. The old controller must belong to another server boot and its lease -must have expired. No native identity, coordinator, result, adapter invocation, +must have expired. Only absent results or the retained before-selection +cancellation fence qualify. No native identity, coordinator, adapter invocation, provider event, or process-launch evidence may exist. Environment cleanup still requires the same receipts. This historical proof permits explicit Retry or a newer saved user message; it does not replay the cancelled input. diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.ts index ef7fb126d5f..9bcadce31cf 100644 --- a/server/src/services/cancelled-native-startup.ts +++ b/server/src/services/cancelled-native-startup.ts @@ -28,14 +28,15 @@ export async function isCancelledNativeStartup(db: Db, run: Run, coordinator: Co if (run.status !== "cancelled" || !run.finishedAt || run.processPid || run.processGroupId || run.processStartedAt || run.sessionIdAfter) return false; const cancellation = run.resultJson?.startupCancellation as Record | undefined; - // Older builds did not retain the cancellation fence. Their immutable + // Older builds could omit the cancellation fence or its unwind marker. Their immutable // native-adapter claim and unresolved preparation stage still prove that // provider dispatch did not begin. Require an expired owner from another // server boot; neither a missing PID nor mutable agent settings is proof. const historicalBeforeSelection = run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && run.executionStage === "preparing" && !run.nativeIssueId && !run.nativeSessionId && !coordinator && claimedAdapterType(run) === "paperclip_runner" && run.errorCode === "operator_interrupted" && - run.resultJson === null && Boolean(run.controllerBootId && run.controllerBootId !== legacyControllerBootId && + (run.resultJson === null || cancellation?.beforeNativeSelection === true) && + Boolean(run.controllerBootId && run.controllerBootId !== legacyControllerBootId && run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt <= new Date()); const beforeSelection = historicalBeforeSelection || run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && !run.nativeSessionId && !coordinator && claimedAdapterType(run) === "paperclip_runner" && diff --git a/server/src/services/explicit-native-continuation.test.ts b/server/src/services/explicit-native-continuation.test.ts index 31b0cb0f4a8..9ddb4a898be 100644 --- a/server/src/services/explicit-native-continuation.test.ts +++ b/server/src/services/explicit-native-continuation.test.ts @@ -598,21 +598,24 @@ const support = await getEmbeddedPostgresTestSupport(); expect(await admit(f)).toMatchObject({ previousRunId: f.sourceRunId }); }); - async function seedHistoricalCancelledPreparation() { + async function seedHistoricalCancelledPreparation(retainedReceipt = false) { const f = await seedCancelledStartup(); await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.sourceRunId)); await db.delete(environmentLeases).where(eq(environmentLeases.heartbeatRunId, f.sourceRunId)); await db.update(heartbeatRuns).set({ runtimeMode: "legacy", runtimeModeResolvedAt: null, nativeIssueId: null, nativeSessionId: null, executionStage: "preparing", errorCode: "operator_interrupted", - runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, resultJson: null, + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + resultJson: retainedReceipt ? { operatorInterrupted: true, stopReason: "cancelled", + startupCancellation: { beforeNativeSelection: true, requestedAt: "2026-09-11T10:00:00Z" } } : null, }).where(eq(heartbeatRuns.id, f.sourceRunId)); await db.update(issueRecoveryActions).set({ cause: "legacy_execution_requires_reconciliation" }) .where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); return f; } - it.each(["message", "retry"])("recovers historical native preparation through an explicit %s", async kind => { - const f = await seedHistoricalCancelledPreparation(); + it.each([false, true].flatMap(receipt => ["message", "retry"].map(kind => ({ receipt, kind }))))( + "recovers historical native preparation through an explicit $kind (receipt: $receipt)", async ({ receipt, kind }) => { + const f = await seedHistoricalCancelledPreparation(receipt); expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ canRetry: true }); await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running" }); const successor = await heartbeatService(db).wakeup(f.agentId, { @@ -625,11 +628,13 @@ const support = await getEmbeddedPostgresTestSupport(); previousRunId: f.sourceRunId, explicitUserContinuation: { commentId: kind === "message" ? f.commentId : null } } }); expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId)); - expect(source).toMatchObject({ status: "cancelled", runtimeMode: "legacy", resultJson: null }); + expect(source).toMatchObject({ status: "cancelled", runtimeMode: "legacy", + resultJson: receipt ? { startupCancellation: { beforeNativeSelection: true } } : null }); }); - it.each(["fresh", "earlier_delivered", "last_delivered"])("resumes saved input after historical native preparation expires across restart exactly once (%s)", async kind => { - const f = await seedHistoricalCancelledPreparation(); + it.each([false, true].flatMap(receipt => ["fresh", "earlier_delivered", "last_delivered"].map(kind => ({ receipt, kind }))))( + "resumes saved input after historical native preparation expires across restart exactly once ($kind, receipt: $receipt)", async ({ receipt, kind }) => { + const f = await seedHistoricalCancelledPreparation(receipt); await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running" }); await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }) .where(eq(heartbeatRuns.id, f.sourceRunId)); @@ -658,14 +663,14 @@ const support = await getEmbeddedPostgresTestSupport(); expect(successors).toHaveLength(1); expect(successors[0].contextSnapshot).toMatchObject({ wakeCommentIds: [f.commentId], explicitUserContinuation: { commentId: f.commentId }, forceFreshSession: true }); - const [receipt] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, waiting.id)); - expect(receipt).toMatchObject({ status: "coalesced", runId: successors[0].id }); + const [queueReceipt] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, waiting.id)); + expect(queueReceipt).toMatchObject({ status: "coalesced", runId: successors[0].id }); expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); }); - it.each(["resolved", "dispatching", "adapter", "missing_boot", "current_boot", "missing_lease", "live_lease", "receipt", "launch", "invoked", "coordinator", "cleanup", "remote"])( - "holds historical native preparation with contradictory or incomplete %s evidence", async kind => { - const f = await seedHistoricalCancelledPreparation(); + it.each([false, true].flatMap(receipt => ["resolved", "dispatching", "adapter", "missing_boot", "current_boot", "missing_lease", "live_lease", "receipt", "launch", "invoked", "coordinator", "cleanup", "remote"].map(kind => ({ receipt, kind }))))( + "holds historical native preparation with contradictory or incomplete $kind evidence (receipt: $receipt)", async ({ receipt, kind }) => { + const f = await seedHistoricalCancelledPreparation(receipt); const patch = kind === "resolved" ? { runtimeModeResolvedAt: new Date() } : kind === "dispatching" ? { executionStage: "dispatching" } : kind === "adapter" ? { runnerProfileJson: { adapterDispatch: { adapterType: "process" } } }