diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 9409d0d608e..17ee6c4f445 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -330,6 +330,10 @@ persisted cancellations of parked retries through the normal guarded cancellatio path if the original owner stops or its final database write fails. They preserve the original cancellation request and never start a provider turn for that work. +Replaying an audited native Stop also fences a later same-run retry. It preserves +the original intent and acknowledgement. That retry fence does not prove that +the provider stopped or that workspace cleanup finished. + Recovery instructions in a wake describe only an active or escalated recovery action. Resolving or cancelling that action removes its repair instructions from subsequent wakes, so the original worker can resume the task. This prompt change diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index fea3271f66d..59d7a94e408 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -99,6 +99,7 @@ import { buildNativeHeartbeatPreparationSpans, createNativeRunTrace } from "./na import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js"; import type { AdapterRuntimeEvent } from "../../adapters/index.js"; import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils"; +import { acknowledgedNativeStopExecutionHasStopped, isAcknowledgedNativeStop } from "../acknowledged-native-stop.js"; vi.mock("node:fs", async (importOriginal) => { const actual = await importOriginal(); @@ -7852,6 +7853,27 @@ describe("native session cancellation", () => { expect(state.persistActivity).not.toHaveBeenCalled(); }); + it.each([ + { companyId: "another-company" }, { runId: "another-run" }, { issueId: "another-issue" }, + { scope: "issue" }, { effects: ["cancel_issue"] }, { intentAuditId: null }, + ])("does not fence a failed retry for a foreign or stale Stop receipt %j", async changed => { + const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop", + ...execution.binding, scope: "run", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], + dispatchState: "acknowledged", dispatched: true, intentAuditId: "earlier-intent-audit", + acknowledgementAuditId: "earlier-ack-audit", ...changed }; + const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure" }; + const persistence = cancellationDb({ status: "failed", coordinator, resultJson: { nativeCancellation: intent } }); + const before = structuredClone(persistence.getResultJson()); + state.cancel.mockClear(); + state.persistActivity.mockClear(); + await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" })).rejects.toThrow(); + expect(persistence.updates).toEqual([]); + expect(coordinator.phase).toBe("retryable_failure"); + expect(persistence.getResultJson()).toEqual(before); + expect(state.cancel).not.toHaveBeenCalled(); + expect(state.persistActivity).not.toHaveBeenCalled(); + }); + it("does not mistake a recovered running attempt for the failed retry that reserved Stop", async () => { const id = "11111111-1111-4111-8111-111111111111"; const persistence = cancellationDb({ status: "running", @@ -7877,6 +7899,39 @@ describe("native session cancellation", () => { expect(persistence.getResultJsonUpdateCount()).toBe(writes); }); + it("fences a failed retry whose earlier Stop is already acknowledged without changing its receipts", async () => { + const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop", + ...execution.binding, scope: "run", reasonCode: "cancellation_run_only", + effects: ["release_run_resources"], dispatchState: "acknowledged", dispatched: true, + intentAuditId: "earlier-intent-audit", acknowledgementAuditId: "earlier-ack-audit", + acknowledgedAt: "2026-01-01T00:00:00.000Z", decisionId: null, priorCoordinatorDecisionId: null }; + const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure", + nextAttemptAt: new Date(Date.now() + 30_000) }; + const persistence = cancellationDb({ status: "failed", coordinator, + resultJson: { nativeCancellation: intent, executionCancellation: { state: "requested", requestedAt: "later-stop" } } }); + const before = structuredClone(persistence.getResultJson()); + state.cancel.mockClear(); + state.persistActivity.mockClear(); + for (let attempt = 0; attempt < 2; attempt++) { + await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" })) + .resolves.toMatchObject({ dispatched: true, auditId: intent.intentAuditId }); + expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null }); + expect(persistence.getResultJson()).toEqual(before); + } + expect(persistence.updates.filter(update => update.table === nativeRunFinalizations)).toHaveLength(1); + expect(persistence.getResultJsonUpdateCount()).toBe(0); + expect(state.cancel).not.toHaveBeenCalled(); + expect(state.persistActivity).not.toHaveBeenCalled(); + const stopped = { id: execution.binding.runId, companyId: execution.binding.companyId, + nativeIssueId: execution.binding.issueId, status: "cancelled", processPid: null, processGroupId: null, + resultJson: { ...persistence.getResultJson(), cancelledByActorType: "user", cancelledByUserId: "operator" } }; + const lease = { provider: "daytona", providerLeaseId: "unconfirmed-provider-resource", status: "active", + releasedAt: null, cleanupStatus: null, metadata: {} }; + const leaseDb = { select: () => ({ from: () => ({ where: async () => [lease] }) }) } as unknown as Db; + expect(isAcknowledgedNativeStop(stopped)).toBe(true); + await expect(acknowledgedNativeStopExecutionHasStopped(leaseDb, stopped)).resolves.toBe(false); + }); + it("uses the reserved caller UUID and permits same-intent and default retries", async () => { const id = "11111111-1111-4111-8111-111111111111"; const persistence = cancellationDb({ resultJson: { startupCancellation: { cancellationRequestId: id } } }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 604bff67972..37abb1b88be 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7541,6 +7541,27 @@ export async function cancelNativeSession( runId, companyId: cancellationContext.companyId, issueId: cancellationContext.issueId, resultJson, requestId: requestedId, scope: options.scope ?? "run", }, coordinator)) throw conflict("Native retry is no longer cancellable"); + const fenceRetryCancellation = async () => { + if (coordinator.phase !== "retryable_failure") return; + await tx + .update(nativeRunFinalizations) + .set({ + phase: "terminal_failure", + nextAttemptAt: null, + leaseOwner: null, + leaseExpiresAt: null, + recoveryState: "blocked", + failureCode: "native_retry_cancelled", + updatedAt: new Date(), + }) + .where( + and( + eq(nativeRunFinalizations.runId, runId), + eq(nativeRunFinalizations.companyId, cancellationContext.companyId), + eq(nativeRunFinalizations.phase, "retryable_failure"), + ), + ); + }; const existing = record(resultJson.nativeCancellation); const existingIntentId = typeof existing.intentId === "string" && existing.intentId.length > 0 @@ -7564,6 +7585,10 @@ export async function cancelNativeSession( : null; if (!existingAuditId) throw new Error("native_cancellation_intent_audit_missing"); + if (retryCancellation && coordinator.phase === "retryable_failure") { + await notifyNativeResultCommit(tx, { companyId: cancellationContext.companyId, runId }); + await fenceRetryCancellation(); + } return { intentId: existingIntentId, auditId: existingAuditId, @@ -7643,24 +7668,7 @@ export async function cancelNativeSession( // Cancellation is also an authority fence for a durable retry whose // preceding provider has already failed. No in-memory session is needed. await notifyNativeResultCommit(tx, { companyId: cancellationContext.companyId, runId }); - await tx - .update(nativeRunFinalizations) - .set({ - phase: "terminal_failure", - nextAttemptAt: null, - leaseOwner: null, - leaseExpiresAt: null, - recoveryState: "blocked", - failureCode: "native_retry_cancelled", - updatedAt: new Date(), - }) - .where( - and( - eq(nativeRunFinalizations.runId, runId), - eq(nativeRunFinalizations.companyId, cancellationContext.companyId), - eq(nativeRunFinalizations.phase, "retryable_failure"), - ), - ); + await fenceRetryCancellation(); intentPublication = activity.publication; return { intentId,