diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 3846a8f8497..6b1224ef3e2 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1302,6 +1302,14 @@ recording the proof and successor lineage. Retained provider files are not edite Bootstrap retries, exact-checkpoint resumes, and fresh replacement sessions share three total provider attempts, including the original attempt. Linked run IDs, controller restarts, and duplicate wakes do not reset this budget. Automatic attempts retain the 30-second delay. Replacement scheduling and predecessor lineage commit together, with one successor per predecessor and admission through the normal task locks, authorization, pause, approval, and budget gates. +When a same-run retry acquires a replacement plugin task lease, the executor +reloads the prior task's durable Runner checkpoint before checking local +lifecycle evidence. This includes retries after a provider turn omitted its +completion result. The existing lease-transfer policy must verify the company, +agent, workspace, retained session, and termination of prior execution. The +checkpoint must match the exact Runner, lease, run, session, turn, and item. +A missing restart claim does not remove these checks or authorize a fresh session. + A prepared remote workspace interrupted before the executor's first durable claim may resume its untouched bootstrap on the same run. This requires attempt and controller generation zero, no prior owner, and no process, provider session, diff --git a/maps/agent-access-map/execution-environments.md b/maps/agent-access-map/execution-environments.md index 210291cf46b..086632e6dd6 100644 --- a/maps/agent-access-map/execution-environments.md +++ b/maps/agent-access-map/execution-environments.md @@ -176,6 +176,15 @@ The task API does not transfer files or reveal guest filesystem paths. This source update does not qualify live agent access, warm retention, or a provider. See the [execution contract](../../doc/plugins/PLUGIN_RUNNER_EXECUTION.md). +Same-run checkpoint recovery also uses the saved lease path after a turn omits +its completion result. It requires verified prior termination, retained session +state, and matching company, agent, workspace, and complete Runner identity. +The [executor regression](../../server/src/services/native-runtime/native-session-executor.test.ts) +checks checkpoint retrieval and identity/termination denials. The +[lease authority fixture](../../server/src/__tests__/environment-task-authority.integration.test.ts) +checks the same-run read through the database-backed task API. These are +controlled regressions; live provider acceptance remains separate. + ## `images` Capability ID: `execution-environments/images`. diff --git a/maps/feature-map/execution-environments.md b/maps/feature-map/execution-environments.md index cbe6739bf34..66d97530d0d 100644 --- a/maps/feature-map/execution-environments.md +++ b/maps/feature-map/execution-environments.md @@ -91,6 +91,12 @@ The [fake-provider integration](../../server/src/__tests__/plugin-runner-task.in checks acquisition, SDK dispatch, scope denial, stop proof and deletion recovery. Its WebSocket is mocked; it does not qualify a live provider or a real Runner handshake. +For a same-run retry on a replacement plugin task, verify that the retained +checkpoint loads before provider startup. The prior execution must have a stop +receipt, and the checkpoint must match the original session identity. Include a +turn that ends without a completion result; a successful initial turn alone does +not exercise this recovery path. Reject missing stop proof and changed identities. + Automated: [execution target](../../server/src/__tests__/environment-execution-target.test.ts) and [custom images](../../server/src/__tests__/environment-custom-images-service.test.ts) cover contracts; [SSH live test](../../server/src/__tests__/environment-live-ssh.test.ts) requires its own external setup. The [environment runtime regression](../../server/src/__tests__/environment-runtime.test.ts) diff --git a/server/src/__tests__/environment-task-authority.integration.test.ts b/server/src/__tests__/environment-task-authority.integration.test.ts index 8671d52b8c8..446c590012f 100644 --- a/server/src/__tests__/environment-task-authority.integration.test.ts +++ b/server/src/__tests__/environment-task-authority.integration.test.ts @@ -111,6 +111,12 @@ databaseSuite("environment task lease authority", () => { const retryLeaseId = randomUUID(); await db.insert(environmentLeases).values({ id: retryLeaseId, companyId, environmentId, heartbeatRunId: thirdRunId, status: "active", provider: prior.provider, providerLeaseId: "retry-task", metadata: pin }); + const retryIdentity = { ...checkpointIdentity, runId: thirdRunId, turnId: "turn", itemId: "item" }; + await expect(resolvePluginTaskResume(db, { companyId, leaseId: retryLeaseId, + priorIdentity: retryIdentity })).resolves.toBe("third-task"); + await expect(runtime.task({ companyId, leaseId: retryLeaseId, operation: { + kind: "read_state", identity: retryIdentity, resumeFrom: "third-task", + } })).resolves.toMatchObject({ kind: "state", state: { ...retryIdentity, lifecycle: "suspended" } }); await expect(runtime.task({ companyId, leaseId: retryLeaseId, operation: { ...submit, projectIds: [], resumeFrom: "third-task" } })).resolves.toMatchObject({ kind: "accepted" }); const [retry] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, retryLeaseId)); await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(), diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 6cd4336a41f..de8105859b1 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -11862,7 +11862,7 @@ describe("runnerd provider runtime wiring", () => { } }); - it.each(["verified", "wrong-state", "no-stop-proof"] as const)("admits only an authorized same-run plugin crash checkpoint: %s", async mode => { + it.each(["verified", "wrong-state", "no-stop-proof", "disposition-retry", "disposition-wrong-state", "disposition-no-stop-proof"] as const)("admits only an authorized same-run plugin checkpoint: %s", async mode => { const current = scopedNativeExecution(`plugin-crash-${mode}`); const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!, runnerInstanceId: "plugin-runner", environmentLeaseId: "old-task-lease", turnId: "old-turn", itemId: "old-item" }; @@ -11871,13 +11871,13 @@ describe("runnerd provider runtime wiring", () => { const journal = JSON.stringify(durableControlPlaneState(identity)); await writeFile(join(root, "control-plane/control-plane-state.json"), journal); const resume = vi.spyOn(environmentTaskRuntime, "resolvePluginTaskResume").mockImplementation(async () => { - if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized"); + if (mode.endsWith("no-stop-proof")) throw new Error("Plugin Runner retained session is not authorized"); return "prior-task"; }); const task = vi.fn(async (operation: any) => { environmentTaskOperationSchema.parse(operation); if (operation.kind === "read_state") return { kind: "state", taskId: "next-task", state: { - ...durableRunnerState(identity, "ready"), ...(mode === "wrong-state" ? { itemId: "other" } : {}), + ...durableRunnerState(identity, "ready"), ...(mode.endsWith("wrong-state") ? { itemId: "other" } : {}), } }; return { kind: "description", launchPreparationVersion: 1, projectAuthCheckVersion: 1, taskId: "next-task", runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; @@ -11886,18 +11886,18 @@ describe("runnerd provider runtime wiring", () => { try { const backend = createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: identity.runnerInstanceId, runnerIngressAuthorized: true, runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "new-task-lease", runnerTask: task }, - restartRecovery: { kind: "resume_dead_runner", runId: current.binding.runId, leaseOwner: "controller", controllerGeneration: 2, - providerAttempt: 2, restartKind: "hard", recoveryRequestId: null } }); - if (mode === "verified") { + ...(mode.startsWith("disposition-") ? {} : { restartRecovery: { kind: "resume_dead_runner" as const, runId: current.binding.runId, leaseOwner: "controller", controllerGeneration: 2, + providerAttempt: 2, restartKind: "hard" as const, recoveryRequestId: null } }) }); + if (mode === "verified" || mode === "disposition-retry") { await expect(backend).resolves.toBeDefined(); expect(state.createBackend).toHaveBeenCalledOnce(); expect(JSON.parse(await readFile(join(root, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "ready" }); } else { - await expect(backend).rejects.toThrow(mode === "wrong-state" ? "checkpoint identity mismatch" : "not authorized"); + await expect(backend).rejects.toThrow(mode.endsWith("wrong-state") ? "checkpoint identity mismatch" : "not authorized"); expect(state.createBackend).not.toHaveBeenCalled(); } expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); - if (mode === "no-stop-proof") expect(task).not.toHaveBeenCalled(); + if (mode.endsWith("no-stop-proof")) expect(task).not.toHaveBeenCalled(); else expect(task).toHaveBeenCalledWith({ kind: "read_state", identity, resumeFrom: "prior-task" }); } finally { resume.mockRestore(); } }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 2dc23a42a76..4817a631bf9 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -4951,13 +4951,14 @@ async function migrateRunnerdStateRootForExecution(input: { throw new Error("runner_state_identity_mismatch: session_scope_mismatch"); } const taskTarget = input.runnerExecutionTarget; - if (input.restartRecovery?.kind === "resume_dead_runner" && durableIdentityMatchesExecution(identity, input.execution) - && taskTarget?.kind === "remote" && taskTarget.transport === "sandbox" && taskTarget.runnerTask && taskTarget.leaseId) { + if (durableIdentityMatchesExecution(identity, input.execution) + && taskTarget?.kind === "remote" && taskTarget.transport === "sandbox" && taskTarget.runnerTask && taskTarget.leaseId + && (input.restartRecovery?.kind === "resume_dead_runner" + || (!input.restartRecovery && identity.environmentLeaseId !== taskTarget.leaseId))) { const resumeFrom = taskTarget.sandboxLeaseAcquisition?.outcome === "resumed" ? undefined : await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId, leaseId: taskTarget.leaseId, priorIdentity: identity }); const result = environmentTaskResultSchema.parse(await taskTarget.runnerTask({ kind: "read_state", identity, resumeFrom })); if (result.kind !== "state") throw new Error("Plugin Runner checkpoint unavailable"); - // The new lease and source termination receipt authorize this exact same-run recovery. savePluginRunnerCheckpoint(scoped, result.state, identity); } if (input.restartRecovery?.kind === "bootstrap_incomplete") {