diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 54c550cc33b..7ab7698cd0d 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1828,6 +1828,21 @@ that classification finishes. waits for the previous executor to finish releasing its task resources. - A verified dead runner starts a replacement from the same durable root and resumes the same provider checkpoint. Only the operating-system PID changes. +- A prepared remote workspace interrupted before the executor's first claim + can continue on the same run without a provider checkpoint. Under the recovery + locks, Paperclip requires attempt and controller generation zero, no prior + controller or lease owner, and no session, process, launch, or provider evidence. + Any existing durable runner root must still prove an untouched bootstrap before + execution starts. A prior claim or contradictory evidence retains the hold. + A server-owned preparation protocol marker is required. It is recorded only + when fresh native identities are created by a server with the ownership fence. + Older unmarked preparations remain held during rolling upgrades; revisiting an + existing run never upgrades that marker from caller-supplied profile data. +- If the original preparation resumes after another execution claimed the run, + it yields without failing the shared run or cleaning up the new owner. An + expired foreign lease still requires normal recovery before takeover. + Recovery leaves this server's active preparation with its current execution. + A detached execution also preserves the current owner's run-scoped gateway tokens. - A runner that died before its first authenticated connection can restart on the same run only when its durable root proves that no provider authority or checkpoint exists. Paperclip quarantines the incomplete root first. diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 6dff82acc7b..e9a7df8bdc4 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1140,6 +1140,25 @@ recording the proof and successor lineage. Retained provider files are not edite Bootstrap retries, exact-checkpoint resumes, and fresh replacement sessions share three total provider attempts, including the original attempt. Linked run IDs, controller restarts, and duplicate wakes do not reset this budget. Automatic attempts retain the 30-second delay. Replacement scheduling and predecessor lineage commit together, with one successor per predecessor and admission through the normal task locks, authorization, pause, approval, and budget gates. +A prepared remote workspace interrupted before the executor's first durable +claim may resume its untouched bootstrap on the same run. This requires attempt +and controller generation zero, no prior owner, and no process, provider session, +checkpoint, or launch evidence. The executor still verifies any durable runner +root before starting. Missing evidence after a prior claim does not qualify. +Classification preserves the provider attempt count; execution consumes the +first attempt through the normal claim and admission gates. + +This path also requires a server-owned preparation fencing marker created with +fresh native run identities. Unmarked preparations from older servers remain +held during rolling upgrades, because those servers cannot safely yield to a +recovery owner. Revisiting an existing preparation cannot add that marker. + +A preparation that resumes after another execution acquired the coordinator +lease yields through the controller-detachment path. It does not fail the shared +run, release its environment, or start another provider. Lease expiry alone does +not authorize that stale preparation to take ownership; normal recovery must +establish a new claim first. + An exact local Codex restart can restore the conversation after losing its active turn. Runnerd records this as `provider_turn_lost_on_restore`, with no invented task result. The admitted restart attempt may send one continuation in that same diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 92a8cddf543..10912e00640 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -139,6 +139,11 @@ proofs, encryption keys, environment variables, provider credentials, command arguments, or an unsanitized stderr stream. Detailed failed-attempt diagnostics remain in the bounded `native_run_finalizations.recovery_history` ledger. +For a prepared remote workspace interrupted before its first executor claim, +that ledger records `remote_execution_never_claimed` with a +`bootstrap_incomplete` disposition. The provider attempt remains zero during +classification; the executor increments it before starting provider work. + For a local Codex turn lost while runnerd was stopped, the canonical `turn.failed` event retains `error.code: "provider_turn_lost_on_restore"` and `error.recoverable: true`. It records the interrupted turn, not a final task diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index 70615f26782..ce2fd3161cf 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -24,6 +24,17 @@ Agent access: `available` — REST / native call_api; CLI issue diagnostics. Policy: Task/run visibility, run ownership and source/action policy; workspace export retry and some operator controls require board. +The server can recover a prepared remote bootstrap only with a durable +never-claimed executor receipt and no contradictory execution evidence. This +automatic classification grants no new agent action or permission; normal +execution admission and durable runner-state verification still apply. +The current server's active preparation remains with its existing execution. +A preparation that loses the coordinator lease cannot fail the shared run or +revoke the new owner's gateway tokens, even after lease expiry. Only normal +recovery can establish a takeover claim; no agent permission is added. +Older unmarked preparations do not qualify for this automatic path during an +upgrade. The fencing marker comes from fresh server-owned identity creation; +caller profile data cannot upgrade an existing preparation. Evidence: `source-only` — [implementation](../../server/src/routes/issues.ts#L9478) at the reviewed commit above. diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 7ca8f4be28b..76e1c5c2025 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -101,6 +101,23 @@ the supported recovery until the source task produces a useful result or a specific escalation. Reload and verify that resolved notices retire and attempt history remains. A fixture pass does not qualify every live harness/provider. +The [native restart integration suite](../../server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts) +also covers a prepared remote workspace interrupted before the executor's first +claim. Real database tests assert one same-run bootstrap claim under concurrent +controllers and reject prior claims, launch/provider evidence, and malformed +workspace references. These cases prove classification, not a live remote +provider restart. For that journey, interrupt a disposable deployment after +workspace preparation and before its first executor claim, then confirm one +provider turn starts on the original run through the normal admission gates. + +The same database suite pauses an original preparation while recovery claims +the remote run. It then verifies that the old preparation yields without opening +a provider or changing the recovery owner, for both live and expired leases. +[Heartbeat boundary tests](../../server/src/services/heartbeat/runtime-execution.test.ts) +verify that the detachment signal transfers recovery ownership before cleanup. +The suite also rejects an older unmarked preparation and verifies that calling +preparation again cannot add the fencing marker from supplied profile data. + For a native workspace owner hold, repeat issue reads and recovery checks after the task becomes done or cancelled. Verify one unchanged action ID, attempt count, and timestamps; task completion must not authorize copyback. Clear only diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index 60efde0bb50..4cee3f2734c 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -176,7 +176,10 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = expect(input.effects.onNativeDispatchStarted).toHaveBeenCalledOnce(); expect(input.effects.onLegacyAdapterEntered).not.toHaveBeenCalled(); expect(await finalizations()).toHaveLength(0); - expectCleanup(input); + if (error instanceof NativeControllerDetachedForRestartError) { + expect(revokeHeartbeatRunGatewayTokens).not.toHaveBeenCalled(); + expect(input.instructions.releaseInstructionCopy).toHaveBeenCalledOnce(); + } else expectCleanup(input); }); it("dispatches native execution with the selected session and persists its usage", async () => { diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 80e4bf183c5..931b81edc59 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -264,6 +264,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe ); } let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; + let preserveRunGatewayTokens = false; const inspectFinalizeWorkspaceBranch = async () => { const workspaceRecord = persistedExecutionWorkspace?.id ? await executionWorkspacesSvc.getStoredById( @@ -986,6 +987,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe // Preserve the provider and its run for the new controller. This // also keeps generic teardown from terminalizing/releasing its lease. input.effects.onNativeSessionResumeScheduled(); + preserveRunGatewayTokens = true; throw adapterErr; } if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) { @@ -1128,7 +1130,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe throw adapterErr; } finally { try { - await revokeHeartbeatRunGatewayTokens({ + if (!preserveRunGatewayTokens) await revokeHeartbeatRunGatewayTokens({ db, companyId: agent.companyId, runId: run.id, diff --git a/server/src/services/heartbeat/runtime-selection.test.ts b/server/src/services/heartbeat/runtime-selection.test.ts index b15146b4e8c..152e30d6966 100644 --- a/server/src/services/heartbeat/runtime-selection.test.ts +++ b/server/src/services/heartbeat/runtime-selection.test.ts @@ -92,7 +92,7 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = if (!selected.selected) throw new Error("Expected selection"); const saved = await read(run.id); expect(saved).toMatchObject({ runtimeMode: "native", driverKind: "codex_app_server", nativePhase: "observed", nativeSessionId: expect.any(String), runnerInstanceId: selected.nativeRunnerInstanceId, - runnerProfileJson: { nativeExecutionInput: selected.nativeExecution, recoveryEventInventoryVersion: 1 } }); + runnerProfileJson: { nativeExecutionInput: selected.nativeExecution, recoveryEventInventoryVersion: 1, nativePreparationFencingVersion: 1 } }); expect(await db.select().from(completionContracts)).toHaveLength(1); expect(selected.getNativeFreshSessionHandoff).toBe(input.session.getFreshSessionHandoff); expect(input.onNativeOwnershipHeld).not.toHaveBeenCalled(); @@ -125,6 +125,24 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = }); }); + it("does not upgrade an older preparation from supplied runtime profile data", async () => { + const { input, run } = await fixture(); + const selected = await selectHeartbeatRuntime(db, input); + if (!selected.selected || !selected.nativeExecution || input.nativeRuntimeResolution.kind !== "native") { + throw new Error("Expected native selection"); + } + const saved = await read(run.id); + const olderProfile = { ...saved.runnerProfileJson }; + delete olderProfile.nativePreparationFencingVersion; + await db.update(heartbeatRuns).set({ runnerProfileJson: olderProfile }).where(eq(heartbeatRuns.id, run.id)); + input.run = await read(run.id); + input.session.persistedNativeExecutionInput = selected.nativeExecution; + input.session.persistedRunnerProfile = { ...olderProfile, nativePreparationFencingVersion: 1 }; + Object.assign(input.nativeRuntimeResolution.profile, { nativePreparationFencingVersion: 1 }); + expect((await selectHeartbeatRuntime(db, input)).selected).toBe(true); + expect((await read(run.id)).runnerProfileJson).not.toHaveProperty("nativePreparationFencingVersion"); + }); + it("restores the immutable admitted input and saved checkpoint when newer context arrives", async () => { const { input, run } = await fixture(); const selected = await selectHeartbeatRuntime(db, input); diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index 5ae76e464bd..27aad33d9ee 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -70,6 +70,7 @@ import { materializeExternalChatQuestionResponseInput, } from "../native-runtime/external-chat-question-response-input.js"; import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY } from "../chat-control-recovery-stop.js"; +import { NATIVE_PREPARATION_FENCING_VERSION } from "../native-runtime/native-active-runs.js"; import type { issueService } from "../issues.js"; import { buildPlanReviewContext } from "../plan-review-context.js"; import { resolvePaperclipRunnerIdleTimeoutMs } from "@paperclipai/adapter-utils"; @@ -932,6 +933,9 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele runnerProfileJson: { ...nativeRuntimeResolution.profile, ...lockedProfile, + nativePreparationFencingVersion: !lockedRun.runnerInstanceId && !lockedRun.nativeSessionId + ? NATIVE_PREPARATION_FENCING_VERSION + : lockedProfile.nativePreparationFencingVersion, ...(lockedProfile.nativeExecutionInput ? {} : { recoveryEventInventoryVersion: 1 }), diff --git a/server/src/services/native-runtime/native-active-runs.ts b/server/src/services/native-runtime/native-active-runs.ts new file mode 100644 index 00000000000..b1b6bc2c47a --- /dev/null +++ b/server/src/services/native-runtime/native-active-runs.ts @@ -0,0 +1,2 @@ +export const activeNativeExecutionRunIds = new Set(); +export const NATIVE_PREPARATION_FENCING_VERSION = 1; diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 96378b4bc2d..886ab2d9849 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1,7 +1,12 @@ import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; -import { recordNativeLocalProcessStop } from "../native-local-process-stop.js"; +import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; +import { + PROCESS_IDENTITY_RECORDED, + PROCESS_START_REQUESTED, + recordNativeLocalProcessStop, +} from "../native-local-process-stop.js"; import { randomUUID } from "node:crypto"; -import { and, desc, eq, inArray, isNull, lte, or, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, isNotNull, isNull, lte, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, @@ -287,6 +292,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: { runnerPidAlive: boolean; runnerGroupAlive: boolean; remoteSandbox?: boolean; + unclaimedBootstrap?: boolean; processStartMatches: boolean; knownProviderProcessAlive?: boolean; knownProviderProcessIdentityAmbiguous?: boolean; @@ -308,6 +314,9 @@ export function classifyNativeRunnerRecoveryEvidence(input: { // inspection; the executor must prove the original lease and authenticate // the surviving runner before it can resume delivery. if (input.remoteSandbox) { + if (input.unclaimedBootstrap === true && !input.hasCheckpoint && !input.hasProviderEvidence) { + return { claimKind: "bootstrap_incomplete", reason: "remote_execution_never_claimed" }; + } return input.hasCheckpoint && input.checkpointIdentityMatches === true && input.hasProviderEvidence @@ -750,8 +759,54 @@ export async function claimNativeRestartRecoveries(input: { const hasProviderEvidence = hasCheckpointProviderIdentity || providerEvents.length > 0; + let unclaimedBootstrap = + remoteWorkspace?.state === "prepared" && + row.run.runnerProfileJson?.nativePreparationFencingVersion === NATIVE_PREPARATION_FENCING_VERSION && + row.coordinator.phase === "observed" && + row.coordinator.attempt === 0 && + row.coordinator.controllerGeneration === 0 && + !row.coordinator.controllerBootId && + !row.coordinator.controllerPid && + !row.coordinator.controllerProcessStartedAt && + !row.coordinator.leaseOwner && + !row.coordinator.leaseExpiresAt && + !row.coordinator.failureDetail?.successorRunId && + !row.run.processPid && + !row.run.processGroupId && + !row.run.processStartedAt && + !row.run.sessionIdBefore && + !row.run.sessionIdAfter && + Boolean(row.run.nativeSessionId && row.run.runnerInstanceId) && + !hasCheckpoint && + !hasProviderEvidence; + if (unclaimedBootstrap) { + if (activeNativeExecutionRunIds.has(row.run.id) && + controller.bootId === (await currentNativeControllerIdentity()).bootId) { + return { kind: "awaiting_evidence", runId: row.run.id, reason: "current_controller_preparing" } as const; + } + const [executionEvent] = await tx + .select({ id: heartbeatRunEvents.id }) + .from(heartbeatRunEvents) + .where(and( + eq(heartbeatRunEvents.companyId, row.run.companyId), + eq(heartbeatRunEvents.runId, row.run.id), + or( + isNotNull(heartbeatRunEvents.sourceEventId), + inArray(heartbeatRunEvents.eventType, [ + "adapter.invoke", + PROCESS_START_REQUESTED, + PROCESS_IDENTITY_RECORDED, + "tool.execution.started", + ]), + ), + )) + .limit(1); + unclaimedBootstrap = !executionEvent; + } + const classification = classifyNativeRunnerRecoveryEvidence({ remoteSandbox, + unclaimedBootstrap, runnerPidAlive, runnerGroupAlive, processStartMatches: exactRunnerIdentity, diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index ac77c21ab4e..76587710106 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -14,6 +14,7 @@ import { companies, createDb, heartbeatRuns, + heartbeatRunEvents, issueRecoveryActions, issues, nativeRunFinalizations, @@ -40,6 +41,7 @@ import { readProcessStartedAt } from "../hot-restart.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; import { buildNativeExecutionInput } from "./native-execution-input.js"; import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; +import { executePaperclipNativeSession, NativeControllerDetachedForRestartError } from "./native-session-executor.js"; const mockCaptureRunFailure = vi.hoisted(() => vi.fn()); vi.mock("../../sentry.js", async () => { @@ -255,7 +257,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( issueId, phase: "observed", }); - return { db, issueId, runId, native }; + const [preparedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + return { db, issueId, runId, native, preparedProfile: preparedRun.runnerProfileJson }; } function transportOptions( @@ -1069,6 +1072,120 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( ]); }); + it("recovers an unclaimed remote startup once without requiring a provider checkpoint", async () => { + const fixture = await seedRun("REMOTE-UNCLAIMED"); + await fixture.db.update(heartbeatRuns).set({ + runnerProfileJson: { ...fixture.preparedProfile, nativeWorkspaceSync: remotePreparedWorkspace() }, + }).where(eq(heartbeatRuns.id, fixture.runId)); + const dispositions = (await Promise.all([successor, { ...successor, bootId: randomUUID() }].map(controller => + claimNativeRestartRecoveries({ db: fixture.db, controller, restartKind: "hard", runIds: [fixture.runId] }), + ))).flat(); + expect(dispositions.filter(row => row.kind === "bootstrap_incomplete")).toEqual([ + expect.objectContaining({ runId: fixture.runId, providerAttempt: 0, controllerGeneration: 1 }), + ]); + expect(dispositions.filter(row => row.kind === "awaiting_evidence")).toHaveLength(1); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run).toMatchObject({ status: "running", nativeSessionId: fixture.native.normalizedSessionId }); + expect(await fixture.db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))).toHaveLength(0); + }); + + it.each([false, true])("yields an original preparation after remote recovery claims its run (expired lease=%s)", async (expired) => { + const fixture = await seedRun(`REMOTE-PREPARATION-${expired}`); + await fixture.db.update(heartbeatRuns).set({ + runnerProfileJson: { ...fixture.preparedProfile, nativeWorkspaceSync: remotePreparedWorkspace() }, + }).where(eq(heartbeatRuns.id, fixture.runId)); + const execution = buildNativeExecutionInput({ + companyId, agentId, runId: fixture.runId, + issue: { id: fixture.issueId, identifier: null, title: "Resume prepared work", description: null, workMode: "standard" }, + taskPrompt: "Resume prepared work", provider: "codex", + workspace: { id: randomUUID(), cwd: runtimeRoot, repoUrl: null, repoRef: null, branchName: null }, + normalizedSessionId: fixture.native.normalizedSessionId, + runtimeContext: nativeRuntimeContextFixture(), + completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", + contract: { revision: "1", objective: "Resume prepared work", criteria: [{ id: "result", requirement: "Complete work" }] } }, + }); + let prepared!: () => void, release!: () => void; + const preparing = new Promise(resolveReady => { prepared = resolveReady; }); + const continuePreparation = new Promise(resolveContinue => { release = resolveContinue; }); + const openSession = vi.fn(async () => { throw new Error("Losing preparation must not open a provider"); }); + const descriptor = vi.fn(async () => { throw new Error("Losing preparation must not configure a provider"); }); + const original = executePaperclipNativeSession({ + db: fixture.db, execution, runnerInstanceId: fixture.native.runnerInstanceId, + backend: { descriptor, openSession }, + preparationSpans: [{ name: "heartbeat.prepare_after_environment", startedAtMs: Date.now(), endedAtMs: Date.now() }], + onEvent: async event => { + if (event.payload?.span === "heartbeat.prepare_after_environment") { + prepared(); + await continuePreparation; + } + }, + }).catch(error => error); + try { + await Promise.race([preparing, original.then(error => { throw error; })]); + expect(await claimNativeRestartRecoveries({ db: fixture.db, restartKind: "hard", runIds: [fixture.runId] })).toEqual([ + { kind: "awaiting_evidence", runId: fixture.runId, reason: "current_controller_preparing" }, + ]); + const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, restartKind: "hard", runIds: [fixture.runId] }); + expect(claim.kind).toBe("bootstrap_incomplete"); + if (expired) await fixture.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + const [before] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)); + release(); + expect(await original).toBeInstanceOf(NativeControllerDetachedForRestartError); + expect(openSession).not.toHaveBeenCalled(); + expect(descriptor).not.toHaveBeenCalled(); + expect((await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)))[0]).toEqual(before); + expect((await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)))[0].status).toBe("running"); + expect(await fixture.db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))).toHaveLength(0); + } finally { + release(); + await original; + } + }); + + function remotePreparedWorkspace() { + return { schema: "paperclip.native-workspace-sync/v3", state: "prepared", + descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, + workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: randomUUID(), + remoteCwd: "/workspace", resourceDisposition: "destroy" }; + } + + it.each(["claimed", "generation", "process", "launch", "provider-event", "checkpoint", "invalid-workspace", "unfenced"])( + "keeps remote startup recovery blocked with contradictory %s evidence", async (kind) => { + const fixture = await seedRun(`REMOTE-${kind}`); + let profile = fixture.preparedProfile; + if (kind === "unfenced") { + profile = { ...profile }; + delete profile.nativePreparationFencingVersion; + await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, fixture.runId)); + const [olderRun] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + await prepareNativeHeartbeatRun({ db: fixture.db, + run: { ...olderRun, runnerProfileJson: { ...profile, nativePreparationFencingVersion: 1 } }, + issue: { id: fixture.issueId, title: "Older preparation", description: null }, environmentLeaseId: "older-lease" }); + const [unchanged] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(unchanged.runnerProfileJson).not.toHaveProperty("nativePreparationFencingVersion"); + profile = unchanged.runnerProfileJson; + } + await fixture.db.update(heartbeatRuns).set({ + runnerProfileJson: { ...profile, nativeWorkspaceSync: kind === "invalid-workspace" ? {} : remotePreparedWorkspace(), + ...(kind === "checkpoint" ? { sessionCheckpoint: {} } : {}) }, + ...(kind === "process" ? { processPid: 2_000_000_001 } : {}), + }).where(eq(heartbeatRuns.id, fixture.runId)); + if (kind === "claimed" || kind === "generation") await fixture.db.update(nativeRunFinalizations) + .set(kind === "claimed" ? { attempt: 1 } : { controllerGeneration: 1 }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + if (kind === "launch" || kind === "provider-event") await fixture.db.insert(heartbeatRunEvents).values({ + companyId, runId: fixture.runId, agentId, seq: 1, stream: "system", level: "info", + eventType: kind === "launch" ? "native.process_start_requested" : "item.updated", + ...(kind === "provider-event" ? { sourceEventId: "provider-event-1" } : {}), + }); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([ + { kind: "blocked", runId: fixture.runId, reason: "ambiguous_remote_recovery_evidence" }, + ]); + }, + ); + it.each([2, 3])("recovers a checkpointed restart interruption within provider budget %s", async (attempt) => { const fixture = await seedRun(`INTERRUPTED-${attempt}`); await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index daa869dd6de..a31043d23f6 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -36,6 +36,7 @@ import { selectRemotePiCompanion } from "./remote-pi-companion.js"; import { readNativeLocalProcessStop, PROCESS_START_REQUESTED } from "../native-local-process-stop.js"; import { remoteLeaseCleanupScope } from "../remote-execution-termination.js"; import { resolveConnectorAssignments, isConnectorSkill } from "../connector-runtime.js"; +import { activeNativeExecutionRunIds } from "./native-active-runs.js"; import { boundedExecutionCleanup, EXECUTION_CONTROL_DEADLINE_MS, @@ -7515,6 +7516,7 @@ export async function executePaperclipNativeSession(input: { resolve: session => resolveStartup(session), }; nativeSessionStartups.set(runId, startup); + activeNativeExecutionRunIds.add(runId); let preparedInput: typeof input = input; let cleanupStagedAttachments: () => Promise = async () => undefined; let sessionScopeId: string | null = null; @@ -7620,6 +7622,7 @@ export async function executePaperclipNativeSession(input: { startup.settle(); if (nativeSessionStartups.get(runId) === startup) { nativeSessionStartups.delete(runId); + activeNativeExecutionRunIds.delete(runId); } if ( ownsSessionScope && @@ -7862,11 +7865,9 @@ async function executePaperclipNativeSessionWithinScope( throw new Error("native_run_already_committed"); if ( coordinator.leaseOwner && - coordinator.leaseOwner !== leaseOwner && - coordinator.leaseExpiresAt && - coordinator.leaseExpiresAt > leaseNow + coordinator.leaseOwner !== leaseOwner ) - throw new Error("native_finalization_lease_busy"); + throw new NativeControllerDetachedForRestartError(); const recovering = input.restartRecovery; if ( recovering && diff --git a/server/src/services/native-runtime/prepare-native-run.ts b/server/src/services/native-runtime/prepare-native-run.ts index 7414101b949..087fcac00a0 100644 --- a/server/src/services/native-runtime/prepare-native-run.ts +++ b/server/src/services/native-runtime/prepare-native-run.ts @@ -7,6 +7,7 @@ import { completionContracts, heartbeatRuns } from "@paperclipai/db"; import { ensureNativeCompletionContract } from "./completion-contracts.js"; import { NATIVE_RUNTIME_RESOLVER_VERSION } from "./runtime-mode.js"; +import { NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY } from "../chat-control-recovery-stop.js"; function record(value: unknown): Record { @@ -136,6 +137,11 @@ export async function prepareNativeHeartbeatRun(input: { turnId, itemId, environmentLeaseId, + ...(!locked.runnerInstanceId && !locked.nativeSessionId + ? { nativePreparationFencingVersion: NATIVE_PREPARATION_FENCING_VERSION } + : Object.hasOwn(record(locked.runnerProfileJson), "nativePreparationFencingVersion") + ? { nativePreparationFencingVersion: record(locked.runnerProfileJson).nativePreparationFencingVersion } + : {}), // This server-owned field is never taken from the supplied profile. // Retain malformed evidence too so admission can fail closed later. ...(Object.hasOwn(