diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index ae94ee4e86f..06445ea65c3 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3836,6 +3836,18 @@ to a replacement during a deploy overlap closes its idle warm sessions within a second, and closes each session that becomes idle after that, so the replacement usually waits only for that close. +The same wait applies before a follow-up resumes a Runner that its saved state +shows as suspended. A suspended state read during the outgoing server's close +does not prove that the close finished. The follow-up starts only when that +server is gone, or when a harness backup that holds the suspended state was +completed after the finished turn. A resume that starts a new Runner uses the +OpenCode launch files the session was admitted with. When no harness backup +carries that admission, it reads the receipt recorded on the session's last +run. A server that leaves new run starts to a replacement also leaves same-run +retries to it, and re-checks every few seconds in case the replacement stops. +A blocked parent whose run waits for its exact same-run resume on a remote +Runner shows that scheduled resume and no Retry. + A fresh sandbox is only requested, not guaranteed: the next acquisition replaces the marked sandbox only when the saved workspace and harness backup allow it. When the next attempt meets an unretired owner again on the same diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index afb8763b56a..0b81445c7a2 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -97,6 +97,35 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => .toMatchObject({ state: "failed", canRetry: true }); }); + it("a blocked parent whose run awaits its exact same-run resume shows retry_scheduled with no Retry", async () => { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Blocked parent", issuePrefix: `B${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Parent agent", adapterType: "paperclip_runner" }); + const [issue] = await db.insert(issues).values({ companyId, title: "Wait for subtasks", + status: "blocked", assigneeAgentId: agentId }).returning(); + await db.insert(issues).values({ companyId, title: "Open subtask", status: "todo", parentId: issue.id, assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issue.id, scopeKind: "issue", + status: "failed", runtimeMode: "native", errorCode: "native_session_interrupted", finishedAt: new Date(), + processPid: 16675, processStartedAt: new Date() }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issue.id)); + const resumeAt = new Date(Date.now() + 30_000); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId, phase: "retryable_failure", + attempt: 1, nextAttemptAt: resumeAt, failureCode: "native_session_interrupted", + failureDetail: { recoverable: true, recoveryMode: "exact_checkpoint_resume", checkpointExists: true } }); + for (const presentation of [(await taskInterruptionPresentations(db, companyId, [issue])).get(issue.id), + await taskInterruptionPresentation(db, issue)]) { + expect(presentation).toMatchObject({ state: "retry_scheduled", canRetry: false, nextRetryAt: resumeAt.toISOString() }); + } + await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, issue.id)); + expect((await taskInterruptionPresentations(db, companyId, [issue])).get(issue.id)) + .toMatchObject({ state: "failed", canRetry: true }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issue.id)); + await db.update(nativeRunFinalizations).set({ failureDetail: { recoverable: true, recoveryMode: "fresh_session" } }) + .where(eq(nativeRunFinalizations.runId, runId)); + expect((await taskInterruptionPresentations(db, companyId, [issue])).get(issue.id)) + .toMatchObject({ state: "failed", canRetry: true }); + }); + it("explains a saved Retry held by unfinished blockers instead of offering Retry again (restart chaos t05)", async () => { const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Waiting parent", issuePrefix: `W${companyId.slice(0, 7)}` }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 7f51a4b089d..4d66ec03d4a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -5,7 +5,7 @@ import { createStorageWaitNotice } from "./heartbeat/storage-wait-notice.js"; import { createStalledQueuedRunCancel } from "./heartbeat/stalled-queued-run.js"; import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; import { isProcessRunAdmissionClosed } from "./heartbeat/run-admission.js"; -import { isYieldingRunStarts, refreshRunStartYieldIfStale, type RunStartHandover } from "./run-start-handover.js"; +import { isYieldingRunStarts, refreshRunStartYieldIfStale, RUN_START_YIELD_RECHECK_MS, type RunStartHandover } from "./run-start-handover.js"; import type { StopInvocationOptions } from "./heartbeat/run-control.js"; import { isContextCompactionRun } from "./context-compaction.js"; import { isUnstartedTaskQuestionAnswer } from "./unstarted-task-question-answer.js"; @@ -1868,6 +1868,10 @@ export function heartbeatService( const dispatchWork: Promise = (async () => { if ((await getSchedulingSuppression()).suppressed) return; if (nativeRecoveryDeadlines.get(runId) !== pending) return; + if (options.runStartHandover ? options.runStartHandover.isYielding() : isYieldingRunStarts()) { + scheduleNativeSessionResumeDispatch(runId, new Date(Date.now() + RUN_START_YIELD_RECHECK_MS), mode); + return; + } nativeRecoveryDeadlines.delete(runId); if (mode === "restart") { await recoverNativeRunsAfterRestart(new Date(), [runId]); diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts index 747a342254f..96a68d605f7 100644 --- a/server/src/services/heartbeat/recovery.test.ts +++ b/server/src/services/heartbeat/recovery.test.ts @@ -33,6 +33,7 @@ import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { hasNativeLocalProcessStop, PROCESS_START_REQUESTED, releaseSettledNativeRunProcess } from "../native-local-process-stop.js"; import { legacyControllerBootId } from "../legacy-controller-lease.js"; import { runBoundedShutdownStep } from "../../shutdown.js"; +import { registerRunStartHandover } from "../run-start-handover.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -693,6 +694,26 @@ describePostgres("heartbeat recovery module ownership", () => { }, ); + it("an outgoing server does not claim a same-run retry once it leaves run starts", async () => { + const child = await ownedProcess(); + const f = await inheritedRunnerRetry(child.pid, "own_turn"); + registerRunStartHandover({ isYielding: () => true, subscribe: () => () => {} } as never); + try { + native.claims.mockClear(); + const deferred = vi.fn(); + await createHeartbeatRecovery(db, f.deps).resumeRetryableNativeProcessSessions({ runId: f.run.id, onDeferredDeadline: deferred }); + expect(native.claims.mock.calls.some(([input]) => input?.runIds?.includes(f.run.id))).toBe(false); + expect(f.deps.executeRun).not.toHaveBeenCalled(); + expect(deferred).toHaveBeenCalledWith(expect.any(Number)); + registerRunStartHandover(null); + await createHeartbeatRecovery(db, f.deps).resumeRetryableNativeProcessSessions({ runId: f.run.id }); + expect(native.claims.mock.calls.some(([input]) => input?.runIds?.includes(f.run.id))).toBe(true); + } finally { + registerRunStartHandover(null); + try { await f.cleanupAll(); } finally { await child.stop(); } + } + }); + it("routes a due plugin-task retry through its existing remote lease", async () => { const child = await ownedProcess(); const f = await remoteRetryFixture(child.pid, "plugin_task"); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index d56ade41b7f..cfb76fcf086 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -137,6 +137,7 @@ import { failedRunRetryBudgetSpent, type HeartbeatRetryDependencies, type create import { serverRestartBudgetSpent } from "../execution-recovery-attempt.js"; import type { CancelRunOptions } from "./run-control.js"; import { requestRestartInterruptedTurnRetry } from "./restart-interrupted-turn.js"; +import { isYieldingRunStarts, RUN_START_YIELD_RECHECK_MS } from "../run-start-handover.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type HeartbeatRetries = ReturnType; @@ -2364,6 +2365,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD } await beforeDispatch?.(); + if (isYieldingRunStarts()) { + if (remoteRetryRunIds.length || claimableNativeRunIds.size) scope.onDeferredDeadline?.(Date.now() + RUN_START_YIELD_RECHECK_MS); + return new Set(); + } const remoteRetryClaims = remoteRetryRunIds.length ? (await recoverNativeRunsAfterRestart(now, remoteRetryRunIds, { skipDelegatedWork: true, trigger: "same_run_retry" })).claims diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index 739a56e3895..6cf7f9b25b5 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -23,7 +23,7 @@ import { NativePredecessorRetirementUnprovenError } from "./predecessor-retireme import { AgentDirectoryPreparationDeferral, deferRunForBusyAgentDirectory, prepareWithAgentDirectoryDeferral } from "../heartbeat/agent-directory-busy.js"; import { AgentDirectoryOperationBusyError } from "../agent-directory-working-copies.js"; import { readNativeControllerRecoveryInventory } from "./native-controller-recovery-inventory.js"; -import { inspectRemoteWarmSessionRecovery } from "./remote-warm-session-recovery.js"; +import { inspectRemoteWarmSessionRecovery, waitForLiveWarmSessionOwner } from "./remote-warm-session-recovery.js"; import { currentProcessSpaceHash } from "../controller-process-identity.js"; import { registerControllerInstanceDb, resetControllerInstanceLeaseForTests } from "../controller-instance-lease.js"; import { readProcessStartedAt } from "../hot-restart.js"; @@ -644,6 +644,26 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d } }); + it("keeps a follow-up waiting for a live outgoing server until that server has checkpointed the session after the turn", async () => { + const { recordOwner, current, fixture } = await seedWarmFollowUpOnOldServer(); + const oldBoot = randomUUID(); + await recordOwner({ controllerBootId: oldBoot, controllerPid: 7, controllerHostHash: "outgoing-container", + controllerProcessStartedAt: new Date(Date.now() - 600_000), leaseOwner: null, leaseExpiresAt: null }); + await db.insert(controllerInstances).values({ bootId: oldBoot, processSpace: "outgoing-container", pid: 7, + leaseExpiresAt: new Date(Date.now() + 90_000) }); + registerControllerInstanceDb(db); + try { + const [prior] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.scope.runId)); + const wait = (checkpointedAt: Date | null) => waitForLiveWarmSessionOwner({ db, execution: current, + priorRunId: fixture.scope.runId, checkpointedAt: async () => checkpointedAt }); + await expect(wait(null)).rejects.toBeInstanceOf(NativeControllerRecoveryWaitingError); + await expect(wait(new Date(prior!.finishedAt!.getTime() - 60_000))).rejects.toBeInstanceOf(NativeControllerRecoveryWaitingError); + await expect(wait(new Date(prior!.finishedAt!.getTime() + 1_000))).resolves.toBeUndefined(); + await db.update(controllerInstances).set({ stoppedAt: new Date() }).where(eq(controllerInstances.bootId, oldBoot)); + await expect(wait(null)).resolves.toBeUndefined(); + } finally { resetControllerInstanceLeaseForTests(); } + }); + it("sends a re-wake after a crash restart to a fresh sandbox when the parked predecessor's warm Runner cannot be adopted", async () => { const fixture = await seed(), runId = randomUUID(), environmentId = randomUUID(), leaseId = randomUUID(); const previous = parseNativeExecutionInput({ diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 685c85c9f8b..c28c1295484 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -7,6 +7,7 @@ import * as runnerCheckpoint from "./native-runner-checkpoint.js"; import * as harnessStorage from "./native-harness-backup-storage.js"; import * as stagedAttachmentRecovery from "./native-staged-attachment-recovery.js"; import * as controllerRecovery from "./native-controller-recovery.js"; +import * as ownerProof from "../owner-proof.js"; import { isSettledRemoteWarmControlState } from "./remote-warm-session-recovery.js"; import * as remoteWarmRecoveryRuntime from "./remote-warm-session-recovery.js"; import * as pluginTaskOwnership from "../plugin-task-ownership.js"; @@ -2936,7 +2937,7 @@ describe("remote provider pack manifest", () => { return JSON.stringify(value); }; - it.each(["opencode", "opencode-new-durable", "opencode-new-durable-reset", "opencode-new-durable-inspect-reset", "opencode-new-durable-provider-reset", "opencode-new-durable-store-failed", "opencode-new-durable-publish-failed", "opencode-new-durable-slow-capture", "opencode-new-durable-ready", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { + it.each(["opencode", "opencode-new-durable", "opencode-new-durable-reset", "opencode-new-durable-inspect-reset", "opencode-new-durable-provider-reset", "opencode-new-durable-store-failed", "opencode-new-durable-publish-failed", "opencode-new-durable-slow-capture", "opencode-new-durable-ready", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-continuation-prior-receipt", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { const firstDurable = provider.startsWith("opencode-new-durable"); // A sandbox connection drops once while the settled state is read out. let checkpointReadResets = provider === "opencode-new-durable-reset" ? 1 : 0; @@ -3206,10 +3207,13 @@ describe("remote provider pack manifest", () => { if (provider === "opencode-continuation-no-receipt") await rm(receiptPath); const cold = provider === "opencode-continuation-cold"; const legacyDurable = provider === "opencode-continuation-legacy-durable"; + const priorReceipt = provider === "opencode-continuation-prior-receipt"; const launchRoot = `/home/runtime-account/.paperclip-runtime/paperclip-runner/provider-artifacts/${installed.digest.slice(7)}`; const artifactReceipt = { schema: "paperclip.native-provider-artifacts.v1" as const, artifactId: randomUUID(), closureDigest: "sha256:" + "9".repeat(64), providerPackDigest: installed.digest, manifest: installed, launchRoot, accountUid: 1000 }; - if (cold || legacyDurable) { + const retainExisting = priorReceipt ? vi.spyOn(providerArtifactStorage, "retainExistingNativeProviderArtifacts").mockResolvedValue(undefined) : null; + if (retainExisting) onTestFinished(() => retainExisting.mockRestore()); + if (cold || legacyDurable || priorReceipt) { if (cold) await rm(receiptPath); const storageMode = vi.spyOn(harnessStorage, "durableNativeHarnessStorage").mockReturnValue({} as never); const backupOwner = vi.spyOn(harnessStorage, "nativeHarnessBackupOwner").mockResolvedValue({} as never); @@ -3228,11 +3232,12 @@ describe("remote provider pack manifest", () => { providerPacks: [{ root: "/opt/paperclip-runner/provider-pack", manifest: next, immutable: true }], }) } : (cold || legacyDurable) && command.args?.[1]?.startsWith("const os=require") ? { ...await execute(command), stdout: JSON.stringify({ home: "/home/runtime-account", uid: 1000 }) } : execute(command) - : legacyDurable ? async (command: { command: string; args?: string[] }) => command.args?.[1]?.startsWith("const os=require") + : legacyDurable || priorReceipt ? async (command: { command: string; args?: string[] }) => command.args?.[1]?.startsWith("const os=require") ? { ...await execute(command), stdout: JSON.stringify({ home: "/home/runtime-account", uid: 1000 }) } : execute(command) : execute; - const continuation = createRunnerdBackend({ db: leaseDb(remoteExecution), execution: remoteExecution, - runnerInstanceId: identity.runnerInstanceId, runnerIngressAuthorized: true, - ...(legacyDurable ? { controllerPersistence: { bootId: randomUUID() } } : {}), + const continuation = createRunnerdBackend({ db: priorReceipt + ? leaseDb(remoteExecution, {}, {}, [], { nativeProviderArtifacts: artifactReceipt }) : leaseDb(remoteExecution), + execution: remoteExecution, runnerInstanceId: identity.runnerInstanceId, runnerIngressAuthorized: true, + ...(legacyDurable || priorReceipt ? { controllerPersistence: { bootId: randomUUID() } } : {}), ...(cold ? { controllerPersistence: { bootId: randomUUID() }, providerPackAdmissionSource: { runId: remoteExecution.binding.runId, receipt: coldAdmission, state: { schema: "paperclip.native-provider-pack-admission-state.v1", state: "verified", admissionEpoch: randomUUID() }, artifacts: artifactReceipt } } : {}), @@ -3240,7 +3245,7 @@ describe("remote provider pack manifest", () => { runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: cold ? "/workspace-new-allocation-lease" : "/workspace", environmentId: "environment", leaseId: "lease", providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true }, runner: { execute: selectedExecute, syncIn, - ...(cold || legacyDurable ? { syncOut: vi.fn() } : {}) } } as never }); + ...(cold || legacyDurable || priorReceipt ? { syncOut: vi.fn() } : {}) } } as never }); if (provider === "opencode-continuation-missing" || provider === "opencode-continuation-no-receipt") { await expect(continuation).rejects.toThrow("image provider pack did not match the bundled release identity"); expect(state.createBackend).toHaveBeenCalledTimes(backendsBefore); @@ -3253,15 +3258,19 @@ describe("remote provider pack manifest", () => { await expect(continued.providerAdmissionPersisted?.()).resolves.toBeUndefined(); expect(providerArtifactStorage.retainNativeProviderArtifacts).not.toHaveBeenCalled(); } + if (priorReceipt) { + await continued.providerAdmissionPersisted?.(); + expect(retainExisting).toHaveBeenCalledWith(expect.objectContaining({ receipt: artifactReceipt })); + } state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ providerPackAuthorityDigest: installed.digest, opencodeProxySha256: installed.payload.artifacts.opencodeProxy.sha256, - opencodeProxyPath: cold ? `${launchRoot}/${installed.payload.artifacts.opencodeProxy.path}` + opencodeProxyPath: cold || priorReceipt ? `${launchRoot}/${installed.payload.artifacts.opencodeProxy.path}` : "/opt/paperclip-runner/provider-pack/dist/cli/opencode-app-server-proxy.cjs", runnerArtifact: { version: transport.runnerArtifact!.version, digest: "sha256:" + createHash("sha256").update(currentDaemon).digest("hex") }, }); - if (cold) { + if (cold || priorReceipt) { expect(providerArtifactStorage.restoreNativeProviderArtifacts).toHaveBeenCalledWith(expect.objectContaining({ companyId: remoteExecution.binding.companyId, receipt: artifactReceipt })); expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ @@ -17712,6 +17721,40 @@ describe("runnerd provider runtime wiring", () => { expect(state.createBackend).not.toHaveBeenCalled(); }); + it("a follow-up that starts within the outgoing server's close window waits for its live lease instead of relaunching the Runner", async () => { + const prior = scopedNativeExecution("close-window"); + prior.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 300_000 }; + const followUp = { ...prior, binding: { ...prior.binding, runId: "run-close-window-follow-up" } }; + const root = await initializeScopedRunnerRoot(prior, "runner-close-window"); + const priorIdentity = { runId: prior.binding.runId, normalizedSessionId: prior.session.normalizedSessionId, + runnerInstanceId: "runner-close-window", environmentLeaseId: "lease-close-window" }; + await mkdir(join(root, "control-plane"), { recursive: true }); + await mkdir(join(root, "runner"), { recursive: true }); + await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(priorIdentity))); + await writeFile(join(root, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(priorIdentity, "suspended"))); + const outgoingAlive = vi.spyOn(ownerProof, "proveOwnerGone").mockResolvedValue({ verdict: "alive", reason: "controller_lease_held" }); + onTestFinished(() => outgoingAlive.mockRestore()); + const defer = vi.spyOn(controllerRecovery, "deferNativeControllerRecovery").mockResolvedValueOnce(); + const onExecutionLeaseAcquired = vi.fn(); + const updates: Array<{ table: unknown; values: Record }> = []; + const db = leaseDb(followUp, { phase: "committed", resultId: "prior-result", controllerBootId: "outgoing-server-boot", + controllerPid: 61422, controllerHostHash: "outgoing-host", controllerProcessStartedAt: new Date(), updatedAt: new Date(), + } as Partial, {}, updates); + state.createBackend.mockClear(); + state.createTransport.mockClear(); + await expect(executePaperclipNativeSession({ db, execution: followUp, runnerInstanceId: "incoming-controller", + useRunnerd: true, onExecutionLeaseAcquired, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + environmentId: "close-window-environment", leaseId: "close-window-lease" }, + })).rejects.toMatchObject({ name: "NativeControllerRecoveryWaitingError" }); + expect(outgoingAlive).toHaveBeenCalledWith(expect.objectContaining({ bootId: "outgoing-server-boot" }), expect.anything()); + expect(defer).toHaveBeenCalledWith(db, expect.objectContaining({ ...followUp.binding, retryAt: expect.any(Date) })); + expect(onExecutionLeaseAcquired).not.toHaveBeenCalled(); + expect(updates.some(update => update.table === nativeRunFinalizations && update.values.attempt !== undefined)).toBe(false); + expect(state.createBackend).not.toHaveBeenCalled(); + expect(state.createTransport).not.toHaveBeenCalled(); + }); + it("rejects overlapping runs for the same runnerd provider session scope", async () => { const first = { ...execution, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7dfc7a616ad..00725c5feeb 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -301,7 +301,7 @@ import { remoteRecoveryProbeReason, runnerStateRequestsFreshBootstrap, } from "./remote-runner-recovery-diagnostics.js"; import { isPluginRpcTimeout, readRemoteRunnerRecoveryProbe, REMOTE_RUNNER_PLUGIN_STALL_BUDGET_MS, REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS, remoteRunnerPluginStall, type RemoteRunnerPluginStall } from "./remote-runner-recovery-probe.js"; -import { describeOwnFailedAttachment, inspectRemoteWarmSessionRecovery, isSettledRemoteWarmControlState, summarizePostTerminalEvents, type RemoteWarmSessionRecovery, unsettledControllerCommands } from "./remote-warm-session-recovery.js"; +import { describeOwnFailedAttachment, inspectRemoteWarmSessionRecovery, isSettledRemoteWarmControlState, summarizePostTerminalEvents, type RemoteWarmSessionRecovery, unsettledControllerCommands, waitForLiveWarmSessionOwner } from "./remote-warm-session-recovery.js"; import { assertNativeWarmIngressRetirement, nativeWarmIngressAllocation, nativeWarmIngressIdentity, type NativeWarmIngressClaim, type NativeWarmIngressOwner } from "./native-warm-ingress-ownership.js"; import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js"; import { emitStartupReuse, environmentStartupReuse } from "@paperclipai/adapter-utils/startup-reuse"; @@ -8273,6 +8273,15 @@ async function restoreDurableHarnessForExecution(db: Db, execution: NativeExecut downloadIdleTimeoutMs: NATIVE_HARNESS_DOWNLOAD_IDLE_TIMEOUT_MS }); } +async function recordedSessionProviderArtifacts(db: Db, execution: NativeExecutionInput, + runId: string): Promise { + const [run] = await db.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, execution.binding.companyId), + eq(heartbeatRuns.agentId, execution.binding.agentId), eq(heartbeatRuns.nativeSessionId, nativeSessionKey(execution)))).limit(1); + const receipt = run?.runnerProfileJson?.nativeProviderArtifacts as NativeProviderArtifactReceipt | undefined; + return receipt?.schema === "paperclip.native-provider-artifacts.v1" ? receipt : undefined; +} + function harnessBackupCandidates(root: string): string[] { const backupRoot = harnessBackupRoot(root); return [resolve(backupRoot, "current"), resolve(backupRoot, "previous")]; @@ -11041,6 +11050,11 @@ async function executePaperclipNativeSessionAttempt( .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, input.execution.binding.runId)); throw new Error(describeOwnFailedAttachment(finalization?.recoveryHistory)); } + await waitForLiveWarmSessionOwner({ db: input.db, execution: input.execution, priorRunId: identity.runId, + checkpointedAt: async () => { + const backup = await verifyNativeHarnessBackupStreamed({ root, execution: input.execution, runnerInstanceId: identity.runnerInstanceId }); + return backup && runnerdAuthorityLifecycle(backup.root, identity) === "suspended" ? new Date(backup.manifest.completedAt) : null; + } }); await logPostTerminalEvents(root, input.onLog); } if (durableIdentityMatchesSession(identity, input.execution) && identity.runId !== input.execution.binding.runId && @@ -16822,6 +16836,10 @@ async function createRunnerdBackendWithinSessionClaim( runnerInstanceId: input.runnerInstanceId }) : null; let providerArtifacts = artifactStorage ? artifactBackup?.manifest.providerArtifacts ?? input.providerPackAdmissionSource?.artifacts as NativeProviderArtifactReceipt | undefined : undefined; + if (artifactStorage && !providerArtifacts && durableBinding !== null && !liveProviderRecovery + && durableIdentityMatchesSession(durableIdentity, input.execution)) { + providerArtifacts = await recordedSessionProviderArtifacts(input.db, input.execution, durableIdentity.runId); + } let legacyProviderContinuity = durableBinding !== null && !providerArtifacts; if (providerArtifacts && artifactStorage && !liveProviderRecovery) await restoreNativeProviderArtifacts({ db: input.db, storage: artifactStorage, companyId: input.execution.binding.companyId, receipt: providerArtifacts, diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.ts b/server/src/services/native-runtime/remote-warm-session-recovery.ts index ebb21f0ac33..bc89dec3b76 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.ts @@ -37,6 +37,28 @@ export async function liveWarmSessionOwnerRetryAt(input: { return new Date(Math.min(now.getTime() + WARM_OWNER_RECHECK_MS, closesBy)); } +export async function waitForLiveWarmSessionOwner(input: { + db: Db; + execution: NativeExecutionInput; + priorRunId: string; + checkpointedAt: () => Promise; +}): Promise { + const { binding } = input.execution; + const [owner] = await input.db.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.runId, input.priorRunId), eq(nativeRunFinalizations.companyId, binding.companyId))).limit(1); + if (!owner) return; + const [prior] = await input.db.select({ finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, input.priorRunId), eq(heartbeatRuns.companyId, binding.companyId), + eq(heartbeatRuns.agentId, binding.agentId))).limit(1); + const finishedAt = prior?.finishedAt ?? null; + const retryAt = await liveWarmSessionOwnerRetryAt({ owner, finishedAt, + idleTimeoutMs: input.execution.session.lifecyclePolicy.idleTimeoutMs ?? null }); + if (!retryAt) return; + const checkpointedAt = await input.checkpointedAt(); + if (checkpointedAt && finishedAt && checkpointedAt >= finishedAt) return; + throw new NativeControllerRecoveryWaitingError(retryAt); +} + export type RemoteWarmSessionRecovery = { identity: Record; checkpoint: PersistedNativeSession; diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index db76c8976e3..ca13eebd909 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -46,6 +46,14 @@ const WAITING_CONTINUATION_MOVE_FORWARD_MS = 15 * 60_000; const NO_WORK_CONTINUATION_HANDOFF_GRACE_MS = 30_000; const NO_WORK_CONTINUATION_CAUSES = new Set(["native_predecessor_retirement_unproven", "native_failed_before_provider_turn"]); +function awaitsExactRemoteResume(issue: Pick, + run: Pick, + coordinator: Pick): boolean { + const detail = coordinator.failureDetail as Record | null; + return issue.status === "blocked" && issue.executionRunId === run.id + && (run.processPid !== null || run.processGroupId !== null) && detail?.recoveryMode === "exact_checkpoint_resume"; +} + type ProjectionIssue = Pick; type ProjectionRun = Pick & @@ -598,7 +606,8 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i const issue = run?.nativeIssueId ? issueById.get(run.nativeIssueId) : undefined; const coordinator = coordinatorByRun.get(candidate.id); const admitted = run && issue && coordinator - && await readNativeSessionResumptionAgent(db, { issue, run, coordinator }).catch(() => null); + && (awaitsExactRemoteResume(issue, run, coordinator) + || await readNativeSessionResumptionAgent(db, { issue, run, coordinator }).catch(() => null)); if (!admitted) candidate.nativeResumeAt = null; })); }