diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index af41f34a222..142e0c6e7e7 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -436,6 +436,7 @@ import { closeIdleWarmNativeSessionsForRestart, WARM_IDLE_CLOSE_MAX_RETRIES, WARM_IDLE_CLOSE_RETRY_MS, + warmRetainedRunnerStopWait, readNativeSessionIdleWork, createGovernedWaitEventObservation, createRemoteRunnerProcessLauncher, @@ -12281,6 +12282,201 @@ describe("native warm session supervision", () => { expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 1, failed: 0 }); }); + it("keeps retrying a failed idle warm close after a failed environment sweep cleared its retry timer", async () => { + const name = "warm-idle-close-retry-after-sweep"; + const current = { ...execution, + binding: { ...execution.binding, runId: name, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 } }, + } as NativeExecutionInputV1; + const close = vi.fn<() => Promise>(async () => { throw new Error("native_runner_checkpoint_owner_lost"); }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: name, normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + const target = { kind: "remote" as const, transport: "sandbox" as const, environmentId: name, + remoteCwd: `/tmp/${name}`, sandboxLeaseAcquisition: { outcome: "created" as const, providerLeaseId: name } }; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + await executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name, runnerExecutionTarget: target }); + await vi.advanceTimersByTimeAsync(20); + expect(close).toHaveBeenCalledTimes(1); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "environment deleted" })) + .toMatchObject({ closed: 0, failed: 1 }); + expect(close).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(WARM_IDLE_CLOSE_RETRY_MS); + expect(close).toHaveBeenCalledTimes(3); + } finally { vi.useRealTimers(); } + close.mockResolvedValue(undefined); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "test cleanup" })).toMatchObject({ closed: 1 }); + }); + + function fakeRetainedRunnerSandbox(input: { runId: string; normalizedSessionId: string; remoteCwd: string; + lifecycle?: string; stopsOn?: "TERM" | "never"; runnerIdCarriers?: number[]; unreadableProcesses?: number }) { + const runnerInstanceId = "runner-instance-retained"; + const sessionDirectory = createHash("sha256").update(input.normalizedSessionId).digest("hex"); + const stateDirectory = `${input.remoteCwd}/.paperclip-runtime/paperclip-runner/sessions/${sessionDirectory}/runner`; + const identity = `retainednonce\n522\n2026-11-16T00:47:54.000Z\n${runnerInstanceId}\nlinux:abcd-1234:100\n`; + const runnerState = { runnerInstanceId, runId: input.runId, normalizedSessionId: input.normalizedSessionId, + lifecycle: input.lifecycle ?? "ready", outbox: [], pendingTerminalDelivery: null, pendingProviderCleanup: null, warmTransition: null }; + const sandbox = { alive: true, signals: [] as string[], labels: [] as string[] }; + const encode = (value: string) => Buffer.from(value).toString("base64"); + const runner = { execute: vi.fn(async (request: { args?: string[] }) => { + const args = request.args ?? []; + const label = args[2] ?? ""; + sandbox.labels.push(label); + if (label === "paperclip-runner-port-holders") { + return { exitCode: 0, timedOut: false, stderr: "", + stdout: `${stateDirectory}/runner-process.identity\t${encode(identity)}\t${encode(JSON.stringify(runnerState))}\n` }; + } + if (label === "paperclip-runner-port-holder-check" || label === "paperclip-runner-exit-check") { + const signal = args[8]; + if (signal) { + sandbox.signals.push(signal); + if (input.stopsOn !== "never") sandbox.alive = false; + return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + } + return { exitCode: sandbox.alive ? 0 : 45, timedOut: false, stdout: "", stderr: "" }; + } + if (label === "paperclip-runner-session-inspect") { + expect(args[4]).toBe(stateDirectory); + return { exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({ + state: { lifecycle: runnerState.lifecycle, runId: input.runId, normalizedSessionId: input.normalizedSessionId, + runnerInstanceId, settled: true }, + markerPid: 522, markerRunnerIdMatchesState: true, procfs: true, unreadableProcesses: input.unreadableProcesses ?? 0, + processes: (input.runnerIdCarriers ?? (sandbox.alive && runnerState.lifecycle !== "suspended" ? [522] : [])) + .map(pid => ({ pid, state: "S", carriesRunnerId: true })), + }) }; + } + return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; + }) }; + return { sandbox, runner }; + } + + async function startRetainedWarmOwner(name: string, close: () => Promise, sandboxInput: Omit[0], "runId" | "normalizedSessionId" | "remoteCwd">) { + const runId = randomUUID(); + const current = { ...execution, + binding: { ...execution.binding, runId, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } }, + } as NativeExecutionInputV1; + const fake = fakeRetainedRunnerSandbox({ runId, normalizedSessionId: name, remoteCwd: `/tmp/${name}`, ...sandboxInput }); + const target = { kind: "remote" as const, transport: "sandbox" as const, environmentId: name, remoteCwd: `/tmp/${name}`, + sandboxLeaseAcquisition: { outcome: "created" as const, providerLeaseId: name }, runner: fake.runner }; + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: name, normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + await expect(executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name, + runnerExecutionTarget: target as never })).resolves.toMatchObject({ exitCode: 0 }); + return { current, ...fake, finishedDb: leaseDb(current, {}, {}, [], {}, "succeeded") }; + } + + const refusedSuspend = () => new NativeSessionCloseUnrecoverableError({ runnerSuspended: false, runnerRetained: true, + providerDrained: true, suspensionState: { commandStatus: "failed", runnerLifecycle: "ready", runnerIdentityMatches: true }, + proofWindowEnd: "suspend_not_completed" }); + + it("stops the retained Runner of a finished warm owner that refused suspend and releases it without a checkpoint when its environment is deleted", async () => { + const name = "warm-retained-runner-refused-suspend"; + const close = vi.fn<() => Promise>(async () => { throw refusedSuspend(); }); + const { sandbox, finishedDb, current } = await startRetainedWarmOwner(name, close, {}); + state.persistActivity.mockClear(); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "environment deleted", + forceStopUnsettled: true, db: finishedDb })).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); + expect(close).toHaveBeenCalledOnce(); + expect(sandbox.signals).toEqual(["TERM"]); + expect(sandbox.alive).toBe(false); + expect(sandbox.labels).toContain("paperclip-runner-exit-check"); + expect(state.persistActivity).toHaveBeenCalledWith(finishedDb, expect.objectContaining({ + action: "native.runner_port_holder_retired", details: expect.objectContaining({ holderRunId: current.binding.runId, outcome: "stopped" }) })); + expect(readNativeSessionIdleWork()).toBe("none"); + }); + + it("keeps a finished warm owner whose retained Runner survives the stop", async () => { + const name = "warm-retained-runner-survives"; + const close = vi.fn<() => Promise>(async () => { throw refusedSuspend(); }); + const { sandbox, finishedDb } = await startRetainedWarmOwner(name, close, { stopsOn: "never" }); + const wait = { ...warmRetainedRunnerStopWait }; + Object.assign(warmRetainedRunnerStopWait, { waitMs: 20, pollMs: 5 }); + try { + expect(await closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "environment deleted", + forceStopUnsettled: true, db: finishedDb })).toMatchObject({ closed: 0, failed: 1 }); + } finally { Object.assign(warmRetainedRunnerStopWait, wait); } + expect(sandbox.signals).toEqual(["TERM", "KILL"]); + expect(sandbox.labels).not.toContain("paperclip-runner-exit-check"); + expect(readNativeSessionIdleWork()).toBe("present"); + close.mockResolvedValue(undefined); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "test cleanup" })).toMatchObject({ closed: 1 }); + }); + + it("never stops the retained Runner of a warm owner whose run has not finished", async () => { + const name = "warm-retained-runner-unfinished"; + const close = vi.fn<() => Promise>(async () => { throw refusedSuspend(); }); + const { sandbox, current } = await startRetainedWarmOwner(name, close, {}); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "environment deleted", + forceStopUnsettled: true, db: leaseDb(current, {}, {}, [], {}, "running") })).toMatchObject({ closed: 0, failed: 1 }); + expect(sandbox.signals).toEqual([]); + close.mockResolvedValue(undefined); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "test cleanup" })).toMatchObject({ closed: 1 }); + }); + + it("logs sandbox Runner evidence for a close failure of a warm session reused by a later run", async () => { + const name = "warm-reused-close-evidence"; + const close = vi.fn<() => Promise>(async () => undefined); + const session = { close }; + const { current, runner, sandbox } = await startRetainedWarmOwner(name, close, {}); + const second = { ...current, binding: { ...current.binding, runId: randomUUID() } } as NativeExecutionInputV1; + state.execute.mockReset().mockImplementationOnce(async options => { + expect(options.existingSession).toBeDefined(); + await options.onSession?.(options.existingSession ?? session); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: name, normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + await expect(executePaperclipNativeSession({ db: leaseDb(second), execution: second, runnerInstanceId: name, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId: name, remoteCwd: `/tmp/${name}`, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: name }, runner } as never })).resolves.toMatchObject({ exitCode: 0 }); + expect(state.execute).toHaveBeenCalledOnce(); + close.mockImplementation(async () => { + throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "marker_missing" }); + }); + sandbox.labels.length = 0; + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 0, failed: 1 }); + expect(sandbox.labels).toContain("paperclip-runner-session-inspect"); + close.mockResolvedValue(undefined); + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 1 }); + }); + + it("releases at controller restart a warm owner whose Runner durably suspended and exited but whose check found a live process without its runner id", async () => { + const name = "warm-suspended-runner-id-missing"; + const close = vi.fn<() => Promise>(async () => { + throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_runner_id_missing" }); + }); + const { sandbox } = await startRetainedWarmOwner(name, close, { lifecycle: "suspended" }); + sandbox.alive = false; + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); + expect(sandbox.signals).toEqual([]); + }); + + it.each([ + ["a live process still carries its runner id", "process_runner_id_missing", [522], 0], + ["a process command line could not be read", "process_runner_id_missing", [], 1], + ["the check reported only a changed identity", "process_identity_changed", [], 0], + ] as const)("keeps a durably suspended warm owner when %s", async (label, reason, carriers, unreadableProcesses) => { + const name = `warm-suspended-kept-${label.replaceAll(" ", "-")}`; + const close = vi.fn<() => Promise>(async () => { + throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason }); + }); + const { sandbox } = await startRetainedWarmOwner(name, close, { lifecycle: "suspended", runnerIdCarriers: [...carriers], + unreadableProcesses }); + sandbox.alive = false; + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 0, failed: 1 }); + close.mockResolvedValue(undefined); + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 1 }); + }); + it.each(["failed", "cancelled"] as const)("retires a %s session even if its warm checkpoint rejects", async (runTerminalState) => { const name = `warm-checkpoint-rejects-${runTerminalState}`; const current = { ...execution, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index cb068a80ddf..c48e91d92e4 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -66,7 +66,9 @@ import { preparePluginRunnerCredentials } from "./plugin-runner-credentials.js"; import { createPluginRunnerTaskOwner, verifyPluginRunnerTaskReattachment } from "./plugin-runner-task.js"; import { findNativeFailedBeforeProviderTurn, NATIVE_FAILED_BEFORE_PROVIDER_TURN_CODE, NativeFailedBeforeProviderTurnError } from "./failed-before-provider-turn.js"; import { findNativePredecessorRetirementUnproven, NATIVE_PREDECESSOR_RETIREMENT_UNPROVEN_CODE, NativePredecessorRetirementDirectoryBusyError, NativePredecessorRetirementUnprovenError } from "./predecessor-retirement.js"; -import { retireSandboxRunnerPortHolders } from "./sandbox-runner-port-holders.js"; +import { + inspectSandboxSessionRunner, retireSandboxRunnerPortHolders, sandboxSessionRunnerExited, type SandboxSessionRunnerInspection, +} from "./sandbox-runner-port-holders.js"; import { AgentDirectoryOperationBusyError } from "../agent-directory-working-copies.js"; import { agentDirectoryBusyRetryDelayMs } from "../heartbeat/agent-directory-busy.js"; import { NativeExecutionContext } from "./native-execution-context.js"; @@ -719,6 +721,8 @@ type WarmNativeSession = { /** The sandbox that holds this owner's Runner, when it runs in one. */ providerLeaseId?: string; sandboxLeaseDb?: Db; + sandboxRunner?: { runner: CommandManagedRuntimeRunner; remoteCwd: string }; + closeFailureEvidenceLogged?: boolean; /** A successor's retirement of this owner failed; it stays registered for its own sandbox. */ retirementUnproven?: boolean; observabilityCompatibilityKey?: string; @@ -832,6 +836,11 @@ async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, finally { await stopping; } } +function warmSandboxRunner(target?: AdapterExecutionTarget | null): WarmNativeSession["sandboxRunner"] { + return target?.kind === "remote" && target.transport === "sandbox" && target.runner && !target.runnerTask + ? { runner: target.runner, remoteCwd: target.remoteCwd } : undefined; +} + function sandboxProviderLeaseId(target?: AdapterExecutionTarget | null): string | null { return target?.kind === "remote" && target.transport === "sandbox" ? target.sandboxLeaseAcquisition?.providerLeaseId ?? null : null; } @@ -1159,12 +1168,88 @@ async function warmRunnerPortTakenBySuccessor(db: Db, entry: WarmNativeSession): async function warmCloseFailureProvesOwnerGone(entry: WarmNativeSession, error: unknown, db = entry.sandboxLeaseDb): Promise { const diagnostic = readRemoteRunnerRecoveryError(error); if (diagnostic?.stage === "process_verification" && diagnostic.reason === "process_not_running") return true; + const runnerIdMissing = diagnostic?.stage === "process_verification" && diagnostic.reason === "process_runner_id_missing"; + const logEvidence = Boolean(entry.sandboxRunner) && !entry.closeFailureEvidenceLogged + && (diagnostic !== null || error instanceof NativeSessionCloseUnrecoverableError); + const inspection = entry.sandboxRunner && (runnerIdMissing || logEvidence) + ? await inspectSandboxSessionRunner({ runner: entry.sandboxRunner.runner, + stateDirectory: remoteRunnerStateDirectory(entry.sandboxRunner.remoteCwd, entry.normalizedSessionId) }) : null; + if (logEvidence) { + entry.closeFailureEvidenceLogged = true; + logger.warn({ runId: entry.runId, issueId: entry.issueId, environmentId: entry.environmentId, + providerLeaseId: entry.providerLeaseId ?? null, ...warmCloseFailureEvidence(entry, error, inspection) }, + "warm native session close failed; sandbox Runner evidence"); + } + if (runnerIdMissing && suspendedRunnerHasExited(entry, inspection)) return true; try { - return db ? await warmSandboxTerminated(db, entry) || await warmRunnerPortTakenBySuccessor(db, entry) : false; + if (db && (await warmSandboxTerminated(db, entry) || await warmRunnerPortTakenBySuccessor(db, entry))) return true; } catch (proofError) { logger.warn({ err: proofError, runId: entry.runId }, "could not read the warm session's sandbox lease"); return false; } + return db ? stopRetainedRunnerOfFinishedOwner(entry, error, db) : false; +} + +function warmCloseFailureEvidence(entry: WarmNativeSession, error: unknown, inspection: SandboxSessionRunnerInspection | null) { + const diagnostic = readRemoteRunnerRecoveryError(error); + const settlement = error instanceof NativeSessionCloseUnrecoverableError ? error.settlement : undefined; + const suspension = settlement?.suspensionState && typeof settlement.suspensionState === "object" + ? settlement.suspensionState as Record : null; + return { + closeStage: diagnostic?.stage ?? null, + closeReason: diagnostic?.reason ?? (error instanceof Error && "code" in error && typeof error.code === "string" ? error.code : null), + closeRunnerLifecycle: diagnostic?.runnerLifecycle ?? suspension?.runnerLifecycle ?? null, + runnerSuspended: settlement?.runnerSuspended ?? null, + runnerRetained: settlement?.runnerRetained ?? null, + suspendCommandStatus: suspension?.commandStatus ?? null, + proofWindowEnd: settlement?.proofWindowEnd ?? null, + sandboxRunner: inspection ? { + lifecycle: inspection.state?.lifecycle ?? null, + settled: inspection.state?.settled ?? null, + identityMatches: inspection.state !== null && inspection.state.runId === entry.runId + && inspection.state.normalizedSessionId === entry.normalizedSessionId && inspection.markerRunnerIdMatchesState, + markerPid: inspection.markerPid, + procfs: inspection.procfs, + unreadableProcesses: inspection.unreadableProcesses, + processes: inspection.processes.slice(0, 20), + } : "unavailable", + }; +} + +function suspendedRunnerHasExited(entry: WarmNativeSession, inspection: SandboxSessionRunnerInspection | null): boolean { + return inspection !== null && inspection.procfs && inspection.unreadableProcesses === 0 && inspection.state?.lifecycle === "suspended" && inspection.state.settled + && inspection.state.runId === entry.runId && inspection.state.normalizedSessionId === entry.normalizedSessionId + && inspection.markerRunnerIdMatchesState && !inspection.processes.some(process => process.carriesRunnerId); +} + +export const warmRetainedRunnerStopWait = { waitMs: 10_000, pollMs: 500 }; + +async function stopRetainedRunnerOfFinishedOwner(entry: WarmNativeSession, error: unknown, db: Db): Promise { + if (!entry.sandboxRunner || !(error instanceof NativeSessionCloseUnrecoverableError) + || error.settlement?.runnerRetained !== true || error.settlement.runnerSuspended === true) return false; + const { runner, remoteCwd } = entry.sandboxRunner; + const runtimeRoot = posix.join(remoteCwd, ".paperclip-runtime", "paperclip-runner"); + const sessionDirectory = createHash("sha256").update(entry.normalizedSessionId).digest("hex"); + try { + await retireSandboxRunnerPortHolders({ db, companyId: entry.companyId, agentId: entry.agentId, runId: entry.runId, + runner, runtimeRoot, ownSessionRoot: "", providerLeaseId: entry.providerLeaseId ?? null, + only: { sessionDirectory, runId: entry.runId }, retireInProcess: async () => false, ...warmRetainedRunnerStopWait }); + } catch (stopError) { + logger.warn({ runId: entry.runId, providerLeaseId: entry.providerLeaseId ?? null, stopFailure: describeWarmCloseFailure(stopError) }, + "could not stop the retained Runner of a finished warm session"); + return false; + } + if (!await sandboxSessionRunnerExited({ runner, runtimeRoot, sessionDirectory })) return false; + logger.warn({ runId: entry.runId, issueId: entry.issueId, environmentId: entry.environmentId, + providerLeaseId: entry.providerLeaseId ?? null }, + "stopped the retained Runner of a finished warm session whose suspend was refused; released it without a checkpoint"); + return true; +} + +function armWarmIdleCloseRetry(sessionId: string, entry: WarmNativeSession): void { + if (warmNativeSessions.get(sessionId) !== entry || entry.released || (entry.idleCloseRetries ?? 0) >= WARM_IDLE_CLOSE_MAX_RETRIES) return; + entry.idleCloseRetries = (entry.idleCloseRetries ?? 0) + 1; + armWarmNativeSessionIdleTimer(sessionId, entry, WARM_IDLE_CLOSE_RETRY_MS); } /** Detach released owners and checkpoint unreleased owners before a controller @@ -1196,6 +1281,7 @@ async function closeIdleWarmNativeSessions(input: { let busy = 0; let failed = 0; const failures: WarmNativeSessionCloseFailure[] = []; + const retryIdleCloseAfterSweep: [string, WarmNativeSession][] = []; let stopped = false; const closeEntry = async (sessionId: string, entry: WarmNativeSession) => { if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId) @@ -1242,6 +1328,7 @@ async function closeIdleWarmNativeSessions(input: { return; } failed += 1; + if (!detach && entry.idleCloseRetries !== undefined) retryIdleCloseAfterSweep.push([sessionId, entry]); const forced = forcedStopFailure === undefined ? "" : `; forced Runner stop failed: ${describeWarmCloseFailure(forcedStopFailure)}`; failures.push({ runId: entry.runId, issueId: entry.issueId, environmentId: entry.environmentId, providerLeaseId: entry.providerLeaseId ?? null, @@ -1276,6 +1363,7 @@ async function closeIdleWarmNativeSessions(input: { const concurrency = input.canRetire ? 1 : WARM_NATIVE_SESSION_CLOSE_CONCURRENCY; await Promise.all(Array.from({ length: Math.min(concurrency, pendingAgents.length) }, closeAgentQueues)); if (failures.length > 0) logger.warn({ reason: input.reason, failures }, "idle warm native sessions did not close"); + for (const [sessionId, entry] of retryIdleCloseAfterSweep) armWarmIdleCloseRetry(sessionId, entry); return { closed, busy, failed, failures }; } @@ -8904,9 +8992,7 @@ function armWarmNativeSessionIdleTimer( forgetWarmNativeSession(sessionId, current); return; } - if ((current.idleCloseRetries ?? 0) >= WARM_IDLE_CLOSE_MAX_RETRIES) return; - current.idleCloseRetries = (current.idleCloseRetries ?? 0) + 1; - armWarmNativeSessionIdleTimer(sessionId, current, WARM_IDLE_CLOSE_RETRY_MS); + armWarmIdleCloseRetry(sessionId, current); }); }) .finally(() => { @@ -12757,6 +12843,8 @@ async function executePaperclipNativeSessionAttempt( existing.ingressOwner = ingress ? { ...input.execution.binding, ingress, controllerBootId: controller.bootId, controllerGeneration } : undefined; existing.session = session; + existing.sandboxRunner = warmSandboxRunner(input.runnerExecutionTarget); + existing.closeFailureEvidenceLogged = false; existing.runId = input.execution.binding.runId; existing.issueId = input.execution.binding.issueId; } else @@ -12774,6 +12862,7 @@ async function executePaperclipNativeSessionAttempt( ? input.runnerExecutionTarget.sandboxLeaseAcquisition?.providerLeaseId : undefined, providerLeaseId: sandboxProviderLeaseId(input.runnerExecutionTarget) ?? undefined, sandboxLeaseDb: input.db, + sandboxRunner: warmSandboxRunner(input.runnerExecutionTarget), runId: input.execution.binding.runId, issueId: input.execution.binding.issueId, instructionWorkingCopy: input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy, diff --git a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts index 5b504458ae2..d872a071ff2 100644 --- a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts +++ b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from "vitest"; import { REMOTE_RECOVERY_MARKER_READ_SCRIPT, REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, REMOTE_RUNNER_EXIT_EVIDENCE_MAX_CHARS, REMOTE_RUNNER_EXIT_EVIDENCE_SCRIPT, RemoteRunnerRecoveryError, boundRemoteRunnerExitEvidence, describeExitedRemoteRunnerReason, - readRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError, + readRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError, remoteRecoveryProbeReason, } from "./remote-runner-recovery-diagnostics.js"; function shellExit(script: string, args: string[]) { @@ -35,7 +35,7 @@ describe("remote recovery diagnostic shell receipts", () => { it.each([ { fingerprint: "linux:abcd-1234:100", code: 45 }, { fingerprint: "linux:abcd-1234:101", code: 47 }, - { fingerprint: "", code: 44 }, + { fingerprint: "", code: 48 }, ])("requires the saved generation before accepting a zombie receipt (%j)", async ({ fingerprint, code }) => { const root = await mkdtemp(join(tmpdir(), "remote-probe-zombie-")); const marker = join(root, "marker"); @@ -53,6 +53,30 @@ describe("remote recovery diagnostic shell receipts", () => { } finally { await rm(root, { recursive: true, force: true }); } }); + it.each([ + { cmdline: ["paperclip-runnerd", "--runner-id", "runner"], code: 0 }, + { cmdline: ["paperclip-runnerd", "--state-dir", "/state"], code: 48 }, + { cmdline: [""], code: 48 }, + ])("reports a live process without its runner id apart from a changed marker (%j)", async ({ cmdline, code }) => { + const root = await mkdtemp(join(tmpdir(), "remote-probe-runner-id-")); + const marker = join(root, "marker"); + try { + await writeFile(marker, `nonce\n${process.pid}\n2026-11-16T00:28:42.758Z\nrunner\n`); + await writeFile(join(root, "cmdline"), cmdline.join("\0")); + await writeFile(join(root, "stat"), `${process.pid} (runnerd) ${["S", ...Array(18).fill("0"), 100].join(" ")}\n`); + await writeFile(join(root, "boot_id"), "abcd-1234\n"); + const script = REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT + .replaceAll('"/proc/" + pid + "/cmdline"', JSON.stringify(join(root, "cmdline"))) + .replaceAll('"/proc/" + pid + "/stat"', JSON.stringify(join(root, "stat"))) + .replaceAll('"/proc/sys/kernel/random/boot_id"', JSON.stringify(join(root, "boot_id"))); + expect(shellExit(script, [marker, "nonce", "runner", String(process.pid), "linux:abcd-1234:100"])).toBe(code); + expect(remoteRecoveryProbeReason("process_verification", 48, false)).toBe("process_runner_id_missing"); + await writeFile(marker, `nonce\n${process.pid}\n2026-11-16T00:28:42.758Z\nother-runner\n`); + expect(shellExit(script, [marker, "nonce", "runner", String(process.pid), "linux:abcd-1234:100"])).toBe(44); + expect(remoteRecoveryProbeReason("process_verification", 44, false)).toBe("process_identity_changed"); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it("distinguishes missing, unsafe and valid marker reads without exposing their contents", async () => { const root = await mkdtemp(join(tmpdir(), "remote-recovery-read-")); const marker = join(root, "marker"); diff --git a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts index 8cc70adc72d..58d95862346 100644 --- a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts +++ b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts @@ -3,7 +3,7 @@ export const REMOTE_RUNNER_RECOVERY_ERROR_CODE = "runner_remote_process_identity const stages = ["runner_state", "identity_marker", "process_verification"] as const; const reasons = [ "marker_missing", "marker_unsafe", "marker_read_failed", "marker_invalid", - "fingerprint_missing", "process_not_running", "process_identity_changed", + "fingerprint_missing", "process_not_running", "process_identity_changed", "process_runner_id_missing", "fingerprint_unavailable", "process_generation_changed", "probe_timeout", "probe_transport_failed", "probe_failed", "state_read_failed", "state_invalid", @@ -219,7 +219,7 @@ catch (error) { if (expectedGeneration) verifyGeneration(); if (!args.includes("--runner-id") || !args.includes(runnerId)) { requireLiveProcess(); - process.exit(44); + process.exit(48); } ' "$pid" "$expected_runner_id"` + ' "${5:-}"', ].join("; "); @@ -238,6 +238,7 @@ export function remoteRecoveryProbeReason(stage: RemoteRunnerRecoveryDiagnostic[ if (exitCode === 45) return "process_not_running"; if (exitCode === 46) return "fingerprint_unavailable"; if (exitCode === 47) return "process_generation_changed"; + if (exitCode === 48) return "process_runner_id_missing"; } return "probe_failed"; } diff --git a/server/src/services/native-runtime/sandbox-runner-port-holders.integration.test.ts b/server/src/services/native-runtime/sandbox-runner-port-holders.integration.test.ts index 7cebee35b59..1ee43e32b5b 100644 --- a/server/src/services/native-runtime/sandbox-runner-port-holders.integration.test.ts +++ b/server/src/services/native-runtime/sandbox-runner-port-holders.integration.test.ts @@ -8,7 +8,7 @@ import { afterEach, expect, it } from "vitest"; import { activityLog, heartbeatRuns } from "@paperclipai/db"; import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; import { NativePredecessorRetirementUnprovenError } from "./predecessor-retirement.js"; -import { retireSandboxRunnerPortHolders } from "./sandbox-runner-port-holders.js"; +import { retireSandboxRunnerPortHolders, sandboxSessionRunnerExited } from "./sandbox-runner-port-holders.js"; const HOLDER_SCRIPT = 'require("node:net").createServer().listen(Number(process.argv[1]), "127.0.0.1", () => process.stdout.write("ready\\n"));'; @@ -121,6 +121,28 @@ describeEmbeddedPostgres("a sandbox Runner port held by another session's Runner expect(await bind(port)).toBe("EADDRINUSE"); }); + it("stops only the named session's finished Runner and then proves it exited", async () => { + const owner = await environment.seedRun("PORT-HOLDER-ONLY-OWNER"); + const other = await environment.seedRun("PORT-HOLDER-ONLY-OTHER", owner.db); + await owner.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, owner.runId)); + await owner.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, other.runId)); + const base = join(environment.current.runtimeRoot, "sandbox-only"); + const runtimeRoot = join(base, ".paperclip-runtime", "paperclip-runner"); + const own = await startHolder({ runtimeRoot, session: "e".repeat(64), runId: owner.runId, port: await freePort() }); + const neighbour = await startHolder({ runtimeRoot, session: "f".repeat(64), runId: other.runId, port: await freePort() }); + const exitedNow = () => sandboxSessionRunnerExited({ runner: confinedRunner(base) as never, runtimeRoot, sessionDirectory: "e".repeat(64) }); + expect(await exitedNow()).toBe(false); + await expect(retireSandboxRunnerPortHolders({ db: owner.db, companyId: environment.current.companyId, + agentId: environment.current.agentId, runId: owner.runId, runner: confinedRunner(base) as never, runtimeRoot, + ownSessionRoot: "", providerLeaseId: "sandbox-1", only: { sessionDirectory: "e".repeat(64), runId: owner.runId }, + retireInProcess: async () => false, waitMs: 3_000, pollMs: 100 })).resolves.toEqual([owner.runId]); + expect(await exited(own)).toBe(true); + expect(await exitedNow()).toBe(true); + expect(neighbour.exitCode).toBeNull(); + expect(await sandboxSessionRunnerExited({ runner: confinedRunner(base) as never, runtimeRoot, sessionDirectory: "f".repeat(64) })).toBe(false); + expect(await sandboxSessionRunnerExited({ runner: confinedRunner(base) as never, runtimeRoot, sessionDirectory: "0".repeat(64) })).toBe(false); + }); + it("ignores this run's own session, moved-aside roots and holders that already exited", async () => { const current = await environment.seedRun("PORT-HOLDER-OWN"); const base = join(environment.current.runtimeRoot, "sandbox-own"); diff --git a/server/src/services/native-runtime/sandbox-runner-port-holders.test.ts b/server/src/services/native-runtime/sandbox-runner-port-holders.test.ts new file mode 100644 index 00000000000..51ccbcb1538 --- /dev/null +++ b/server/src/services/native-runtime/sandbox-runner-port-holders.test.ts @@ -0,0 +1,67 @@ +import { spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { inspectSandboxSessionRunner, SANDBOX_SESSION_RUNNER_INSPECT_SCRIPT } from "./sandbox-runner-port-holders.js"; + +const localRunner = (script: string) => ({ execute: async (request: { command: string; args?: string[] }) => { + const args = [...request.args ?? []]; + args[3] = script; + const result = spawnSync(request.command, args, { encoding: "utf8", timeout: 10_000 }); + return { exitCode: result.status, timedOut: result.error !== undefined, stdout: result.stdout ?? "", stderr: result.stderr ?? "" }; +} }); + +async function fakeProcess(proc: string, pid: number, argv: string[], state = "S") { + await mkdir(join(proc, String(pid)), { recursive: true }); + await writeFile(join(proc, String(pid), "cmdline"), `${argv.join("\0")}\0`); + await writeFile(join(proc, String(pid), "stat"), `${pid} (${argv[0]?.split("/").pop()}) ${state} 1 ${pid} ${pid}\n`); +} + +describe("sandbox session Runner inspection", () => { + it("reports the durable state, the marker pid and each process carrying the runner id, never the nonce or command lines", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-inspect-")); + const proc = join(root, "proc"); + const state = join(root, "runner"); + try { + await mkdir(join(proc, "self"), { recursive: true }); + await writeFile(join(proc, "self", "cmdline"), "node\0"); + await mkdir(state, { recursive: true }); + await writeFile(join(state, "runner-process.identity"), "private-nonce\n522\n2026-11-16T00:47:54.000Z\nrunner-1\nlinux:abcd:100\n"); + await writeFile(join(state, "runner-state.json"), JSON.stringify({ runnerInstanceId: "runner-1", runId: "run-1", + normalizedSessionId: "session-1", lifecycle: "suspended", outbox: [], pendingTerminalDelivery: null, secret: "private-token" })); + await fakeProcess(proc, 522, ["/opt/bin/paperclip-runnerd", "--runner-id", "runner-1", "--token", "private-token"]); + await fakeProcess(proc, 600, ["/opt/bin/paperclip-runnerd", "--runner-id", "runner-2"], "Z"); + await fakeProcess(proc, 700, ["sleep", "100"]); + const script = SANDBOX_SESSION_RUNNER_INSPECT_SCRIPT.replaceAll("'/proc", `'${proc}`); + const inspect = () => inspectSandboxSessionRunner({ runner: localRunner(script) as never, stateDirectory: state }); + const inspection = await inspect(); + expect(inspection).toEqual({ + state: { lifecycle: "suspended", runId: "run-1", normalizedSessionId: "session-1", runnerInstanceId: "runner-1", settled: true }, + markerPid: 522, markerRunnerIdMatchesState: true, procfs: true, unreadableProcesses: 0, + processes: [{ pid: 522, state: "S", carriesRunnerId: true }, { pid: 600, state: "Z", carriesRunnerId: false }], + }); + expect(JSON.stringify(inspection)).not.toMatch(/private/); + await rm(join(proc, "522"), { recursive: true }); + await writeFile(join(state, "runner-state.json"), JSON.stringify({ runnerInstanceId: "runner-1", runId: "run-1", + normalizedSessionId: "session-1", lifecycle: "suspended", outbox: [{ seq: 1 }], pendingTerminalDelivery: null })); + expect(await inspect()).toMatchObject({ state: { settled: false }, unreadableProcesses: 0, + processes: [{ pid: 600, carriesRunnerId: false }] }); + await mkdir(join(proc, "800", "cmdline"), { recursive: true }); + expect(await inspect()).toMatchObject({ procfs: true, unreadableProcesses: 1, processes: [{ pid: 600 }] }); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it("fails closed without a process table and never reads state through a symlink", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-inspect-noproc-")); + const state = join(root, "runner"); + try { + await mkdir(state, { recursive: true }); + await writeFile(join(root, "elsewhere.json"), JSON.stringify({ lifecycle: "suspended", runnerInstanceId: "runner-1" })); + await symlink(join(root, "elsewhere.json"), join(state, "runner-state.json")); + const script = SANDBOX_SESSION_RUNNER_INSPECT_SCRIPT.replaceAll("'/proc", `'${join(root, "missing-proc")}`); + expect(await inspectSandboxSessionRunner({ runner: localRunner(script) as never, stateDirectory: state })).toEqual({ + state: null, markerPid: null, markerRunnerIdMatchesState: false, procfs: false, unreadableProcesses: 0, processes: [] }); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); diff --git a/server/src/services/native-runtime/sandbox-runner-port-holders.ts b/server/src/services/native-runtime/sandbox-runner-port-holders.ts index 69361d5dfbf..c8784150756 100644 --- a/server/src/services/native-runtime/sandbox-runner-port-holders.ts +++ b/server/src/services/native-runtime/sandbox-runner-port-holders.ts @@ -1,3 +1,4 @@ +import { posix } from "node:path"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, type Db } from "@paperclipai/db"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; @@ -68,22 +69,102 @@ export function parseSandboxRunnerHolders(stdout: string): SandboxRunnerHolder[] type HolderProbe = "alive" | "gone" | "unproven"; -async function probeHolder(runner: CommandManagedRuntimeRunner, holder: SandboxRunnerHolder, - signal?: "TERM" | "KILL"): Promise { +async function checkHolder(runner: CommandManagedRuntimeRunner, holder: SandboxRunnerHolder, label: string, + signal?: "TERM" | "KILL"): Promise { const result = await runner.execute({ command: "sh", args: ["-c", REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT + (signal ? '; kill -"$6" "$pid"' : ""), - "paperclip-runner-port-holder-check", holder.identityPath, holder.nonce, holder.runnerInstanceId, String(holder.pid), + label, holder.identityPath, holder.nonce, holder.runnerInstanceId, String(holder.pid), holder.processStartFingerprint ?? "", ...(signal ? [signal] : [])], bypassSession: true, timeoutMs: 10_000, }).catch(() => null); - if (!result || result.timedOut) return "unproven"; - if (result.exitCode === 0) return "alive"; - if (result.exitCode === 45 || result.exitCode === 47) return "gone"; + return !result || result.timedOut ? null : result.exitCode; +} + +async function probeHolder(runner: CommandManagedRuntimeRunner, holder: SandboxRunnerHolder, + signal?: "TERM" | "KILL"): Promise { + const exitCode = await checkHolder(runner, holder, "paperclip-runner-port-holder-check", signal); + if (exitCode === 0) return "alive"; + if (exitCode === 45 || exitCode === 47) return "gone"; return "unproven"; } +async function listSandboxRunnerHolders(runner: CommandManagedRuntimeRunner, runtimeRoot: string, ownSessionRoot: string) { + const listed = await runner.execute({ + command: "sh", + args: ["-c", SANDBOX_RUNNER_HOLDER_LIST_SCRIPT, "paperclip-runner-port-holders", runtimeRoot, ownSessionRoot], + bypassSession: true, + timeoutMs: 15_000, + }).catch(() => null); + return { listed, holders: !listed || listed.exitCode !== 0 || listed.timedOut ? null : parseSandboxRunnerHolders(listed.stdout) }; +} + +function holderSessionDirectory(holder: SandboxRunnerHolder): string { + return posix.basename(posix.dirname(posix.dirname(holder.identityPath))); +} + +export async function sandboxSessionRunnerExited(input: { + runner: CommandManagedRuntimeRunner; + runtimeRoot: string; + sessionDirectory: string; +}): Promise { + const { holders } = await listSandboxRunnerHolders(input.runner, input.runtimeRoot, ""); + const holder = holders?.find(candidate => holderSessionDirectory(candidate) === input.sessionDirectory); + return holder !== undefined && await checkHolder(input.runner, holder, "paperclip-runner-exit-check") === 45; +} + +export const SANDBOX_SESSION_RUNNER_INSPECT_SCRIPT = [ + "const fs=require('fs'),path=require('path');", + "const dir=process.argv[1];", + "const readSafe=f=>{try{if(fs.lstatSync(f).isSymbolicLink())return null;return fs.readFileSync(f,'utf8')}catch{return null}};", + "let state=null;try{const v=JSON.parse(readSafe(path.join(dir,'runner-state.json')));if(v&&typeof v==='object'&&!Array.isArray(v))state=v}catch{}", + "const marker=(readSafe(path.join(dir,'runner-process.identity'))||'').split(String.fromCharCode(10));", + "const runnerId=marker[3]||null,markerPid=/^[1-9][0-9]*$/.test(marker[1]||'')?Number(marker[1]):null;", + "const str=v=>typeof v==='string'?v:null;", + "const out={state:state&&{lifecycle:str(state.lifecycle),runId:str(state.runId),normalizedSessionId:str(state.normalizedSessionId),", + "runnerInstanceId:str(state.runnerInstanceId),settled:Array.isArray(state.outbox)&&state.outbox.length===0&&state.pendingTerminalDelivery==null", + "&&state.pendingProviderCleanup==null&&state.warmTransition==null},markerPid,", + "markerRunnerIdMatchesState:runnerId!==null&&state!==null&&state.runnerInstanceId===runnerId,procfs:false,unreadableProcesses:0,processes:[]};", + "let entries=null;try{entries=fs.readdirSync('/proc')}catch{}", + "if(entries&&fs.existsSync('/proc/self/cmdline')){out.procfs=true;for(const d of entries){", + "if(String(Number(d))!==d||Number(d)===process.pid||Number(d)===process.ppid)continue;", + "let a;try{a=fs.readFileSync('/proc/'+d+'/cmdline','utf8').split(String.fromCharCode(0))}catch(e){if(e.code!=='ENOENT'&&e.code!=='ESRCH')out.unreadableProcesses++;continue}", + "const carries=runnerId!==null&&a.includes('--runner-id')&&a.includes(runnerId);", + "if(!carries&&(a[0]||'').split('/').pop()!=='paperclip-runnerd')continue;", + "let st=null;try{const s=fs.readFileSync('/proc/'+d+'/stat','utf8');st=s.slice(s.lastIndexOf(')')+2).split(' ')[0]||null}catch{}", + "out.processes.push({pid:Number(d),state:st,carriesRunnerId:carries})}}", + "out.processes.sort((x,y)=>x.pid-y.pid);process.stdout.write(JSON.stringify(out));", +].join(""); + +export type SandboxSessionRunnerInspection = { + state: { lifecycle: string | null; runId: string | null; normalizedSessionId: string | null; + runnerInstanceId: string | null; settled: boolean } | null; + markerPid: number | null; + markerRunnerIdMatchesState: boolean; + procfs: boolean; + unreadableProcesses: number; + processes: Array<{ pid: number; state: string | null; carriesRunnerId: boolean }>; +}; + +export async function inspectSandboxSessionRunner(input: { + runner: CommandManagedRuntimeRunner; + stateDirectory: string; +}): Promise { + const result = await input.runner.execute({ + command: "sh", + args: ["-c", 'exec node -e "$1" "$2"', "paperclip-runner-session-inspect", SANDBOX_SESSION_RUNNER_INSPECT_SCRIPT, + input.stateDirectory], + bypassSession: true, + timeoutMs: 5_000, + }).catch(() => null); + if (!result || result.timedOut || result.exitCode !== 0) return null; + try { + const parsed = JSON.parse(result.stdout.trim()) as SandboxSessionRunnerInspection; + return parsed && typeof parsed === "object" && Array.isArray(parsed.processes) ? parsed : null; + } catch { return null; } +} + async function waitUntilGone(runner: CommandManagedRuntimeRunner, holder: SandboxRunnerHolder, waitMs: number, pollMs: number) { const deadline = Date.now() + waitMs; for (;;) { @@ -104,24 +185,21 @@ export async function retireSandboxRunnerPortHolders(input: { ownSessionRoot: string; providerLeaseId: string | null; adoptedRunId?: string | null; + only?: { sessionDirectory: string; runId: string }; retireInProcess: (holderRunId: string) => Promise; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; waitMs?: number; pollMs?: number; }): Promise { - const listed = await input.runner.execute({ - command: "sh", - args: ["-c", SANDBOX_RUNNER_HOLDER_LIST_SCRIPT, "paperclip-runner-port-holders", input.runtimeRoot, input.ownSessionRoot], - bypassSession: true, - timeoutMs: 15_000, - }).catch(() => null); - if (!listed || listed.exitCode !== 0 || listed.timedOut) { + const { listed, holders } = await listSandboxRunnerHolders(input.runner, input.runtimeRoot, input.ownSessionRoot); + if (!holders) { throw new NativePredecessorRetirementUnprovenError(null, input.providerLeaseId, new Error(`runner_port_holder_inventory_unavailable: exit=${listed?.exitCode ?? "none"} timedOut=${listed?.timedOut ?? false}`)); } const retired: string[] = []; - for (const holder of parseSandboxRunnerHolders(listed.stdout)) { - if (holder.runId === input.runId || (holder.runId !== null && holder.runId === input.adoptedRunId)) continue; + for (const holder of holders) { + if (input.only ? holderSessionDirectory(holder) !== input.only.sessionDirectory || holder.runId !== input.only.runId + : holder.runId === input.runId || (holder.runId !== null && holder.runId === input.adoptedRunId)) continue; const initial = await probeHolder(input.runner, holder); if (initial === "gone") continue; const unproven = (reason: string) => new NativePredecessorRetirementUnprovenError(holder.runId, input.providerLeaseId,