diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 228954f5e81..8d7a5e16a82 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -361,6 +361,13 @@ runner-sourced event or controller checkpoint, records `remote_runner_never_launched` with the same disposition and budget. The executor records `native.process_start_requested` before a backend may spawn a runner, so its absence (with no checkpoint) proves nothing reached a runner. +A remote run that a graceful drain suspended just after its runner opened the +provider session records `remote_session_opened_without_turn` with the same +disposition. Its Postgres controller record must hold only `run.prepare` and +`session.open`, with every committed event a session-open event and no event +missing. Before that same-run bootstrap relaunches in the reused sandbox, the +executor stops the interrupted runner it can verify there, so the new runner +does not fail with `runner_ingress_bind_conflict`. The same rule covers a remote run that its executor already settled as a same-run `bootstrap_retry` (for example when a shutdown closed the local runtime listener during startup) and that is due when the server restarts. diff --git a/server/src/services/native-runtime/native-bootstrap-proof.ts b/server/src/services/native-runtime/native-bootstrap-proof.ts index bf7660c5ccb..85338dbf122 100644 --- a/server/src/services/native-runtime/native-bootstrap-proof.ts +++ b/server/src/services/native-runtime/native-bootstrap-proof.ts @@ -27,6 +27,45 @@ export function runnerdStateIsUnconnectedBootstrap(state: Record): boolean { + if (runnerdStateIsIncompleteBootstrap(state)) return true; + if (!RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(state.schema))) return false; + if (typeof state.connectionCount !== "number" || !Number.isSafeInteger(state.connectionCount) || + state.connectionCount < 0) return false; + if (state.warmTransition != null || state.completedWarmTransition != null || + state.stagedRunAttachment != null) return false; + if (record(state.bootstrapHistory).adopted === true) return false; + if (!Array.isArray(state.commands) || !Array.isArray(state.committedEvents)) return false; + const commands = state.commands.map(record); + if (!commands.every(command => RUNNERD_BOOTSTRAP_COMMAND_TYPES.has(String(command.type)) && + (command.status === "pending" || command.status === "completed"))) return false; + const events = state.committedEvents.map(record); + // Every event since the first must still be here, in order, so none is hidden. + if (typeof state.ackedSourceSeq !== "number" || !Number.isSafeInteger(state.ackedSourceSeq) || + state.ackedSourceSeq < 0 || state.ackedSourceSeq > events.length) return false; + return events.every((event, index) => event.sourceSeq === index + 1 && + SESSION_OPEN_EVENT_TYPES.has(String(event.eventType))); +} + /** * A provider turn can start only after session.open completes and the * controller issues a turn command. A root whose controller journal stops diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index b72a164a1cf..e2e15b1b010 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -12,7 +12,7 @@ import { notifyInstructionCollection } from "../instruction-collection-notificat import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js"; import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader, - nativeRunnerCheckpointProvesIncompleteBootstrap } from "./native-runner-checkpoint.js"; + nativeRunnerCheckpointRemoteBootstrapProof } from "./native-runner-checkpoint.js"; import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; @@ -369,6 +369,11 @@ export async function evaluateNativeProviderProcesses(input: { } /** Run event types that can carry a provider process identity. */ +/** Projected events a runner emits while opening its provider session, before any turn. */ +const SESSION_OPEN_PROVIDER_EVENT_TYPES: ReadonlySet = new Set([ + "harness.ready", "session.started", "session.resumed", "session.updated", +]); + export const NATIVE_PROVIDER_PROCESS_EVENT_TYPES = [ "harness.ready", "session.started", @@ -487,7 +492,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: { unclaimedBootstrap?: boolean; /** Durable evidence proves a claimed remote bootstrap reached no provider work. */ remoteBootstrapProvenIncomplete?: boolean; - remoteBootstrapIncompleteReason?: "remote_runner_never_launched"; + remoteBootstrapIncompleteReason?: "remote_runner_never_launched" | "remote_session_opened_without_turn"; processStartMatches: boolean; knownProviderProcessAlive?: boolean; knownProviderProcessIdentityAmbiguous?: boolean; @@ -518,7 +523,8 @@ export function classifyNativeRunnerRecoveryEvidence(input: { if (input.unclaimedBootstrap === true && !input.hasCheckpoint && !input.hasProviderEvidence) { return { claimKind: "bootstrap_incomplete", reason: "remote_execution_never_claimed" }; } - if (input.remoteBootstrapProvenIncomplete === true && !input.hasCheckpoint && !input.hasProviderEvidence) { + // Callers prove provider evidence is absent or only session-open events. + if (input.remoteBootstrapProvenIncomplete === true && !input.hasCheckpoint) { // Each same-run bootstrap is another provider attempt; keep restarts bounded. if (input.resumeChargesAttempt !== false && (input.providerAttempt ?? 0) >= 3) return { claimKind: null, reason: "execution_recovery_budget_exhausted" }; @@ -1070,6 +1076,7 @@ export async function claimNativeRestartRecoveries(input: { const providerEvents = await tx .select({ id: heartbeatRunEvents.id, + eventType: heartbeatRunEvents.eventType, payload: heartbeatRunEvents.payload, }) .from(heartbeatRunEvents) @@ -1158,12 +1165,22 @@ export async function claimNativeRestartRecoveries(input: { // writes every command to Postgres before sending it, so that checkpoint // is the complete record of what reached the sandbox runner. When it // proves no provider work, bootstrap again instead of holding the task. - const remoteBootstrapProvenIncomplete = !unclaimedBootstrap && remoteWorkspace !== null && !pluginTask - && Boolean(durableAuthority) && stagedAttachment === null && !hasCheckpoint && !hasProviderEvidence + // A graceful drain can also suspend a run whose runner had just opened its + // provider session: session-open events may already be projected here, + // but no session snapshot or turn exists. Those events alone do not make + // the checkpoint proof ambiguous; any turn or provider event does. + const providerEvidenceIsSessionOpenOnly = !hasCheckpointProviderIdentity + && providerEvents.every(event => SESSION_OPEN_PROVIDER_EVENT_TYPES.has(event.eventType)); + const remoteBootstrapProof = !unclaimedBootstrap && remoteWorkspace !== null && !pluginTask + && Boolean(durableAuthority) && stagedAttachment === null && !hasCheckpoint && providerEvidenceIsSessionOpenOnly && Boolean(row.run.runnerInstanceId && row.run.nativeSessionId) - && await nativeRunnerCheckpointProvesIncompleteBootstrap(tx, { + ? await nativeRunnerCheckpointRemoteBootstrapProof(tx, { companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id, - }, { runnerInstanceId: row.run.runnerInstanceId!, normalizedSessionId: row.run.nativeSessionId! }); + }, { runnerInstanceId: row.run.runnerInstanceId!, normalizedSessionId: row.run.nativeSessionId! }) + : null; + // Projected session-open events need the stronger session-opened proof. + const remoteBootstrapProvenIncomplete = remoteBootstrapProof !== null + && (!hasProviderEvidence || remoteBootstrapProof === "session_opened_without_turn"); // Read before this claim appends its own history entry. const cleanDrainSuspension = suspendedByCleanDrain(row.coordinator); @@ -1171,7 +1188,8 @@ export async function claimNativeRestartRecoveries(input: { remoteSandbox: remoteSandbox || Boolean(pluginTask && durableAuthority), unclaimedBootstrap, remoteBootstrapProvenIncomplete: remoteBootstrapProvenIncomplete || remoteRunnerNeverLaunched, - remoteBootstrapIncompleteReason: remoteRunnerNeverLaunched ? "remote_runner_never_launched" : undefined, + remoteBootstrapIncompleteReason: remoteRunnerNeverLaunched ? "remote_runner_never_launched" + : remoteBootstrapProof === "session_opened_without_turn" ? "remote_session_opened_without_turn" : undefined, runnerPidAlive, runnerGroupAlive, processStartMatches: exactRunnerIdentity, diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index bbb673df149..c39886e6a6a 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -6,7 +6,7 @@ import { inspectDurablePrpCheckpointHeader, inspectUnclaimedBootstrapCheckpoint, import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js"; import { legacyControllerBootId } from "../legacy-controller-lease.js"; import { assertNativeSourceCheckpointConsumer, lockSourceRecoveryCoordinators } from "./native-source-recovery-ownership.js"; -import { runnerdStateIsIncompleteBootstrap } from "./native-bootstrap-proof.js"; +import { runnerdStateHasNoProviderTurn, runnerdStateIsIncompleteBootstrap } from "./native-bootstrap-proof.js"; type Binding = { companyId: string; issueId: string; runId: string }; export const NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES = 1024 * 1024; @@ -261,49 +261,59 @@ export async function loadNativeUnclaimedBootstrapCheckpoint( } const BOOTSTRAP_CHECKPOINT_PARTS = new Set(["header", "cursor", "command:1", "command:2"]); +/** Session-open events are few; a longer journal is never a bootstrap. */ +const MAX_BOOTSTRAP_CHECKPOINT_EVENTS = 16; + +export type NativeRemoteBootstrapProof = "bootstrap_incomplete" | "session_opened_without_turn"; /** * A remote runner's controller state lives only in this checkpoint, and every * command is written here before the controller sends it. Once the previous * controller's lease has expired, the checkpoint is therefore a complete record - * of what reached the sandbox runner. Returns true only when that record holds - * the bootstrap commands alone (run.prepare, an unfinished session.open) with - * no committed or acknowledged event, so no provider turn can exist. The + * of what reached the sandbox runner. Returns which proof holds: + * `bootstrap_incomplete` when it holds only run.prepare and an unfinished + * session.open with no committed or acknowledged event, or + * `session_opened_without_turn` when session.open completed and every committed + * event is a session-open event. Either way no provider turn can exist. The * sandbox runner journal is not on this host; the proof uses the controller * record alone, as for a local root whose runner never journaled. */ -export async function nativeRunnerCheckpointProvesIncompleteBootstrap( +export async function nativeRunnerCheckpointRemoteBootstrapProof( db: Pick, binding: Binding, expected: { runnerInstanceId: string; normalizedSessionId: string }, -): Promise { +): Promise { try { + const limit = BOOTSTRAP_CHECKPOINT_PARTS.size + MAX_BOOTSTRAP_CHECKPOINT_EVENTS; const parts = await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints).where(and( eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId), - eq(nativeRunnerCheckpoints.runId, binding.runId))).limit(BOOTSTRAP_CHECKPOINT_PARTS.size + 1); - if (!parts.length || parts.length > BOOTSTRAP_CHECKPOINT_PARTS.size - || parts.some(row => !BOOTSTRAP_CHECKPOINT_PARTS.has(row.part))) return false; + eq(nativeRunnerCheckpoints.runId, binding.runId))).limit(limit + 1); + const eventParts = parts.map(row => row.part).filter(part => /^event:\d+$/.test(part)); + if (!parts.length || parts.length > limit + || parts.some(row => !BOOTSTRAP_CHECKPOINT_PARTS.has(row.part) && !/^event:\d+$/.test(row.part))) return null; const checkpoint = await loadNativeRunnerCheckpoint(db, binding); if (!checkpoint || checkpoint.predecessor !== null || checkpoint.recoveryEvidence.runnerInstanceId !== expected.runnerInstanceId - || checkpoint.recoveryEvidence.normalizedSessionId !== expected.normalizedSessionId) return false; + || checkpoint.recoveryEvidence.normalizedSessionId !== expected.normalizedSessionId) return null; // The decoder above validated these canonical v2 records. Rebuild the - // controller state in memory; the part check admits no event records. + // controller state in memory without writing files under recovery locks. const read = (part: string) => { const value = checkpoint.records.get(part); return value === undefined ? undefined : JSON.parse(value) as Record; }; const header = read("header"); const cursor = read("cursor"); - if (header?.schema !== "paperclip.runner.controller-checkpoint.v2" || !cursor) return false; + if (header?.schema !== "paperclip.runner.controller-checkpoint.v2" || !cursor) return null; const state: Record = { ...(header.state as Record), ackedSourceSeq: cursor.ackedSourceSeq, commands: ["command:1", "command:2"].map(read).filter(command => command !== undefined), - committedEvents: [], + committedEvents: eventParts.map(part => Number(part.slice("event:".length))).sort((a, b) => a - b) + .map(seq => read(`event:${seq}`)), }; - return (state.identity as { runId?: unknown } | undefined)?.runId === binding.runId - && runnerdStateIsIncompleteBootstrap(state); + if ((state.identity as { runId?: unknown } | undefined)?.runId !== binding.runId) return null; + if (runnerdStateIsIncompleteBootstrap(state)) return "bootstrap_incomplete"; + return runnerdStateHasNoProviderTurn(state) ? "session_opened_without_turn" : null; } catch { - return false; + return null; } } diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 4744f0523dc..73044a9a9f8 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -62,6 +62,7 @@ import { NativeControllerDetachedForRestartError, restoreRemoteBootstrapCheckpointForRecovery, runnerdStateProvesIncompleteBootstrap, + runnerdStateProvesNoRemoteProviderTurn, } from "./native-session-executor.js"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { @@ -2366,6 +2367,72 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( restartKind: "hard", runIds: [fixture.runId] })).toEqual([{ kind: "blocked", runId: fixture.runId, reason }]); }); + // RC-29d: a graceful drain suspended the run just after its sandbox runner + // opened the provider session. The controller record holds a completed + // session.open and session-open events, but no turn was ever sent. + const openSession = (core: DurablePrpControlPlane, eventTypes = ["session.started", "session.capabilities.updated", + "session.goal.snapshot"]) => { + const [prepare, open] = core.store.state.commands; + prepare!.status = "completed"; + prepare!.result = { status: "prepared" } as never; + open!.status = "completed"; + open!.result = { status: "started", processId: 608, providerSessionId: "provider-session" } as never; + core.store.state.commandDeliveryCounts = { [prepare!.commandId]: 1, [open!.commandId]: 1 }; + const { runId, normalizedSessionId, runnerInstanceId, turnId, itemId } = core.store.state.identity; + core.store.state.committedEvents = eventTypes.map((eventType, index) => ({ sourceSeq: index + 1, + sourceEventId: `runner-session-event-${index + 1}`, eventType, priority: 1, deliveryCount: 1, logicalEffectCount: 1, + envelope: { kind: "event", payload: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", + sourceInstanceId: runnerInstanceId, sourceEventId: `runner-session-event-${index + 1}`, sourceSeq: index + 1, runId, + normalizedSessionId, turnId, itemId, eventType, priority: 1, emittedAt: "2026-11-07T00:49:52.000Z", payload: {} } }, + })) as never; + core.store.state.ackedSourceSeq = eventTypes.length; + }; + + it("restarts a drained remote bootstrap whose runner opened its session but never got a turn", async () => { + const fixture = await seedClaimedRemoteBootstrap("SESSION-OPENED", { mutate: core => openSession(core) }); + // The dying controller may already have projected the session-open event. + await fixture.db.insert(heartbeatRunEvents).values({ companyId, runId: fixture.runId, agentId, seq: 902, + stream: "system", level: "info", eventType: "session.started" }); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "bootstrap_incomplete", runId: fixture.runId, controllerGeneration: 2 })]); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator.recoveryHistory.at(-1)).toMatchObject({ disposition: "bootstrap_incomplete", + reason: "remote_session_opened_without_turn" }); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run).toMatchObject({ status: "running", errorCode: null }); + }); + + it("lets the executor prove a drained remote session had no turn from its restored checkpoint", async () => { + const fixture = await seedClaimedRemoteBootstrap("SESSION-OPENED-EXECUTOR", { mutate: core => openSession(core) }); + const root = resolve(runtimeRoot, `opened-${fixture.runId}`); + try { + expect(await restoreNativeRunnerCheckpoint(fixture.db, { companyId, issueId: fixture.issueId, runId: fixture.runId }, + root)).not.toBeNull(); + // The local proof cannot see a sandbox provider process; the remote proof + // relies on the complete controller record instead. + expect(runnerdStateProvesIncompleteBootstrap(root)).toBe(false); + expect(runnerdStateProvesNoRemoteProviderTurn(root)).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it.each<[string, Parameters[1], { eventType: string } | null]>([ + ["a turn was issued", { mutate: core => { openSession(core); + core.queueCommand("turn.start", { prompt: "Prior work must stay held." }); } }, null], + ["a turn event was committed", { mutate: core => openSession(core, ["session.started", "turn.started"]) }, null], + ["an acknowledged event is missing", { mutate: core => { openSession(core); core.store.state.ackedSourceSeq = 4; } }, null], + ["a provider turn event was projected", { mutate: core => openSession(core) }, { eventType: "turn.started" }], + ])("keeps a drained remote session held when %s", async (label, options, projected) => { + const fixture = await seedClaimedRemoteBootstrap(`OPENED-${label.replace(/\W+/g, "-")}`, options); + if (projected) await fixture.db.insert(heartbeatRunEvents).values({ companyId, runId: fixture.runId, agentId, seq: 902, + stream: "system", level: "info", ...projected }); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ kind: "blocked", runId: fixture.runId })]); + }); + // RC-29b: a restart while the executor was still preparing a claimed Daytona // run, before it asked for a runner process. The executor records // native.process_start_requested (and only then may a backend spawn or a diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 9a775f099fc..93f8c455e77 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -424,6 +424,7 @@ import { REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, REMOTE_RUNNER_AUTHORITY_ARCHIVE_SCRIPT, verifyRemoteRunnerReattachment, + stopStaleRemoteRunnerBeforeBootstrap, readRemoteProviderPackManifest, prepareRemoteProviderPackOnTarget, resolveRemoteProviderPackSource, @@ -597,6 +598,46 @@ describe("remote controller restart adoption", () => { execute.mockResolvedValueOnce({ stdout: "", stderr: "", exitCode: 45, timedOut: false }); expect(await adopted.isAlive()).toBe(false); }); + describe("stale runner stop before a same-run remote bootstrap", () => { + const execution = { binding: { runId: "run" }, session: { normalizedSessionId: "session" } } as never; + const signalled = (execute: ReturnType["execute"]) => execute.mock.calls + .map(([request]) => request.args.at(-1)).filter(signal => signal === "TERM" || signal === "KILL"); + it("stops the interrupted runner with this run's identity before relaunch", async () => { + const { execute, target } = fixture(); + // The runner exits on SIGTERM: every later process check finds it gone. + const live = execute.getMockImplementation()!; + let terminated = false; + execute.mockImplementation(async request => { + if (terminated && request.args[1]!.includes("expected_pid")) return { stdout: "", stderr: "", exitCode: 45, timedOut: false }; + if (request.args.at(-1) === "TERM") terminated = true; + return live(request); + }); + await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, + { waitMs: 1_000, pollMs: 1 })).resolves.toBe("stopped"); + expect(signalled(execute)).toEqual(["TERM"]); + }); + it("escalates to SIGKILL when the runner ignores SIGTERM", async () => { + const { execute, target } = fixture(); + await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, + { waitMs: 20, pollMs: 1 })).resolves.toBe("killed"); + expect(signalled(execute)).toEqual(["TERM", "KILL"]); + }); + it("signals nothing when the sandbox runner belongs to another identity", async () => { + const { execute, target } = fixture({ runnerInstanceId: "different" }); + await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, + { waitMs: 20, pollMs: 1 })).resolves.toBe("no_prior_runner"); + expect(signalled(execute)).toEqual([]); + }); + it("signals nothing when no prior runner is alive", async () => { + const { execute, target } = fixture(); + const live = execute.getMockImplementation()!; + execute.mockImplementation(async request => request.args[1]!.includes("expected_pid") + ? { stdout: "", stderr: "", exitCode: 45, timedOut: false } : live(request)); + await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, + { waitMs: 20, pollMs: 1 })).resolves.toBe("no_prior_runner"); + expect(signalled(execute)).toEqual([]); + }); + }); it.each([ "runId", "normalizedSessionId", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index bab52898a4a..7f69d7ef5cf 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -90,7 +90,7 @@ import { createNativeProviderFailureObservation } from "./native-provider-failur import { nativeStartupRetryExhaustedAction, nativeStartupRetryWait, planNativeStartupRetry } from "./native-startup-retry.js"; import { admitUnclaimedNativeBootstrapRetry } from "./native-bootstrap-retry.js"; import { RUNNERD_CONTROL_PLANE_STATE_SCHEMAS, runnerdStateIsIncompleteBootstrap, - runnerdStateIsOpenedSessionBeforeTurn, runnerdStateIsUnconnectedBootstrap } from "./native-bootstrap-proof.js"; + runnerdStateHasNoProviderTurn, runnerdStateIsOpenedSessionBeforeTurn, runnerdStateIsUnconnectedBootstrap } from "./native-bootstrap-proof.js"; import { createNativeDeliverableTextObservation } from "./native-deliverable-text.js"; import { createNativeGitHubAccess, type NativeGitHubAccess } from "./native-github-access.js"; import { resolveGitHubOperationCredentials } from "../github-operation-credentials.js"; @@ -5659,7 +5659,14 @@ async function migrateRunnerdStateRootForExecution(input: { // A local runner's process ids belong to this host, so a fresh provider // session it opened before any turn can be proven gone here. const localProcesses = !input.runnerExecutionTarget || input.runnerExecutionTarget.kind === "local"; - if (runnerdStateProvesIncompleteBootstrap(scoped, { localProcesses })) { + const target = input.runnerExecutionTarget; + const remoteSandbox = target?.kind === "remote" && target.transport === "sandbox" && !target.runnerTask; + // A remote controller record restored from Postgres is complete once its + // lease expired. A runner that opened its session but got no turn there + // holds no provider work (RC-29d). + if (runnerdStateProvesIncompleteBootstrap(scoped, { localProcesses }) + || (remoteSandbox && runnerdStateProvesNoRemoteProviderTurn(scoped))) { + if (remoteSandbox) await stopStaleRemoteRunnerBeforeBootstrap({ ...input, identity }); quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); return; } @@ -6178,6 +6185,74 @@ export function runnerdStateProvesIncompleteBootstrap( } } +const STALE_REMOTE_RUNNER_STOP_WAIT_MS = 10_000; + +/** + * The same-run bootstrap reuses the run's sandbox. A runner from the + * interrupted attempt can still be alive there (its reconnect grace is + * minutes) and still hold the runner ingress port, so the new runner would fail + * with runner_ingress_bind_conflict. Stop that exact runner first, using the + * same verified identity marker and process check as reattachment, which only + * signals a process whose identity matches this run. Best effort: if no prior + * runner can be verified there is nothing to stop, and a remaining conflict is + * still caught by the bounded same-run bootstrap retry. + */ +export async function stopStaleRemoteRunnerBeforeBootstrap(input: { + execution: NativeExecutionInput; + runnerExecutionTarget?: AdapterExecutionTarget | null; + identity: Record; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; +}, options: { + verify?: typeof verifyRemoteRunnerReattachment; + waitMs?: number; + pollMs?: number; +} = {}): Promise<"no_prior_runner" | "stopped" | "killed" | "stop_failed"> { + const target = input.runnerExecutionTarget; + if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner) return "no_prior_runner"; + const providerLeaseId = target.sandboxLeaseAcquisition?.providerLeaseId; + if (!providerLeaseId) return "no_prior_runner"; + const runId = input.execution.binding.runId; + const verify = options.verify ?? verifyRemoteRunnerReattachment; + let runner: Awaited>; + try { + runner = await verify({ + claim: { runId, remote: { providerLeaseId, remoteCwd: target.remoteCwd } }, + target, identity: input.identity, runId, normalizedSessionId: nativeSessionKey(input.execution), + }); + } catch { + // No runner with this run's exact identity and generation is alive here. + return "no_prior_runner"; + } + try { + let outcome: "stopped" | "killed" = "stopped"; + if (await runner.signal("SIGTERM")) { + const deadline = Date.now() + (options.waitMs ?? STALE_REMOTE_RUNNER_STOP_WAIT_MS); + while (Date.now() < deadline && await runner.isAlive()) { + await new Promise(done => setTimeout(done, options.pollMs ?? 500)); + } + if (await runner.isAlive()) { + await runner.signal("SIGKILL"); + outcome = "killed"; + } + } + await input.onLog?.("stdout", `[paperclip-runner] Stopped the interrupted runner ${runner.pid} before restarting the bootstrap.\n`); + return outcome; + } catch (error) { + await input.onLog?.("stderr", `[paperclip-runner] Could not stop the interrupted runner before restarting the bootstrap: ${ + error instanceof Error ? error.message : String(error)}\n`); + return "stop_failed"; + } +} + +/** The remote controller record proves its runner was never sent a turn. */ +export function runnerdStateProvesNoRemoteProviderTurn(root: string): boolean { + try { + return runnerdStateHasNoProviderTurn(readRunnerdControlPlaneState(root)); + } catch { + return false; + } +} + function runnerdStateProvesUnconnectedBootstrap(root: string): boolean { try { return runnerdStateIsUnconnectedBootstrap(readRunnerdControlPlaneState(root));