diff --git a/server/src/services/native-runtime/native-bootstrap-proof.test.ts b/server/src/services/native-runtime/native-bootstrap-proof.test.ts new file mode 100644 index 00000000000..9c2b25d4377 --- /dev/null +++ b/server/src/services/native-runtime/native-bootstrap-proof.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { attemptLaunchesFreshBootstrap } from "./native-bootstrap-proof.js"; + +describe("attemptLaunchesFreshBootstrap", () => { + it("accepts a first attempt", () => { + expect(attemptLaunchesFreshBootstrap(undefined)).toBe(true); + expect(attemptLaunchesFreshBootstrap(null)).toBe(true); + }); + + // The restart stops and quarantines the unfinished bootstrap, then launches a + // new one on the same run. A transient endpoint failure of that new bootstrap + // is as retryable as one on a first attempt. + it("accepts a same-run restart of an incomplete bootstrap", () => { + expect(attemptLaunchesFreshBootstrap({ kind: "bootstrap_incomplete" })).toBe(true); + }); + + it.each(["reattach_existing_runner", "reattach_remote_runner", "resume_dead_runner"])( + "refuses a %s recovery, which continues an existing Runner or provider session", + kind => { expect(attemptLaunchesFreshBootstrap({ kind })).toBe(false); }, + ); +}); diff --git a/server/src/services/native-runtime/native-bootstrap-proof.ts b/server/src/services/native-runtime/native-bootstrap-proof.ts index 90d169be4cc..acc47705c07 100644 --- a/server/src/services/native-runtime/native-bootstrap-proof.ts +++ b/server/src/services/native-runtime/native-bootstrap-proof.ts @@ -17,6 +17,17 @@ function record(value: unknown): Record { : {}; } +/** + * Whether this attempt launches its own fresh Runner bootstrap, so that its + * unclaimed controller checkpoint can prove a startup failure retryable. A + * `bootstrap_incomplete` restart does: it stops and quarantines the unfinished + * bootstrap, then launches a new one on the same run. Other recoveries continue + * an existing Runner or provider session. + */ +export function attemptLaunchesFreshBootstrap(restartRecovery: { kind: string } | null | undefined): boolean { + return !restartRecovery || restartRecovery.kind === "bootstrap_incomplete"; +} + /** The runner never connected, so every bootstrap command is still undelivered. */ export function runnerdStateIsUnconnectedBootstrap(state: Record): boolean { const commands = Array.isArray(state.commands) ? state.commands.map(record) : []; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 8c6c95f78a2..2ff0253168c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -109,7 +109,7 @@ import { createNativeProviderFailureObservation } from "./native-provider-failur import { explainNativeTurnFailure } from "./native-run-failure-explanation.js"; import { nativeStartupRetryExhaustedAction, nativeStartupRetryWait, planNativeStartupRetry } from "./native-startup-retry.js"; import { admitUnclaimedNativeBootstrapRetry } from "./native-bootstrap-retry.js"; -import { RUNNERD_CONTROL_PLANE_STATE_SCHEMAS, runnerdStateIsIncompleteBootstrap, +import { attemptLaunchesFreshBootstrap, RUNNERD_CONTROL_PLANE_STATE_SCHEMAS, runnerdStateIsIncompleteBootstrap, runnerdStateHasNoProviderTurn, runnerdStateIsOpenedSessionBeforeTurn, runnerdStateIsUnconnectedBootstrap } from "./native-bootstrap-proof.js"; import { createNativeDeliverableTextObservation } from "./native-deliverable-text.js"; import { reconcileControllerJournalWithRunner, refreshRetiredWarmOwnerProjection } from "./warm-controller-journal.js"; @@ -12995,7 +12995,7 @@ async function executePaperclipNativeSessionAttempt( : nativeRemoteStartupRejectionCode(classifiedFailureCode, error, input.runnerExecutionTarget?.kind === "remote", previousStartupSignature); const bootstrap = input.useRunnerd && input.runnerExecutionTarget?.kind === "remote" - && !input.restartRecovery && !retainedTransition && !remoteWarmRecovery + && attemptLaunchesFreshBootstrap(input.restartRecovery) && !retainedTransition && !remoteWarmRecovery ? { binding: input.execution.binding, identity: { runnerInstanceId: effectiveRunnerInstanceId, diff --git a/server/src/services/native-runtime/native-startup-retry.integration.test.ts b/server/src/services/native-runtime/native-startup-retry.integration.test.ts index f32a6bfd65f..7f420fc2c6b 100644 --- a/server/src/services/native-runtime/native-startup-retry.integration.test.ts +++ b/server/src/services/native-runtime/native-startup-retry.integration.test.ts @@ -35,7 +35,7 @@ import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js"; import { DurablePrpControlPlane, NativeProviderTerminalFailure } from "../../vendor/paperclip-runner/index.js"; import { createNativeRunnerExecutionDriver } from "./native-runner-execution-driver.js"; -import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; +import { currentNativeControllerIdentity, type NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; import { environmentRunnerIngressErrorData, JsonRpcCallError, PluginEnvironmentRunnerIngressError } from "@paperclipai/plugin-sdk"; import { accountRunCost } from "../run-cost-accounting.js"; @@ -290,6 +290,7 @@ describe("native provider startup retry", () => { it.each([ { name: "transient 502", status: 502, retry: true }, + { name: "transient 502 after a bootstrap_incomplete restart", status: 502, retry: true, restart: true }, { name: "credential 401", status: 401, retry: false }, { name: "permission 403", status: 403, retry: false }, { name: "invalid request 400", status: 400, retry: false }, @@ -305,6 +306,19 @@ describe("native provider startup retry", () => { ])("settles ingress failure safely: $name", async options => { const seeded = await seedRun(await createAgent()); if (options.finalAttempt) await db.update(nativeRunFinalizations).set({ attempt: 2 }).where(eq(nativeRunFinalizations.runId, seeded.runId)); + // A server restart interrupted the first bootstrap. The recovery scan's + // claim makes this attempt launch a fresh bootstrap on the same run. + let restartRecovery: NativeRestartRecoveryClaim | undefined; + if (options.restart) { + const controller = await currentNativeControllerIdentity(); + restartRecovery = { kind: "bootstrap_incomplete", runId: seeded.runId, leaseOwner: `restart-${seeded.runId}`, + controllerGeneration: 2, providerAttempt: 0, restartKind: "hard", recoveryRequestId: null }; + await db.update(nativeRunFinalizations).set({ leaseOwner: restartRecovery.leaseOwner, + leaseExpiresAt: new Date(Date.now() + 10 * 60_000), controllerBootId: controller.bootId, + controllerPid: controller.pid, controllerHostHash: controller.hostHash ?? null, + controllerProcessStartedAt: controller.processStartedAt, controllerGeneration: 2, + recoveryState: "bootstrap_incomplete" }).where(eq(nativeRunFinalizations.runId, seeded.runId)); + } const root = mkdtempSync(join(tmpdir(), "native-ingress-bootstrap-")); const server = createServer((_request, response) => { response.writeHead(options.status, { "content-type": "text/html" }); @@ -386,6 +400,7 @@ describe("native provider startup retry", () => { try { await expect(executePaperclipNativeSession({ db, execution: seeded.execution, runnerInstanceId: seeded.runnerInstanceId, useRunnerd: true, + ...(restartRecovery ? { leaseOwner: restartRecovery.leaseOwner, restartRecovery } : {}), runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner: {} } as never, backend: { descriptor: async () => { throw new Error("unexpected descriptor"); }, openSession: async () => { throw new Error("unexpected provider session"); } },