diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index c5801b8d8ec..ae94ee4e86f 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3823,6 +3823,25 @@ in the sandbox, and only the killed server could have adopted it. A refusal because the new run itself is no longer admitted (stopped, reassigned, paused or no longer owned by this controller) stays an ordinary failure. +The finished run's finalization lease ends at commit, but its warm session +stays with the server process that ran it. Before live adoption, the new run +proves that process gone with the shared owner proof: a dead PID in the same +process space, or a stopped or fenced controller instance lease in another one. +While that server is still alive, the run defers with +`native_controller_recovery_waiting_for_owner` and checks again every few +seconds. It never opens a second controller on the same Runner. The wait ends +when that server exits, closes the session, or the finished run's warm window +(its idle timeout plus two minutes) ends. A server that leaves new run starts +to a replacement during a deploy overlap closes its idle warm sessions within +a second, and closes each session that becomes idle after that, so the +replacement usually waits only for that close. + +A fresh sandbox is only requested, not guaranteed: the next acquisition +replaces the marked sandbox only when the saved workspace and harness backup +allow it. When the next attempt meets an unretired owner again on the same +sandbox, that attempt stops the automatic retries. It fails without another +fallback, and the task shows a Retry. + A follow-up on a warm session starts with its predecessor's Runner process identity. That inherited identity alone does not count as a process of the follow-up when it proves that it left the workspace unchanged. The identity diff --git a/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts b/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts index a99d4ca48c1..0a6c76312d5 100644 --- a/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts +++ b/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts @@ -44,7 +44,7 @@ type AdmissionFailure = { remaining: number; turnNeverIssued: boolean; environme sessionRoots: string[] }; function nativeBackend(db: ReturnType, options: { failDescriptorOnce?: { failed: boolean }; - unretiredPredecessorOnce?: { failed: boolean; environmentId: string; taskBlocked?: boolean }; turnMessages?: Map; + unretiredPredecessorOnce?: { failed: boolean; environmentId: string; taskBlocked?: boolean; repeat?: string[] }; turnMessages?: Map; admissionFailure?: AdmissionFailure } = {}): NonNullable { return execution => { @@ -87,7 +87,8 @@ function nativeBackend(db: ReturnType, options: { failDescripto throw Object.assign(new Error("could not serialize access due to concurrent update"), { name: "PostgresError", code: "40001" }); } const unretired = options.unretiredPredecessorOnce; - if (unretired && !unretired.failed) { + if (unretired && (!unretired.failed || unretired.repeat)) { + unretired.repeat?.push(execution.binding.runId); // Staging: this run resumed the task's reused sandbox, where // the previous run's warm Runner could not be retired before any // provider work. The executor reports that after its coordinator claim. @@ -242,6 +243,41 @@ describe.skipIf(!support.supported)("heartbeat native pre-dispatch failure retry } }, 90_000); + it("stops retrying with a truthful failure when the next attempt met the same unretired sandbox instead of a fresh one", async () => { + const { companyId } = fixture.current; + const db = createDb(fixture.current.temporary.connectionString, { maxConnections: 4 }); + const [environment] = await db.insert(environments).values({ name: `reused-${randomUUID()}`, driver: "sandbox" }).returning(); + const attempts: string[] = []; + const heartbeat = heartbeatService(db, { nativeSessionBackendFactory: nativeBackend(db, + { unretiredPredecessorOnce: { failed: false, environmentId: environment!.id, repeat: attempts } }) }); + try { + const { issueId, runId } = await startTask(db, heartbeat); + const [first] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.retryOfRunId, runId))); + expect(first).toMatchObject({ scheduledRetryReason: "interrupted_preparation", scheduledRetryAttempt: 1 }); + expect(await heartbeat.promoteDueScheduledRetries()).toMatchObject({ promoted: 1 }); + await heartbeat.resumeQueuedRuns(); + await waitForCondition("the second attempt to end", async () => + (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, first!.id)))[0]?.status === "failed"); + await waitForCondition("the second attempt to settle", () => heartbeat.getTaskDrainStatus().quiescent); + expect(attempts).toEqual([runId, first!.id]); + const [second] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, first!.id)); + expect(second?.resultJson ?? {}).not.toHaveProperty("predecessorRetirementFallback"); + expect(second?.error ?? "").not.toMatch(/fresh sandbox/i); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.retryOfRunId, first!.id)))).toHaveLength(0); + expect(await heartbeat.promoteDueScheduledRetries()).toMatchObject({ promoted: 0 }); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(await taskInterruptionPresentation(db, issue!)).toMatchObject({ canRetry: true }); + } finally { + await heartbeat.drainActiveRunExecutions(); + await closeIdleWarmNativeSessionsForRestart(); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, ["queued", "scheduled_retry"]))); + await db.$client.end(); + } + }, 90_000); + it.each([ { title: "continues a follow-up on a blocked task whose predecessor could not be retired", blockedAfter: false }, { title: "does not continue a task that was blocked only after its run failed", blockedAfter: true }, diff --git a/server/src/services/heartbeat/predecessor-retirement-fallback.ts b/server/src/services/heartbeat/predecessor-retirement-fallback.ts index 648ce5aee87..354406f9da3 100644 --- a/server/src/services/heartbeat/predecessor-retirement-fallback.ts +++ b/server/src/services/heartbeat/predecessor-retirement-fallback.ts @@ -4,7 +4,7 @@ import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema } from import { persistActivity } from "../activity-log.js"; import { hasNativeExecutionStartEvidence } from "../native-runtime/native-restart-recovery.js"; import { - findNativePredecessorRetirementUnproven, REPLACE_BEFORE_REUSE_KEY, REPLACE_BEFORE_REUSE_SCHEMA, + findNativePredecessorRetirementUnproven, readPredecessorRetirementFallback, REPLACE_BEFORE_REUSE_KEY, REPLACE_BEFORE_REUSE_SCHEMA, type PredecessorRetirementFallback, type ReplaceBeforeReuseMarker, } from "../native-runtime/predecessor-retirement.js"; import { FAILED_BEFORE_PROVIDER_TURN_KEY, FAILED_BEFORE_PROVIDER_TURN_SCHEMA, findNativeFailedBeforeProviderTurn, @@ -47,6 +47,7 @@ export async function preparePredecessorRetirementFallback(db: Db, run: Run, err : eq(environmentLeases.leasePolicy, "reuse_by_environment"), isNotNull(environmentLeases.providerLeaseId), )).for("update")).filter(lease => !failure.providerLeaseId || lease.providerLeaseId === failure.providerLeaseId); if (leases.length === 0 && !beforeProviderTurn) return null; + if (await replacementAlreadyMissed(tx as unknown as Db, current, leases.map(lease => lease.providerLeaseId!))) return null; if (beforeProviderTurn) await tx.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({ [FAILED_BEFORE_PROVIDER_TURN_KEY]: { schema: FAILED_BEFORE_PROVIDER_TURN_SCHEMA, cause: beforeProviderTurn.code, proof: beforeProviderTurn.proof, technicalCause: beforeProviderTurn.technicalCause } })}::jsonb` }) @@ -68,6 +69,15 @@ export async function preparePredecessorRetirementFallback(db: Db, run: Run, err }); } +async function replacementAlreadyMissed(db: Db, run: Pick, + providerLeaseIds: string[]): Promise { + if (!run.retryOfRunId || providerLeaseIds.length === 0) return false; + const [source] = await db.select({ resultJson: heartbeatRuns.resultJson }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.retryOfRunId))); + const replaced = readPredecessorRetirementFallback(source?.resultJson)?.replacedProviderLeaseIds ?? []; + return providerLeaseIds.some(id => replaced.includes(id)); +} + /** * After the failed run ended: hand its prepared remote workspace on at the * saved version, but only through the shared unchanged-files proof (no runner diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index e91afae1b21..739a56e3895 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -10,7 +10,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { agentTaskSessions, agents, companies, createDb, environmentLeases, environments, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, nativeRunnerCheckpoints } from "@paperclipai/db"; +import { agentTaskSessions, agents, companies, controllerInstances, createDb, environmentLeases, environments, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, nativeRunnerCheckpoints } from "@paperclipai/db"; import { DurablePrpControlPlane, parseNativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js"; import { createIsolatedTestHome } from "../../__tests__/helpers/isolated-test-home.js"; @@ -24,6 +24,9 @@ import { AgentDirectoryPreparationDeferral, deferRunForBusyAgentDirectory, prepa import { AgentDirectoryOperationBusyError } from "../agent-directory-working-copies.js"; import { readNativeControllerRecoveryInventory } from "./native-controller-recovery-inventory.js"; import { inspectRemoteWarmSessionRecovery } from "./remote-warm-session-recovery.js"; +import { currentProcessSpaceHash } from "../controller-process-identity.js"; +import { registerControllerInstanceDb, resetControllerInstanceLeaseForTests } from "../controller-instance-lease.js"; +import { readProcessStartedAt } from "../hot-restart.js"; describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable discovery and import", () => { let database: Awaited>; @@ -535,6 +538,112 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "duplicate", runIds: [runId], now: deadline })).toEqual([]); }); + async function seedWarmFollowUpOnOldServer() { + const fixture = await seed(), runId = randomUUID(), environmentId = randomUUID(); + const previous = parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, + binding: { companyId, agentId, issueId: fixture.scope.issueId, runId: fixture.scope.runId, executionWorkspaceId: randomUUID() }, + task: { identifier: "NCD-9", title: "Answer the follow-up", description: null, prompt: "Continue.", workMode: "standard" }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: fixture.scope.normalizedSessionId, driverKind: "codex_app_server", + protocolVersion: 1, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 600_000 } }, + completionContract: { id: randomUUID(), sha256: "fixture", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Continue.", criteria: [{ id: "objective", requirement: "Save the result." }] } }, + credentialBindings: [], interactionResponses: [], + }); + const current = { ...previous, binding: { ...previous.binding, runId } }; + const checkpoint: PersistedNativeSession = { ...fixture.snapshot, driverKind: "codex_app_server", + providerSessionId: "warm-provider-session", activeTurnId: null, pendingRuntimeRequests: [] }; + await db.update(heartbeatRuns).set({ finishedAt: new Date(), runnerProfileJson: { ...fixture.profile, nativeExecutionInput: previous, + sessionCheckpoint: checkpoint, nativeControllerRecovery: createNativeControllerRecovery(checkpoint, compatibility, + { runId: fixture.scope.runId, runnerInstanceId: fixture.native.runnerInstanceId, normalizedSessionId: fixture.scope.normalizedSessionId }) } }) + .where(eq(heartbeatRuns.id, fixture.scope.runId)); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: fixture.scope.issueId, + nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native", + runnerProfileJson: { nativeExecutionInput: current, sessionCheckpoint: { ...checkpoint, identity: { ...checkpoint.identity, runId } } } }); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: fixture.scope.issueId, runId, phase: "observed" }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, fixture.scope.issueId)); + await db.insert(environments).values({ id: environmentId, name: `Warm ${environmentId}`, driver: "sandbox" }); + await db.insert(environmentLeases).values({ companyId, environmentId, issueId: fixture.scope.issueId, + heartbeatRunId: fixture.scope.runId, provider: "daytona", providerLeaseId: "warm-sandbox", status: "released", + metadata: { remoteCwd: "/workspace" } }); + const identity = { runId: fixture.scope.runId, normalizedSessionId: fixture.scope.normalizedSessionId, + runnerInstanceId: fixture.native.runnerInstanceId, environmentLeaseId: fixture.native.environmentLeaseId, + turnId: fixture.native.turnId, itemId: fixture.native.itemId }; + const inspect = () => inspectRemoteWarmSessionRecovery({ db, execution: current, identity, + target: { kind: "remote", transport: "sandbox", environmentId, leaseId: randomUUID(), providerKey: "daytona", + remoteCwd: "/workspace", sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "warm-sandbox" } } as never, + readControlState: () => ({ identity, commands: [{ type: "turn.start", status: "completed" }], + committedEvents: [{ eventType: "run.terminal", envelope: identity }] }), + matchesExecution: saved => saved.binding.runId === fixture.scope.runId, + loadCheckpoint: () => checkpoint, canReuseProcess: () => true, + }); + const recordOwner = (owner: Partial) => db.update(nativeRunFinalizations) + .set(owner).where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); + return { fixture, runId, current, identity, inspect, recordOwner }; + } + + it("waits while the outgoing server that ran the finished turn still holds its warm session, then takes it over once that server exits", async () => { + const oldServer = spawn(process.execPath, ["-e", "setInterval(() => {}, 60_000)"], { stdio: "ignore" }); + try { + await new Promise(resolve => oldServer.once("spawn", resolve)); + const startedAt = await readProcessStartedAt(oldServer.pid!); + expect(startedAt).toBeTruthy(); + const { inspect, recordOwner, runId, current } = await seedWarmFollowUpOnOldServer(); + await recordOwner({ controllerBootId: randomUUID(), controllerPid: oldServer.pid!, controllerHostHash: currentProcessSpaceHash(), + controllerProcessStartedAt: new Date(startedAt!), leaseOwner: null, leaseExpiresAt: null }); + const before = Date.now(); + const waiting = await inspect().catch(error => error); + expect(waiting).toBeInstanceOf(NativeControllerRecoveryWaitingError); + expect(waiting.retryAt.getTime()).toBeGreaterThan(before); + expect(waiting.retryAt.getTime()).toBeLessThanOrEqual(Date.now() + 10_000); + await deferNativeControllerRecovery(db, { ...current.binding, retryAt: waiting.retryAt }); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]) + .toMatchObject({ phase: "retryable_failure", attempt: 0, failureCode: NATIVE_RETAINED_OWNER_WAIT, resultId: null }); + const [task] = await db.select().from(issues).where(eq(issues.executionRunId, runId)); + expect(task?.status).toBe("in_progress"); + + const exited = new Promise(resolve => oldServer.once("exit", resolve)); + oldServer.kill("SIGKILL"); + await exited; + const recovered = await inspect(); + expect(recovered?.identity.runId).not.toBe(runId); + expect(recovered?.reuseProcess).toBe(true); + } finally { + oldServer.kill("SIGKILL"); + } + }); + + it("waits for a live outgoing server in another process space until its controller lease ends", async () => { + const { inspect, recordOwner } = await seedWarmFollowUpOnOldServer(); + const oldBoot = randomUUID(); + await recordOwner({ controllerBootId: oldBoot, controllerPid: 7, controllerHostHash: "outgoing-container", + controllerProcessStartedAt: new Date(Date.now() - 600_000), leaseOwner: null, leaseExpiresAt: null }); + await db.insert(controllerInstances).values({ bootId: oldBoot, processSpace: "outgoing-container", pid: 7, + leaseExpiresAt: new Date(Date.now() + 90_000) }); + registerControllerInstanceDb(db); + try { + await expect(inspect()).rejects.toBeInstanceOf(NativeControllerRecoveryWaitingError); + await db.update(controllerInstances).set({ stoppedAt: new Date() }).where(eq(controllerInstances.bootId, oldBoot)); + await expect(inspect()).resolves.not.toBeNull(); + } finally { resetControllerInstanceLeaseForTests(); } + }); + + it("stops waiting for a live outgoing server once the finished turn's warm window has closed", async () => { + const oldServer = spawn(process.execPath, ["-e", "setInterval(() => {}, 60_000)"], { stdio: "ignore" }); + try { + await new Promise(resolve => oldServer.once("spawn", resolve)); + const startedAt = await readProcessStartedAt(oldServer.pid!); + const { inspect, recordOwner, fixture } = await seedWarmFollowUpOnOldServer(); + await recordOwner({ controllerBootId: randomUUID(), controllerPid: oldServer.pid!, controllerHostHash: currentProcessSpaceHash(), + controllerProcessStartedAt: new Date(startedAt!), leaseOwner: null, leaseExpiresAt: null }); + await db.update(heartbeatRuns).set({ finishedAt: new Date(Date.now() - 3_600_000) }).where(eq(heartbeatRuns.id, fixture.scope.runId)); + await expect(inspect()).resolves.not.toBeNull(); + } finally { + oldServer.kill("SIGKILL"); + } + }); + it("sends a re-wake after a crash restart to a fresh sandbox when the parked predecessor's warm Runner cannot be adopted", async () => { const fixture = await seed(), runId = randomUUID(), environmentId = randomUUID(), leaseId = randomUUID(); const previous = parseNativeExecutionInput({ diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 81aba3bf4c1..cb08e585392 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -22,6 +22,7 @@ import { configuredEnvironment, configuredEnvironmentProjection } from "../../ve import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { enableReattachPhase, isReattachPhaseActive, joinReattachPhase, onReattachPhaseEnd, resetReattachPhaseForTests, type ReattachPhaseSummary } from "../heartbeat/run-admission.js"; import { createSystemRecovery, failSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure } from "../system-recovery-state.js"; +import { registerRunStartHandover } from "../run-start-handover.js"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { once } from "node:events"; import { PgDialect } from "drizzle-orm/pg-core"; @@ -11465,6 +11466,41 @@ describe("native warm session supervision", () => { } }); + it.each(["before", "after"] as const)("closes a warm session promptly when a replacement server starts this server's runs (%s the turn ends)", async when => { + const identity = `warm-handoff-${when}`; + const warmExecution = { ...execution, binding: { ...execution.binding, runId: identity, executionWorkspaceId: `${identity}-workspace` }, + session: { ...execution.session, normalizedSessionId: identity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 600_000 } }, + } as NativeExecutionInputV1; + const session = { close: vi.fn(async () => undefined) }; + state.execute.mockReset().mockImplementation(async (options) => { + await options.onSession?.(session); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: identity, normalizedSessionId: identity, providerSessionId: identity, + driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + const listeners = new Set<() => void>(); + let leaving = when === "before"; + registerRunStartHandover({ isLeavingRunStarts: () => leaving, subscribe: (listener: () => void) => { + listeners.add(listener); + return () => { listeners.delete(listener); }; + } } as unknown as Parameters[0]); + try { + await executePaperclipNativeSession({ db: leaseDb(warmExecution), execution: warmExecution, runnerInstanceId: identity, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId: identity, remoteCwd: `/tmp/${identity}` } }); + if (when === "after") { + await new Promise(resolve => setTimeout(resolve, 1_500)); + expect(session.close).not.toHaveBeenCalled(); + leaving = true; + for (const listener of listeners) listener(); + } + await vi.waitFor(() => expect(session.close).toHaveBeenCalledTimes(1), { timeout: 3_000 }); + await vi.waitFor(() => expect(readNativeSessionIdleWork()).toBe("none")); + } finally { + registerRunStartHandover(null); + await closeWarmNativeSessionsForEnvironment({ environmentId: identity, reason: "test cleanup" }); + } + }); + it("a follow-up that arrives while the idle timeout is closing the warm session waits for that close instead of failing busy", async () => { const identity = "warm-idle-close-follow-up"; const warmExecution = { ...execution, binding: { ...execution.binding, runId: identity, executionWorkspaceId: `${identity}-workspace` }, @@ -18955,7 +18991,7 @@ describe("runnerd provider runtime wiring", () => { it.each((["codex", "claude"] as const).flatMap(provider => [ ...[[false, false, false, false], [true, false, false, false], [false, true, false, false], [false, false, true, false], ...(provider === "codex" ? [[false, false, false, true]] : [])] .map(([unknown, shutdown, journalFailure, deadline]) => [provider, unknown, shutdown, journalFailure, deadline, "idle timeout"] as const), - ...(["controller restart", "detach failure", "environment deletion"] as const) + ...(["controller restart", "detach failure", "environment deletion", "replacement hand-off"] as const) .map(retirement => [provider, false, false, false, false, retirement] as const), ]))( "releases settled %s only under its current owner and retires its cache (unknown=%s, shutdown=%s, journalFailure=%s, deadline=%s, retirement=%s)", async (provider, unknown, shutdown, journalFailure, deadline, retirement) => { @@ -18965,6 +19001,8 @@ describe("runnerd provider runtime wiring", () => { process.env.PAPERCLIP_HOME = isolatedStateDirectory; const current = scopedNativeExecution(`released-${provider}-${unknown}`); current.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: retirement === "idle timeout" ? unknown ? 750 : 250 : 60_000 }; + if (retirement === "replacement hand-off") registerRunStartHandover({ isLeavingRunStarts: () => true, + subscribe: () => () => undefined } as unknown as Parameters[0]); if (provider === "claude") current.provider = { kind: "acpx", agent: "claude", model: "fixture", permissionMode: "deny-all" } as never; const source = { runnerInstanceId: "released-runner", environmentLeaseId: "released-lease", runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!, turnId: "settled-turn", itemId: "settled-item" }; @@ -19119,10 +19157,10 @@ describe("runnerd provider runtime wiring", () => { } detachControllerForRestart.mockResolvedValue(undefined); } - if (retirement !== "idle timeout") { + if (retirement !== "idle timeout" && retirement !== "replacement hand-off") { expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); - } else await vi.waitFor(() => expect(detachControllerForRestart).toHaveBeenCalledOnce()); + } else await vi.waitFor(() => expect(detachControllerForRestart).toHaveBeenCalledOnce(), { timeout: 3_000 }); expect(readNativeSessionIdleWork()).toBe("none"); expect(close).not.toHaveBeenCalled(); expect(task.mock.calls.map(([operation]) => (operation as { kind: string }).kind)).toEqual(["describe"]); @@ -19157,6 +19195,7 @@ describe("runnerd provider runtime wiring", () => { expect(JSON.parse(await readFile(path, "utf8")).releasedRunner).toEqual(replacement.releasedRunner); } finally { detachControllerForRestart.mockResolvedValue(undefined); + registerRunStartHandover(null); allowRelease(); allowPhysicalStop(); await execute.catch(() => undefined); await closeIdleWarmNativeSessionsForRestart(); owners.mockRestore(); if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; } diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 3f33a94c005..914ceded34a 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -41,6 +41,7 @@ import { notifyExternalAgentSettlement } from "../../modules/external-agents/ind import { notifyNativeResultCommit } from "../native-result-commit-notifications.js"; import { releaseRetainedNativeSessionCleanup } from "../retained-native-session-cleanup-notifications.js"; import { registerIdleLocalWorkProbe, type IdleLocalWork } from "../idle-local-work.js"; +import { isLeavingRunStartsToReplacement, subscribeRunStartHandoverChanges } from "../run-start-handover.js"; import type { NativeCleanupOutcome } from "./retained-native-session-cleanup.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; @@ -930,6 +931,21 @@ export function readNativeSessionIdleWork(): IdleLocalWork { } registerIdleLocalWorkProbe(readNativeSessionIdleWork); +export const WARM_HANDOFF_CLOSE_DELAY_MS = 1_000; + +function warmIdleCloseDelay(idleTimeoutMs: number): number { + return isLeavingRunStartsToReplacement() ? Math.min(idleTimeoutMs, WARM_HANDOFF_CLOSE_DELAY_MS) : idleTimeoutMs; +} + +subscribeRunStartHandoverChanges(() => { + if (!isLeavingRunStartsToReplacement()) return; + for (const [sessionId, entry] of warmNativeSessions) { + if (entry.busy || entry.idleTimer === null) continue; + clearTimeout(entry.idleTimer); + armWarmNativeSessionIdleTimer(sessionId, entry, WARM_HANDOFF_CLOSE_DELAY_MS); + } +}); + /** * How long a new run's admission waits for the warm session to stop being busy. * A follow-up comment can wake the next run within milliseconds of the previous @@ -8775,7 +8791,7 @@ async function releaseWarmNativeSession( } finally { entry.busy = false; } return; } - armWarmNativeSessionIdleTimer(sessionId, entry, idleTimeoutMs); + armWarmNativeSessionIdleTimer(sessionId, entry, warmIdleCloseDelay(idleTimeoutMs)); } function armWarmNativeSessionIdleTimer( @@ -13774,7 +13790,7 @@ async function executePaperclipNativeSessionAttempt( if (current()) { delete entry.finishRelease; entry.busy = false; - armWarmNativeSessionIdleTimer(scope, entry, lifecyclePolicy.idleTimeoutMs); + armWarmNativeSessionIdleTimer(scope, entry, warmIdleCloseDelay(lifecyclePolicy.idleTimeoutMs)); } } }; diff --git a/server/src/services/native-runtime/predecessor-retirement.ts b/server/src/services/native-runtime/predecessor-retirement.ts index 8882a12f3e1..b3b397be9ec 100644 --- a/server/src/services/native-runtime/predecessor-retirement.ts +++ b/server/src/services/native-runtime/predecessor-retirement.ts @@ -21,7 +21,7 @@ export class NativePredecessorRetirementUnprovenError extends Error { cause: unknown, ) { super(`The previous run's session on this sandbox could not be retired (${ - cause instanceof Error ? cause.message : String(cause)}). The next attempt starts on a fresh sandbox.`, { cause }); + cause instanceof Error ? cause.message : String(cause)}).`, { cause }); this.name = "NativePredecessorRetirementUnprovenError"; } } diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.ts b/server/src/services/native-runtime/remote-warm-session-recovery.ts index 65c4bd151e9..ebb21f0ac33 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.ts @@ -6,6 +6,36 @@ import { currentNativeControllerIdentity, evaluateNativeControllerTakeover, reco import { NativeControllerRecoveryWaitingError, readNativeControllerRecovery } from "./native-controller-recovery.js"; import { NativePredecessorRetirementUnprovenError } from "./predecessor-retirement.js"; import { readyPortableTaskSource } from "../plugin-task-ownership.js"; +import { PAPERCLIP_RUNNER_IDLE_TIMEOUT_MAX_MS } from "@paperclipai/adapter-utils"; +import { currentControllerBootId } from "../controller-process-identity.js"; +import { currentProcessProbe, proveOwnerGone } from "../owner-proof.js"; + +export const WARM_OWNER_RECHECK_MS = 5_000; +const WARM_OWNER_CLOSE_MARGIN_MS = 2 * 60_000; + +export async function liveWarmSessionOwnerRetryAt(input: { + owner: Pick; + finishedAt: Date | null; + idleTimeoutMs: number | null; + now?: Date; +}): Promise { + const { owner } = input; + const now = input.now ?? new Date(); + if (!owner.controllerBootId || owner.controllerBootId === currentControllerBootId()) return null; + const releasedAt = (input.finishedAt ?? owner.updatedAt).getTime(); + const closesBy = releasedAt + (input.idleTimeoutMs ?? PAPERCLIP_RUNNER_IDLE_TIMEOUT_MAX_MS) + WARM_OWNER_CLOSE_MARGIN_MS; + if (closesBy <= now.getTime()) return null; + const proof = await proveOwnerGone({ + kind: "local", scope: "native_warm_session", generation: null, host: null, + processSpace: owner.controllerHostHash, processSpaceUnrecorded: owner.controllerHostHash == null, + bootId: owner.controllerBootId, pid: owner.controllerPid, + processStartedAt: owner.controllerProcessStartedAt?.toISOString() ?? null, + touchedAt: owner.updatedAt.toISOString(), + }, { ...currentProcessProbe(), readOnly: true }); + if (proof.verdict !== "alive") return null; + return new Date(Math.min(now.getTime() + WARM_OWNER_RECHECK_MS, closesBy)); +} export type RemoteWarmSessionRecovery = { identity: Record; @@ -176,6 +206,9 @@ export async function inspectRemoteWarmSessionRecovery(input: { throw new NativeControllerRecoveryWaitingError(prior.owner.leaseExpiresAt); } if (prior.owner.phase !== "committed" || prior.owner.leaseOwner) return deny("owner_unsettled"); + const liveOwnerRetryAt = await liveWarmSessionOwnerRetryAt({ owner: prior.owner, finishedAt: prior.run.finishedAt, + idleTimeoutMs: previous.session?.lifecyclePolicy?.idleTimeoutMs ?? null }); + if (liveOwnerRetryAt) throw new NativeControllerRecoveryWaitingError(liveOwnerRetryAt); if (sourceProviderLeaseId === providerLeaseId) settledPredecessor = { runId: identity.runId, providerLeaseId }; const checkpoint = input.loadCheckpoint(previous); if (!checkpoint || checkpoint.identity.runId !== identity.runId || diff --git a/server/src/services/run-start-handover.ts b/server/src/services/run-start-handover.ts index 1fef8c812ac..4109a9e75d8 100644 --- a/server/src/services/run-start-handover.ts +++ b/server/src/services/run-start-handover.ts @@ -347,6 +347,7 @@ export function createRunStartHandover(input: { refresh, /** This server holds a hand-over an older server may be reading. */ isHandingOver: () => handingOver(), + isLeavingRunStarts: () => yieldingTo !== null, /** A hand-over involves this server (either side). Pure read, no refresh. */ isActive: () => handingOver() || yieldingTo !== null, subscribe(listener: () => void) { @@ -387,6 +388,10 @@ export function refreshRunStartYieldIfStale(): Promise | null { return current?.refreshIfStale() ?? null; } +export function isLeavingRunStartsToReplacement(): boolean { + return current?.isLeavingRunStarts() ?? false; +} + /** Either side of a hand-over is in progress here; a pure read for listeners. */ export function isRunStartHandoverActive(): boolean { return current?.isActive() ?? false;