diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index b4b8437a284..65f279c1661 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1892,6 +1892,14 @@ Only successful turns retain a live warm provider session. A structured failed or cancelled result must retire that session and collect its managed files before heartbeat stops the reusable sandbox. A later retry can resume the same sandbox without inheriting the stopped provider transport. + +An idle warm owner proves only the exact prior run in the same company, session, +workspace and environment. Mutable configuration compatibility is checked +separately: changed settings retire the owned provider before a replacement can +execute. A copied older owner record, an active or missing owner, and a different +environment cannot use this in-memory proof. Database terminal status alone is +not proof that an unsuspended remote runner can be adopted. + The shared lease-release boundary rechecks the durable run status for recovery and ordinary teardown. Successful workspace copy-back alone must not keep a failed turn's sandbox running. diff --git a/doc/plans/2026-10-15-native-warm-owner-proof.md b/doc/plans/2026-10-15-native-warm-owner-proof.md new file mode 100644 index 00000000000..ac74ccb8a35 --- /dev/null +++ b/doc/plans/2026-10-15-native-warm-owner-proof.md @@ -0,0 +1,38 @@ +# Exact warm-owner proof across native configuration changes + +A follow-up can fail with `prior_owner_terminal_state_indeterminate` before +provider execution even when its prior heartbeat succeeded. The old predicate +required the next configuration digest to match before recognizing the idle +owner; this prevented the existing compatibility step from retiring a process +whose configuration changed. + +Keep the session cache, durable-root admission and compatibility retirement in +their current coordinator. They share process and instruction-copy ownership; +this fix does not move their lifetime or weaken their gates. Cache each actual +owner's heartbeat run ID and require it to match the durable prior-run identity. +Keep company/session/workspace/environment checks and the terminal database +check. Let the later compatibility step decide reuse versus retirement, and +continue to reject unconfirmed retirement before another provider executes. + +Verification is through the production session entry point with scripted native +backends and durable local owner files. The six existing selected controls pass. +The initial three-case reproduction has one pass and two failures at the same +prior-owner guard; after the fix all three pass. Eight focused controls now cover +unchanged reuse through three turns, changed settings, changed runtime context, +failed retirement, stale recorded identity, active owner, missing owner and a +different environment. The runtime-context case uses the current v5 input and +fails on the baseline at the same prior-owner guard before passing the repair. Replacing the exact +run-ID comparison with an any-owner check makes the stale-owner control fail; +the source is restored afterward. + +The first complete affected executor and resume suites passed 764 cases before +the additional v5 runtime-context case. Full typecheck initially found an +optional-argument error in the test database wrapper; after that correction, +full typecheck and build passed. The v5 fixture expansion then exposed a legacy +v1-only type on the shared database fixture. Its implementation only uses the +common binding, so it now accepts the full native input union. After integrating the separate idle-expiry admission repair from #16432, the +complete executor and resume suites passed 767 cases with zero selected skips. +Full repository typecheck and build passed on the same frozen code and tests. Scripted-provider results do not establish real Daytona recovery, +managed subscription behavior or hosted latency. Recheck the failed hosted +continuation after normal deployment, preserving its existing recovery evidence. +The broader response-latency target remains open independently of this repair. diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index fac9f420114..572e126883b 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -35,6 +35,7 @@ import { type Db, } from "@paperclipai/db"; import { + canonicalNativeRuntimeContextDigest, CURSOR_DISTRIBUTION_PINS, QUALIFIED_ACPX_PROFILES, acpxRuntimeSessionDirectoryName, @@ -5158,7 +5159,7 @@ type LeaseCoordinator = { }; function leaseDb( - boundExecution: NativeExecutionInputV1 = execution, + boundExecution: NativeExecutionInput = execution, coordinatorOverrides: Partial = {}, runResultJson: Record = {}, updates: Array<{ table: unknown; values: Record }> = [], @@ -9727,7 +9728,7 @@ describe("runnerd provider runtime wiring", () => { }; } - function priorNativeRunDb(priorExecution: NativeExecutionInputV1, status = "succeeded"): Db { + function priorNativeRunDb(priorExecution: NativeExecutionInput, status = "succeeded"): Db { return { select: () => ({ from: () => ({ where: () => ({ limit: async () => [{ status, @@ -9736,7 +9737,7 @@ describe("runnerd provider runtime wiring", () => { } as unknown as Db; } - async function initializeScopedRunnerRoot(current: NativeExecutionInputV1, runnerInstanceId: string) { + async function initializeScopedRunnerRoot(current: NativeExecutionInput, runnerInstanceId: string) { state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId }); @@ -9771,6 +9772,107 @@ describe("runnerd provider runtime wiring", () => { await rm(isolatedStateDirectory, { recursive: true, force: true }); }); + it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => { + const first = { ...scopedNativeExecution(`warm-admission-${scenario}`), + schema: "paperclip.native-execution-input.v5" as const, + provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const }, + executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() }; + first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 }; + const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` }, + session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner"].includes(scenario) ? 60_000 : 90_000 } } }; + if (scenario === "runtime-context") { + next.runtimeContext = { ...first.runtimeContext, + mcp: { ...first.runtimeContext.mcp, digest: "1".repeat(64) } }; + next.runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(next.runtimeContext); + } + const environmentId = `warm-admission-${scenario}`; + const target = { kind: "remote" as const, transport: "sandbox" as const, environmentId, + remoteCwd: "/home/daytona/workspace", runner: { execute: vi.fn() } }; + const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId, + runnerInstanceId: `runner-${scenario}`, environmentLeaseId: first.binding.executionWorkspaceId }; + const root = await initializeScopedRunnerRoot(first, identity.runnerInstanceId); + await writeRunnerAuthority(root, identity, "ready"); + const order: string[] = []; + let failRetirement = scenario === "retirement-failed"; + const firstSession = { close: vi.fn(async () => { + order.push("retire"); + if (failRetirement) throw new Error("retirement unconfirmed"); + await writeRunnerAuthority(root, identity, "suspended"); + }) }; + const nextSession = { close: vi.fn(async () => undefined) }; + const result = { result: { summary: "done" }, terminal: { runTerminalState: "succeeded" }, + turnId: "turn", normalizedSessionId: first.session.normalizedSessionId, + providerSessionId: "provider", driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.(firstSession); + return result; + }).mockImplementationOnce(async options => { + order.push("execute-next"); + expect(options.existingSession).toBe(scenario === "unchanged" ? firstSession : undefined); + await options.onSession?.(scenario === "unchanged" ? firstSession : nextSession); + return result; + }); + const older = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-older` } }; + const nextDb = leaseDb(next); + const ordinarySelect = nextDb.select.bind(nextDb); + nextDb.select = ((fields?: Record) => fields?.status === heartbeatRuns.status + && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson + ? priorNativeRunDb(scenario === "stale-owner" ? older : first, scenario === "active-owner" ? "running" : "succeeded").select(fields as never) + : ordinarySelect(fields as never)) as Db["select"]; + try { + await executePaperclipNativeSession({ db: leaseDb(first), execution: first, + runnerInstanceId: identity.runnerInstanceId, useRunnerd: true, runnerExecutionTarget: target as never }); + if (scenario === "stale-owner") await writeRunnerAuthority(root, { ...identity, runId: older.binding.runId }, "ready"); + if (scenario === "missing-owner") { + await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "fixture owner loss" }); + await writeRunnerAuthority(root, identity, "ready"); + order.length = 0; + firstSession.close.mockClear(); + } + const continuation = executePaperclipNativeSession({ db: nextDb, execution: next, + runnerInstanceId: `runner-next-${scenario}`, useRunnerd: true, + runnerExecutionTarget: (scenario === "foreign-environment" ? { ...target, environmentId: `${environmentId}-other` } : target) as never }); + if (["stale-owner", "active-owner", "missing-owner", "foreign-environment"].includes(scenario)) { + await expect(continuation).rejects.toThrow(scenario === "active-owner" + ? "prior_owner_active" : "prior_owner_terminal_state_indeterminate"); + expect(state.execute).toHaveBeenCalledOnce(); + expect(firstSession.close).not.toHaveBeenCalled(); + expect(order).toEqual([]); + } else if (scenario === "retirement-failed") { + await expect(continuation).rejects.toThrow("retirement unconfirmed"); + expect(state.execute).toHaveBeenCalledOnce(); + expect(order).toEqual(["retire"]); + } else { + await expect(continuation).resolves.toMatchObject({ exitCode: 0 }); + expect(order).toEqual(scenario === "unchanged" ? ["execute-next"] : ["retire", "execute-next"]); + if (scenario === "unchanged") { + await writeRunnerAuthority(root, { ...identity, runId: next.binding.runId }, "ready"); + const third = { ...next, binding: { ...next.binding, runId: `${next.binding.runId}-third` } }; + const thirdDb = leaseDb(third); + const thirdSelect = thirdDb.select.bind(thirdDb); + thirdDb.select = ((fields?: Record) => fields?.status === heartbeatRuns.status + && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson + ? priorNativeRunDb(next).select(fields as never) + : thirdSelect(fields as never)) as Db["select"]; + state.execute.mockImplementationOnce(async options => { + expect(options.existingSession).toBe(firstSession); + await options.onSession?.(firstSession); + return result; + }); + await expect(executePaperclipNativeSession({ db: thirdDb, execution: third, + runnerInstanceId: `runner-third-${scenario}`, useRunnerd: true, + runnerExecutionTarget: target as never })).resolves.toMatchObject({ exitCode: 0 }); + expect(firstSession.close).not.toHaveBeenCalled(); + expect(state.execute).toHaveBeenCalledTimes(3); + } + } + } finally { + failRetirement = false; + await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "test cleanup" }); + } + }); + it.each([ ["open", "before-close"], ["open", "during-close"], diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 04dcb09e8af..f90d6a55060 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -435,6 +435,7 @@ export type NativeInstructionWorkingCopy = { }; type WarmNativeSession = { + runId: string; instructionWorkingCopy?: NativeInstructionWorkingCopy; // Keep the admitted physical root independent of the per-run collection // capability, which instruction preparation adopts before final admission. @@ -4244,7 +4245,7 @@ async function verifyPriorRunnerdStateForSessionScope(input: { identity: RunnerdDurableIdentity; execution: NativeExecutionInput; allowVerifiedBackup: boolean; - allowRetainedWarmRunner: boolean; + retainedWarmOwnerRunId?: string; }): Promise { let priorRun: { status: string; @@ -4291,7 +4292,7 @@ async function verifyPriorRunnerdStateForSessionScope(input: { input.identity, ); if ( - input.allowRetainedWarmRunner && + input.retainedWarmOwnerRunId === input.identity.runId && (lifecycle === "not_suspended" || // A remote runner keeps runner-state.json in the sandbox rather than // beside the controller's PRP journal. During a genuinely warm handoff @@ -4753,7 +4754,7 @@ async function migrateRunnerdStateRootForExecution(input: { db: Db; execution: NativeExecutionInput; allowVerifiedBackup: boolean; - allowRetainedWarmRunner: boolean; + retainedWarmOwnerRunId?: string; allowLocalRecovery: boolean; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; restartRecovery?: NativeRestartRecoveryClaim; @@ -4770,7 +4771,7 @@ async function migrateRunnerdStateRootForExecution(input: { }); if ( input.allowLocalRecovery && - !input.allowRetainedWarmRunner && + !input.retainedWarmOwnerRunId && !input.restartRecovery ) { await recoverQuiescentRunnerdState({ ...input, scoped }); @@ -4843,7 +4844,7 @@ async function migrateRunnerdStateRootForExecution(input: { identity, execution: input.execution, allowVerifiedBackup: input.allowVerifiedBackup, - allowRetainedWarmRunner: input.allowRetainedWarmRunner, + retainedWarmOwnerRunId: input.retainedWarmOwnerRunId, }); if ( verification !== "verified" && @@ -4885,7 +4886,7 @@ async function migrateRunnerdStateRootForExecution(input: { identity, execution: input.execution, allowVerifiedBackup: input.allowVerifiedBackup, - allowRetainedWarmRunner: input.allowRetainedWarmRunner, + retainedWarmOwnerRunId: input.retainedWarmOwnerRunId, }); if ( verification !== "verified" && @@ -5406,27 +5407,17 @@ function nativeSessionConfigDigest( .digest("hex")}`; } -function hasIdleWarmNativeSessionOwner(input: { +function idleWarmNativeSessionOwnerRunId(input: { execution: NativeExecutionInput; runnerExecutionTarget?: AdapterExecutionTarget | null; -}): boolean { - if (input.execution.session.lifecyclePolicy.mode !== "warm") return false; +}): string | undefined { + if (input.execution.session.lifecyclePolicy.mode !== "warm") return undefined; const entry = warmNativeSessions.get(nativeSessionScopeKey(input.execution)); - if (!entry || (entry.busy && entry.instructionPreparationRunId !== input.execution.binding.runId)) return false; - // A verified idle owner proves the checkpoint belongs to this session even - // when its process must later rotate to a new run-scoped broker capability. - // Local environments also have an id. Compare the same environment binding - // stored at acquisition instead of treating every local target as unbound. + if (!entry || (entry.busy && entry.instructionPreparationRunId !== input.execution.binding.runId)) return undefined; const environmentId = input.runnerExecutionTarget?.environmentId ?? null; - return ( - entry.companyId === input.execution.binding.companyId && - entry.environmentId === environmentId && - entry.configDigest === - nativeSessionConfigDigest( - input.execution, - input.runnerExecutionTarget?.kind ?? "local", - ) - ); + return entry.companyId === input.execution.binding.companyId && entry.environmentId === environmentId + ? entry.runId + : undefined; } function nativeHarnessEnvironmentFingerprint( @@ -7762,7 +7753,7 @@ async function executePaperclipNativeSessionWithinScope( // suspended. Only the exact idle in-process owner may rotate that // prior-run authority; after a hard restart the map is empty and the // durable-state verifier continues to require a suspended runner. - allowRetainedWarmRunner: hasIdleWarmNativeSessionOwner(input), + retainedWarmOwnerRunId: idleWarmNativeSessionOwnerRunId(input), allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote", onLog: input.onLog, restartRecovery: input.restartRecovery, @@ -8864,8 +8855,10 @@ async function executePaperclipNativeSessionWithinScope( throw new Error("native_session_supervisor_busy"); } existing.session = session; + existing.runId = input.execution.binding.runId; } else warmNativeSessions.set(warmSessionId, { + runId: input.execution.binding.runId, instructionWorkingCopy: input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy, instructionRoot: input.instructionWorkingCopy?.root, agentId: input.execution.binding.agentId, @@ -11220,7 +11213,7 @@ export async function createRunnerdBackend(input: { db: input.db, execution: input.execution, allowVerifiedBackup: input.runnerExecutionTarget?.kind === "remote", - allowRetainedWarmRunner: false, + retainedWarmOwnerRunId: undefined, allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote", onLog: input.onLog, restartRecovery: input.restartRecovery,