Release and resume retained plugin Runner sessions in Core (#19611)

This commit is contained in:
Nicky Leach authored and GitHub committed 2026-11-06 20:45:02 -08:00
1 parent 9efee7ee16
commit b21884bdcf
7 files changed
+965 -80

No files matched your search

@@ -45,7 +45,7 @@ import { notifyExecutionRunWork } from "../execution-control-notifications.js";
import { createRunUsageRecorder } from "../usage-receipts.js";
import { registerPluginTaskCredentialHandoff, type PluginTaskCredentialWritebackBinding } from "../plugin-task-credential-writeback.js";
import { resolvePluginTaskResume } from "../environment-task-runtime.js";
import { environmentTaskResultSchema } from "@paperclipai/plugin-sdk";
import { environmentTaskOperationSchema, environmentTaskResultSchema } from "@paperclipai/plugin-sdk";
import { preparePluginRunnerCredentials } from "./plugin-runner-credentials.js";
import { createPluginRunnerTaskOwner, verifyPluginRunnerTaskReattachment } from "./plugin-runner-task.js";
import { NativeExecutionContext } from "./native-execution-context.js";
@@ -669,6 +669,8 @@ type WarmNativeSession = {
companyId: string;
environmentId: string | null;
busy: boolean;
released?: boolean;
finishRelease?: () => Promise<void>;
closeOnReleaseReason?: string;
idleTimer: ReturnType<typeof setTimeout> | null;
lastActivityAt: string;
@@ -716,7 +718,9 @@ const pluginTaskSessionOwners = new WeakMap<NativeSession, ReturnType<typeof cre
const closingWarmNativeSessions = new Map<string, Promise<void>>();
export function readNativeSessionIdleWork(): IdleLocalWork {
return warmNativeSessions.size !== 0 || closingWarmNativeSessions.size !== 0 ? "present" : "none";
return closingWarmNativeSessions.size !== 0 || [...warmNativeSessions.values()].some(entry =>
entry.busy || entry.preparingRunId || entry.instructionPreparationRunId || entry.closeOnReleaseReason || !entry.released)
? "present" : "none";
}
registerIdleLocalWorkProbe(readNativeSessionIdleWork);
@@ -876,7 +880,8 @@ async function closeIdleWarmNativeSessions(input: {
if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) {
continue;
}
if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)) {
if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)
|| (input.canRetire && (entry.instructionPreparationRunId || closingWarmNativeSessions.has(sessionId)))) {
// A busy turn can complete while another idle session is checkpointing.
// Fence that entry now so its eventual release cannot leave a new idle
// owner behind after the shutdown sweep has already passed it.
@@ -884,6 +889,8 @@ async function closeIdleWarmNativeSessions(input: {
busy += 1;
continue;
}
// The host owns retention after release; a quiet local handle needs no retirement for sleep.
if (input.canRetire && entry.released && !entry.closeOnReleaseReason) continue;
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
// Keep a fenced owner until both retirement and stopped collection succeed.
// Failure must not make a live directory eligible for a replacement owner.
@@ -1927,6 +1934,10 @@ class SessionAuthorityEpoch {
if (!isNativeUsageAccountingEvent(event)) return;
this.#assertCurrent();
if (event.runId === this.runId) throw new Error("native_recovery_usage_source_unproven");
if (event.runId === this.#drainingRun?.runId) {
await this.acceptAccountingEvent(event);
return;
}
await capture();
this.#assertCurrent();
}
@@ -7324,27 +7335,92 @@ function legacyNativeSessionCheckpointPath(
);
}
type ReleasedWarmRunner = { source: DurableRecoveryIdentity; taskId: string; scopeDigest: string };
function releasedWarmScope(input: Parameters<typeof executePaperclipNativeSession>[0]): string {
const target = input.runnerExecutionTarget;
return nativeSha256({ companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId,
environmentId: target?.kind === "remote" ? target.environmentId ?? null : null, workspace: nativeSessionWorkspaceScope(input.execution),
cwd: input.execution.workspace.cwd, remoteCwd: target?.kind === "remote" ? target.remoteCwd : null,
projectIds: target?.kind === "remote" && target.transport === "sandbox" ? target.runnerTaskProjectIds ?? [] : [],
accountIdentity: input.managedAiAccountIdentity ?? null, credentialIdentity: input.managedAiCredentialIdentity ?? null });
}
function requireReleasedWarmScope(input: Parameters<typeof executePaperclipNativeSession>[0],
proof: ReleasedWarmRunner | undefined, identity: unknown): DurableRecoveryIdentity {
const target = input.runnerExecutionTarget;
if (!proof || target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask
|| proof.taskId !== target.sandboxLeaseAcquisition?.providerLeaseId
|| proof.scopeDigest !== releasedWarmScope(input) || canonicalJson(proof.source) !== canonicalJson(identity)) {
throw new Error("native_session_released_owner_scope_mismatch");
}
return proof.source;
}
function readWarmNativeEnvelope(execution: NativeExecutionInput): Record<string, unknown> | null {
const path = nativeSessionCheckpointPath(execution);
if (!existsSync(path)) return null;
const envelope = record(JSON.parse(readBoundedNativeFile(path, NATIVE_WARM_CHECKPOINT_MAX_BYTES,
"native_session_supervisor_checkpoint_too_large").toString("utf8")));
if (envelope.schema !== "paperclip.native-session-supervisor.v1") throw new Error("native_session_supervisor_checkpoint_mismatch");
return envelope;
}
function releasedWarmRunner(envelope: Record<string, unknown> | null): ReleasedWarmRunner | undefined {
if (!envelope || envelope.releasedRunner === undefined) return undefined;
const proof = record(envelope.releasedRunner), source = record(proof.source);
const snapshot = record(envelope.snapshot), identity = record(snapshot.identity);
for (const key of ["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"]) {
if (typeof source[key] !== "string" || !source[key]) throw new Error("native_session_released_source_mismatch");
}
if (typeof proof.taskId !== "string" || !proof.taskId || typeof proof.scopeDigest !== "string"
|| identity.runId !== source.runId || identity.sessionId !== source.normalizedSessionId) {
throw new Error("native_session_released_source_mismatch");
}
return proof as ReleasedWarmRunner;
}
function writeWarmNativeEnvelope(execution: NativeExecutionInput, envelope: Record<string, unknown>): void {
const path = nativeSessionCheckpointPath(execution);
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
writeFileSync(temporary, JSON.stringify(envelope), { encoding: "utf8", mode: 0o600 });
renameSync(temporary, path);
chmodSync(path, 0o600);
}
function updateReleasedWarmRunner(execution: NativeExecutionInput, source: DurableRecoveryIdentity,
taskId: string, proof: ReleasedWarmRunner | null): void {
const envelope = readWarmNativeEnvelope(execution);
const previous = releasedWarmRunner(envelope);
if (!envelope) {
if (proof) throw new Error("native_session_supervisor_checkpoint_missing");
return;
}
if (!proof && !previous) return;
const identity = record(record(envelope.snapshot).identity);
if (identity.companyId !== execution.binding.companyId || identity.agentId !== execution.binding.agentId
|| identity.runId !== source.runId || identity.sessionId !== source.normalizedSessionId
|| (previous && (previous.taskId !== taskId || canonicalJson(previous.source) !== canonicalJson(source)))) {
throw new Error("native_session_released_source_mismatch");
}
if (proof) envelope.releasedRunner = proof;
else delete envelope.releasedRunner;
writeWarmNativeEnvelope(execution, envelope);
}
function persistWarmNativeCheckpoint(
execution: NativeExecutionInput,
configDigest: string,
snapshot: PersistedNativeSession,
processConfigDigest?: string,
): void {
const path = nativeSessionCheckpointPath(execution);
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
writeFileSync(
temporary,
JSON.stringify({
schema: "paperclip.native-session-supervisor.v1",
configDigest,
processConfigDigest,
updatedAt: new Date().toISOString(),
snapshot,
}),
{ encoding: "utf8", mode: 0o600 },
);
renameSync(temporary, path);
chmodSync(path, 0o600);
const previous = releasedWarmRunner(readWarmNativeEnvelope(execution));
writeWarmNativeEnvelope(execution, {
schema: "paperclip.native-session-supervisor.v1", configDigest, processConfigDigest,
updatedAt: new Date().toISOString(), snapshot,
...(previous && previous.source.runId === snapshot.identity.runId
&& previous.source.normalizedSessionId === snapshot.identity.sessionId ? { releasedRunner: previous } : {}),
});
}
function needsCoreGitHubAccess(input: Pick<Parameters<typeof executePaperclipNativeSession>[0], "managedGitHub" | "runnerExecutionTarget">): boolean {
@@ -7497,10 +7573,17 @@ function armWarmNativeSessionIdleTimer(
return;
}
current.busy = true;
current.closeOnReleaseReason = "warm native session idle timeout";
const closing = closeWarmNativeSession(current, current.closeOnReleaseReason)
const released = current.released === true;
if (!released) current.closeOnReleaseReason = "warm native session idle timeout";
const closing = (released ? (async () => {
if (!current.session.detachControllerForRestart) throw new Error("native_session_released_detach_unavailable");
await current.session.detachControllerForRestart();
await current.githubAccess?.stop();
})() : closeWarmNativeSession(current, current.closeOnReleaseReason!))
.then(() => { forgetWarmNativeSession(sessionId, current); })
.catch(() => {})
.catch(() => {
if (released) current.closeOnReleaseReason = "released Runner local detach is unconfirmed";
})
.finally(() => {
current.busy = false;
if (closingWarmNativeSessions.get(sessionId) === closing) closingWarmNativeSessions.delete(sessionId);
@@ -9098,29 +9181,8 @@ export async function executePaperclipNativeSession(input: {
startupUnsettled = !(await startup.executionContext.drain());
startup.executionContext.finish();
}
if (ownsPhysicalIngress && physicalIngressKey && executingNativeIngressAllocations.get(physicalIngressKey) === ownerToken) {
executingNativeIngressAllocations.delete(physicalIngressKey);
}
if (executingNativeOwnerScopes.get(ownerScope) === ownerToken) {
executingNativeOwnerScopes.delete(ownerScope);
}
startup.resolve(null);
startup.settle();
if (nativeSessionStartups.get(runId) === startup) {
nativeSessionStartups.delete(runId);
setNativeExecutionActive(runId, false);
}
if (
ownsSessionScope &&
sessionScopeId !== null &&
executingRunnerdSessionScopes.get(sessionScopeId) ===
input.execution.binding.runId
) {
executingRunnerdSessionScopes.delete(sessionScopeId);
}
// Preparation can fail before the execution loop owns its normal cleanup.
// Remove the settled startup before yielding, so detachment cannot reacquire
// its old resolved session while renewal cleanup drains.
// Join preparation cleanup before release; keep cancellation reachable
// until the exact retained owner has finished the handover.
await startup.executionLeaseRenewal?.stop().catch(() => undefined);
if (startup.unstartedClaim && !nativeRunsDetachingForRestart.has(runId)) {
await releaseUnstartedNativeExecutionClaim({ db: input.db, runId, companyId: input.execution.binding.companyId,
@@ -9128,9 +9190,11 @@ export async function executePaperclipNativeSession(input: {
logger.warn({ err: error, runId }, "Native preparation claim release failed; the control deadline will settle it");
});
}
let stagedCleanupFailed = false;
try {
await cleanupStagedAttachments();
} catch (cleanupError) {
stagedCleanupFailed = true;
// Cleanup is a confidentiality incident, but it occurs after the native
// provider may already have completed the turn. Reclassifying that turn
// as failed could replay provider side effects. Emit a private runtime
@@ -9152,6 +9216,38 @@ export async function executePaperclipNativeSession(input: {
companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId,
}, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId,
issueId: input.execution.binding.issueId });
const pendingRelease = ownsSessionScope && sessionScopeId
&& executingRunnerdSessionScopes.get(sessionScopeId) === runId
? warmNativeSessions.get(sessionScopeId) : undefined;
try {
if (pendingRelease?.runId === runId && pendingRelease.finishRelease) {
if (executionFailure !== undefined || startupUnsettled || stagedCleanupFailed) {
pendingRelease.closeOnReleaseReason = "warm native session final cleanup failed";
}
await pendingRelease.finishRelease();
}
} finally {
if (ownsPhysicalIngress && physicalIngressKey && executingNativeIngressAllocations.get(physicalIngressKey) === ownerToken) {
executingNativeIngressAllocations.delete(physicalIngressKey);
}
if (executingNativeOwnerScopes.get(ownerScope) === ownerToken) {
executingNativeOwnerScopes.delete(ownerScope);
}
if (
ownsSessionScope &&
sessionScopeId !== null &&
executingRunnerdSessionScopes.get(sessionScopeId) ===
input.execution.binding.runId
) {
executingRunnerdSessionScopes.delete(sessionScopeId);
}
startup.resolve(null);
startup.settle();
if (nativeSessionStartups.get(runId) === startup) {
nativeSessionStartups.delete(runId);
setNativeExecutionActive(runId, false);
}
}
if (startupUnsettled) throw new NativeStartupCleanupPendingError(startup.executionContext!.id,
startup.executionContext!.deadlineAtMs, startup.leaseOwner!, startup.executionContext!.pluginSupported,
input.runnerExecutionTarget?.kind === "remote" ? input.runnerExecutionTarget.leaseId ?? undefined : undefined);
@@ -9270,6 +9366,9 @@ async function executePaperclipNativeSessionWithinScope(
}
let retainedTransition: VerifiedWarmTransitionBinding | undefined;
let remoteWarmRecovery: RemoteWarmSessionRecovery | undefined;
const cachedReleasedOwner = warmNativeSessions.get(nativeSessionScopeKey(input.execution));
if (cachedReleasedOwner?.released) requireReleasedWarmScope(input, releasedWarmRunner(readWarmNativeEnvelope(input.execution)),
readRunnerdDurableIdentity(scopedRunnerdStateRoot(input.execution)));
if (input.useRunnerd && !input.restartRecovery && !input.backend &&
input.runnerExecutionTarget?.kind === "remote" &&
input.execution.session.lifecyclePolicy.mode === "warm" &&
@@ -9295,6 +9394,7 @@ async function executePaperclipNativeSessionWithinScope(
}
if (durableIdentityMatchesSession(identity, input.execution) && identity.runId !== input.execution.binding.runId &&
runnerdAuthorityLifecycleWithVerifiedBackup({ root, identity, execution: input.execution, allowVerifiedBackup: true }) !== "suspended") {
let releasedSource: DurableRecoveryIdentity | undefined;
remoteWarmRecovery = await inspectRemoteWarmSessionRecovery({
db: input.db, execution: input.execution, target: input.runnerExecutionTarget, identity,
readControlState: () => record(JSON.parse(readBoundedNativeFile(
@@ -9302,7 +9402,13 @@ async function executePaperclipNativeSessionWithinScope(
"runner_durable_identity_too_large").toString("utf8"))),
matchesExecution: previous => nativeSessionScopeKey(previous) === nativeSessionScopeKey(input.execution) &&
nativeSessionConfigDigest(previous, "remote") === nativeSessionConfigDigest(input.execution, "remote"),
loadCheckpoint: previous => loadWarmNativeCheckpoint(previous, nativeSessionConfigDigest(previous, "remote"), "remote"),
loadCheckpoint: previous => {
const proof = releasedWarmRunner(readWarmNativeEnvelope(previous));
if (proof) {
releasedSource = requireReleasedWarmScope(input, proof, identity);
}
return loadWarmNativeCheckpoint(previous, nativeSessionConfigDigest(previous, "remote"), "remote");
},
canReuseProcess: previous => {
const expected = warmNativeProcessConfigDigest(input);
if (!expected) return false;
@@ -9311,6 +9417,7 @@ async function executePaperclipNativeSessionWithinScope(
return envelope.processConfigDigest === expected;
},
});
remoteWarmRecovery.releasedSource = releasedSource;
}
}
if (input.useRunnerd) {
@@ -9517,7 +9624,7 @@ async function executePaperclipNativeSessionWithinScope(
const target = input.runnerExecutionTarget;
const retained = warmNativeSessions.get(nativeSessionScopeKey(input.execution));
if (!input.restartRecovery && target?.kind === "remote" && target.transport === "sandbox" && target.runnerTask && target.leaseId &&
retained?.pluginTaskId && idleWarmNativeSessionOwnerRunId(input) !== undefined &&
((retained?.pluginTaskId && idleWarmNativeSessionOwnerRunId(input) !== undefined) || remoteWarmRecovery?.releasedSource) &&
(input.execution.provider.kind === "codex" || (input.execution.provider.kind === "acpx" && input.execution.provider.agent === "claude"))) {
claimedStartupContext = new NativeExecutionContext(input.execution, Math.min(executionLeaseTtlMs,
coldAdmissionTimeoutMs(input.execution.provider.kind, input.execution.provider.kind === "acpx" ? input.execution.provider.agent : undefined, false)));
@@ -10216,7 +10323,9 @@ async function executePaperclipNativeSessionWithinScope(
};
if (warmSessionId !== null && warmConfigDigest !== null && !input.restartRecovery) {
const entry = warmNativeSessions.get(warmSessionId);
if (entry) {
if (entry?.released) requireReleasedWarmScope(input, releasedWarmRunner(readWarmNativeEnvelope(input.execution)),
readRunnerdDurableIdentity(scopedRunnerdStateRoot(input.execution)));
if (entry || remoteWarmRecovery?.releasedSource) {
if (input.execution.provider.kind === "codex" ||
(input.execution.provider.kind === "acpx" && input.execution.provider.agent === "claude")) {
executionContext = claimedStartupContext ?? new NativeExecutionContext(input.execution,
@@ -10225,10 +10334,10 @@ async function executePaperclipNativeSessionWithinScope(
startupForAdmission!.executionContext = executionContext;
if (startupForAdmission!.stopRequested) executionContext.abort(new Error("native_session_cancelled"));
}
if (entry.preparingRunId && entry.preparingRunId !== input.execution.binding.runId) throw new Error("native_session_supervisor_busy");
entry.preparingRunId = undefined;
if (entry?.preparingRunId && entry.preparingRunId !== input.execution.binding.runId) throw new Error("native_session_supervisor_busy");
if (entry) entry.preparingRunId = undefined;
const taskTarget=input.runnerExecutionTarget;
if (entry.pluginTaskId && taskTarget?.kind === "remote" && taskTarget.transport === "sandbox" && taskTarget.runnerTask) {
if ((entry?.pluginTaskId || remoteWarmRecovery?.releasedSource) && taskTarget?.kind === "remote" && taskTarget.transport === "sandbox" && taskTarget.runnerTask) {
const description=environmentTaskResultSchema.parse(await (executionContext
? executionContext.wait(() => taskTarget.runnerTask!({kind:"describe"}, { executionContext }))
: taskTarget.runnerTask({kind:"describe"})));
@@ -10237,6 +10346,8 @@ async function executePaperclipNativeSessionWithinScope(
if (executionContext && description.executionContextVersion === 1) executionContext.pluginSupported = true;
else if (executionContext) await finishStartupContextForFallback();
}
}
if (entry) {
// Old run-scoped environments still require process replacement. A
// session-owned broker can change run authority without replacing it.
const hasBrokerCapability = Boolean(
@@ -10301,6 +10412,7 @@ async function executePaperclipNativeSessionWithinScope(
entry.instructionWorkingCopy = input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy;
entry.busy = true;
entry.ownerToken = warmSessionOwnerToken;
entry.released = false;
entry.environmentId =
input.runnerExecutionTarget?.environmentId ?? null;
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
@@ -10329,7 +10441,7 @@ async function executePaperclipNativeSessionWithinScope(
);
}
}
if ((claimedStartupContext && executionContext !== claimedStartupContext) || (executionContext && !existingWarmSession)) {
if ((claimedStartupContext && executionContext !== claimedStartupContext) || (executionContext && !existingWarmSession && !remoteWarmRecovery?.releasedSource)) {
await finishStartupContextForFallback();
}
async function resolvePendingGovernedWait() {
@@ -10497,9 +10609,10 @@ async function executePaperclipNativeSessionWithinScope(
})
: null;
if (remoteWarmRecovery) {
await remoteWarmRecovery.revalidate(leaseOwner);
if (executionContext) await executionContext.wait(() => remoteWarmRecovery.revalidate(leaseOwner));
else await remoteWarmRecovery.revalidate(leaseOwner);
const recovered = await runnerdBackend?.recoverSession?.(remoteWarmRecovery.checkpoint, {
signal: new AbortController().signal,
signal: executionContext?.signal ?? new AbortController().signal,
});
if (!recovered?.recovered || !recovered.session) throw new Error("runner_state_identity_mismatch: remote_warm_session_unproven");
if (remoteWarmRecovery.reuseProcess) {
@@ -10509,6 +10622,7 @@ async function executePaperclipNativeSessionWithinScope(
persistedWarmSession = null;
} else {
await recovered.session.close({ reason: "recovered warm session requires current process configuration" });
if (executionContext) await finishStartupContextForFallback();
// Closing an externally persisted controller leaves an identity marker,
// not its attachment template. The next backend must rotate the saved
// PRP authority from the committed suspension checkpoint.
@@ -10710,8 +10824,10 @@ async function executePaperclipNativeSessionWithinScope(
rmSync(resolve(scopedRunnerdStateRoot(input.execution), "plugin-runner-state.json"), { force: true });
}
if (session && session === existingWarmSession) remoteWarmRecoveryPending = false;
if (session && input.managedAiCredentialHome) {
managedCredentialSession = runnerdBackend?.bindManagedSession(session) ?? session;
if (session && (input.managedAiCredentialHome || (input.runnerExecutionTarget?.kind === "remote"
&& input.runnerExecutionTarget.transport === "sandbox" && input.runnerExecutionTarget.runnerTask))) {
const bound = runnerdBackend?.bindManagedSession(session, input.managedAiCredentialHome ? undefined : { releaseResumeOnly: true }) ?? session;
if (input.managedAiCredentialHome) managedCredentialSession = bound;
}
releaseRegisteredGoalController();
liveQuestions.close();
@@ -11662,11 +11778,101 @@ async function executePaperclipNativeSessionWithinScope(
if (collectInstructions && ownedSession) {
ownedSession.closeOnReleaseReason = "registered instruction edits require stopped-provider collection";
}
await timeController("warm_release", () => releaseWarmNativeSession(
warmSessionId,
warmSessionOwnerToken,
lifecyclePolicy.idleTimeoutMs,
!retainSession,
if (retainSession && ownedSession?.ownerToken === warmSessionOwnerToken && !ownedSession.closeOnReleaseReason
&& pluginTaskSessionOwners.get(ownedSession.session)?.driver.releaseResume) {
const scope = warmSessionId, entry = ownedSession, token = warmSessionOwnerToken;
entry.finishRelease = async () => {
const current = () => warmNativeSessions.get(scope) === entry && entry.ownerToken === token;
if (!current()) return;
try {
if (nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) entry.closeOnReleaseReason = "warm native session cancelled";
if (entry.closeOnReleaseReason) {
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
forgetWarmNativeSession(scope, entry);
return;
}
const deadline = Date.now() + Math.min(lifecyclePolicy.idleTimeoutMs,
coldAdmissionTimeoutMs(input.execution.provider.kind,
input.execution.provider.kind === "acpx" ? input.execution.provider.agent : undefined, false));
let released: DurableRecoveryIdentity | false | undefined;
for (;;) {
let gateTimer: ReturnType<typeof setTimeout> | undefined;
try {
const gate = new Promise<never>((_resolve, reject) => {
const check = () => {
if (!current() || Date.now() >= deadline || entry.closeOnReleaseReason
|| nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) {
reject(new Error("native_session_release_reconciliation_stopped"));
} else gateTimer = setTimeout(check, Math.min(25, deadline - Date.now()));
};
check();
});
// Observe late replies, but only this reservation may publish their result.
released = await Promise.race([Promise.resolve().then(() => {
if (!current() || Date.now() >= deadline || entry.closeOnReleaseReason
|| nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) {
throw new Error("native_session_release_reconciliation_stopped");
}
return entry.session.release?.();
}), gate]);
break;
}
catch {
if (!current()) return;
if (Date.now() >= deadline || entry.closeOnReleaseReason
|| nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) {
entry.closeOnReleaseReason ??= "warm Runner release reconciliation expired";
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
forgetWarmNativeSession(scope, entry);
return;
}
await new Promise(resolve => setTimeout(resolve, 250));
}
finally { clearTimeout(gateTimer); }
}
if (!current()) return;
if (nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) entry.closeOnReleaseReason = "warm native session cancelled";
if (entry.closeOnReleaseReason) {
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
forgetWarmNativeSession(scope, entry);
return;
}
if (released !== false && released !== undefined) {
try {
const operation = environmentTaskOperationSchema.parse({ kind: "release", identity: released });
const target = input.runnerExecutionTarget;
if (operation.kind !== "release" || operation.identity.runId !== input.execution.binding.runId
|| operation.identity.normalizedSessionId !== nativeSessionKey(input.execution)
|| operation.identity.runnerInstanceId !== effectiveRunnerInstanceId
|| target?.kind !== "remote" || operation.identity.environmentLeaseId !== (durableRunnerBinding?.environmentLeaseId ?? target.leaseId)
|| target.transport !== "sandbox" || !entry.pluginTaskId
|| entry.pluginTaskId !== target.sandboxLeaseAcquisition?.providerLeaseId) {
throw new Error("native_session_released_source_mismatch");
}
const source = operation.identity;
updateReleasedWarmRunner(input.execution, source, entry.pluginTaskId,
{ source, taskId: entry.pluginTaskId, scopeDigest: releasedWarmScope(input) });
} catch {
entry.closeOnReleaseReason = "released Runner source proof could not be verified or saved";
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
forgetWarmNativeSession(scope, entry);
return;
}
entry.released = true;
}
} catch {
// The business turn already settled. An unknown release stays owned and active.
logger.warn({ runId: input.execution.binding.runId }, "Retained Runner release remains unconfirmed");
} finally {
if (current()) {
delete entry.finishRelease;
entry.busy = false;
armWarmNativeSessionIdleTimer(scope, entry, lifecyclePolicy.idleTimeoutMs);
}
}
};
} else await timeController("warm_release", () => releaseWarmNativeSession(
warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, !retainSession,
));
if (collectInstructions && !collectedByOwner) await timeController("instruction_collection", () => instructionCopy!.collectStopped());
}
@@ -13741,7 +13947,7 @@ async function prepareRemoteExternalRunner({ input, target, runner, root, identi
}
type ManagedRunnerdBackend = NativeSessionBackend & {
bindManagedSession(session: NativeSession): NativeSession;
bindManagedSession(session: NativeSession, options?: { releaseResumeOnly: true }): NativeSession;
bindDiagnosticSession?(session: NativeSession): void;
sessionCheckpointPersisted?(snapshot: PersistedNativeSession): void;
providerAdmissionPersisted?(): Promise<void>;
@@ -13941,8 +14147,11 @@ async function createRunnerdBackendWithinSessionClaim(
} : undefined,
onRecoveryAccountingEvent: input.onAccountingEvent ? async (event: PrpEvent, source: Parameters<NonNullable<typeof recoveryUsage>>[1]) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current || !recoveryUsage) throw new Error("native_recovery_usage_source_unproven");
await current.acceptRecoveryAccountingEvent(event, () => recoveryUsage(event, source));
if (!current) throw new Error("native_recovery_usage_source_unproven");
await current.acceptRecoveryAccountingEvent(event, () => {
if (!recoveryUsage) throw new Error("native_recovery_usage_source_unproven");
return recoveryUsage(event, source);
});
} : undefined,
};
const effectiveRunnerInstanceId =
@@ -14003,6 +14212,10 @@ async function createRunnerdBackendWithinSessionClaim(
onCredentials: input.execution.session.lifecyclePolicy.mode === "warm" ? undefined : credentials.onCredentials,
warmTaskVersion: description.warmTaskVersion,
releaseResumeVersion: description.releaseResumeVersion,
authorityRunId: description.releaseResumeVersion === 1 ? input.execution.binding.runId : undefined,
releasedSource: input.remoteWarmRecovery?.releasedSource,
onResumed: async source => updateReleasedWarmRunner(input.execution, source, description.taskId, null),
onRetired: async source => updateReleasedWarmRunner(input.execution, source, description.taskId, null),
adoptIdentity: adoptionIdentity ?? (adopting && prior ? {
runnerInstanceId: prior.runnerInstanceId, environmentLeaseId: prior.environmentLeaseId,
runId: prior.runId, normalizedSessionId: prior.normalizedSessionId,
@@ -14031,6 +14244,7 @@ async function createRunnerdBackendWithinSessionClaim(
});
}, onDispatched: input.onTaskDispatched,
};
if (input.remoteWarmRecovery?.releasedSource && description.releaseResumeVersion !== 1) throw new Error("Plugin Runner release/resume unavailable");
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin warm task capability or account unavailable");
const owner = retainedOwner ?? createPluginRunnerTaskOwner(ownerInput);
if (retainedOwner) retainedOwner.rebind(ownerInput);
@@ -14107,7 +14321,10 @@ async function createRunnerdBackendWithinSessionClaim(
sessionToolAuthorityEpochs.set(sessionScopeId, authorityEpoch);
return { ...backend, descriptor: () => backend.descriptor(), openSession: session => backend.openSession(session),
recoverSession: (snapshot, options) => backend.recoverSession!(snapshot, options),
bindManagedSession: session => { pluginTaskSessionOwners.set(session, owner); return session; } };
bindManagedSession: (session, options) => {
if (!options?.releaseResumeOnly || description.releaseResumeVersion === 1) pluginTaskSessionOwners.set(session, owner);
return session;
} };
}
if (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8") {
if (target.kind === "remote" && input.dotWorkspaceRoot) throw new Error("dot_remote_workspace_access_unavailable");