mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 01:50:13 +01:00
Release and resume retained plugin Runner sessions in Core (#19611)
This commit is contained in:
1 parent
9efee7ee16
commit
b21884bdcf
7 files changed
+965
-80
No files matched your search
@@ -45,7 +45,7 @@ import { notifyExecutionRunWork } from "../execution-control-notifications.js";
|
||||
import { createRunUsageRecorder } from "../usage-receipts.js";
|
||||
import { registerPluginTaskCredentialHandoff, type PluginTaskCredentialWritebackBinding } from "../plugin-task-credential-writeback.js";
|
||||
import { resolvePluginTaskResume } from "../environment-task-runtime.js";
|
||||
import { environmentTaskResultSchema } from "@paperclipai/plugin-sdk";
|
||||
import { environmentTaskOperationSchema, environmentTaskResultSchema } from "@paperclipai/plugin-sdk";
|
||||
import { preparePluginRunnerCredentials } from "./plugin-runner-credentials.js";
|
||||
import { createPluginRunnerTaskOwner, verifyPluginRunnerTaskReattachment } from "./plugin-runner-task.js";
|
||||
import { NativeExecutionContext } from "./native-execution-context.js";
|
||||
@@ -669,6 +669,8 @@ type WarmNativeSession = {
|
||||
companyId: string;
|
||||
environmentId: string | null;
|
||||
busy: boolean;
|
||||
released?: boolean;
|
||||
finishRelease?: () => Promise<void>;
|
||||
closeOnReleaseReason?: string;
|
||||
idleTimer: ReturnType<typeof setTimeout> | null;
|
||||
lastActivityAt: string;
|
||||
@@ -716,7 +718,9 @@ const pluginTaskSessionOwners = new WeakMap<NativeSession, ReturnType<typeof cre
|
||||
const closingWarmNativeSessions = new Map<string, Promise<void>>();
|
||||
|
||||
export function readNativeSessionIdleWork(): IdleLocalWork {
|
||||
return warmNativeSessions.size !== 0 || closingWarmNativeSessions.size !== 0 ? "present" : "none";
|
||||
return closingWarmNativeSessions.size !== 0 || [...warmNativeSessions.values()].some(entry =>
|
||||
entry.busy || entry.preparingRunId || entry.instructionPreparationRunId || entry.closeOnReleaseReason || !entry.released)
|
||||
? "present" : "none";
|
||||
}
|
||||
registerIdleLocalWorkProbe(readNativeSessionIdleWork);
|
||||
|
||||
@@ -876,7 +880,8 @@ async function closeIdleWarmNativeSessions(input: {
|
||||
if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) {
|
||||
continue;
|
||||
}
|
||||
if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)) {
|
||||
if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)
|
||||
|| (input.canRetire && (entry.instructionPreparationRunId || closingWarmNativeSessions.has(sessionId)))) {
|
||||
// A busy turn can complete while another idle session is checkpointing.
|
||||
// Fence that entry now so its eventual release cannot leave a new idle
|
||||
// owner behind after the shutdown sweep has already passed it.
|
||||
@@ -884,6 +889,8 @@ async function closeIdleWarmNativeSessions(input: {
|
||||
busy += 1;
|
||||
continue;
|
||||
}
|
||||
// The host owns retention after release; a quiet local handle needs no retirement for sleep.
|
||||
if (input.canRetire && entry.released && !entry.closeOnReleaseReason) continue;
|
||||
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
|
||||
// Keep a fenced owner until both retirement and stopped collection succeed.
|
||||
// Failure must not make a live directory eligible for a replacement owner.
|
||||
@@ -1927,6 +1934,10 @@ class SessionAuthorityEpoch {
|
||||
if (!isNativeUsageAccountingEvent(event)) return;
|
||||
this.#assertCurrent();
|
||||
if (event.runId === this.runId) throw new Error("native_recovery_usage_source_unproven");
|
||||
if (event.runId === this.#drainingRun?.runId) {
|
||||
await this.acceptAccountingEvent(event);
|
||||
return;
|
||||
}
|
||||
await capture();
|
||||
this.#assertCurrent();
|
||||
}
|
||||
@@ -7324,27 +7335,92 @@ function legacyNativeSessionCheckpointPath(
|
||||
);
|
||||
}
|
||||
|
||||
type ReleasedWarmRunner = { source: DurableRecoveryIdentity; taskId: string; scopeDigest: string };
|
||||
|
||||
function releasedWarmScope(input: Parameters<typeof executePaperclipNativeSession>[0]): string {
|
||||
const target = input.runnerExecutionTarget;
|
||||
return nativeSha256({ companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId,
|
||||
environmentId: target?.kind === "remote" ? target.environmentId ?? null : null, workspace: nativeSessionWorkspaceScope(input.execution),
|
||||
cwd: input.execution.workspace.cwd, remoteCwd: target?.kind === "remote" ? target.remoteCwd : null,
|
||||
projectIds: target?.kind === "remote" && target.transport === "sandbox" ? target.runnerTaskProjectIds ?? [] : [],
|
||||
accountIdentity: input.managedAiAccountIdentity ?? null, credentialIdentity: input.managedAiCredentialIdentity ?? null });
|
||||
}
|
||||
|
||||
function requireReleasedWarmScope(input: Parameters<typeof executePaperclipNativeSession>[0],
|
||||
proof: ReleasedWarmRunner | undefined, identity: unknown): DurableRecoveryIdentity {
|
||||
const target = input.runnerExecutionTarget;
|
||||
if (!proof || target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask
|
||||
|| proof.taskId !== target.sandboxLeaseAcquisition?.providerLeaseId
|
||||
|| proof.scopeDigest !== releasedWarmScope(input) || canonicalJson(proof.source) !== canonicalJson(identity)) {
|
||||
throw new Error("native_session_released_owner_scope_mismatch");
|
||||
}
|
||||
return proof.source;
|
||||
}
|
||||
|
||||
function readWarmNativeEnvelope(execution: NativeExecutionInput): Record<string, unknown> | null {
|
||||
const path = nativeSessionCheckpointPath(execution);
|
||||
if (!existsSync(path)) return null;
|
||||
const envelope = record(JSON.parse(readBoundedNativeFile(path, NATIVE_WARM_CHECKPOINT_MAX_BYTES,
|
||||
"native_session_supervisor_checkpoint_too_large").toString("utf8")));
|
||||
if (envelope.schema !== "paperclip.native-session-supervisor.v1") throw new Error("native_session_supervisor_checkpoint_mismatch");
|
||||
return envelope;
|
||||
}
|
||||
|
||||
function releasedWarmRunner(envelope: Record<string, unknown> | null): ReleasedWarmRunner | undefined {
|
||||
if (!envelope || envelope.releasedRunner === undefined) return undefined;
|
||||
const proof = record(envelope.releasedRunner), source = record(proof.source);
|
||||
const snapshot = record(envelope.snapshot), identity = record(snapshot.identity);
|
||||
for (const key of ["runnerInstanceId", "environmentLeaseId", "runId", "normalizedSessionId", "turnId", "itemId"]) {
|
||||
if (typeof source[key] !== "string" || !source[key]) throw new Error("native_session_released_source_mismatch");
|
||||
}
|
||||
if (typeof proof.taskId !== "string" || !proof.taskId || typeof proof.scopeDigest !== "string"
|
||||
|| identity.runId !== source.runId || identity.sessionId !== source.normalizedSessionId) {
|
||||
throw new Error("native_session_released_source_mismatch");
|
||||
}
|
||||
return proof as ReleasedWarmRunner;
|
||||
}
|
||||
|
||||
function writeWarmNativeEnvelope(execution: NativeExecutionInput, envelope: Record<string, unknown>): void {
|
||||
const path = nativeSessionCheckpointPath(execution);
|
||||
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
||||
writeFileSync(temporary, JSON.stringify(envelope), { encoding: "utf8", mode: 0o600 });
|
||||
renameSync(temporary, path);
|
||||
chmodSync(path, 0o600);
|
||||
}
|
||||
|
||||
function updateReleasedWarmRunner(execution: NativeExecutionInput, source: DurableRecoveryIdentity,
|
||||
taskId: string, proof: ReleasedWarmRunner | null): void {
|
||||
const envelope = readWarmNativeEnvelope(execution);
|
||||
const previous = releasedWarmRunner(envelope);
|
||||
if (!envelope) {
|
||||
if (proof) throw new Error("native_session_supervisor_checkpoint_missing");
|
||||
return;
|
||||
}
|
||||
if (!proof && !previous) return;
|
||||
const identity = record(record(envelope.snapshot).identity);
|
||||
if (identity.companyId !== execution.binding.companyId || identity.agentId !== execution.binding.agentId
|
||||
|| identity.runId !== source.runId || identity.sessionId !== source.normalizedSessionId
|
||||
|| (previous && (previous.taskId !== taskId || canonicalJson(previous.source) !== canonicalJson(source)))) {
|
||||
throw new Error("native_session_released_source_mismatch");
|
||||
}
|
||||
if (proof) envelope.releasedRunner = proof;
|
||||
else delete envelope.releasedRunner;
|
||||
writeWarmNativeEnvelope(execution, envelope);
|
||||
}
|
||||
|
||||
function persistWarmNativeCheckpoint(
|
||||
execution: NativeExecutionInput,
|
||||
configDigest: string,
|
||||
snapshot: PersistedNativeSession,
|
||||
processConfigDigest?: string,
|
||||
): void {
|
||||
const path = nativeSessionCheckpointPath(execution);
|
||||
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
||||
writeFileSync(
|
||||
temporary,
|
||||
JSON.stringify({
|
||||
schema: "paperclip.native-session-supervisor.v1",
|
||||
configDigest,
|
||||
processConfigDigest,
|
||||
updatedAt: new Date().toISOString(),
|
||||
snapshot,
|
||||
}),
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
renameSync(temporary, path);
|
||||
chmodSync(path, 0o600);
|
||||
const previous = releasedWarmRunner(readWarmNativeEnvelope(execution));
|
||||
writeWarmNativeEnvelope(execution, {
|
||||
schema: "paperclip.native-session-supervisor.v1", configDigest, processConfigDigest,
|
||||
updatedAt: new Date().toISOString(), snapshot,
|
||||
...(previous && previous.source.runId === snapshot.identity.runId
|
||||
&& previous.source.normalizedSessionId === snapshot.identity.sessionId ? { releasedRunner: previous } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function needsCoreGitHubAccess(input: Pick<Parameters<typeof executePaperclipNativeSession>[0], "managedGitHub" | "runnerExecutionTarget">): boolean {
|
||||
@@ -7497,10 +7573,17 @@ function armWarmNativeSessionIdleTimer(
|
||||
return;
|
||||
}
|
||||
current.busy = true;
|
||||
current.closeOnReleaseReason = "warm native session idle timeout";
|
||||
const closing = closeWarmNativeSession(current, current.closeOnReleaseReason)
|
||||
const released = current.released === true;
|
||||
if (!released) current.closeOnReleaseReason = "warm native session idle timeout";
|
||||
const closing = (released ? (async () => {
|
||||
if (!current.session.detachControllerForRestart) throw new Error("native_session_released_detach_unavailable");
|
||||
await current.session.detachControllerForRestart();
|
||||
await current.githubAccess?.stop();
|
||||
})() : closeWarmNativeSession(current, current.closeOnReleaseReason!))
|
||||
.then(() => { forgetWarmNativeSession(sessionId, current); })
|
||||
.catch(() => {})
|
||||
.catch(() => {
|
||||
if (released) current.closeOnReleaseReason = "released Runner local detach is unconfirmed";
|
||||
})
|
||||
.finally(() => {
|
||||
current.busy = false;
|
||||
if (closingWarmNativeSessions.get(sessionId) === closing) closingWarmNativeSessions.delete(sessionId);
|
||||
@@ -9098,29 +9181,8 @@ export async function executePaperclipNativeSession(input: {
|
||||
startupUnsettled = !(await startup.executionContext.drain());
|
||||
startup.executionContext.finish();
|
||||
}
|
||||
if (ownsPhysicalIngress && physicalIngressKey && executingNativeIngressAllocations.get(physicalIngressKey) === ownerToken) {
|
||||
executingNativeIngressAllocations.delete(physicalIngressKey);
|
||||
}
|
||||
if (executingNativeOwnerScopes.get(ownerScope) === ownerToken) {
|
||||
executingNativeOwnerScopes.delete(ownerScope);
|
||||
}
|
||||
startup.resolve(null);
|
||||
startup.settle();
|
||||
if (nativeSessionStartups.get(runId) === startup) {
|
||||
nativeSessionStartups.delete(runId);
|
||||
setNativeExecutionActive(runId, false);
|
||||
}
|
||||
if (
|
||||
ownsSessionScope &&
|
||||
sessionScopeId !== null &&
|
||||
executingRunnerdSessionScopes.get(sessionScopeId) ===
|
||||
input.execution.binding.runId
|
||||
) {
|
||||
executingRunnerdSessionScopes.delete(sessionScopeId);
|
||||
}
|
||||
// Preparation can fail before the execution loop owns its normal cleanup.
|
||||
// Remove the settled startup before yielding, so detachment cannot reacquire
|
||||
// its old resolved session while renewal cleanup drains.
|
||||
// Join preparation cleanup before release; keep cancellation reachable
|
||||
// until the exact retained owner has finished the handover.
|
||||
await startup.executionLeaseRenewal?.stop().catch(() => undefined);
|
||||
if (startup.unstartedClaim && !nativeRunsDetachingForRestart.has(runId)) {
|
||||
await releaseUnstartedNativeExecutionClaim({ db: input.db, runId, companyId: input.execution.binding.companyId,
|
||||
@@ -9128,9 +9190,11 @@ export async function executePaperclipNativeSession(input: {
|
||||
logger.warn({ err: error, runId }, "Native preparation claim release failed; the control deadline will settle it");
|
||||
});
|
||||
}
|
||||
let stagedCleanupFailed = false;
|
||||
try {
|
||||
await cleanupStagedAttachments();
|
||||
} catch (cleanupError) {
|
||||
stagedCleanupFailed = true;
|
||||
// Cleanup is a confidentiality incident, but it occurs after the native
|
||||
// provider may already have completed the turn. Reclassifying that turn
|
||||
// as failed could replay provider side effects. Emit a private runtime
|
||||
@@ -9152,6 +9216,38 @@ export async function executePaperclipNativeSession(input: {
|
||||
companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId,
|
||||
}, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId,
|
||||
issueId: input.execution.binding.issueId });
|
||||
const pendingRelease = ownsSessionScope && sessionScopeId
|
||||
&& executingRunnerdSessionScopes.get(sessionScopeId) === runId
|
||||
? warmNativeSessions.get(sessionScopeId) : undefined;
|
||||
try {
|
||||
if (pendingRelease?.runId === runId && pendingRelease.finishRelease) {
|
||||
if (executionFailure !== undefined || startupUnsettled || stagedCleanupFailed) {
|
||||
pendingRelease.closeOnReleaseReason = "warm native session final cleanup failed";
|
||||
}
|
||||
await pendingRelease.finishRelease();
|
||||
}
|
||||
} finally {
|
||||
if (ownsPhysicalIngress && physicalIngressKey && executingNativeIngressAllocations.get(physicalIngressKey) === ownerToken) {
|
||||
executingNativeIngressAllocations.delete(physicalIngressKey);
|
||||
}
|
||||
if (executingNativeOwnerScopes.get(ownerScope) === ownerToken) {
|
||||
executingNativeOwnerScopes.delete(ownerScope);
|
||||
}
|
||||
if (
|
||||
ownsSessionScope &&
|
||||
sessionScopeId !== null &&
|
||||
executingRunnerdSessionScopes.get(sessionScopeId) ===
|
||||
input.execution.binding.runId
|
||||
) {
|
||||
executingRunnerdSessionScopes.delete(sessionScopeId);
|
||||
}
|
||||
startup.resolve(null);
|
||||
startup.settle();
|
||||
if (nativeSessionStartups.get(runId) === startup) {
|
||||
nativeSessionStartups.delete(runId);
|
||||
setNativeExecutionActive(runId, false);
|
||||
}
|
||||
}
|
||||
if (startupUnsettled) throw new NativeStartupCleanupPendingError(startup.executionContext!.id,
|
||||
startup.executionContext!.deadlineAtMs, startup.leaseOwner!, startup.executionContext!.pluginSupported,
|
||||
input.runnerExecutionTarget?.kind === "remote" ? input.runnerExecutionTarget.leaseId ?? undefined : undefined);
|
||||
@@ -9270,6 +9366,9 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
}
|
||||
let retainedTransition: VerifiedWarmTransitionBinding | undefined;
|
||||
let remoteWarmRecovery: RemoteWarmSessionRecovery | undefined;
|
||||
const cachedReleasedOwner = warmNativeSessions.get(nativeSessionScopeKey(input.execution));
|
||||
if (cachedReleasedOwner?.released) requireReleasedWarmScope(input, releasedWarmRunner(readWarmNativeEnvelope(input.execution)),
|
||||
readRunnerdDurableIdentity(scopedRunnerdStateRoot(input.execution)));
|
||||
if (input.useRunnerd && !input.restartRecovery && !input.backend &&
|
||||
input.runnerExecutionTarget?.kind === "remote" &&
|
||||
input.execution.session.lifecyclePolicy.mode === "warm" &&
|
||||
@@ -9295,6 +9394,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
}
|
||||
if (durableIdentityMatchesSession(identity, input.execution) && identity.runId !== input.execution.binding.runId &&
|
||||
runnerdAuthorityLifecycleWithVerifiedBackup({ root, identity, execution: input.execution, allowVerifiedBackup: true }) !== "suspended") {
|
||||
let releasedSource: DurableRecoveryIdentity | undefined;
|
||||
remoteWarmRecovery = await inspectRemoteWarmSessionRecovery({
|
||||
db: input.db, execution: input.execution, target: input.runnerExecutionTarget, identity,
|
||||
readControlState: () => record(JSON.parse(readBoundedNativeFile(
|
||||
@@ -9302,7 +9402,13 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
"runner_durable_identity_too_large").toString("utf8"))),
|
||||
matchesExecution: previous => nativeSessionScopeKey(previous) === nativeSessionScopeKey(input.execution) &&
|
||||
nativeSessionConfigDigest(previous, "remote") === nativeSessionConfigDigest(input.execution, "remote"),
|
||||
loadCheckpoint: previous => loadWarmNativeCheckpoint(previous, nativeSessionConfigDigest(previous, "remote"), "remote"),
|
||||
loadCheckpoint: previous => {
|
||||
const proof = releasedWarmRunner(readWarmNativeEnvelope(previous));
|
||||
if (proof) {
|
||||
releasedSource = requireReleasedWarmScope(input, proof, identity);
|
||||
}
|
||||
return loadWarmNativeCheckpoint(previous, nativeSessionConfigDigest(previous, "remote"), "remote");
|
||||
},
|
||||
canReuseProcess: previous => {
|
||||
const expected = warmNativeProcessConfigDigest(input);
|
||||
if (!expected) return false;
|
||||
@@ -9311,6 +9417,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
return envelope.processConfigDigest === expected;
|
||||
},
|
||||
});
|
||||
remoteWarmRecovery.releasedSource = releasedSource;
|
||||
}
|
||||
}
|
||||
if (input.useRunnerd) {
|
||||
@@ -9517,7 +9624,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
const target = input.runnerExecutionTarget;
|
||||
const retained = warmNativeSessions.get(nativeSessionScopeKey(input.execution));
|
||||
if (!input.restartRecovery && target?.kind === "remote" && target.transport === "sandbox" && target.runnerTask && target.leaseId &&
|
||||
retained?.pluginTaskId && idleWarmNativeSessionOwnerRunId(input) !== undefined &&
|
||||
((retained?.pluginTaskId && idleWarmNativeSessionOwnerRunId(input) !== undefined) || remoteWarmRecovery?.releasedSource) &&
|
||||
(input.execution.provider.kind === "codex" || (input.execution.provider.kind === "acpx" && input.execution.provider.agent === "claude"))) {
|
||||
claimedStartupContext = new NativeExecutionContext(input.execution, Math.min(executionLeaseTtlMs,
|
||||
coldAdmissionTimeoutMs(input.execution.provider.kind, input.execution.provider.kind === "acpx" ? input.execution.provider.agent : undefined, false)));
|
||||
@@ -10216,7 +10323,9 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
};
|
||||
if (warmSessionId !== null && warmConfigDigest !== null && !input.restartRecovery) {
|
||||
const entry = warmNativeSessions.get(warmSessionId);
|
||||
if (entry) {
|
||||
if (entry?.released) requireReleasedWarmScope(input, releasedWarmRunner(readWarmNativeEnvelope(input.execution)),
|
||||
readRunnerdDurableIdentity(scopedRunnerdStateRoot(input.execution)));
|
||||
if (entry || remoteWarmRecovery?.releasedSource) {
|
||||
if (input.execution.provider.kind === "codex" ||
|
||||
(input.execution.provider.kind === "acpx" && input.execution.provider.agent === "claude")) {
|
||||
executionContext = claimedStartupContext ?? new NativeExecutionContext(input.execution,
|
||||
@@ -10225,10 +10334,10 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
startupForAdmission!.executionContext = executionContext;
|
||||
if (startupForAdmission!.stopRequested) executionContext.abort(new Error("native_session_cancelled"));
|
||||
}
|
||||
if (entry.preparingRunId && entry.preparingRunId !== input.execution.binding.runId) throw new Error("native_session_supervisor_busy");
|
||||
entry.preparingRunId = undefined;
|
||||
if (entry?.preparingRunId && entry.preparingRunId !== input.execution.binding.runId) throw new Error("native_session_supervisor_busy");
|
||||
if (entry) entry.preparingRunId = undefined;
|
||||
const taskTarget=input.runnerExecutionTarget;
|
||||
if (entry.pluginTaskId && taskTarget?.kind === "remote" && taskTarget.transport === "sandbox" && taskTarget.runnerTask) {
|
||||
if ((entry?.pluginTaskId || remoteWarmRecovery?.releasedSource) && taskTarget?.kind === "remote" && taskTarget.transport === "sandbox" && taskTarget.runnerTask) {
|
||||
const description=environmentTaskResultSchema.parse(await (executionContext
|
||||
? executionContext.wait(() => taskTarget.runnerTask!({kind:"describe"}, { executionContext }))
|
||||
: taskTarget.runnerTask({kind:"describe"})));
|
||||
@@ -10237,6 +10346,8 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
if (executionContext && description.executionContextVersion === 1) executionContext.pluginSupported = true;
|
||||
else if (executionContext) await finishStartupContextForFallback();
|
||||
}
|
||||
}
|
||||
if (entry) {
|
||||
// Old run-scoped environments still require process replacement. A
|
||||
// session-owned broker can change run authority without replacing it.
|
||||
const hasBrokerCapability = Boolean(
|
||||
@@ -10301,6 +10412,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
entry.instructionWorkingCopy = input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy;
|
||||
entry.busy = true;
|
||||
entry.ownerToken = warmSessionOwnerToken;
|
||||
entry.released = false;
|
||||
entry.environmentId =
|
||||
input.runnerExecutionTarget?.environmentId ?? null;
|
||||
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
|
||||
@@ -10329,7 +10441,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
);
|
||||
}
|
||||
}
|
||||
if ((claimedStartupContext && executionContext !== claimedStartupContext) || (executionContext && !existingWarmSession)) {
|
||||
if ((claimedStartupContext && executionContext !== claimedStartupContext) || (executionContext && !existingWarmSession && !remoteWarmRecovery?.releasedSource)) {
|
||||
await finishStartupContextForFallback();
|
||||
}
|
||||
async function resolvePendingGovernedWait() {
|
||||
@@ -10497,9 +10609,10 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
})
|
||||
: null;
|
||||
if (remoteWarmRecovery) {
|
||||
await remoteWarmRecovery.revalidate(leaseOwner);
|
||||
if (executionContext) await executionContext.wait(() => remoteWarmRecovery.revalidate(leaseOwner));
|
||||
else await remoteWarmRecovery.revalidate(leaseOwner);
|
||||
const recovered = await runnerdBackend?.recoverSession?.(remoteWarmRecovery.checkpoint, {
|
||||
signal: new AbortController().signal,
|
||||
signal: executionContext?.signal ?? new AbortController().signal,
|
||||
});
|
||||
if (!recovered?.recovered || !recovered.session) throw new Error("runner_state_identity_mismatch: remote_warm_session_unproven");
|
||||
if (remoteWarmRecovery.reuseProcess) {
|
||||
@@ -10509,6 +10622,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
persistedWarmSession = null;
|
||||
} else {
|
||||
await recovered.session.close({ reason: "recovered warm session requires current process configuration" });
|
||||
if (executionContext) await finishStartupContextForFallback();
|
||||
// Closing an externally persisted controller leaves an identity marker,
|
||||
// not its attachment template. The next backend must rotate the saved
|
||||
// PRP authority from the committed suspension checkpoint.
|
||||
@@ -10710,8 +10824,10 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
rmSync(resolve(scopedRunnerdStateRoot(input.execution), "plugin-runner-state.json"), { force: true });
|
||||
}
|
||||
if (session && session === existingWarmSession) remoteWarmRecoveryPending = false;
|
||||
if (session && input.managedAiCredentialHome) {
|
||||
managedCredentialSession = runnerdBackend?.bindManagedSession(session) ?? session;
|
||||
if (session && (input.managedAiCredentialHome || (input.runnerExecutionTarget?.kind === "remote"
|
||||
&& input.runnerExecutionTarget.transport === "sandbox" && input.runnerExecutionTarget.runnerTask))) {
|
||||
const bound = runnerdBackend?.bindManagedSession(session, input.managedAiCredentialHome ? undefined : { releaseResumeOnly: true }) ?? session;
|
||||
if (input.managedAiCredentialHome) managedCredentialSession = bound;
|
||||
}
|
||||
releaseRegisteredGoalController();
|
||||
liveQuestions.close();
|
||||
@@ -11662,11 +11778,101 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
if (collectInstructions && ownedSession) {
|
||||
ownedSession.closeOnReleaseReason = "registered instruction edits require stopped-provider collection";
|
||||
}
|
||||
await timeController("warm_release", () => releaseWarmNativeSession(
|
||||
warmSessionId,
|
||||
warmSessionOwnerToken,
|
||||
lifecyclePolicy.idleTimeoutMs,
|
||||
!retainSession,
|
||||
if (retainSession && ownedSession?.ownerToken === warmSessionOwnerToken && !ownedSession.closeOnReleaseReason
|
||||
&& pluginTaskSessionOwners.get(ownedSession.session)?.driver.releaseResume) {
|
||||
const scope = warmSessionId, entry = ownedSession, token = warmSessionOwnerToken;
|
||||
entry.finishRelease = async () => {
|
||||
const current = () => warmNativeSessions.get(scope) === entry && entry.ownerToken === token;
|
||||
if (!current()) return;
|
||||
try {
|
||||
if (nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) entry.closeOnReleaseReason = "warm native session cancelled";
|
||||
if (entry.closeOnReleaseReason) {
|
||||
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
|
||||
forgetWarmNativeSession(scope, entry);
|
||||
return;
|
||||
}
|
||||
const deadline = Date.now() + Math.min(lifecyclePolicy.idleTimeoutMs,
|
||||
coldAdmissionTimeoutMs(input.execution.provider.kind,
|
||||
input.execution.provider.kind === "acpx" ? input.execution.provider.agent : undefined, false));
|
||||
let released: DurableRecoveryIdentity | false | undefined;
|
||||
for (;;) {
|
||||
let gateTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const gate = new Promise<never>((_resolve, reject) => {
|
||||
const check = () => {
|
||||
if (!current() || Date.now() >= deadline || entry.closeOnReleaseReason
|
||||
|| nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) {
|
||||
reject(new Error("native_session_release_reconciliation_stopped"));
|
||||
} else gateTimer = setTimeout(check, Math.min(25, deadline - Date.now()));
|
||||
};
|
||||
check();
|
||||
});
|
||||
// Observe late replies, but only this reservation may publish their result.
|
||||
released = await Promise.race([Promise.resolve().then(() => {
|
||||
if (!current() || Date.now() >= deadline || entry.closeOnReleaseReason
|
||||
|| nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) {
|
||||
throw new Error("native_session_release_reconciliation_stopped");
|
||||
}
|
||||
return entry.session.release?.();
|
||||
}), gate]);
|
||||
break;
|
||||
}
|
||||
catch {
|
||||
if (!current()) return;
|
||||
if (Date.now() >= deadline || entry.closeOnReleaseReason
|
||||
|| nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) {
|
||||
entry.closeOnReleaseReason ??= "warm Runner release reconciliation expired";
|
||||
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
|
||||
forgetWarmNativeSession(scope, entry);
|
||||
return;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 250));
|
||||
}
|
||||
finally { clearTimeout(gateTimer); }
|
||||
}
|
||||
if (!current()) return;
|
||||
if (nativeSessionStartups.get(input.execution.binding.runId)?.stopRequested) entry.closeOnReleaseReason = "warm native session cancelled";
|
||||
if (entry.closeOnReleaseReason) {
|
||||
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
|
||||
forgetWarmNativeSession(scope, entry);
|
||||
return;
|
||||
}
|
||||
if (released !== false && released !== undefined) {
|
||||
try {
|
||||
const operation = environmentTaskOperationSchema.parse({ kind: "release", identity: released });
|
||||
const target = input.runnerExecutionTarget;
|
||||
if (operation.kind !== "release" || operation.identity.runId !== input.execution.binding.runId
|
||||
|| operation.identity.normalizedSessionId !== nativeSessionKey(input.execution)
|
||||
|| operation.identity.runnerInstanceId !== effectiveRunnerInstanceId
|
||||
|| target?.kind !== "remote" || operation.identity.environmentLeaseId !== (durableRunnerBinding?.environmentLeaseId ?? target.leaseId)
|
||||
|| target.transport !== "sandbox" || !entry.pluginTaskId
|
||||
|| entry.pluginTaskId !== target.sandboxLeaseAcquisition?.providerLeaseId) {
|
||||
throw new Error("native_session_released_source_mismatch");
|
||||
}
|
||||
const source = operation.identity;
|
||||
updateReleasedWarmRunner(input.execution, source, entry.pluginTaskId,
|
||||
{ source, taskId: entry.pluginTaskId, scopeDigest: releasedWarmScope(input) });
|
||||
} catch {
|
||||
entry.closeOnReleaseReason = "released Runner source proof could not be verified or saved";
|
||||
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
|
||||
forgetWarmNativeSession(scope, entry);
|
||||
return;
|
||||
}
|
||||
entry.released = true;
|
||||
}
|
||||
} catch {
|
||||
// The business turn already settled. An unknown release stays owned and active.
|
||||
logger.warn({ runId: input.execution.binding.runId }, "Retained Runner release remains unconfirmed");
|
||||
} finally {
|
||||
if (current()) {
|
||||
delete entry.finishRelease;
|
||||
entry.busy = false;
|
||||
armWarmNativeSessionIdleTimer(scope, entry, lifecyclePolicy.idleTimeoutMs);
|
||||
}
|
||||
}
|
||||
};
|
||||
} else await timeController("warm_release", () => releaseWarmNativeSession(
|
||||
warmSessionId, warmSessionOwnerToken, lifecyclePolicy.idleTimeoutMs, !retainSession,
|
||||
));
|
||||
if (collectInstructions && !collectedByOwner) await timeController("instruction_collection", () => instructionCopy!.collectStopped());
|
||||
}
|
||||
@@ -13741,7 +13947,7 @@ async function prepareRemoteExternalRunner({ input, target, runner, root, identi
|
||||
}
|
||||
|
||||
type ManagedRunnerdBackend = NativeSessionBackend & {
|
||||
bindManagedSession(session: NativeSession): NativeSession;
|
||||
bindManagedSession(session: NativeSession, options?: { releaseResumeOnly: true }): NativeSession;
|
||||
bindDiagnosticSession?(session: NativeSession): void;
|
||||
sessionCheckpointPersisted?(snapshot: PersistedNativeSession): void;
|
||||
providerAdmissionPersisted?(): Promise<void>;
|
||||
@@ -13941,8 +14147,11 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
} : undefined,
|
||||
onRecoveryAccountingEvent: input.onAccountingEvent ? async (event: PrpEvent, source: Parameters<NonNullable<typeof recoveryUsage>>[1]) => {
|
||||
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
|
||||
if (!current || !recoveryUsage) throw new Error("native_recovery_usage_source_unproven");
|
||||
await current.acceptRecoveryAccountingEvent(event, () => recoveryUsage(event, source));
|
||||
if (!current) throw new Error("native_recovery_usage_source_unproven");
|
||||
await current.acceptRecoveryAccountingEvent(event, () => {
|
||||
if (!recoveryUsage) throw new Error("native_recovery_usage_source_unproven");
|
||||
return recoveryUsage(event, source);
|
||||
});
|
||||
} : undefined,
|
||||
};
|
||||
const effectiveRunnerInstanceId =
|
||||
@@ -14003,6 +14212,10 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
onCredentials: input.execution.session.lifecyclePolicy.mode === "warm" ? undefined : credentials.onCredentials,
|
||||
warmTaskVersion: description.warmTaskVersion,
|
||||
releaseResumeVersion: description.releaseResumeVersion,
|
||||
authorityRunId: description.releaseResumeVersion === 1 ? input.execution.binding.runId : undefined,
|
||||
releasedSource: input.remoteWarmRecovery?.releasedSource,
|
||||
onResumed: async source => updateReleasedWarmRunner(input.execution, source, description.taskId, null),
|
||||
onRetired: async source => updateReleasedWarmRunner(input.execution, source, description.taskId, null),
|
||||
adoptIdentity: adoptionIdentity ?? (adopting && prior ? {
|
||||
runnerInstanceId: prior.runnerInstanceId, environmentLeaseId: prior.environmentLeaseId,
|
||||
runId: prior.runId, normalizedSessionId: prior.normalizedSessionId,
|
||||
@@ -14031,6 +14244,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
});
|
||||
}, onDispatched: input.onTaskDispatched,
|
||||
};
|
||||
if (input.remoteWarmRecovery?.releasedSource && description.releaseResumeVersion !== 1) throw new Error("Plugin Runner release/resume unavailable");
|
||||
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin warm task capability or account unavailable");
|
||||
const owner = retainedOwner ?? createPluginRunnerTaskOwner(ownerInput);
|
||||
if (retainedOwner) retainedOwner.rebind(ownerInput);
|
||||
@@ -14107,7 +14321,10 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
sessionToolAuthorityEpochs.set(sessionScopeId, authorityEpoch);
|
||||
return { ...backend, descriptor: () => backend.descriptor(), openSession: session => backend.openSession(session),
|
||||
recoverSession: (snapshot, options) => backend.recoverSession!(snapshot, options),
|
||||
bindManagedSession: session => { pluginTaskSessionOwners.set(session, owner); return session; } };
|
||||
bindManagedSession: (session, options) => {
|
||||
if (!options?.releaseResumeOnly || description.releaseResumeVersion === 1) pluginTaskSessionOwners.set(session, owner);
|
||||
return session;
|
||||
} };
|
||||
}
|
||||
if (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8") {
|
||||
if (target.kind === "remote" && input.dotWorkspaceRoot) throw new Error("dot_remote_workspace_access_unavailable");
|
||||
|
||||
Reference in new issue
Block a user