Preserve plugin session continuity through warm execution and checkpoint recovery (#17820)

This commit is contained in:
Nicky Leach authored and GitHub committed 2026-10-26 20:25:10 -07:00
1 parent dcfc2acfa8
commit ea727dc3c2
23 files changed
+433 -82

No files matched your search

@@ -85,10 +85,11 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
throw new Error("Plugin-provided Runner execution identity mismatch");
}
if (runner && resumeFrom) {
const sources = await db.select({ runId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }).from(environmentLeases).where(and(
const inspectCheckpoint = operation.kind === "read_state";
const sources = await db.select({ id: environmentLeases.id, runId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.provider, lease.provider!),
eq(environmentLeases.providerLeaseId, resumeFrom),
sql`${environmentLeases.metadata}->>'runnerTaskSessionRetained' = 'true'`,
inspectCheckpoint ? undefined : sql`${environmentLeases.metadata}->>'runnerTaskSessionRetained' = 'true'`,
checkpointIdentity ? eq(environmentLeases.heartbeatRunId, checkpointIdentity.runId) : undefined,
)).limit(2);
const priorRunId = sources.length === 1 ? sources[0]!.runId : null;
@@ -98,7 +99,20 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
companyId: input.companyId, leaseId: lease.id,
priorIdentity: { runnerInstanceId: runner.runnerId, environmentLeaseId: runner.leaseId,
runId: priorRunId, normalizedSessionId: runner.sessionId },
inspectCheckpoint,
})) throw new Error("Plugin Runner resume identity mismatch");
// Older providers misclassified suspended checkpoints. A fresh provider
// attestation may repair availability, but never supplies stop/owner authority.
if (inspectCheckpoint && sources[0]!.metadata?.runnerTaskSessionRetained !== true) {
const status = await executeEnvironmentTask(db, workers, { companyId: input.companyId,
leaseId: sources[0]!.id, operation: { kind: "status" } });
if (status.kind !== "status" || status.executionStopped !== true || status.sessionStateAvailable !== true ||
resumeFrom !== await resolvePluginTaskResume(db, { companyId: input.companyId, leaseId: lease.id,
priorIdentity: { runnerInstanceId: runner.runnerId, environmentLeaseId: runner.leaseId,
runId: priorRunId, normalizedSessionId: runner.sessionId } })) {
throw new Error("Plugin Runner retained session is not authorized");
}
}
}
if (metadata.driver !== "plugin" || typeof metadata.pluginId !== "string" || typeof metadata.driverKey !== "string") {
throw new Error("Plugin-provided Runner execution lease has no pinned plugin driver");
@@ -380,6 +394,8 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor
export async function resolvePluginTaskResume(db: Db, input: {
companyId: string; leaseId: string;
priorIdentity: { runnerInstanceId: string; environmentLeaseId: string; runId: string; normalizedSessionId: string };
/** Read-only discovery; launching still requires an attested retained checkpoint. */
inspectCheckpoint?: boolean;
}): Promise<string> {
const rows = await db.select({ lease: environmentLeases, run: heartbeatRuns }).from(environmentLeases)
.innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId), eq(heartbeatRuns.companyId, input.companyId)))
@@ -389,7 +405,8 @@ export async function resolvePluginTaskResume(db: Db, input: {
// A retained PRP lease is logical; later provider tasks have different physical lease IDs.
const candidates = rows.filter(row => {
const identity = row.lease.metadata?.runnerTaskIdentity as Record<string, unknown> | undefined;
return row.lease.id !== input.leaseId && row.run.id === input.priorIdentity.runId && row.lease.metadata?.runnerTaskSessionRetained === true &&
return row.lease.id !== input.leaseId && row.run.id === input.priorIdentity.runId &&
(input.inspectCheckpoint || row.lease.metadata?.runnerTaskSessionRetained === true) &&
identity?.runnerId === input.priorIdentity.runnerInstanceId && identity.leaseId === input.priorIdentity.environmentLeaseId &&
identity.sessionId === input.priorIdentity.normalizedSessionId && identity.runId === input.priorIdentity.runId &&
!rows.some(successor => successor.lease.id !== input.leaseId &&