Checkpoint warm native sessions before stopping their computer owners

Close the exact run for stop-and-retain and default stopping paths, retain failed explicit cleanup fences, and settle Stop requests arriving during or after warm retention. Prevent stale warm returns from overwriting a released or rebound lease.

Co-authored-by: Codex <noreply@openai.com>

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
committed 2026-10-11 08:59:05 -05:00
1 parent a681637461
commit 46cbfc67a3
6 files changed
+246 -24

No files matched your search

@@ -671,6 +671,34 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(workerManager.call).not.toHaveBeenCalled();
});
it.each(["released", "rebound"])("does not publish stale warm retention after the lease was %s", async (change) => {
const { runId, reusableLease } = await seedReusablePluginSandboxLease();
const realFactory = environmentsModule.environmentService;
const receipt = { proof: "exact-owner-stopped" };
let changed = false;
const factory = vi.spyOn(environmentsModule, "environmentService").mockImplementation(database => {
const real = realFactory(database);
return { ...real, getById: async (id: string) => {
const result = await real.getById(id);
if (!changed) {
changed = true;
if (change === "released") await real.releaseLease(reusableLease.id, "released", { cleanupStatus: "success", remoteExecutionTermination: receipt });
else await database.update(environmentLeases).set({ providerLeaseId: "new-owner", metadata: { computerOwner: { ownerId: "new-owner", generation: 2 } } }).where(eq(environmentLeases.id, reusableLease.id));
}
return result;
} };
});
try {
const released = await environmentRuntimeService(db).releaseRunLeases(runId, "released", undefined, "keep_running");
expect(released).toEqual([]);
const actual = await realFactory(db).getLeaseById(reusableLease.id);
if (change === "released") {
expect(actual).toMatchObject({ status: "released", metadata: { remoteExecutionTermination: receipt } });
expect(actual?.releasedAt).not.toBeNull();
} else expect(actual).toMatchObject({ status: "active", providerLeaseId: "new-owner", metadata: { computerOwner: { generation: 2 } } });
} finally { factory.mockRestore(); }
});
it("stops a reusable sandbox and keeps its lease resumable", async () => {
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
const workerManager = {
+17 -5
View File
@@ -4121,11 +4121,23 @@ export function environmentRuntimeService(
providerResourceDisposition === "keep_running" &&
leaseSnapshot.leasePolicy === "reuse_by_environment"
) {
const lease = await environmentsSvc.releaseLease(
leaseRow.id,
"retained",
{ cleanupStatus: "success" },
);
// A concurrent stop or owner replacement outranks this stale warm
// return. Never erase its termination receipt or restore retention.
const now = new Date();
const [retained] = await db.update(environmentLeases).set({
status: "retained", cleanupStatus: "success", releasedAt: null,
lastUsedAt: now, updatedAt: now,
metadata: sql`${environmentLeases.metadata} - 'remoteExecutionTermination'`,
}).where(and(
eq(environmentLeases.id, leaseRow.id),
eq(environmentLeases.companyId, leaseRow.companyId),
eq(environmentLeases.heartbeatRunId, heartbeatRunId),
eq(environmentLeases.status, "active"),
sql`${environmentLeases.environmentId} is not distinct from ${leaseRow.environmentId}`,
sql`${environmentLeases.providerLeaseId} is not distinct from ${leaseRow.providerLeaseId}`,
sql`${environmentLeases.metadata} is not distinct from ${leaseRow.metadata === null ? null : JSON.stringify(leaseRow.metadata)}::jsonb`,
)).returning();
const lease = retained ? toEnvironmentLeaseSnapshot(retained) : null;
if (lease) {
released.push({
environment,
@@ -148,6 +148,43 @@ describe("heartbeat run-control module boundary", () => {
expect(database.access).not.toHaveBeenCalled();
});
it.each(["stop_and_retain", undefined] as const)("closes the exact warm run before stopping its lease: %s", async (disposition) => {
const database = guardedDatabase();
const deps = callbacks(database.db);
deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run);
let finishClose!: () => void;
deps.options.closeWarmNativeSessionsForRun.mockImplementation(() => new Promise(resolve => {
finishClose = () => resolve({ closed: 1, busy: 0, failed: 0 });
}));
const release = createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({
runId: "stopping", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: disposition,
});
await vi.waitFor(() => expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith(expect.objectContaining({ runId: "stopping" })));
expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();
finishClose();
await release;
expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledOnce();
});
it.each(["busy", "failed"] as const)("defers stop-and-retain while exact warm cleanup is %s", async (outcome) => {
const database = guardedDatabase();
const deps = callbacks(database.db);
deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run);
deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: outcome === "busy" ? 1 : 0, failed: outcome === "failed" ? 1 : 0 });
await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "stopping", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "stop_and_retain" });
expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();
expect(database.access).not.toHaveBeenCalled();
});
it("leaves successful keep-running ownership warm", async () => {
const database = guardedDatabase();
const deps = callbacks(database.db);
deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run);
await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "warm", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "keep_running" });
expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled();
expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ providerResourceDisposition: "keep_running" }));
});
it("defers destruction when a warm session remains busy", async () => {
const database = guardedDatabase();
const deps = callbacks(database.db);
+7 -5
View File
@@ -289,24 +289,26 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
input.providerResourceDisposition,
status,
);
if (providerResourceDisposition === "destroy") {
if (providerResourceDisposition !== "keep_running") {
const closeResult = await (
options.closeWarmNativeSessionsForRun ??
closeWarmNativeSessionsForRun
)({
runId: input.runId,
reason: "terminal heartbeat run destroyed its environment lease",
reason: providerResourceDisposition === "destroy"
? "terminal heartbeat run destroyed its environment lease"
: "terminal heartbeat run stopped its environment lease",
}).catch((err) => {
logger.warn(
{ err, runId: input.runId },
"failed to close warm native sessions before environment lease destruction",
"failed to close warm native sessions before environment lease stop",
);
return { closed: 0, busy: 0, failed: 1 };
});
// A failed remote checkpoint cannot veto destruction of the isolated
// sandbox after the run stopped. Provider destruction supplies the exit
// proof; it does not turn the interrupted checkpoint into a success.
const remoteLeases = closeResult.failed > 0 && leaseOwnerRun &&
const remoteLeases = providerResourceDisposition === "destroy" && closeResult.failed > 0 && leaseOwnerRun &&
["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status)
? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId),
@@ -318,7 +320,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) {
logger.warn(
{ runId: input.runId, warmNativeSessions: closeResult },
"deferred environment lease destruction until warm native sessions close",
"deferred environment lease stop until warm native sessions close",
);
return;
}
@@ -7876,6 +7876,116 @@ describe("native warm session supervision", () => {
await closeWarmNativeSessionsForRun({ runId: owners[1]!.runId, reason: "fixture cleanup" });
});
it.each(["provider active", "retaining"])("settles a requested computer Stop before physical retirement when %s", async (when) => {
const name = `computer-stop-${when}`;
const current = { ...execution, binding: { ...execution.binding, runId: name, agentId: name, executionWorkspaceId: name },
session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1;
const order: string[] = [];
const close = vi.fn(async () => { order.push("checkpoint settled"); });
const stop = async () => {
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "stop exact run" })).toMatchObject({ busy: 1, closed: 0 });
expect(close).not.toHaveBeenCalled();
};
const retainWarm = vi.fn(async () => { if (when === "retaining") await stop(); });
const retire = vi.fn(async (options?: { beforeStop?: () => Promise<void> }) => {
await options?.beforeStop?.(); order.push("physical retirement"); return true;
});
const target = { kind: "remote", transport: "computer", environmentId: name, leaseId: name,
remoteCwd: `/home/user/${name}`, listenerPort: 45111,
resourceAuthority: { kind: "computer-owner", computerId: name, ownerId: name, generation: 1 },
fileAuthority: { kind: "remote-persistent", placementId: name, root: `/home/user/${name}`, agentHome: `/home/user/${name}/home` },
retainWarm, retire, inspectProcess: async () => ({ running: true, claim: {} }), launch: async () => ({}),
computerTool: { command: "cua-driver", args: ["mcp"] } };
state.execute.mockReset().mockImplementationOnce(async options => {
await options.onSession?.({ close });
if (when === "provider active") await stop();
return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: name,
normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1",
nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
});
try {
await executePaperclipNativeSession({ db: leaseDb(current, {}, {}, [], {}, "running", [], { provider: "boat", providerLeaseId: name }),
execution: current, runnerInstanceId: name, runnerExecutionTarget: target as never });
expect(order).toEqual(["checkpoint settled", "physical retirement"]);
expect(close).toHaveBeenCalledOnce();
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat stop" })).toEqual({ closed: 0, busy: 0, failed: 0 });
} finally { await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" }); }
});
it.each([false, true])("drains a Stop after warm release before clearing execution scope; close fails=%s", async (fails) => {
const name = `late-warm-stop-${fails}`;
const current = { ...execution, binding: { ...execution.binding, runId: name, executionWorkspaceId: name },
session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1;
const close = vi.fn(async () => { if (fails) throw new Error("checkpoint not settled"); });
state.execute.mockReset().mockImplementationOnce(async options => {
await options.onSession?.({ close });
return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: name,
normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1",
nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
});
let requested = false;
let stopResult: Awaited<ReturnType<typeof closeWarmNativeSessionsForRun>> | undefined;
let closedBeforeScopeExit = false;
try {
const result = await executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name, useRunnerd: true,
onEvent: async event => {
if (event.eventType === "run.performance.span" && event.payload?.span === "task.run.measured" && !requested) {
requested = true;
stopResult = await closeWarmNativeSessionsForRun({ runId: name, reason: "late exact stop" });
closedBeforeScopeExit = close.mock.calls.length > 0;
}
} });
expect(requested).toBe(true);
expect(stopResult).toMatchObject({ busy: 1 });
expect(closedBeforeScopeExit).toBe(false);
expect(result.exitCode).toBe(0);
expect(close).toHaveBeenCalledOnce();
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat exact stop" }))
.toEqual({ closed: 0, busy: 0, failed: fails ? 1 : 0 });
} finally {
close.mockResolvedValue(undefined);
await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" });
}
});
it.each(["failed turn", "environment", "idle timeout"])("keeps a late explicit Stop fenced when an implicit %s close fails", async (kind) => {
const name = `explicit-during-close-${kind}`;
const current = { ...execution, binding: { ...execution.binding, runId: name, executionWorkspaceId: name },
session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1;
let entered!: () => void, rejectClose!: (error: Error) => void;
const started = new Promise<void>(resolve => { entered = resolve; });
const pendingClose = new Promise<void>((_resolve, reject) => { rejectClose = reject; });
const failure = new Error("implicit checkpoint failed");
const close = vi.fn(async () => { entered(); await pendingClose; });
state.execute.mockReset().mockImplementationOnce(async options => {
await options.onSession?.({ close });
return { result: { summary: "settled" }, terminal: { runTerminalState: kind === "failed turn" ? "failed" : "succeeded" }, turnId: name,
normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1",
nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
});
if (kind === "idle timeout") vi.useFakeTimers();
const running = executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name,
runnerExecutionTarget: { kind: "local", environmentId: name } }).catch(error => error);
const closing = kind === "environment"
? running.then(() => closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "implicit environment cleanup" }))
: running;
try {
if (kind === "idle timeout") { await running; await vi.advanceTimersByTimeAsync(60_000); }
await started;
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "explicit Stop during close" })).toMatchObject({ busy: 1 });
rejectClose(failure);
await closing;
if (kind === "idle timeout") await vi.advanceTimersByTimeAsync(0);
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat explicit Stop" })).toEqual({ closed: 0, busy: 0, failed: 1 });
} finally {
rejectClose(failure);
await closing;
close.mockResolvedValue(undefined);
await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" });
if (kind === "idle timeout") vi.useRealTimers();
}
});
it.each(["model", "credential", "tools", "read-only"])("replaces a computer warm runner after %s configuration admits a fresh owner", async (change) => {
const name = `computer-config-change-${change}`;
const current = { ...execution, binding: { ...execution.binding, runId: `${name}-one`, agentId: name, executionWorkspaceId: name },
@@ -8513,12 +8623,12 @@ describe("native warm session supervision", () => {
...execution,
binding: {
...execution.binding,
runId: "run-warm-restart-release",
executionWorkspaceId: "workspace-warm-restart-release",
runId: `run-warm-restart-release-${checkpointFails}`,
executionWorkspaceId: `workspace-warm-restart-release-${checkpointFails}`,
},
session: {
...execution.session,
normalizedSessionId: "session-warm-restart-release",
normalizedSessionId: `session-warm-restart-release-${checkpointFails}`,
lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 },
},
} as NativeExecutionInputV1;
@@ -8544,7 +8654,8 @@ describe("native warm session supervision", () => {
const running = executePaperclipNativeSession({
db: leaseDb(warmExecution),
execution: warmExecution,
runnerInstanceId: "runner-warm-restart-release",
runnerInstanceId: `runner-warm-restart-release-${checkpointFails}`,
useRunnerd: true,
}).then((result) => { settled = true; return result; });
try {
await vi.waitFor(() => expect(close).toHaveBeenCalledExactlyOnceWith({ reason: "controller restart" }));
@@ -8554,7 +8665,12 @@ describe("native warm session supervision", () => {
if (checkpointFails) await expect(running).rejects.toBe(checkpointError);
else await running;
}
await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 0, busy: 0, failed: 0 });
expect(close).toHaveBeenCalledOnce();
await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 0, busy: 0, failed: checkpointFails ? 1 : 0 });
if (checkpointFails) {
close.mockResolvedValue(undefined);
await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toMatchObject({ closed: 1 });
}
});
it("preserves the active turn when a warm checkpoint resumes the same run", async () => {
@@ -483,6 +483,8 @@ type WarmNativeSession = {
environmentId: string | null;
busy: boolean;
closeOnReleaseReason?: string;
closeAfterExecution?: boolean;
explicitCloseRequested?: boolean;
idleTimer: ReturnType<typeof setTimeout> | null;
lastActivityAt: string;
};
@@ -504,6 +506,7 @@ class ComputerOwnerSupersededError extends Error {
}
export async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, preserveInstructionsForRunId?: string) {
entry.closeAfterExecution = false;
let closeFailure: unknown;
let closeCompleted = false;
let closeState: WarmRetirementDiagnostic["closeState"] = "not_started";
@@ -709,7 +712,7 @@ export async function closeWarmNativeSessionsForEnvironment(input: {
/** Cancel one attempt without closing other agents on its computer. */
export async function closeWarmNativeSessionsForRun(input: { runId: string; reason: string }) {
return closeIdleWarmNativeSessions(input);
return closeIdleWarmNativeSessions({ ...input, closeBusyOnRelease: true });
}
/** Suspend idle owners and persist their remote backup before a controller
@@ -737,11 +740,15 @@ async function closeIdleWarmNativeSessions(input: {
if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) {
continue;
}
if (input.closeBusyOnRelease) entry.explicitCloseRequested = true;
if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)) {
// A busy turn can complete while another idle session is checkpointing.
// Fence that entry now so its eventual release cannot leave a new idle
// owner behind after the shutdown sweep has already passed it.
if (input.closeBusyOnRelease) entry.closeOnReleaseReason = input.reason;
if (input.closeBusyOnRelease) {
entry.closeOnReleaseReason = input.reason;
if (executingRunnerdSessionScopes.has(sessionId)) entry.closeAfterExecution = true;
}
busy += 1;
continue;
}
@@ -759,7 +766,8 @@ async function closeIdleWarmNativeSessions(input: {
if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId);
closed += 1;
} catch {
if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
// A later requested Stop must not mistake failed cleanup for no owner.
if (!entry.explicitCloseRequested && !entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
failed += 1;
} finally {
entry.busy = false;
@@ -6362,10 +6370,10 @@ async function releaseWarmNativeSession(
// A late completion or cleanup callback from an older execution must never
// release a replacement that has since claimed the same logical session.
if (!entry || entry.ownerToken !== ownerToken) return;
entry.busy = false;
entry.busy = true;
entry.lastActivityAt = new Date().toISOString();
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
if (failed || entry.closeOnReleaseReason !== undefined) {
const closeRequested = async () => {
const requireCleanClose = entry.closeOnReleaseReason !== undefined;
entry.closeOnReleaseReason ??= "warm native session failed";
entry.busy = true;
@@ -6373,13 +6381,25 @@ async function releaseWarmNativeSession(
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId);
} catch (error) {
if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
if (requireCleanClose) throw error;
// An explicit Stop can arrive while an implicit close is pending.
// Consult the current request, not only the pre-await close reason.
if (!requireCleanClose && !entry.explicitCloseRequested && !entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
if (requireCleanClose || entry.explicitCloseRequested) throw error;
} finally { entry.busy = false; }
};
if (failed || entry.closeOnReleaseReason !== undefined) {
await closeRequested();
return;
}
// Durable shared-computer ownership must be retained before completion is acknowledged.
await entry.computerTarget?.retainWarm(idleTimeoutMs);
try { await entry.computerTarget?.retainWarm(idleTimeoutMs); }
finally { entry.busy = false; }
// Stop may arrive while durable retention is being written. It still owns
// protocol close before any physical retirement, even after a successful turn.
if (entry.closeOnReleaseReason !== undefined) {
await closeRequested();
return;
}
entry.idleTimer = setTimeout(() => {
const current = warmNativeSessions.get(sessionId);
if (current !== entry || current.busy) return;
@@ -6395,7 +6415,7 @@ async function releaseWarmNativeSession(
current.idleTimer = null;
return;
}
if (!current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId);
if (!current.explicitCloseRequested && !current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId);
})
.finally(() => { current.busy = false; });
}, idleTimeoutMs);
@@ -7877,6 +7897,13 @@ export async function executePaperclipNativeSession(input: {
input.execution.binding.runId
) {
executingRunnerdSessionScopes.delete(sessionScopeId);
const entry = warmNativeSessions.get(sessionScopeId);
// A stop arriving after warm release but before this scope clears cannot
// close in parallel with execution. Drain that exact request now, retaining
// failed cleanup without reclassifying the committed provider result.
if (entry?.runId === runId && entry.closeAfterExecution && entry.closeOnReleaseReason !== undefined) {
await closeWarmNativeSessionsForRun({ runId, reason: entry.closeOnReleaseReason });
}
}
try {
await cleanupStagedAttachments();