mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-18 05:23:39 +02:00
Checkpoint warm native sessions before stopping their computer owners
Close the exact run for stop-and-retain and default stopping paths, retain failed explicit cleanup fences, and settle Stop requests arriving during or after warm retention. Prevent stale warm returns from overwriting a released or rebound lease. Co-authored-by: Codex <noreply@openai.com> Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
a681637461
commit
46cbfc67a3
6 files changed
+246
-24
No files matched your search
@@ -671,6 +671,34 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
expect(workerManager.call).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["released", "rebound"])("does not publish stale warm retention after the lease was %s", async (change) => {
|
||||
const { runId, reusableLease } = await seedReusablePluginSandboxLease();
|
||||
const realFactory = environmentsModule.environmentService;
|
||||
const receipt = { proof: "exact-owner-stopped" };
|
||||
let changed = false;
|
||||
const factory = vi.spyOn(environmentsModule, "environmentService").mockImplementation(database => {
|
||||
const real = realFactory(database);
|
||||
return { ...real, getById: async (id: string) => {
|
||||
const result = await real.getById(id);
|
||||
if (!changed) {
|
||||
changed = true;
|
||||
if (change === "released") await real.releaseLease(reusableLease.id, "released", { cleanupStatus: "success", remoteExecutionTermination: receipt });
|
||||
else await database.update(environmentLeases).set({ providerLeaseId: "new-owner", metadata: { computerOwner: { ownerId: "new-owner", generation: 2 } } }).where(eq(environmentLeases.id, reusableLease.id));
|
||||
}
|
||||
return result;
|
||||
} };
|
||||
});
|
||||
try {
|
||||
const released = await environmentRuntimeService(db).releaseRunLeases(runId, "released", undefined, "keep_running");
|
||||
expect(released).toEqual([]);
|
||||
const actual = await realFactory(db).getLeaseById(reusableLease.id);
|
||||
if (change === "released") {
|
||||
expect(actual).toMatchObject({ status: "released", metadata: { remoteExecutionTermination: receipt } });
|
||||
expect(actual?.releasedAt).not.toBeNull();
|
||||
} else expect(actual).toMatchObject({ status: "active", providerLeaseId: "new-owner", metadata: { computerOwner: { generation: 2 } } });
|
||||
} finally { factory.mockRestore(); }
|
||||
});
|
||||
|
||||
it("stops a reusable sandbox and keeps its lease resumable", async () => {
|
||||
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
|
||||
const workerManager = {
|
||||
|
||||
@@ -4121,11 +4121,23 @@ export function environmentRuntimeService(
|
||||
providerResourceDisposition === "keep_running" &&
|
||||
leaseSnapshot.leasePolicy === "reuse_by_environment"
|
||||
) {
|
||||
const lease = await environmentsSvc.releaseLease(
|
||||
leaseRow.id,
|
||||
"retained",
|
||||
{ cleanupStatus: "success" },
|
||||
);
|
||||
// A concurrent stop or owner replacement outranks this stale warm
|
||||
// return. Never erase its termination receipt or restore retention.
|
||||
const now = new Date();
|
||||
const [retained] = await db.update(environmentLeases).set({
|
||||
status: "retained", cleanupStatus: "success", releasedAt: null,
|
||||
lastUsedAt: now, updatedAt: now,
|
||||
metadata: sql`${environmentLeases.metadata} - 'remoteExecutionTermination'`,
|
||||
}).where(and(
|
||||
eq(environmentLeases.id, leaseRow.id),
|
||||
eq(environmentLeases.companyId, leaseRow.companyId),
|
||||
eq(environmentLeases.heartbeatRunId, heartbeatRunId),
|
||||
eq(environmentLeases.status, "active"),
|
||||
sql`${environmentLeases.environmentId} is not distinct from ${leaseRow.environmentId}`,
|
||||
sql`${environmentLeases.providerLeaseId} is not distinct from ${leaseRow.providerLeaseId}`,
|
||||
sql`${environmentLeases.metadata} is not distinct from ${leaseRow.metadata === null ? null : JSON.stringify(leaseRow.metadata)}::jsonb`,
|
||||
)).returning();
|
||||
const lease = retained ? toEnvironmentLeaseSnapshot(retained) : null;
|
||||
if (lease) {
|
||||
released.push({
|
||||
environment,
|
||||
|
||||
@@ -148,6 +148,43 @@ describe("heartbeat run-control module boundary", () => {
|
||||
expect(database.access).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["stop_and_retain", undefined] as const)("closes the exact warm run before stopping its lease: %s", async (disposition) => {
|
||||
const database = guardedDatabase();
|
||||
const deps = callbacks(database.db);
|
||||
deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run);
|
||||
let finishClose!: () => void;
|
||||
deps.options.closeWarmNativeSessionsForRun.mockImplementation(() => new Promise(resolve => {
|
||||
finishClose = () => resolve({ closed: 1, busy: 0, failed: 0 });
|
||||
}));
|
||||
const release = createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({
|
||||
runId: "stopping", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: disposition,
|
||||
});
|
||||
await vi.waitFor(() => expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith(expect.objectContaining({ runId: "stopping" })));
|
||||
expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();
|
||||
finishClose();
|
||||
await release;
|
||||
expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each(["busy", "failed"] as const)("defers stop-and-retain while exact warm cleanup is %s", async (outcome) => {
|
||||
const database = guardedDatabase();
|
||||
const deps = callbacks(database.db);
|
||||
deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run);
|
||||
deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: outcome === "busy" ? 1 : 0, failed: outcome === "failed" ? 1 : 0 });
|
||||
await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "stopping", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "stop_and_retain" });
|
||||
expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();
|
||||
expect(database.access).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves successful keep-running ownership warm", async () => {
|
||||
const database = guardedDatabase();
|
||||
const deps = callbacks(database.db);
|
||||
deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run);
|
||||
await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "warm", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "keep_running" });
|
||||
expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled();
|
||||
expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ providerResourceDisposition: "keep_running" }));
|
||||
});
|
||||
|
||||
it("defers destruction when a warm session remains busy", async () => {
|
||||
const database = guardedDatabase();
|
||||
const deps = callbacks(database.db);
|
||||
|
||||
@@ -289,24 +289,26 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
input.providerResourceDisposition,
|
||||
status,
|
||||
);
|
||||
if (providerResourceDisposition === "destroy") {
|
||||
if (providerResourceDisposition !== "keep_running") {
|
||||
const closeResult = await (
|
||||
options.closeWarmNativeSessionsForRun ??
|
||||
closeWarmNativeSessionsForRun
|
||||
)({
|
||||
runId: input.runId,
|
||||
reason: "terminal heartbeat run destroyed its environment lease",
|
||||
reason: providerResourceDisposition === "destroy"
|
||||
? "terminal heartbeat run destroyed its environment lease"
|
||||
: "terminal heartbeat run stopped its environment lease",
|
||||
}).catch((err) => {
|
||||
logger.warn(
|
||||
{ err, runId: input.runId },
|
||||
"failed to close warm native sessions before environment lease destruction",
|
||||
"failed to close warm native sessions before environment lease stop",
|
||||
);
|
||||
return { closed: 0, busy: 0, failed: 1 };
|
||||
});
|
||||
// A failed remote checkpoint cannot veto destruction of the isolated
|
||||
// sandbox after the run stopped. Provider destruction supplies the exit
|
||||
// proof; it does not turn the interrupted checkpoint into a success.
|
||||
const remoteLeases = closeResult.failed > 0 && leaseOwnerRun &&
|
||||
const remoteLeases = providerResourceDisposition === "destroy" && closeResult.failed > 0 && leaseOwnerRun &&
|
||||
["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status)
|
||||
? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, input.companyId),
|
||||
@@ -318,7 +320,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) {
|
||||
logger.warn(
|
||||
{ runId: input.runId, warmNativeSessions: closeResult },
|
||||
"deferred environment lease destruction until warm native sessions close",
|
||||
"deferred environment lease stop until warm native sessions close",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -7876,6 +7876,116 @@ describe("native warm session supervision", () => {
|
||||
await closeWarmNativeSessionsForRun({ runId: owners[1]!.runId, reason: "fixture cleanup" });
|
||||
});
|
||||
|
||||
it.each(["provider active", "retaining"])("settles a requested computer Stop before physical retirement when %s", async (when) => {
|
||||
const name = `computer-stop-${when}`;
|
||||
const current = { ...execution, binding: { ...execution.binding, runId: name, agentId: name, executionWorkspaceId: name },
|
||||
session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1;
|
||||
const order: string[] = [];
|
||||
const close = vi.fn(async () => { order.push("checkpoint settled"); });
|
||||
const stop = async () => {
|
||||
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "stop exact run" })).toMatchObject({ busy: 1, closed: 0 });
|
||||
expect(close).not.toHaveBeenCalled();
|
||||
};
|
||||
const retainWarm = vi.fn(async () => { if (when === "retaining") await stop(); });
|
||||
const retire = vi.fn(async (options?: { beforeStop?: () => Promise<void> }) => {
|
||||
await options?.beforeStop?.(); order.push("physical retirement"); return true;
|
||||
});
|
||||
const target = { kind: "remote", transport: "computer", environmentId: name, leaseId: name,
|
||||
remoteCwd: `/home/user/${name}`, listenerPort: 45111,
|
||||
resourceAuthority: { kind: "computer-owner", computerId: name, ownerId: name, generation: 1 },
|
||||
fileAuthority: { kind: "remote-persistent", placementId: name, root: `/home/user/${name}`, agentHome: `/home/user/${name}/home` },
|
||||
retainWarm, retire, inspectProcess: async () => ({ running: true, claim: {} }), launch: async () => ({}),
|
||||
computerTool: { command: "cua-driver", args: ["mcp"] } };
|
||||
state.execute.mockReset().mockImplementationOnce(async options => {
|
||||
await options.onSession?.({ close });
|
||||
if (when === "provider active") await stop();
|
||||
return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: name,
|
||||
normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1",
|
||||
nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
|
||||
});
|
||||
try {
|
||||
await executePaperclipNativeSession({ db: leaseDb(current, {}, {}, [], {}, "running", [], { provider: "boat", providerLeaseId: name }),
|
||||
execution: current, runnerInstanceId: name, runnerExecutionTarget: target as never });
|
||||
expect(order).toEqual(["checkpoint settled", "physical retirement"]);
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat stop" })).toEqual({ closed: 0, busy: 0, failed: 0 });
|
||||
} finally { await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" }); }
|
||||
});
|
||||
|
||||
it.each([false, true])("drains a Stop after warm release before clearing execution scope; close fails=%s", async (fails) => {
|
||||
const name = `late-warm-stop-${fails}`;
|
||||
const current = { ...execution, binding: { ...execution.binding, runId: name, executionWorkspaceId: name },
|
||||
session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1;
|
||||
const close = vi.fn(async () => { if (fails) throw new Error("checkpoint not settled"); });
|
||||
state.execute.mockReset().mockImplementationOnce(async options => {
|
||||
await options.onSession?.({ close });
|
||||
return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: name,
|
||||
normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1",
|
||||
nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
|
||||
});
|
||||
let requested = false;
|
||||
let stopResult: Awaited<ReturnType<typeof closeWarmNativeSessionsForRun>> | undefined;
|
||||
let closedBeforeScopeExit = false;
|
||||
try {
|
||||
const result = await executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name, useRunnerd: true,
|
||||
onEvent: async event => {
|
||||
if (event.eventType === "run.performance.span" && event.payload?.span === "task.run.measured" && !requested) {
|
||||
requested = true;
|
||||
stopResult = await closeWarmNativeSessionsForRun({ runId: name, reason: "late exact stop" });
|
||||
closedBeforeScopeExit = close.mock.calls.length > 0;
|
||||
}
|
||||
} });
|
||||
expect(requested).toBe(true);
|
||||
expect(stopResult).toMatchObject({ busy: 1 });
|
||||
expect(closedBeforeScopeExit).toBe(false);
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat exact stop" }))
|
||||
.toEqual({ closed: 0, busy: 0, failed: fails ? 1 : 0 });
|
||||
} finally {
|
||||
close.mockResolvedValue(undefined);
|
||||
await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["failed turn", "environment", "idle timeout"])("keeps a late explicit Stop fenced when an implicit %s close fails", async (kind) => {
|
||||
const name = `explicit-during-close-${kind}`;
|
||||
const current = { ...execution, binding: { ...execution.binding, runId: name, executionWorkspaceId: name },
|
||||
session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1;
|
||||
let entered!: () => void, rejectClose!: (error: Error) => void;
|
||||
const started = new Promise<void>(resolve => { entered = resolve; });
|
||||
const pendingClose = new Promise<void>((_resolve, reject) => { rejectClose = reject; });
|
||||
const failure = new Error("implicit checkpoint failed");
|
||||
const close = vi.fn(async () => { entered(); await pendingClose; });
|
||||
state.execute.mockReset().mockImplementationOnce(async options => {
|
||||
await options.onSession?.({ close });
|
||||
return { result: { summary: "settled" }, terminal: { runTerminalState: kind === "failed turn" ? "failed" : "succeeded" }, turnId: name,
|
||||
normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1",
|
||||
nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
|
||||
});
|
||||
if (kind === "idle timeout") vi.useFakeTimers();
|
||||
const running = executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name,
|
||||
runnerExecutionTarget: { kind: "local", environmentId: name } }).catch(error => error);
|
||||
const closing = kind === "environment"
|
||||
? running.then(() => closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "implicit environment cleanup" }))
|
||||
: running;
|
||||
try {
|
||||
if (kind === "idle timeout") { await running; await vi.advanceTimersByTimeAsync(60_000); }
|
||||
await started;
|
||||
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "explicit Stop during close" })).toMatchObject({ busy: 1 });
|
||||
rejectClose(failure);
|
||||
await closing;
|
||||
if (kind === "idle timeout") await vi.advanceTimersByTimeAsync(0);
|
||||
expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat explicit Stop" })).toEqual({ closed: 0, busy: 0, failed: 1 });
|
||||
} finally {
|
||||
rejectClose(failure);
|
||||
await closing;
|
||||
close.mockResolvedValue(undefined);
|
||||
await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" });
|
||||
if (kind === "idle timeout") vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["model", "credential", "tools", "read-only"])("replaces a computer warm runner after %s configuration admits a fresh owner", async (change) => {
|
||||
const name = `computer-config-change-${change}`;
|
||||
const current = { ...execution, binding: { ...execution.binding, runId: `${name}-one`, agentId: name, executionWorkspaceId: name },
|
||||
@@ -8513,12 +8623,12 @@ describe("native warm session supervision", () => {
|
||||
...execution,
|
||||
binding: {
|
||||
...execution.binding,
|
||||
runId: "run-warm-restart-release",
|
||||
executionWorkspaceId: "workspace-warm-restart-release",
|
||||
runId: `run-warm-restart-release-${checkpointFails}`,
|
||||
executionWorkspaceId: `workspace-warm-restart-release-${checkpointFails}`,
|
||||
},
|
||||
session: {
|
||||
...execution.session,
|
||||
normalizedSessionId: "session-warm-restart-release",
|
||||
normalizedSessionId: `session-warm-restart-release-${checkpointFails}`,
|
||||
lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 },
|
||||
},
|
||||
} as NativeExecutionInputV1;
|
||||
@@ -8544,7 +8654,8 @@ describe("native warm session supervision", () => {
|
||||
const running = executePaperclipNativeSession({
|
||||
db: leaseDb(warmExecution),
|
||||
execution: warmExecution,
|
||||
runnerInstanceId: "runner-warm-restart-release",
|
||||
runnerInstanceId: `runner-warm-restart-release-${checkpointFails}`,
|
||||
useRunnerd: true,
|
||||
}).then((result) => { settled = true; return result; });
|
||||
try {
|
||||
await vi.waitFor(() => expect(close).toHaveBeenCalledExactlyOnceWith({ reason: "controller restart" }));
|
||||
@@ -8554,7 +8665,12 @@ describe("native warm session supervision", () => {
|
||||
if (checkpointFails) await expect(running).rejects.toBe(checkpointError);
|
||||
else await running;
|
||||
}
|
||||
await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 0, busy: 0, failed: 0 });
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 0, busy: 0, failed: checkpointFails ? 1 : 0 });
|
||||
if (checkpointFails) {
|
||||
close.mockResolvedValue(undefined);
|
||||
await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toMatchObject({ closed: 1 });
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves the active turn when a warm checkpoint resumes the same run", async () => {
|
||||
|
||||
@@ -483,6 +483,8 @@ type WarmNativeSession = {
|
||||
environmentId: string | null;
|
||||
busy: boolean;
|
||||
closeOnReleaseReason?: string;
|
||||
closeAfterExecution?: boolean;
|
||||
explicitCloseRequested?: boolean;
|
||||
idleTimer: ReturnType<typeof setTimeout> | null;
|
||||
lastActivityAt: string;
|
||||
};
|
||||
@@ -504,6 +506,7 @@ class ComputerOwnerSupersededError extends Error {
|
||||
}
|
||||
|
||||
export async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, preserveInstructionsForRunId?: string) {
|
||||
entry.closeAfterExecution = false;
|
||||
let closeFailure: unknown;
|
||||
let closeCompleted = false;
|
||||
let closeState: WarmRetirementDiagnostic["closeState"] = "not_started";
|
||||
@@ -709,7 +712,7 @@ export async function closeWarmNativeSessionsForEnvironment(input: {
|
||||
|
||||
/** Cancel one attempt without closing other agents on its computer. */
|
||||
export async function closeWarmNativeSessionsForRun(input: { runId: string; reason: string }) {
|
||||
return closeIdleWarmNativeSessions(input);
|
||||
return closeIdleWarmNativeSessions({ ...input, closeBusyOnRelease: true });
|
||||
}
|
||||
|
||||
/** Suspend idle owners and persist their remote backup before a controller
|
||||
@@ -737,11 +740,15 @@ async function closeIdleWarmNativeSessions(input: {
|
||||
if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) {
|
||||
continue;
|
||||
}
|
||||
if (input.closeBusyOnRelease) entry.explicitCloseRequested = true;
|
||||
if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)) {
|
||||
// A busy turn can complete while another idle session is checkpointing.
|
||||
// Fence that entry now so its eventual release cannot leave a new idle
|
||||
// owner behind after the shutdown sweep has already passed it.
|
||||
if (input.closeBusyOnRelease) entry.closeOnReleaseReason = input.reason;
|
||||
if (input.closeBusyOnRelease) {
|
||||
entry.closeOnReleaseReason = input.reason;
|
||||
if (executingRunnerdSessionScopes.has(sessionId)) entry.closeAfterExecution = true;
|
||||
}
|
||||
busy += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -759,7 +766,8 @@ async function closeIdleWarmNativeSessions(input: {
|
||||
if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId);
|
||||
closed += 1;
|
||||
} catch {
|
||||
if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
|
||||
// A later requested Stop must not mistake failed cleanup for no owner.
|
||||
if (!entry.explicitCloseRequested && !entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
|
||||
failed += 1;
|
||||
} finally {
|
||||
entry.busy = false;
|
||||
@@ -6362,10 +6370,10 @@ async function releaseWarmNativeSession(
|
||||
// A late completion or cleanup callback from an older execution must never
|
||||
// release a replacement that has since claimed the same logical session.
|
||||
if (!entry || entry.ownerToken !== ownerToken) return;
|
||||
entry.busy = false;
|
||||
entry.busy = true;
|
||||
entry.lastActivityAt = new Date().toISOString();
|
||||
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
|
||||
if (failed || entry.closeOnReleaseReason !== undefined) {
|
||||
const closeRequested = async () => {
|
||||
const requireCleanClose = entry.closeOnReleaseReason !== undefined;
|
||||
entry.closeOnReleaseReason ??= "warm native session failed";
|
||||
entry.busy = true;
|
||||
@@ -6373,13 +6381,25 @@ async function releaseWarmNativeSession(
|
||||
await closeWarmNativeSession(entry, entry.closeOnReleaseReason);
|
||||
if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId);
|
||||
} catch (error) {
|
||||
if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
|
||||
if (requireCleanClose) throw error;
|
||||
// An explicit Stop can arrive while an implicit close is pending.
|
||||
// Consult the current request, not only the pre-await close reason.
|
||||
if (!requireCleanClose && !entry.explicitCloseRequested && !entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId);
|
||||
if (requireCleanClose || entry.explicitCloseRequested) throw error;
|
||||
} finally { entry.busy = false; }
|
||||
};
|
||||
if (failed || entry.closeOnReleaseReason !== undefined) {
|
||||
await closeRequested();
|
||||
return;
|
||||
}
|
||||
// Durable shared-computer ownership must be retained before completion is acknowledged.
|
||||
await entry.computerTarget?.retainWarm(idleTimeoutMs);
|
||||
try { await entry.computerTarget?.retainWarm(idleTimeoutMs); }
|
||||
finally { entry.busy = false; }
|
||||
// Stop may arrive while durable retention is being written. It still owns
|
||||
// protocol close before any physical retirement, even after a successful turn.
|
||||
if (entry.closeOnReleaseReason !== undefined) {
|
||||
await closeRequested();
|
||||
return;
|
||||
}
|
||||
entry.idleTimer = setTimeout(() => {
|
||||
const current = warmNativeSessions.get(sessionId);
|
||||
if (current !== entry || current.busy) return;
|
||||
@@ -6395,7 +6415,7 @@ async function releaseWarmNativeSession(
|
||||
current.idleTimer = null;
|
||||
return;
|
||||
}
|
||||
if (!current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId);
|
||||
if (!current.explicitCloseRequested && !current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId);
|
||||
})
|
||||
.finally(() => { current.busy = false; });
|
||||
}, idleTimeoutMs);
|
||||
@@ -7877,6 +7897,13 @@ export async function executePaperclipNativeSession(input: {
|
||||
input.execution.binding.runId
|
||||
) {
|
||||
executingRunnerdSessionScopes.delete(sessionScopeId);
|
||||
const entry = warmNativeSessions.get(sessionScopeId);
|
||||
// A stop arriving after warm release but before this scope clears cannot
|
||||
// close in parallel with execution. Drain that exact request now, retaining
|
||||
// failed cleanup without reclassifying the committed provider result.
|
||||
if (entry?.runId === runId && entry.closeAfterExecution && entry.closeOnReleaseReason !== undefined) {
|
||||
await closeWarmNativeSessionsForRun({ runId, reason: entry.closeOnReleaseReason });
|
||||
}
|
||||
}
|
||||
try {
|
||||
await cleanupStagedAttachments();
|
||||
|
||||
Reference in new issue
Block a user