mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-24 18:19:05 +02:00
fix: require stop-only cleanup for every retained sandbox
Persist generic stop intent before provider dispatch, use the exact pinned StopLease capability for release and restart recovery, and retain saved allocations when capability or stop confirmation is unavailable. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
d884e1ab04
commit
ea10ee8994
5 files changed
+199
-27
No files matched your search
@@ -483,14 +483,14 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
if (method === "environmentResumeLease") return {
|
||||
providerLeaseId: warmProviderLeaseId, metadata: { remoteCwd: "/workspace" },
|
||||
};
|
||||
if (method === "environmentDestroyLease" || method === "environmentReleaseLease") return {
|
||||
if (method === "environmentDestroyLease" || method === "environmentStopLease" || method === "environmentReleaseLease") return {
|
||||
providerLeaseId: input.providerLeaseId, state: method === "environmentDestroyLease" ? "destroyed" : "stopped",
|
||||
};
|
||||
throw new Error(`Unexpected lifecycle method: ${method}`);
|
||||
});
|
||||
const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
|
||||
isRunning: () => true, call,
|
||||
getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }),
|
||||
getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }),
|
||||
} as unknown as PluginWorkerManager });
|
||||
const input = {
|
||||
companyId: seeded.companyId, agentId: seeded.agentId, issueId,
|
||||
@@ -531,11 +531,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
});
|
||||
const workerManager = {
|
||||
isRunning: () => true, call,
|
||||
getWorker: () => ({ supportedMethods: ["environmentReleaseLease"] }),
|
||||
getWorker: () => ({ supportedMethods: ["environmentStopLease"] }),
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
||||
await runtimeWithPlugin.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true);
|
||||
expect(call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.objectContaining({
|
||||
expect(call).toHaveBeenCalledWith(pluginId, "environmentStopLease", expect.objectContaining({
|
||||
companyId, providerLeaseId: reusableLease.providerLeaseId, cancelActiveWork: true,
|
||||
}), expect.any(Number));
|
||||
expect(call).toHaveBeenCalledOnce();
|
||||
@@ -544,6 +544,66 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["release_only", "stopped", "stop_failed", "unconfirmed", "restart", "competing_owner", "missing_pin", "late_receipt"])(
|
||||
"never dispatches destructive release for an untagged stop-and-retain request: %s", async outcome => {
|
||||
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
||||
const lease = seeded.reusableLease;
|
||||
await db.update(environmentLeases).set({ leasePolicy: "ephemeral", metadata: {
|
||||
...lease.metadata, reuseLease: false, ...(outcome === "missing_pin" ? { pluginId: undefined } : {}),
|
||||
} }).where(eq(environmentLeases.id, lease.id));
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
let supported = !["release_only", "restart"].includes(outcome);
|
||||
let calls = 0;
|
||||
const call = vi.fn(async (_id: string, method: string) => {
|
||||
if (method !== "environmentStopLease") return { providerLeaseId: lease.providerLeaseId, state: "destroyed" };
|
||||
if (++calls === 1 && outcome === "stop_failed") throw new Error("injected stop failure");
|
||||
if (calls === 1 && outcome === "unconfirmed") return undefined;
|
||||
if (outcome === "late_receipt") await db.update(environmentLeases).set({
|
||||
status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true },
|
||||
}).where(eq(environmentLeases.id, lease.id));
|
||||
return { providerLeaseId: lease.providerLeaseId, state: "stopped" };
|
||||
});
|
||||
const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: {
|
||||
isRunning: () => true, call,
|
||||
getWorker: () => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", ...(supported ? ["environmentStopLease"] : [])] }),
|
||||
} as unknown as PluginWorkerManager });
|
||||
if (outcome === "competing_owner") await environmentService(db).acquireLease({
|
||||
companyId: seeded.companyId, environmentId: seeded.environment.id, heartbeatRunId: null,
|
||||
leasePolicy: "ephemeral", provider: lease.provider, providerLeaseId: lease.providerLeaseId,
|
||||
});
|
||||
await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain");
|
||||
expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true);
|
||||
if (outcome === "late_receipt") {
|
||||
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "active", heartbeatRunId: null, metadata: { newOwner: true } });
|
||||
return;
|
||||
}
|
||||
if (outcome === "missing_pin") {
|
||||
expect(call).not.toHaveBeenCalled();
|
||||
await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
||||
expect(call).not.toHaveBeenCalled();
|
||||
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup" });
|
||||
return;
|
||||
}
|
||||
if (["release_only", "restart", "competing_owner"].includes(outcome)) {
|
||||
expect(call).not.toHaveBeenCalled();
|
||||
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup", metadata: {
|
||||
sandboxStopAndRetain: { pluginId: seeded.pluginId, runId: seeded.runId, providerLeaseId: lease.providerLeaseId },
|
||||
} });
|
||||
await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
||||
expect(call).not.toHaveBeenCalled();
|
||||
if (outcome === "competing_owner") return;
|
||||
supported = true;
|
||||
}
|
||||
if (["release_only", "stop_failed", "unconfirmed", "restart"].includes(outcome)) {
|
||||
await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
||||
}
|
||||
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: null,
|
||||
metadata: { remoteExecutionTermination: { state: "stopped", providerLeaseId: lease.providerLeaseId },
|
||||
sandboxStopAndRetainReceipt: { method: "environmentStopLease", runId: seeded.runId, pluginId: seeded.pluginId } } });
|
||||
expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("reports an unwired manager separately from a stopped sandbox worker", async () => {
|
||||
const { companyId, environment, runId } = await seedReusablePluginSandboxLease();
|
||||
const input = {
|
||||
@@ -609,13 +669,14 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
const workerManager = {
|
||||
isRunning: vi.fn((id: string) => id === pluginId),
|
||||
call: vi.fn(async (_pluginId: string, method: string) => {
|
||||
if (method === "environmentReleaseLease") return undefined;
|
||||
if (method === "environmentStopLease") return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" };
|
||||
throw new Error(`Unexpected plugin method while stopping lease: ${method}`);
|
||||
}),
|
||||
getWorker: vi.fn(() => ({
|
||||
supportedMethods: [
|
||||
"environmentResumeLease",
|
||||
"environmentReleaseLease",
|
||||
"environmentStopLease",
|
||||
"environmentDestroyLease",
|
||||
],
|
||||
})),
|
||||
@@ -635,7 +696,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
});
|
||||
expect(workerManager.call).toHaveBeenCalledWith(
|
||||
pluginId,
|
||||
"environmentReleaseLease",
|
||||
"environmentStopLease",
|
||||
expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }),
|
||||
expect.any(Number),
|
||||
);
|
||||
@@ -663,12 +724,12 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
if (method === "environmentResumeLease") return {
|
||||
providerLeaseId: "projectless-1", metadata: { remoteCwd: "/workspace" },
|
||||
};
|
||||
if (method === "environmentReleaseLease") return undefined;
|
||||
if (method === "environmentStopLease") return { providerLeaseId: "projectless-1", state: "stopped" };
|
||||
throw new Error(`Unexpected projectless lease method: ${method}`);
|
||||
});
|
||||
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: {
|
||||
isRunning: () => true, call,
|
||||
getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }),
|
||||
getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }),
|
||||
} as unknown as PluginWorkerManager });
|
||||
const first = await runtimeWithPlugin.acquireRunLease({
|
||||
companyId: seeded.companyId, environment: seeded.environment, issueId,
|
||||
@@ -841,7 +902,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
},
|
||||
};
|
||||
}
|
||||
if (method === "environmentReleaseLease") return undefined;
|
||||
if (method === "environmentStopLease") return { providerLeaseId: "sandbox-exact-resume", state: "stopped" };
|
||||
if (method === "environmentResumeLease") {
|
||||
return {
|
||||
providerLeaseId: "sandbox-exact-resume",
|
||||
@@ -860,6 +921,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
supportedMethods: [
|
||||
"environmentResumeLease",
|
||||
"environmentReleaseLease",
|
||||
"environmentStopLease",
|
||||
"environmentDestroyLease",
|
||||
],
|
||||
})),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, settleStopOnlyCleanup, stopOnlyCleanupKey } from "./sandbox-stop-and-retain.js";
|
||||
import { readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
|
||||
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { hasNativeWorkspaceExportResume, readNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js";
|
||||
import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -2573,8 +2574,8 @@ function createSandboxEnvironmentDriver(
|
||||
},
|
||||
|
||||
async retryPendingSandboxTeardown(input) {
|
||||
const exportResume = hasNativeWorkspaceExportResume(input.lease);
|
||||
const resumeIntent = readNativeWorkspaceExportResume(input.lease);
|
||||
const exportResume = hasStopOnlyCleanup(input.lease);
|
||||
const resumeIntent = readStopOnlyCleanup(input.lease);
|
||||
const assertExportResumeOwnership = async () => {
|
||||
if (!resumeIntent) throw new Error("Workspace export resume ownership is invalid.");
|
||||
const [current] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, input.lease.id)).limit(1);
|
||||
@@ -2584,8 +2585,8 @@ function createSandboxEnvironmentDriver(
|
||||
)).limit(1);
|
||||
if (!current || current.status !== "pending_cleanup" || current.companyId !== input.lease.companyId
|
||||
|| current.metadata?.pendingCleanupAttemptId !== input.lease.metadata?.pendingCleanupAttemptId
|
||||
|| readNativeWorkspaceExportResume(current)?.requestId !== resumeIntent.requestId
|
||||
|| readNativeWorkspaceExportResume(current)?.pluginId !== resumeIntent.pluginId || otherOwner) {
|
||||
|| readStopOnlyCleanup(current)?.requestId !== resumeIntent.requestId
|
||||
|| readStopOnlyCleanup(current)?.pluginId !== resumeIntent.pluginId || otherOwner) {
|
||||
throw new Error("Workspace export resume ownership changed before cleanup.");
|
||||
}
|
||||
};
|
||||
@@ -2648,12 +2649,12 @@ function createSandboxEnvironmentDriver(
|
||||
throw new Error("Workspace export recovery requires verified stop-only cleanup.");
|
||||
}
|
||||
await assertExportResumeOwnership();
|
||||
const receipt = await pluginWorkerManager.call(pluginId, "environmentStopLease", {
|
||||
const receipt = await runLeaseReleaseWithRunParent(input.lease.id, () => pluginWorkerManager.call(pluginId, "environmentStopLease", {
|
||||
driverKey: recordedProvider, companyId: input.lease.companyId,
|
||||
environmentId: input.lease.environmentId ?? "", issueId: input.lease.issueId,
|
||||
config: workerConfig, providerLeaseId: input.lease.providerLeaseId,
|
||||
leaseMetadata: input.lease.metadata ?? {}, cancelActiveWork: true, resourceDisposition: "stop_and_retain",
|
||||
}, Math.min(resolvePluginSandboxRpcTimeoutMs(workerConfig) ?? 60_000, 60_000));
|
||||
}, Math.min(resolvePluginSandboxRpcTimeoutMs(workerConfig) ?? 60_000, 60_000)));
|
||||
if (remoteTerminationReceipt(input.lease, receipt)?.state !== "stopped") {
|
||||
throw new Error("Workspace export recovery did not confirm the retained sandbox stopped.");
|
||||
}
|
||||
@@ -2722,7 +2723,7 @@ function createSandboxEnvironmentDriver(
|
||||
// teardown runs, throws its own "no worker manager" error, and counts
|
||||
// toward the cap.
|
||||
if (!pluginWorkerManager) return true;
|
||||
if (hasNativeWorkspaceExportResume(input.lease)) {
|
||||
if (hasStopOnlyCleanup(input.lease)) {
|
||||
const pinnedPluginId = readString(input.lease.metadata?.pluginId);
|
||||
if (!pinnedPluginId) return true; // Permanent invalid intent, never a by-key fallback.
|
||||
const pinned = await resolvePluginSandboxProviderDriverById({ db, pluginId: pinnedPluginId, driverKey: recordedProvider });
|
||||
@@ -3215,6 +3216,9 @@ function createSandboxEnvironmentDriver(
|
||||
async function releasePluginBackedSandboxLease(
|
||||
input: EnvironmentDriverReleaseInput,
|
||||
): Promise<EnvironmentLease | null> {
|
||||
if (input.resourceDisposition === "stop_and_retain") {
|
||||
throw new Error("Stop-only sandbox cleanup requires a durable stop intent.");
|
||||
}
|
||||
const metadata = input.lease.metadata ?? {};
|
||||
const pluginId = readString(metadata.pluginId);
|
||||
const providerKey = readString(metadata.provider);
|
||||
@@ -3243,7 +3247,6 @@ function createSandboxEnvironmentDriver(
|
||||
providerLeaseId: input.lease.providerLeaseId,
|
||||
leaseMetadata: metadata,
|
||||
...(input.cancelActiveWork ? { cancelActiveWork: true } : {}),
|
||||
...(input.resourceDisposition ? { resourceDisposition: input.resourceDisposition } : {}),
|
||||
}, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig))),
|
||||
);
|
||||
termination = remoteTerminationReceipt(input.lease, receipt);
|
||||
@@ -3928,7 +3931,7 @@ export function environmentRuntimeService(
|
||||
and(
|
||||
eq(environmentLeases.heartbeatRunId, heartbeatRunId),
|
||||
or(eq(environmentLeases.status, "active"), and(eq(environmentLeases.status, "pending_cleanup"),
|
||||
sql`${environmentLeases.metadata} ? 'nativeWorkspaceExportResume'`)),
|
||||
sql`(${environmentLeases.metadata} ? 'nativeWorkspaceExportResume' or ${environmentLeases.metadata} ? 'sandboxStopAndRetain')`)),
|
||||
),
|
||||
);
|
||||
if (leaseRows.length === 0) {
|
||||
@@ -3950,7 +3953,14 @@ export function environmentRuntimeService(
|
||||
const completed = await releaseCompletedNativeWorkspaceExportRetention(db, leaseSnapshot);
|
||||
if (completed) leaseSnapshot = toEnvironmentLeaseSnapshot(completed);
|
||||
}
|
||||
if (hasNativeWorkspaceExportResume(leaseSnapshot)) {
|
||||
if (providerResourceDisposition === "stop_and_retain" && !hasStopOnlyCleanup(leaseSnapshot)
|
||||
&& getLeaseDriverKey(leaseSnapshot, environment) === "sandbox") {
|
||||
const prepared = await prepareSandboxStopAndRetain(db, leaseSnapshot);
|
||||
if (!prepared) continue;
|
||||
leaseSnapshot = toEnvironmentLeaseSnapshot(prepared);
|
||||
}
|
||||
if (hasStopOnlyCleanup(leaseSnapshot)) {
|
||||
const intentKey = stopOnlyCleanupKey(leaseSnapshot);
|
||||
const attemptId = randomUUID();
|
||||
const [claimed] = await db.update(environmentLeases).set({
|
||||
status: "pending_cleanup", cleanupStatus: "failed", releasedAt: new Date(),
|
||||
@@ -3959,7 +3969,7 @@ export function environmentRuntimeService(
|
||||
}).where(and(eq(environmentLeases.id, leaseRow.id), eq(environmentLeases.heartbeatRunId, heartbeatRunId),
|
||||
inArray(environmentLeases.status, ["active", "pending_cleanup"]),
|
||||
sql`coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') = 'false'`,
|
||||
sql`${environmentLeases.metadata}->'nativeWorkspaceExportResume' = ${JSON.stringify(leaseRow.metadata?.nativeWorkspaceExportResume)}::jsonb`,
|
||||
sql`${environmentLeases.metadata}->${intentKey} = ${JSON.stringify(leaseSnapshot.metadata?.[intentKey])}::jsonb`,
|
||||
)).returning();
|
||||
if (!claimed) continue;
|
||||
const snapshot = toEnvironmentLeaseSnapshot(claimed);
|
||||
@@ -3967,11 +3977,11 @@ export function environmentRuntimeService(
|
||||
const driver = requireDriverKey(getLeaseDriverKey(snapshot, environment));
|
||||
if (!driver.retryPendingSandboxTeardown) throw new Error("Workspace export recovery requires stop-only cleanup.");
|
||||
const receipt = await driver.retryPendingSandboxTeardown({ environment, lease: snapshot });
|
||||
const lease = await settleNativeWorkspaceExportResume(db, snapshot, { attemptId, receipt });
|
||||
const lease = await settleStopOnlyCleanup(db, snapshot, { attemptId, receipt });
|
||||
if (lease && environment) released.push({ environment, lease: toEnvironmentLeaseSnapshot(lease),
|
||||
leaseContext: { executionWorkspaceId: lease.executionWorkspaceId, executionWorkspaceMode: null } });
|
||||
} catch (error) {
|
||||
await settleNativeWorkspaceExportResume(db, snapshot, { attemptId });
|
||||
await settleStopOnlyCleanup(db, snapshot, { attemptId });
|
||||
throw error;
|
||||
}
|
||||
continue;
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
NativeWorkspaceFinalizationOwnershipLostError,
|
||||
type NativeWorkspaceFinalizationOwnership,
|
||||
} from "./native-runtime/native-workspace-finalization-ownership.js";
|
||||
import { hasNativeWorkspaceExportResume, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js";
|
||||
import { hasStopOnlyCleanup, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js";
|
||||
import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result";
|
||||
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
|
||||
import { externalConversationStateSql, nonIdleSlackIssueCondition } from "./slack-conversation-state.js";
|
||||
@@ -18584,7 +18584,7 @@ export function heartbeatService(
|
||||
// An interrupted export resume also uses recorded cleanup, but its tagged
|
||||
// intent permits verified stop-and-retain only, never sandbox destruction.
|
||||
const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral";
|
||||
const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasNativeWorkspaceExportResume(lease);
|
||||
const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease);
|
||||
|
||||
// Do not consume a finite cleanup attempt while the provider plugin is
|
||||
// briefly unavailable. A plugin worker restart, a plugin reload, or a
|
||||
@@ -18653,8 +18653,8 @@ export function heartbeatService(
|
||||
environment,
|
||||
lease,
|
||||
});
|
||||
const released = hasNativeWorkspaceExportResume(lease)
|
||||
? await settleNativeWorkspaceExportResume(db, lease, { attemptId: claimed, receipt })
|
||||
const released = hasStopOnlyCleanup(lease)
|
||||
? await settleStopOnlyCleanup(db, lease, { attemptId: claimed, receipt })
|
||||
: await environmentsSvc.releaseLease(lease.id, "expired", {
|
||||
expectedPendingCleanupAttemptId: claimed,
|
||||
cleanupStatus: "success",
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql, type SQL } from "drizzle-orm";
|
||||
import { environmentLeases, type Db } from "@paperclipai/db";
|
||||
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { hasNativeWorkspaceExportResume, readNativeWorkspaceExportResume, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js";
|
||||
|
||||
type Lease = Pick<typeof environmentLeases.$inferSelect,
|
||||
"id" | "companyId" | "heartbeatRunId" | "provider" | "providerLeaseId" | "metadata">;
|
||||
export const SANDBOX_STOP_AND_RETAIN_KEY = "sandboxStopAndRetain";
|
||||
|
||||
export function hasStopOnlyCleanup(lease: Pick<Lease, "metadata">): boolean {
|
||||
return hasNativeWorkspaceExportResume(lease)
|
||||
|| Object.prototype.hasOwnProperty.call(lease.metadata ?? {}, SANDBOX_STOP_AND_RETAIN_KEY);
|
||||
}
|
||||
|
||||
export function stopOnlyCleanupKey(lease: Pick<Lease, "metadata">) {
|
||||
return hasNativeWorkspaceExportResume(lease) ? "nativeWorkspaceExportResume" : SANDBOX_STOP_AND_RETAIN_KEY;
|
||||
}
|
||||
|
||||
export function readStopOnlyCleanup(lease: Lease) {
|
||||
if (hasNativeWorkspaceExportResume(lease)) return readNativeWorkspaceExportResume(lease);
|
||||
const value = lease.metadata?.[SANDBOX_STOP_AND_RETAIN_KEY];
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const intent = value as Record<string, unknown>;
|
||||
if (intent.schema !== "paperclip.sandbox-stop-and-retain.v1"
|
||||
|| intent.companyId !== lease.companyId || intent.runId !== lease.heartbeatRunId || !lease.heartbeatRunId
|
||||
|| intent.leaseId !== lease.id || intent.provider !== lease.provider || !lease.provider
|
||||
|| intent.providerLeaseId !== lease.providerLeaseId || !lease.providerLeaseId
|
||||
|| typeof intent.requestId !== "string" || !intent.requestId
|
||||
|| typeof intent.pluginId !== "string" || !intent.pluginId || intent.pluginId !== lease.metadata?.pluginId) return null;
|
||||
return { ...intent, requestId: intent.requestId, pluginId: intent.pluginId };
|
||||
}
|
||||
|
||||
/** Persist before provider dispatch so an older worker or a restart cannot turn
|
||||
* an explicit stop into ordinary destructive release. Invalid pins stay pending. */
|
||||
export async function prepareSandboxStopAndRetain(db: Db, lease: Lease) {
|
||||
const requestId = randomUUID(), now = new Date();
|
||||
const intent = { schema: "paperclip.sandbox-stop-and-retain.v1", requestId,
|
||||
companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id,
|
||||
provider: lease.provider, providerLeaseId: lease.providerLeaseId, pluginId: lease.metadata?.pluginId };
|
||||
const [updated] = await db.update(environmentLeases).set({
|
||||
status: "pending_cleanup", cleanupStatus: "failed", failureReason: "sandbox_stop_pending", releasedAt: now, updatedAt: now,
|
||||
metadata: sql`(coalesce(${environmentLeases.metadata}, '{}'::jsonb) - 'remoteExecutionTermination' - 'sandboxStopAndRetainReceipt') || ${JSON.stringify({
|
||||
[SANDBOX_STOP_AND_RETAIN_KEY]: intent, pendingCleanupAttemptId: requestId,
|
||||
pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0,
|
||||
pendingCleanupRetryAfterMs: 0, pendingCleanupRetryAttempts: 0, pendingCleanupRetryCapWarned: false,
|
||||
})}::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId),
|
||||
eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), eq(environmentLeases.status, "active"),
|
||||
sql`${environmentLeases.provider} is not distinct from ${lease.provider}`,
|
||||
sql`${environmentLeases.providerLeaseId} is not distinct from ${lease.providerLeaseId}`,
|
||||
sql`${environmentLeases.metadata} is not distinct from ${lease.metadata === null ? null : JSON.stringify(lease.metadata)}::jsonb`,
|
||||
)).returning();
|
||||
return updated ?? null;
|
||||
}
|
||||
|
||||
export async function settleStopOnlyCleanup(db: Db, lease: Lease, options: { attemptId: string; receipt?: unknown }) {
|
||||
if (hasNativeWorkspaceExportResume(lease)) return settleNativeWorkspaceExportResume(db, lease, options);
|
||||
const intent = readStopOnlyCleanup(lease);
|
||||
if (!intent) return null;
|
||||
const receipt = remoteTerminationReceipt(lease, options.receipt), stopped = receipt?.state === "stopped";
|
||||
const now = new Date();
|
||||
const removeIntent: SQL = stopped ? sql`- 'sandboxStopAndRetain'` : sql``;
|
||||
const [updated] = await db.update(environmentLeases).set({
|
||||
status: stopped ? "released" : "pending_cleanup", cleanupStatus: stopped ? "success" : "failed",
|
||||
failureReason: stopped ? null : "sandbox_stop_pending",
|
||||
releasedAt: now, lastUsedAt: now, updatedAt: now,
|
||||
metadata: sql`(${environmentLeases.metadata} - 'remoteExecutionTermination' ${removeIntent}) || ${JSON.stringify({
|
||||
...(stopped ? { remoteExecutionTermination: receipt, sandboxStopAndRetainReceipt: {
|
||||
schema: "paperclip.sandbox-stop-and-retain-receipt.v1", requestId: intent.requestId,
|
||||
companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id,
|
||||
provider: lease.provider, providerLeaseId: lease.providerLeaseId, pluginId: intent.pluginId,
|
||||
method: "environmentStopLease", confirmedAt: receipt.confirmedAt,
|
||||
} } : {}),
|
||||
pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0,
|
||||
})}::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId),
|
||||
eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), eq(environmentLeases.provider, lease.provider!),
|
||||
eq(environmentLeases.providerLeaseId, lease.providerLeaseId!), eq(environmentLeases.status, "pending_cleanup"),
|
||||
sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${options.attemptId}`,
|
||||
sql`${environmentLeases.metadata}->'sandboxStopAndRetain'->>'requestId' = ${intent.requestId}`,
|
||||
sql`${environmentLeases.metadata}->>'pluginId' = ${intent.pluginId}`,
|
||||
)).returning();
|
||||
return updated ?? null;
|
||||
}
|
||||
Reference in new issue
Block a user