Files
PaperClipAI/server/src/services/sandbox-creation-operations.ts
T

286 lines
19 KiB
TypeScript

import { collectSecretRefPaths, isUuidSecretRef, parseSecretRefBindingObject, readConfigValueAtPath } from "./json-schema-secret-refs.js";
import { createHash, randomUUID } from "node:crypto";
import { and, eq, lte, or, sql } from "drizzle-orm";
import { currentControllerProcess } from "./controller-process-identity.js";
import { companies, environments, heartbeatRuns, sandboxCreationOperations, type Db } from "@paperclipai/db";
import { readEnvironmentCreationCleanupError, type PluginEnvironmentLease, type PluginEnvironmentLeaseObservation } from "@paperclipai/plugin-sdk";
import { assertEnvironmentCompanyBinding, environmentService } from "./environments.js";
import { ENVIRONMENT_CLEANUP_RETRY_MS, recoverableSandboxCreationCondition } from "./environment-cleanup-state.js";
import { logger } from "../middleware/logger.js";
import { notifyEnvironmentCleanup, writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
import { crashPoint } from "@paperclipai/shared/crash-points";
export type SandboxCreationOperation = typeof sandboxCreationOperations.$inferSelect;
// A live creator may still be waiting for the provider, and a provider may still
// materialize an aborted request. Absence is conclusive only after this delay.
const CREATION_SETTLE_FLOOR_MS = 30 * 60_000;
function creationAbsenceInconclusive(operation: SandboxCreationOperation): boolean {
const settleAt = Date.parse(typeof operation.metadata.creationSettleAt === "string" ? operation.metadata.creationSettleAt : "");
return operation.metadata.creationControllerLost === true && Number.isFinite(settleAt) && Date.now() < settleAt;
}
export function recoverableTaskCreation(operation: Pick<SandboxCreationOperation, "metadata">): boolean {
const metadata = operation.metadata;
return metadata.runnerTaskLease === true && metadata.creationOperationVersion === 1 &&
typeof metadata.runnerTaskCreationContextId === "string" && metadata.runnerTaskCreationContextId.length > 0 &&
metadata.providerMetadata !== null && typeof metadata.providerMetadata === "object" && !Array.isArray(metadata.providerMetadata);
}
/** Plugin allocations can return their exact creation lease after a lost reply. */
export function recoverablePluginCreation(operation: Pick<SandboxCreationOperation, "metadata">): boolean {
return recoverableTaskCreation(operation) || (operation.metadata.driver === "plugin" && operation.metadata.purpose === "login");
}
type CreationOwner = {
companyId: string;
environmentId: string;
executionWorkspaceId?: string | null;
issueId?: string | null;
heartbeatRunId?: string | null;
provider: string;
pluginId?: string;
driverKey: string;
config: Record<string, unknown>;
metadata: Record<string, unknown>;
timeoutMs?: number;
requestKey?: string;
configSchema?: Record<string, unknown> | null;
resolvedConfig?: Record<string, unknown>;
};
export async function createOwnedSandbox(
db: Db, owner: CreationOwner,
create: (identity: { id: string; runId: string }) => Promise<PluginEnvironmentLease>,
): Promise<PluginEnvironmentLease> {
const credentialHashes: Record<string, string> = {};
for (const path of collectSecretRefPaths(owner.configSchema)) {
const value = readConfigValueAtPath(owner.config, path);
if (value != null && value !== "" && !(typeof value === "string" && isUuidSecretRef(value)) && !parseSecretRefBindingObject(value)) {
throw new Error("Sandbox creation requires saved secret references");
}
if (value != null && value !== "") {
const resolved = readConfigValueAtPath(owner.resolvedConfig ?? {}, path);
if (typeof resolved !== "string") throw new Error("Sandbox creation credential identity is unavailable");
credentialHashes[path] = createHash("sha256").update(resolved).digest("hex");
}
}
const id = randomUUID();
const runId = owner.heartbeatRunId ?? randomUUID();
const settleAt = new Date(Date.now() + Math.max(CREATION_SETTLE_FLOOR_MS, (owner.timeoutMs ?? 0) + 60_000));
const creationController = await currentControllerProcess();
const operation = await db.transaction(async tx => {
await notifyEnvironmentCleanup(tx, { companyId: owner.companyId, runId: owner.heartbeatRunId ?? undefined });
await assertEnvironmentCompanyBinding(tx as unknown as Db, owner.environmentId, owner.companyId);
const [inserted] = await tx.insert(sandboxCreationOperations).values({
id, companyId: owner.companyId, environmentId: owner.environmentId, originalEnvironmentId: owner.environmentId,
runId, requestKey: owner.requestKey ?? "run", issueId: owner.issueId, executionWorkspaceId: owner.executionWorkspaceId,
provider: owner.provider, pluginId: owner.pluginId, driverKey: owner.driverKey,
config: owner.config, metadata: { ...owner.metadata, creationHeartbeatRunId: owner.heartbeatRunId ?? null, creationCredentialHashes: credentialHashes,
creationController, creationSettleAt: settleAt.toISOString() },
nextAttemptAt: settleAt,
}).onConflictDoNothing().returning();
if (inserted) return inserted;
const [existing] = await tx.select().from(sandboxCreationOperations).where(and(
eq(sandboxCreationOperations.companyId, owner.companyId), eq(sandboxCreationOperations.originalEnvironmentId, owner.environmentId),
eq(sandboxCreationOperations.runId, runId), eq(sandboxCreationOperations.requestKey, owner.requestKey ?? "run"),
));
if (!existing) throw new Error("Sandbox creation ownership was not recorded");
return existing;
});
if (operation.id !== id) {
if (operation.state !== "attached" || !operation.leaseId) throw new Error("Sandbox creation requires reconciliation before another attempt");
const existing = await environmentService(db).getLeaseById(operation.leaseId);
if (!existing || existing.status !== "active" || !existing.providerLeaseId) throw new Error("Sandbox creation has already transferred ownership");
return { providerLeaseId: existing.providerLeaseId, expiresAt: existing.expiresAt?.toISOString(),
metadata: { ...existing.metadata, creationOperationId: operation.id } };
}
try {
crashPoint("sandbox.create.after_intent_before_provider");
const lease = await create({ id, runId });
crashPoint("sandbox.create.after_provider_before_lease");
return { ...lease, metadata: { ...lease.metadata, creationOperationId: id } };
} catch (error) {
if (recoverablePluginCreation(operation)) {
try {
await writeEnvironmentCleanup(db, tx => tx.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(), updatedAt: new Date() })
.where(and(eq(sandboxCreationOperations.id, id), recoverableSandboxCreationCondition())),
{ companyId: operation.companyId, runId: owner.heartbeatRunId ?? undefined });
} catch {
logger.warn({ errorKind: "task_creation_recovery_wakeup_failed", operationId: id, companyId: operation.companyId },
"Task creation recovery retains its saved deadline");
}
}
throw error;
}
}
/** Keep an already returned resource identity even when task attachment fails.
* This enriches the existing owner; it never transfers or reopens ownership. */
export async function recordSandboxCreationResource(db: Db, input: {
id: string; companyId: string; environmentId: string; providerLeaseId: string;
providerMetadata?: Record<string, unknown>;
}): Promise<void> {
await writeEnvironmentCleanup(db, async tx => {
const [recorded] = await tx.update(sandboxCreationOperations).set({
nextAttemptAt: sql`case when ${sandboxCreationOperations.state} = 'needs_reconciliation' then clock_timestamp() else ${sandboxCreationOperations.nextAttemptAt} end`,
metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || ${JSON.stringify({ creationProviderLeaseId: input.providerLeaseId, ...(input.providerMetadata ? { creationProviderMetadata: input.providerMetadata } : {}) })}::jsonb`,
updatedAt: new Date(),
}).where(and(eq(sandboxCreationOperations.id, input.id), eq(sandboxCreationOperations.companyId, input.companyId),
eq(sandboxCreationOperations.originalEnvironmentId, input.environmentId),
or(sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' is null`,
sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' = ${input.providerLeaseId}`),
)).returning({ id: sandboxCreationOperations.id });
if (!recorded) throw new Error("Sandbox creation resource identity changed");
}, { companyId: input.companyId });
}
export async function queueSandboxCreationCleanup(db: Db, input: {
id: string; companyId: string; environmentId: string; providerLeaseId: string;
providerMetadata?: Record<string, unknown>;
}): Promise<void> {
await db.transaction(async tx => {
await tx.select({ id: environments.id }).from(environments).where(eq(environments.id, input.environmentId)).for("update");
const [operation] = await tx.select().from(sandboxCreationOperations).where(and(eq(sandboxCreationOperations.id, input.id),
eq(sandboxCreationOperations.companyId, input.companyId), eq(sandboxCreationOperations.originalEnvironmentId, input.environmentId))).for("update");
if (!operation || (operation.metadata.creationProviderLeaseId && operation.metadata.creationProviderLeaseId !== input.providerLeaseId)) {
throw new Error("Sandbox creation cleanup ownership changed");
}
if (["attached", "cleanup_pending", "resolved"].includes(operation.state)) return;
const contextId = operation.metadata.runnerTaskLease === true && typeof operation.metadata.runnerTaskCreationContextId === "string"
? operation.metadata.runnerTaskCreationContextId : null;
if (operation.state !== "pending" && !(operation.state === "needs_reconciliation" && contextId &&
operation.metadata.creationProviderLeaseId === input.providerLeaseId)) throw new Error("Sandbox creation cleanup ownership changed");
// The non-null runId also correlates agentless creation attempts. Only the
// original optional heartbeat reference belongs in the lease's foreign key.
// Older task intents predate that explicit field; retain their relationship
// only when an actual same-company heartbeat row still exists.
const heartbeatReference = Object.hasOwn(operation.metadata, "creationHeartbeatRunId")
? operation.metadata.creationHeartbeatRunId : operation.runId;
const [heartbeat] = typeof heartbeatReference === "string"
? await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, heartbeatReference), eq(heartbeatRuns.companyId, operation.companyId),
)).for("key share")
: [];
const heartbeatRunId = heartbeat?.id ?? null;
await notifyEnvironmentCleanup(tx, { companyId: operation.companyId, runId: heartbeatRunId ?? undefined });
const lease = await environmentService(tx as unknown as Db).insertPendingCleanupLease({
companyId: operation.companyId, environmentId: operation.originalEnvironmentId,
heartbeatRunId, issueId: operation.issueId, executionWorkspaceId: operation.executionWorkspaceId,
provider: operation.provider, providerLeaseId: input.providerLeaseId,
metadata: { ...operation.metadata, creationOperationId: operation.id, environmentId: operation.originalEnvironmentId,
providerMetadata: input.providerMetadata ?? operation.metadata.creationProviderMetadata ?? {},
...(operation.metadata.runnerTaskLease === true ? { runnerTaskRetiring: true } : {}),
...(contextId ? { runnerTaskStartupCleanup: { contextId, requiresPreparationStop: false } } : {}) },
failureReason: operation.metadata.purpose === "login" ? "interrupted_login_creation" : "interrupted_task_creation",
});
await tx.update(sandboxCreationOperations).set({ state: "cleanup_pending", leaseId: lease.id, updatedAt: new Date(),
metadata: { ...operation.metadata, creationProviderLeaseId: input.providerLeaseId,
creationProviderMetadata: input.providerMetadata ?? operation.metadata.creationProviderMetadata ?? {} },
}).where(eq(sandboxCreationOperations.id, operation.id));
});
}
export async function recoverSandboxCreations(db: Db,
inspect: (operation: SandboxCreationOperation) => Promise<PluginEnvironmentLeaseObservation | null>,
onTaskRecovered?: (operation: SandboxCreationOperation) => void | Promise<void>,
): Promise<void> {
// After a restart, an intent whose creator is proven gone cannot still be in
// flight there: the shared ownership path makes it due now. Absence stays
// inconclusive until the settle deadline (creationAbsenceInconclusive).
const { settleOwnableKind } = await import("./recovery/ownership-invariant-report.js");
await settleOwnableKind(db, "sandbox_creation_intent").catch(() => {
logger.warn({ errorKind: "sandbox_creation_intent_settle_failed" }, "Sandbox creation intents keep their saved deadlines");
});
const due = await db.select().from(sandboxCreationOperations).where(and(
recoverableSandboxCreationCondition(), lte(sandboxCreationOperations.nextAttemptAt, new Date()),
)).limit(100);
for (const candidate of due) {
try {
if (candidate.metadata.runnerTaskLease === true && typeof candidate.metadata.runnerTaskCreationContextId === "string" &&
typeof candidate.metadata.creationProviderLeaseId === "string") {
await queueSandboxCreationCleanup(db, { id: candidate.id, companyId: candidate.companyId,
environmentId: candidate.originalEnvironmentId, providerLeaseId: candidate.metadata.creationProviderLeaseId });
if (recoverableTaskCreation(candidate)) await onTaskRecovered?.(candidate);
continue;
}
const [existingCompany] = await db.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).limit(1);
const observation = existingCompany ? await inspect(candidate) : null;
if (existingCompany && recoverablePluginCreation(candidate) && observation?.creationLease) {
const lease = observation.creationLease;
// The inspect call is scoped to this creation operation. Providers may
// assign a different resource ID; require the receipt/envelope to agree,
// then let the durable resource record reject any changed identity.
if (typeof lease.providerLeaseId !== "string" || lease.providerLeaseId.length === 0 || lease.providerLeaseId !== observation.providerLeaseId ||
!lease.metadata || typeof lease.metadata !== "object" || Array.isArray(lease.metadata)) {
throw new Error("Plugin allocation creation recovery returned foreign ownership");
}
const identity = { id: candidate.id, companyId: candidate.companyId, environmentId: candidate.originalEnvironmentId,
providerLeaseId: lease.providerLeaseId, providerMetadata: lease.metadata };
await recordSandboxCreationResource(db, identity);
await queueSandboxCreationCleanup(db, identity);
await onTaskRecovered?.(candidate);
continue;
}
if (existingCompany && candidate.metadata.purpose === "login" && observation?.state === "destroyed" && observation.providerLeaseId === candidate.id) {
const settledAt = Date.parse(typeof candidate.metadata.creationSettleAt === "string" ? candidate.metadata.creationSettleAt : "");
if (!Number.isFinite(settledAt) || Date.now() < settledAt) throw new Error("Login allocation creation has not settled yet");
await writeEnvironmentCleanup(db, tx => tx.update(sandboxCreationOperations).set({ state: "resolved", updatedAt: new Date() })
.where(and(eq(sandboxCreationOperations.id, candidate.id), eq(sandboxCreationOperations.state, "pending"))), { companyId: candidate.companyId });
continue;
}
if (existingCompany && recoverablePluginCreation(candidate)) throw new Error("Plugin allocation creation identity is not available yet");
const cleanup = observation?.creationCleanup ? readEnvironmentCreationCleanupError({ data: {
schema: "paperclip/environment-creation-cleanup/v1", cleanup: observation.creationCleanup,
} }) : null;
if (cleanup && (cleanup.companyId !== candidate.companyId || cleanup.environmentId !== candidate.originalEnvironmentId ||
cleanup.runId !== candidate.runId || cleanup.attemptId !== candidate.id || cleanup.providerLeaseId !== observation!.providerLeaseId)) {
throw new Error("Sandbox creation recovery returned foreign ownership");
}
// An early lookup after a lost creator may precede provider materialization.
if (existingCompany && !cleanup && creationAbsenceInconclusive(candidate)) {
throw new Error("Sandbox creation has not settled yet");
}
await db.transaction(async tx => {
const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).for("key share");
await tx.select({ id: environments.id }).from(environments).where(eq(environments.id, candidate.originalEnvironmentId)).for("update");
const [operation] = await tx.select().from(sandboxCreationOperations)
.where(eq(sandboxCreationOperations.id, candidate.id)).for("update");
if (!operation || operation.state !== "pending") return;
await notifyEnvironmentCleanup(tx, { companyId: operation.companyId });
if (!company || !cleanup) {
logger.warn({ errorKind: "sandbox_creation_needs_reconciliation", operationId: operation.id, companyId: operation.companyId },
"Sandbox creation could not be identified; operator reconciliation is required");
await tx.update(sandboxCreationOperations).set({ state: "needs_reconciliation", updatedAt: new Date(),
metadata: { ...operation.metadata, creationReconciliationReason: company ? "creation_identity_unavailable" : "company_deleted" },
})
.where(eq(sandboxCreationOperations.id, operation.id));
return;
}
const lease = await environmentService(tx as unknown as Db).insertPendingCleanupLease({
companyId: operation.companyId, environmentId: operation.originalEnvironmentId,
provider: operation.provider, providerLeaseId: cleanup.providerLeaseId,
metadata: { ...operation.metadata, creationOperationId: operation.id, failedCreateCleanup: cleanup },
failureReason: "interrupted_sandbox_creation",
});
await tx.update(sandboxCreationOperations).set({ state: "cleanup_pending", leaseId: lease.id, updatedAt: new Date() })
.where(eq(sandboxCreationOperations.id, operation.id));
});
} catch {
const retry = await db.transaction(async tx => {
const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).for("key share");
await notifyEnvironmentCleanup(tx, { companyId: candidate.companyId });
await tx.update(sandboxCreationOperations).set(company
? { nextAttemptAt: new Date(Date.now() + ENVIRONMENT_CLEANUP_RETRY_MS), updatedAt: new Date() }
: { state: "needs_reconciliation", updatedAt: new Date(),
metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || '{"creationReconciliationReason":"company_deleted"}'::jsonb` })
.where(and(eq(sandboxCreationOperations.id, candidate.id), recoverableSandboxCreationCondition()));
return Boolean(company);
});
logger.warn({ errorKind: retry ? "sandbox_creation_recovery_failed" : "sandbox_creation_needs_reconciliation", operationId: candidate.id, companyId: candidate.companyId },
retry ? "Sandbox creation recovery will retry at its saved deadline" : "Sandbox creation company was deleted; operator reconciliation is required");
}
}
}