mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-02 08:28:48 +01:00
Merge origin/master and retain both Cloud skill fixture contracts
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
commit
7db456e235
58 files changed
+2225
-1922
No files matched your search
@@ -23,7 +23,8 @@ export function assertHarnessTestPassed(result: AdapterEnvironmentTestResult) {
|
||||
const incomplete = !verified && (result.status === "warn" || result.checks.some(check => check.code.includes("hello_probe")));
|
||||
if (result.status === "fail" || authenticationFailed || incomplete) {
|
||||
const has = (suffix: string) => result.checks.some(check => check.code.endsWith(suffix));
|
||||
const code = authenticationFailed ? "harness_auth_failed"
|
||||
const code = result.checks.some(check => check.level === "error" && check.retryable === true) ? "test_environment_unavailable"
|
||||
: authenticationFailed ? "harness_auth_failed"
|
||||
: has("_hello_probe_model_unavailable") || has("_model_invalid") ? "harness_model_unavailable"
|
||||
: has("_command_unresolvable") ? "harness_command_unavailable"
|
||||
: has("_cwd_invalid") ? "harness_cwd_invalid"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { environmentLeases, type Db } from "@paperclipai/db";
|
||||
import { environmentLeases, sandboxCreationOperations, type Db } from "@paperclipai/db";
|
||||
import { hasDeferredEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
|
||||
export const ENVIRONMENT_CLEANUP_RETRY_MS = 5 * 60_000;
|
||||
@@ -20,7 +20,10 @@ export async function nextEnvironmentCleanupAt(db: Db, backoffMs = ENVIRONMENT_C
|
||||
const [row] = await db.select({ next: sql<string | null>`min(greatest(
|
||||
extract(epoch from ${environmentLeases.updatedAt}) * 1000 + ${backoffMs},
|
||||
${pendingCleanupDeadlineSql()}))` }).from(environmentLeases).where(eq(environmentLeases.status, "pending_cleanup"));
|
||||
const saved = row?.next == null ? null : Number(row.next);
|
||||
const [creation] = await db.select({ next: sql<string | null>`min(extract(epoch from ${sandboxCreationOperations.nextAttemptAt}) * 1000)` })
|
||||
.from(sandboxCreationOperations).where(eq(sandboxCreationOperations.state, "pending"));
|
||||
const deadlines = [row?.next, creation?.next].filter((value): value is string => value != null).map(Number);
|
||||
const saved = deadlines.length ? Math.min(...deadlines) : null;
|
||||
const buffered = hasDeferredEnvironmentCleanup(db) ? Date.now() + backoffMs : null;
|
||||
return saved === null ? buffered : buffered === null ? saved : Math.min(saved, buffered);
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
import { HttpError } from "../errors.js";
|
||||
import { captureEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js";
|
||||
import { createOwnedSandbox, recoverSandboxCreations } from "./sandbox-creation-operations.js";
|
||||
import { dueRunLeaseReleaseCondition, pendingRunLeaseReleaseCondition } from "./environment-lease-release-state.js";
|
||||
import { flushDeferredEnvironmentCleanup, registerDeferredEnvironmentCleanup, writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
import { executeEnvironmentTask, pluginEnvironmentSupportsTasks, settleDispatchedEnvironmentTask } from "./environment-task-runtime.js";
|
||||
@@ -11,7 +14,6 @@ import type { NativeWorkspaceFinalizationOwnership } from "./native-runtime/nati
|
||||
import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, settleStopOnlyCleanup, stopOnlyCleanupKey } from "./sandbox-stop-and-retain.js";
|
||||
import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
|
||||
import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js";
|
||||
import { captureEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js";
|
||||
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
@@ -93,7 +95,7 @@ import {
|
||||
resolvePluginExecuteRpcTimeoutMs,
|
||||
resumePluginEnvironmentLease,
|
||||
} from "./plugin-environment-driver.js";
|
||||
import { collectSecretRefPaths } from "./json-schema-secret-refs.js";
|
||||
import { collectSecretRefPaths, readConfigValueAtPath } from "./json-schema-secret-refs.js";
|
||||
import { buildWorkspaceRealizationRecordFromDriverInput } from "./workspace-realization.js";
|
||||
import {
|
||||
createSandboxOrphanCleanupSpool,
|
||||
@@ -698,39 +700,6 @@ export interface EnvironmentRuntimeDriver {
|
||||
flushDeferredOrphanCleanups?(): Promise<{ recovered: number; pending: number }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The acquire provisioned a remote sandbox, the lease insert rejected it, the
|
||||
* compensating teardown failed, and every retry of the durable pending-cleanup
|
||||
* write also failed. A live sandbox now has no lease row and no cleanup record,
|
||||
* so no sweep can find it. The acquire throws this error so the failure stays visible
|
||||
* and never resolves to a clean rejection. The `cause` field holds the original
|
||||
* lease insert rejection. The `cleanupWriteError` field holds the write failure.
|
||||
*/
|
||||
export class SandboxOrphanCleanupWriteError extends Error {
|
||||
readonly provider: string;
|
||||
readonly providerLeaseId: string | null;
|
||||
readonly cleanupWriteError: unknown;
|
||||
|
||||
constructor(input: {
|
||||
provider: string;
|
||||
providerLeaseId: string | null;
|
||||
cause: unknown;
|
||||
cleanupWriteError: unknown;
|
||||
}) {
|
||||
super(
|
||||
`Sandbox provider "${input.provider}" leaked lease ` +
|
||||
`"${input.providerLeaseId ?? "unknown"}": the acquire could not tear it down ` +
|
||||
"and could not record a durable pending-cleanup row. The live sandbox has no " +
|
||||
"durable cleanup state and needs a manual teardown.",
|
||||
{ cause: input.cause },
|
||||
);
|
||||
this.name = "SandboxOrphanCleanupWriteError";
|
||||
this.provider = input.provider;
|
||||
this.providerLeaseId = input.providerLeaseId;
|
||||
this.cleanupWriteError = input.cleanupWriteError;
|
||||
}
|
||||
}
|
||||
|
||||
/** A reusable sandbox could not be resumed, but has not been proven lost. */
|
||||
export class ReusableSandboxResumeError extends Error {
|
||||
readonly provider: string;
|
||||
@@ -785,17 +754,6 @@ export type ProviderResourceDisposition =
|
||||
const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS = 5_000;
|
||||
const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS = 100;
|
||||
|
||||
// The durable pending-cleanup write is the only automated way to find a leaked
|
||||
// sandbox. A single insert can lose to a transient database fault (a dropped
|
||||
// connection, a lock timeout, a serialization conflict). So the acquire retries
|
||||
// the write a few times with a short backoff before it gives up. A later
|
||||
// attempt can still land the durable row that a sweep finds.
|
||||
const DEFAULT_SANDBOX_ORPHAN_CLEANUP_WRITE_ATTEMPTS = 3;
|
||||
const DEFAULT_SANDBOX_ORPHAN_CLEANUP_WRITE_BACKOFF_MS = 100;
|
||||
|
||||
// A prolonged database outage after a failed teardown can buffer many orphan
|
||||
// records in-process. The bound stops that buffer from growing without limit. A
|
||||
// full buffer keeps the error log as the last durable handle for a new orphan.
|
||||
const DEFAULT_DEFERRED_ORPHAN_CLEANUP_BUFFER_LIMIT = 256;
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
@@ -1311,8 +1269,6 @@ function createSandboxEnvironmentDriver(
|
||||
pluginWorkerManager?: PluginWorkerManager;
|
||||
pluginWorkerReadyTimeoutMs?: number;
|
||||
pluginWorkerReadyPollMs?: number;
|
||||
pendingCleanupWriteAttempts?: number;
|
||||
pendingCleanupWriteBackoffMs?: number;
|
||||
deferredOrphanCleanupBufferLimit?: number;
|
||||
orphanCleanupSpool?: SandboxOrphanCleanupSpool;
|
||||
orphanCleanupSpoolDir?: string;
|
||||
@@ -1321,16 +1277,6 @@ function createSandboxEnvironmentDriver(
|
||||
const pluginWorkerManager = options.pluginWorkerManager;
|
||||
const pluginWorkerReadyTimeoutMs = options.pluginWorkerReadyTimeoutMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS;
|
||||
const pluginWorkerReadyPollMs = options.pluginWorkerReadyPollMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS;
|
||||
// The retry count is at least one attempt, so a zero or negative override
|
||||
// never skips the durable write. The backoff is at least zero milliseconds.
|
||||
const pendingCleanupWriteAttempts = Math.max(
|
||||
1,
|
||||
options.pendingCleanupWriteAttempts ?? DEFAULT_SANDBOX_ORPHAN_CLEANUP_WRITE_ATTEMPTS,
|
||||
);
|
||||
const pendingCleanupWriteBackoffMs = Math.max(
|
||||
0,
|
||||
options.pendingCleanupWriteBackoffMs ?? DEFAULT_SANDBOX_ORPHAN_CLEANUP_WRITE_BACKOFF_MS,
|
||||
);
|
||||
// The buffer holds at least one orphan, so a zero or negative override never
|
||||
// drops every buffered record.
|
||||
const deferredOrphanCleanupBufferLimit = Math.max(
|
||||
@@ -1339,15 +1285,6 @@ function createSandboxEnvironmentDriver(
|
||||
);
|
||||
const environmentsSvc = environmentService(db);
|
||||
|
||||
// A live sandbox whose teardown failed needs a durable `pending_cleanup` row,
|
||||
// so a sweep can find and release it. When every synchronous write attempt
|
||||
// fails, the database is down but the process still runs. So the driver keeps
|
||||
// the orphan record in this in-process buffer, and a later cleanup sweep
|
||||
// flushes it back to the database. The buffer closes the common window where
|
||||
// the database recovers after the synchronous attempts but while the process
|
||||
// still runs. The `pending_cleanup` failure reason matches the synchronous
|
||||
// write, so a flushed row and a synchronous row are indistinguishable to the
|
||||
// sweep.
|
||||
const deferredOrphanCleanups: DeferredOrphanCleanupRecord[] = [];
|
||||
// Keep the token through queue splices, failed flushes and buffer overflow.
|
||||
// Release only when the database durably owns cleanup of this exact orphan.
|
||||
@@ -1363,16 +1300,6 @@ function createSandboxEnvironmentDriver(
|
||||
};
|
||||
const DEFERRED_ORPHAN_CLEANUP_FAILURE_REASON = "acquire_rejected_teardown_failed";
|
||||
|
||||
// The in-process buffer alone loses an orphan on a restart. So the driver also
|
||||
// persists each buffered orphan to this durable local spool. The spool keeps
|
||||
// the record on the local disk, next to the other durable server state, so a
|
||||
// restart still finds it. The first flush after a restart loads the spool back
|
||||
// into the buffer, so the sweep re-inserts the durable row. The spool holds no
|
||||
// secret value: it persists the same sanitized metadata the database row
|
||||
// holds, so a restart-recovered record still authorizes the teardown but never
|
||||
// exposes a credential. Only a crash during the outage before the durable
|
||||
// spool write lands loses the record; the error log stays the last handle for
|
||||
// that narrow case.
|
||||
const orphanCleanupSpool =
|
||||
options.orphanCleanupSpool ?? createSandboxOrphanCleanupSpool(options.orphanCleanupSpoolDir);
|
||||
|
||||
@@ -1460,143 +1387,6 @@ function createSandboxEnvironmentDriver(
|
||||
return orphanFlush;
|
||||
};
|
||||
|
||||
// Build the safe diagnostic fields for an orphan record. The provider
|
||||
// identifiers are the only fields any diagnostic log emits, and they carry no
|
||||
// secret. A caught write exception never enters a log: it can carry a
|
||||
// credential in its name, code, message, cause, or stack.
|
||||
const orphanDiagnosticFields = (record: DeferredOrphanCleanupRecord) => ({
|
||||
errorKind: SANDBOX_ORPHAN_CLEANUP_WRITE_ERROR_KIND,
|
||||
provider: record.provider,
|
||||
providerLeaseId: record.providerLeaseId,
|
||||
companyId: record.companyId,
|
||||
environmentId: record.environmentId,
|
||||
executionWorkspaceId: record.executionWorkspaceId,
|
||||
issueId: record.issueId,
|
||||
heartbeatRunId: record.heartbeatRunId,
|
||||
});
|
||||
|
||||
// Try to write the durable `pending_cleanup` row for an orphan sandbox, with a
|
||||
// few retries. Report the new lease id on success, or a null lease id and the
|
||||
// last write error on total failure. This function never buffers, never logs
|
||||
// at error level, and never throws. The caller decides whether a total failure
|
||||
// is fatal, because a later successful teardown removes the orphan and needs
|
||||
// no durable row.
|
||||
//
|
||||
// The row carries the provider lease id and the same resolution metadata a
|
||||
// normal lease holds, so a later cleanup sweep finds and releases the orphan.
|
||||
// One atomic insert writes the row directly in the terminal `pending_cleanup`
|
||||
// state, and the insert skips the company-binding assertion, so it records the
|
||||
// orphan even for a foreign-bound environment. A transient database fault (a
|
||||
// dropped connection, a lock timeout, a serialization conflict) can reject one
|
||||
// insert but clear on the next, so the write retries a few times with a short
|
||||
// backoff.
|
||||
const tryWriteDurablePendingCleanup = async (
|
||||
record: DeferredOrphanCleanupRecord,
|
||||
): Promise<{ leaseId: string | null; lastError: unknown }> => {
|
||||
const diagnosticFields = orphanDiagnosticFields(record);
|
||||
let lastCleanupWriteError: unknown;
|
||||
for (let attempt = 1; attempt <= pendingCleanupWriteAttempts; attempt += 1) {
|
||||
try {
|
||||
const lease = await environmentsSvc.insertPendingCleanupLease({
|
||||
companyId: record.companyId,
|
||||
environmentId: record.environmentId,
|
||||
executionWorkspaceId: record.executionWorkspaceId,
|
||||
issueId: record.issueId,
|
||||
heartbeatRunId: record.heartbeatRunId,
|
||||
provider: record.provider,
|
||||
providerLeaseId: record.providerLeaseId,
|
||||
metadata: record.metadata,
|
||||
failureReason: DEFERRED_ORPHAN_CLEANUP_FAILURE_REASON,
|
||||
});
|
||||
// The durable row exists now, so a sweep can find and release the orphan.
|
||||
return { leaseId: lease.id, lastError: undefined };
|
||||
} catch (cleanupWriteError) {
|
||||
lastCleanupWriteError = cleanupWriteError;
|
||||
if (attempt < pendingCleanupWriteAttempts) {
|
||||
// The write failed, but attempts remain. Log the retry at warn level
|
||||
// and back off, then try again. A later attempt can still land the
|
||||
// durable row.
|
||||
logger.warn(
|
||||
{ ...diagnosticFields, attempt, maxAttempts: pendingCleanupWriteAttempts },
|
||||
"sandbox orphan cleanup write failed; retrying the durable pending-cleanup write",
|
||||
);
|
||||
await delay(pendingCleanupWriteBackoffMs * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { leaseId: null, lastError: lastCleanupWriteError };
|
||||
};
|
||||
|
||||
// Release the durable `pending_cleanup` row after a successful inline teardown.
|
||||
// The teardown removed the orphan sandbox, so the row is no longer needed. The
|
||||
// release moves the row to the terminal `expired` state with a `success`
|
||||
// cleanup status, the same state the sweep records for a successful teardown.
|
||||
//
|
||||
// The release is best-effort. If it fails, the row stays `pending_cleanup`, so
|
||||
// a later sweep runs the idempotent teardown on the already-gone sandbox and
|
||||
// releases the row itself. A failed release never loses or leaks the orphan.
|
||||
const releaseCleanedUpOrphanRow = async (
|
||||
leaseId: string,
|
||||
diagnosticFields: Record<string, unknown>,
|
||||
receipt: unknown,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const lease = await environmentsSvc.getLeaseById(leaseId);
|
||||
await environmentsSvc.releaseLease(leaseId, "expired", {
|
||||
...(lease ? { remoteExecutionTermination: remoteTerminationReceipt(lease, receipt) } : {}),
|
||||
cleanupStatus: "success",
|
||||
failureReason: "acquire_rejected_teardown_succeeded",
|
||||
});
|
||||
} catch {
|
||||
// The caught release exception never enters a log: it can carry a
|
||||
// credential in its name, code, message, cause, or stack.
|
||||
logger.warn(
|
||||
diagnosticFields,
|
||||
"could not release the pending-cleanup row after a successful orphan teardown; a later sweep reconciles the released row",
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Neither the durable write nor the inline teardown worked, so the orphan
|
||||
// sandbox is live with no durable row. Persist the orphan to the durable spool
|
||||
// and buffer it in-process, so a later cleanup-sweep flush lands the durable
|
||||
// row. The spool survives a restart, so the flush recovers the orphan even
|
||||
// after a crash. Then log the plain provider identifiers at error level, so an
|
||||
// operator keeps a handle to tear the sandbox down by hand. Only a crash before
|
||||
// the spool write lands loses the record, so the error log is the last handle
|
||||
// for that narrow case. Throw `SandboxOrphanCleanupWriteError`, which the
|
||||
// caller propagates. The error keeps the original insert rejection as its
|
||||
// `cause`, so the real reason the acquire failed stays visible in the error
|
||||
// chain.
|
||||
const escalateUnwrittenOrphan = async (
|
||||
record: DeferredOrphanCleanupRecord,
|
||||
cause: unknown,
|
||||
cleanupWriteError: unknown,
|
||||
): Promise<never> => {
|
||||
trackOrphanIdleWork(record);
|
||||
const persisted = await orphanCleanupSpool.append(record);
|
||||
const buffered = enqueueDeferredOrphanCleanup(record);
|
||||
logger.error(
|
||||
{
|
||||
...orphanDiagnosticFields(record),
|
||||
persisted,
|
||||
buffered,
|
||||
deferredBufferSize: deferredOrphanCleanups.length,
|
||||
},
|
||||
persisted
|
||||
? "sandbox orphan cleanup write failed; persisted the orphan to the durable spool for a later cleanup-sweep flush"
|
||||
: buffered
|
||||
? "sandbox orphan cleanup write failed and the durable spool write failed; buffered the orphan in-process only for a later cleanup-sweep flush"
|
||||
: "sandbox orphan cleanup write failed and neither the durable spool nor the in-process buffer kept the orphan; live sandbox needs a manual teardown",
|
||||
);
|
||||
throw new SandboxOrphanCleanupWriteError({
|
||||
provider: record.provider,
|
||||
providerLeaseId: record.providerLeaseId,
|
||||
cause,
|
||||
cleanupWriteError,
|
||||
});
|
||||
};
|
||||
|
||||
// Resolve an uncertain creation to a verified provider ID, journal it, then
|
||||
// authorize deletion. This makes a later missing-ID response conclusive even
|
||||
// when the destructive RPC reply or the final lease update was lost.
|
||||
@@ -1628,68 +1418,7 @@ function createSandboxEnvironmentDriver(
|
||||
}
|
||||
};
|
||||
|
||||
// Clean up an orphan after a rejected insert or uncertain provider creation.
|
||||
// The acquire provisioned a remote sandbox, then the insert rejected (a
|
||||
// foreign-company binding), so no lease row tracks the live sandbox. This
|
||||
// handler records the durable `pending_cleanup` row FIRST, before the inline
|
||||
// teardown, then tears the sandbox down.
|
||||
//
|
||||
// The write goes first on purpose. The conditional insert just returned a
|
||||
// definitive rejection, so the database is reachable at this instant. An
|
||||
// inline teardown can run long, and the database can drop during it. So a
|
||||
// write after the teardown can fail and leave the orphan only in memory, which
|
||||
// a restart loses. With the write first, the durable row lands while the
|
||||
// database is proven up, so a crash after it still leaves a row that a sweep
|
||||
// finds and releases.
|
||||
//
|
||||
// The teardown then runs. On success the orphan is gone, so the handler
|
||||
// releases the durable row. On a failed teardown the row stays for the sweep.
|
||||
// If the durable write itself failed and the teardown also failed, the handler
|
||||
// buffers the orphan and throws `SandboxOrphanCleanupWriteError`. A successful
|
||||
// teardown after a failed write leaves no orphan, so it needs no error.
|
||||
const cleanUpRejectedOrphanSandbox = async (input: {
|
||||
record: DeferredOrphanCleanupRecord;
|
||||
cause: unknown;
|
||||
canTeardown: boolean;
|
||||
teardown: (pendingLeaseId: string | null) => Promise<unknown>;
|
||||
}): Promise<boolean> => {
|
||||
const durable = await tryWriteDurablePendingCleanup(input.record);
|
||||
let teardownFailed = !input.canTeardown;
|
||||
let receipt: unknown;
|
||||
if (!teardownFailed) {
|
||||
try {
|
||||
receipt = await input.teardown(durable.leaseId);
|
||||
} catch {
|
||||
teardownFailed = true;
|
||||
}
|
||||
}
|
||||
if (!teardownFailed) {
|
||||
// The teardown removed the orphan, so drop the durable row if we wrote one.
|
||||
if (durable.leaseId !== null) {
|
||||
await releaseCleanedUpOrphanRow(durable.leaseId, orphanDiagnosticFields(input.record), receipt);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
// The teardown failed, so the orphan sandbox is still live.
|
||||
if (durable.leaseId !== null) {
|
||||
// The durable row already tracks the orphan, so a sweep finds and releases
|
||||
// it. The caller rethrows the original insert rejection.
|
||||
return false;
|
||||
}
|
||||
return await escalateUnwrittenOrphan(input.record, input.cause, durable.lastError);
|
||||
};
|
||||
|
||||
// The run-time exec parent context, held per lease id. A plugin sandbox
|
||||
// provider can open a persistent session on the first command and delete it
|
||||
// on lease release. The session open runs inside `execute`, under the run
|
||||
// parent (the same context the `sandbox.exec` span reads). The session delete
|
||||
// runs inside the lease-release RPC, which the run orchestrator calls after
|
||||
// the run, outside that scope. Without a parent the host mints no
|
||||
// `traceparent` and drops the provider `session.teardown` span. So `execute`
|
||||
// records the exec parent here, and the release paths replay it around the
|
||||
// release RPC. The host still mints and validates the `traceparent` itself;
|
||||
// the value only widens which host calls carry a run parent. An entry is
|
||||
// removed on release, so the map holds at most one context per live lease.
|
||||
const runExecParentByLeaseId = new Map<string, StartupSpanContext>();
|
||||
|
||||
// Run a lease-release RPC under the lease's recorded exec parent context, and
|
||||
@@ -2128,12 +1857,19 @@ function createSandboxEnvironmentDriver(
|
||||
});
|
||||
}
|
||||
}
|
||||
const acquisitionRunId = input.heartbeatRunId ?? randomUUID();
|
||||
const acquiredLease = providerLease ?? await (async () => {
|
||||
const acquiredLease = providerLease ?? await createOwnedSandbox(db, {
|
||||
companyId: input.companyId, environmentId: input.environment.id,
|
||||
heartbeatRunId: input.heartbeatRunId, issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId,
|
||||
provider: parsed.config.provider, pluginId: pluginProvider.resolved.plugin.id, driverKey: parsed.config.provider,
|
||||
resolvedConfig: { ...parsed.config },
|
||||
configSchema: pluginProvider.resolved.driver.configSchema as Record<string, unknown> | undefined,
|
||||
config: sandboxConfigForLeaseMetadata(storedConfig), requestKey: createHash("sha256").update(JSON.stringify(storedConfig)).digest("hex"),
|
||||
metadata: { ...sandboxConfigForLeaseMetadata(storedConfig), driver: input.environment.driver,
|
||||
pluginId: pluginProvider.resolved.plugin.id, pluginKey: pluginProvider.resolved.plugin.pluginKey, sandboxProviderPlugin: true },
|
||||
timeoutMs: resolvePluginSandboxRpcTimeoutMs(workerConfig, pluginProvider.resolved.driver.defaultAcquireTimeoutMs),
|
||||
}, async identity => {
|
||||
try {
|
||||
return await pluginWorkerManager.call(
|
||||
pluginProvider.resolved.plugin.id,
|
||||
"environmentAcquireLease",
|
||||
return await pluginWorkerManager.call(pluginProvider.resolved.plugin.id, "environmentAcquireLease",
|
||||
{
|
||||
driverKey: parsed.config.provider,
|
||||
companyId: input.companyId,
|
||||
@@ -2143,7 +1879,8 @@ function createSandboxEnvironmentDriver(
|
||||
// Plugin SDK requires a string; ad-hoc test leases use a fresh
|
||||
// UUID so providers that validate or persist the runId still see
|
||||
// a well-formed identifier.
|
||||
runId: acquisitionRunId,
|
||||
runId: identity.runId,
|
||||
creationOperationId: identity.id,
|
||||
workspaceMode: input.executionWorkspaceMode ?? undefined,
|
||||
agentId: input.agentId ?? undefined,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? undefined,
|
||||
@@ -2162,62 +1899,18 @@ function createSandboxEnvironmentDriver(
|
||||
? { requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt) }
|
||||
: {}),
|
||||
},
|
||||
resolvePluginSandboxRpcTimeoutMs(
|
||||
workerConfig,
|
||||
pluginProvider.resolved.driver.defaultAcquireTimeoutMs,
|
||||
),
|
||||
);
|
||||
resolvePluginSandboxRpcTimeoutMs(workerConfig, pluginProvider.resolved.driver.defaultAcquireTimeoutMs));
|
||||
} catch (error) {
|
||||
captureEnvironmentAcquisitionDiagnostic(error, {
|
||||
companyId: input.companyId, environmentId: input.environment.id, runId: acquisitionRunId,
|
||||
}, null);
|
||||
const cleanup = readEnvironmentCreationCleanupError(error);
|
||||
// The authenticated worker may identify its uncertain allocation,
|
||||
// but cannot redirect cleanup to another company, environment, or run.
|
||||
if (cleanup && cleanup.companyId === input.companyId &&
|
||||
cleanup.environmentId === input.environment.id && cleanup.runId === acquisitionRunId) {
|
||||
if (error instanceof JsonRpcCallError) captureEnvironmentAcquisitionDiagnostic(error, {
|
||||
companyId: input.companyId, environmentId: input.environment.id, runId: acquisitionRunId,
|
||||
}, readEnvironmentAcquisitionDiagnostic(error));
|
||||
const cleanupMetadata = {
|
||||
...sandboxConfigForLeaseMetadata(storedConfig),
|
||||
failedCreateCleanup: cleanup,
|
||||
driver: input.environment.driver,
|
||||
pluginId: pluginProvider.resolved.plugin.id,
|
||||
pluginKey: pluginProvider.resolved.plugin.pluginKey,
|
||||
sandboxProviderPlugin: true,
|
||||
};
|
||||
// Record before retrying the provider. Existing pending-cleanup
|
||||
// recovery (including its durable spool) survives controller loss.
|
||||
const cleanupConfirmed = await cleanUpRejectedOrphanSandbox({
|
||||
record: {
|
||||
companyId: input.companyId, environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
||||
issueId: input.issueId ?? null, heartbeatRunId: input.heartbeatRunId ?? null,
|
||||
provider: parsed.config.provider, providerLeaseId: cleanup.providerLeaseId,
|
||||
metadata: cleanupMetadata,
|
||||
},
|
||||
cause: error,
|
||||
canTeardown: pluginWorkerManager.isRunning(pluginProvider.resolved.plugin.id),
|
||||
teardown: (pendingLeaseId) => destroyWithCreationObservation({
|
||||
leaseId: pendingLeaseId, metadata: cleanupMetadata,
|
||||
teardown: (metadata) => pluginWorkerManager.call(pluginProvider.resolved.plugin.id, "environmentDestroyLease", {
|
||||
driverKey: parsed.config.provider, companyId: input.companyId,
|
||||
environmentId: input.environment.id, issueId: input.issueId,
|
||||
config: workerConfig, providerLeaseId: cleanup.providerLeaseId,
|
||||
leaseMetadata: metadata,
|
||||
}, resolvePluginSandboxRpcTimeoutMs(workerConfig)),
|
||||
}),
|
||||
});
|
||||
if (cleanupConfirmed) {
|
||||
throw new Error("Sandbox creation failed; allocated sandbox cleanup was confirmed.", { cause: error });
|
||||
}
|
||||
}
|
||||
// A cleanup record is not a usable lease. Keep acquisition failed,
|
||||
// even when its compensating deletion succeeds on this attempt.
|
||||
const diagnostic = error instanceof JsonRpcCallError && cleanup?.companyId === input.companyId &&
|
||||
cleanup.environmentId === input.environment.id && cleanup.runId === identity.runId
|
||||
? readEnvironmentAcquisitionDiagnostic(error) : null;
|
||||
captureEnvironmentAcquisitionDiagnostic(error, {
|
||||
companyId: input.companyId, environmentId: input.environment.id, runId: identity.runId,
|
||||
}, diagnostic);
|
||||
throw error;
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
// Ad-hoc test leases are never publishable for reuse: storing them
|
||||
// as `reuse_by_environment` would let a concurrent heartbeat resume
|
||||
@@ -2275,82 +1968,34 @@ function createSandboxEnvironmentDriver(
|
||||
: {}),
|
||||
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
|
||||
};
|
||||
try {
|
||||
return await environmentsSvc.acquireLease({
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
issueId: input.issueId,
|
||||
heartbeatRunId: input.heartbeatRunId,
|
||||
assertCompanyBinding: input.assertCompanyBinding,
|
||||
leasePolicy: resolvedLeasePolicy,
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: acquiredLease.providerLeaseId,
|
||||
expiresAt: providerAttestedLeaseExpiry(
|
||||
input.requestedExpiresAt,
|
||||
acquiredLease.expiresAt ? new Date(acquiredLease.expiresAt) : undefined,
|
||||
),
|
||||
metadata: pluginLeaseMetadata,
|
||||
reusesReusableLeaseId:
|
||||
providerLease &&
|
||||
reusableLease?.heartbeatRunId === input.heartbeatRunId
|
||||
? reusableLease.id
|
||||
: null,
|
||||
replacesReusableLeaseId:
|
||||
providerLease &&
|
||||
reusableLease?.heartbeatRunId !== input.heartbeatRunId
|
||||
? reusableLease?.id
|
||||
: null,
|
||||
});
|
||||
} catch (error) {
|
||||
// The conditional lease insert rejected, so no lease row exists. A
|
||||
// managed reconciliation can bind the environment to another company
|
||||
// between the route guard and this insert, so the insert fails closed
|
||||
// with `environment_company_mismatch`. This call already provisioned the
|
||||
// remote plugin sandbox above, so tear it down now. Without this step
|
||||
// the rejected insert leaks a live sandbox that no lease row tracks. The
|
||||
// Claude login runs on a plugin-backed sandbox, so this path is the one
|
||||
// the login uses. Do not tear down a reused sandbox that an earlier
|
||||
// lease still owns.
|
||||
if (!reusableLease || acquiredLease.providerLeaseId !== reusableLease.providerLeaseId) {
|
||||
// Record the durable pending-cleanup row before the teardown, then
|
||||
// tear the sandbox down. On a successful teardown the handler releases
|
||||
// the row. If the durable write and the teardown both fail, the
|
||||
// handler throws a `SandboxOrphanCleanupWriteError` that carries the
|
||||
// original rejection as its cause.
|
||||
await cleanUpRejectedOrphanSandbox({
|
||||
record: {
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
||||
issueId: input.issueId ?? null,
|
||||
heartbeatRunId: input.heartbeatRunId ?? null,
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: acquiredLease.providerLeaseId,
|
||||
metadata: pluginLeaseMetadata,
|
||||
},
|
||||
cause: error,
|
||||
canTeardown: pluginWorkerManager.isRunning(pluginProvider.resolved.plugin.id),
|
||||
teardown: async () => {
|
||||
return await pluginWorkerManager.call(
|
||||
pluginProvider.resolved.plugin.id,
|
||||
"environmentDestroyLease",
|
||||
{
|
||||
driverKey: parsed.config.provider,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
issueId: input.issueId,
|
||||
config: workerConfig,
|
||||
providerLeaseId: acquiredLease.providerLeaseId,
|
||||
leaseMetadata: acquiredLease.metadata ?? undefined,
|
||||
},
|
||||
resolvePluginSandboxRpcTimeoutMs(workerConfig),
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return await environmentsSvc.acquireLease({
|
||||
creationOperationId: providerLease ? undefined : acquiredLease.metadata?.creationOperationId as string | undefined,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
issueId: input.issueId,
|
||||
heartbeatRunId: input.heartbeatRunId,
|
||||
assertCompanyBinding: input.assertCompanyBinding,
|
||||
leasePolicy: resolvedLeasePolicy,
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: acquiredLease.providerLeaseId,
|
||||
expiresAt: providerAttestedLeaseExpiry(
|
||||
input.requestedExpiresAt,
|
||||
acquiredLease.expiresAt ? new Date(acquiredLease.expiresAt) : undefined,
|
||||
),
|
||||
metadata: pluginLeaseMetadata,
|
||||
reusesReusableLeaseId:
|
||||
providerLease &&
|
||||
reusableLease?.heartbeatRunId === input.heartbeatRunId
|
||||
? reusableLease.id
|
||||
: null,
|
||||
replacesReusableLeaseId:
|
||||
providerLease &&
|
||||
reusableLease?.heartbeatRunId !== input.heartbeatRunId
|
||||
? reusableLease?.id
|
||||
: null,
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
// Built-in sandbox provider path. Same guard as the plugin-backed path:
|
||||
@@ -2441,10 +2086,16 @@ function createSandboxEnvironmentDriver(
|
||||
|
||||
let providerLease;
|
||||
try {
|
||||
providerLease = await acquireSandboxProviderLease({
|
||||
providerLease = await createOwnedSandbox(db, {
|
||||
companyId: input.companyId, environmentId: input.environment.id, heartbeatRunId: input.heartbeatRunId,
|
||||
issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId, provider: parsed.config.provider,
|
||||
driverKey: parsed.config.provider, config: sandboxConfigForLeaseMetadata(parsed.config),
|
||||
requestKey: createHash("sha256").update(JSON.stringify(parsed.config)).digest("hex"),
|
||||
metadata: { ...sandboxConfigForLeaseMetadata(parsed.config), driver: input.environment.driver },
|
||||
}, identity => acquireSandboxProviderLease({
|
||||
config: parsed.config,
|
||||
environmentId: input.environment.id,
|
||||
heartbeatRunId: input.heartbeatRunId ?? randomUUID(),
|
||||
heartbeatRunId: identity.runId,
|
||||
issueId: input.issueId,
|
||||
agentId: input.agentId,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
@@ -2452,7 +2103,7 @@ function createSandboxEnvironmentDriver(
|
||||
// Forward a caller deadline so the provider configures a provider-side
|
||||
// expiry at or before it and returns the real provider expiry.
|
||||
requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt),
|
||||
});
|
||||
}));
|
||||
} catch (error) {
|
||||
if (reusableLease) {
|
||||
throw new ReusableSandboxResumeError({
|
||||
@@ -2521,72 +2172,35 @@ function createSandboxEnvironmentDriver(
|
||||
: {}),
|
||||
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
|
||||
};
|
||||
try {
|
||||
return await environmentsSvc.acquireLease({
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
issueId: input.issueId,
|
||||
heartbeatRunId: input.heartbeatRunId,
|
||||
assertCompanyBinding: input.assertCompanyBinding,
|
||||
leasePolicy: resolvedLeasePolicy,
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: providerLease.providerLeaseId,
|
||||
expiresAt: providerAttestedLeaseExpiry(
|
||||
input.requestedExpiresAt,
|
||||
providerLease.expiresAt ? new Date(providerLease.expiresAt) : undefined,
|
||||
),
|
||||
metadata: builtinLeaseMetadata,
|
||||
reusesReusableLeaseId:
|
||||
reusableLease &&
|
||||
providerLease.providerLeaseId === reusableLease.providerLeaseId &&
|
||||
reusableLease.heartbeatRunId === input.heartbeatRunId
|
||||
? reusableLease.id
|
||||
: null,
|
||||
replacesReusableLeaseId:
|
||||
reusableLease &&
|
||||
providerLease.providerLeaseId === reusableLease.providerLeaseId &&
|
||||
reusableLease.heartbeatRunId !== input.heartbeatRunId
|
||||
? reusableLease.id
|
||||
: null,
|
||||
});
|
||||
} catch (error) {
|
||||
// The conditional lease insert rejected, so no lease row exists. A managed
|
||||
// reconciliation can bind the environment to another company between the
|
||||
// route guard and this insert, so the insert fails closed with
|
||||
// `environment_company_mismatch`. This call already provisioned the remote
|
||||
// sandbox above, so release it now. Without this teardown the rejected
|
||||
// insert leaks a live sandbox that no lease row tracks. Do not tear down a
|
||||
// reused sandbox that an earlier lease still owns.
|
||||
if (!reusableLease || providerLease.providerLeaseId !== reusableLease.providerLeaseId) {
|
||||
// Record the durable pending-cleanup row before the teardown, then
|
||||
// release the remote sandbox. On a successful teardown the handler
|
||||
// releases the row. If the durable write and the teardown both fail,
|
||||
// the handler throws a `SandboxOrphanCleanupWriteError` that carries the
|
||||
// original rejection as its cause.
|
||||
await cleanUpRejectedOrphanSandbox({
|
||||
record: {
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
||||
issueId: input.issueId ?? null,
|
||||
heartbeatRunId: input.heartbeatRunId ?? null,
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: providerLease.providerLeaseId,
|
||||
metadata: builtinLeaseMetadata,
|
||||
},
|
||||
cause: error,
|
||||
canTeardown: true,
|
||||
teardown: async () => {
|
||||
await destroySandboxProviderLease({
|
||||
config: parsed.config,
|
||||
providerLeaseId: providerLease.providerLeaseId,
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return await environmentsSvc.acquireLease({
|
||||
creationOperationId: providerLease.metadata?.creationOperationId as string | undefined,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
issueId: input.issueId,
|
||||
heartbeatRunId: input.heartbeatRunId,
|
||||
assertCompanyBinding: input.assertCompanyBinding,
|
||||
leasePolicy: resolvedLeasePolicy,
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: providerLease.providerLeaseId,
|
||||
expiresAt: providerAttestedLeaseExpiry(
|
||||
input.requestedExpiresAt,
|
||||
providerLease.expiresAt ? new Date(providerLease.expiresAt) : undefined,
|
||||
),
|
||||
metadata: builtinLeaseMetadata,
|
||||
reusesReusableLeaseId:
|
||||
reusableLease &&
|
||||
providerLease.providerLeaseId === reusableLease.providerLeaseId &&
|
||||
reusableLease.heartbeatRunId === input.heartbeatRunId
|
||||
? reusableLease.id
|
||||
: null,
|
||||
replacesReusableLeaseId:
|
||||
reusableLease &&
|
||||
providerLease.providerLeaseId === reusableLease.providerLeaseId &&
|
||||
reusableLease.heartbeatRunId !== input.heartbeatRunId
|
||||
? reusableLease.id
|
||||
: null,
|
||||
});
|
||||
},
|
||||
|
||||
async releaseRunLease(input) {
|
||||
@@ -2719,7 +2333,7 @@ function createSandboxEnvironmentDriver(
|
||||
`Sandbox provider "${recordedProvider}" needs a plugin worker manager for cleanup, but none is available.`,
|
||||
);
|
||||
}
|
||||
const pinnedPluginId = exportResume ? readString(input.lease.metadata?.pluginId) : null;
|
||||
const pinnedPluginId = exportResume || input.lease.metadata?.creationOperationId ? readString(input.lease.metadata?.pluginId) : null;
|
||||
if (exportResume && !pinnedPluginId) throw new Error("Workspace export cleanup has no recorded provider plugin.");
|
||||
const pluginProvider = await resolveSandboxProviderPlugin({ provider: recordedProvider, ...(pinnedPluginId ? { pluginId: pinnedPluginId } : {}) });
|
||||
if (pluginProvider.state !== "running") {
|
||||
@@ -3534,6 +3148,7 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([
|
||||
"remoteCwd",
|
||||
"shellCommand",
|
||||
"sandboxProviderPlugin",
|
||||
"creationOperationId",
|
||||
"sandboxLeaseAcquisition",
|
||||
"nativeHarnessBackup",
|
||||
"nativeWorkspaceSync",
|
||||
@@ -3698,13 +3313,20 @@ function createPluginEnvironmentDriver(
|
||||
resumed = undefined;
|
||||
}
|
||||
}
|
||||
const providerLease = resumed ?? await workerManager.call(plugin.id, "environmentAcquireLease", {
|
||||
const providerLease = resumed ?? await createOwnedSandbox(db, {
|
||||
companyId: input.companyId, environmentId: input.environment.id, heartbeatRunId: input.heartbeatRunId,
|
||||
issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId,
|
||||
provider: `plugin:${parsed.config.pluginKey}:${parsed.config.driverKey}`, pluginId: plugin.id, driverKey: parsed.config.driverKey,
|
||||
config: {}, timeoutMs: rpcTimeoutMs, requestKey: fingerprint,
|
||||
metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: parsed.config.pluginKey, driverKey: parsed.config.driverKey },
|
||||
}, identity => workerManager.call(plugin.id, "environmentAcquireLease", {
|
||||
driverKey: parsed.config.driverKey,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
issueId: input.issueId,
|
||||
config: parsed.config.driverConfig,
|
||||
runId: input.heartbeatRunId ?? randomUUID(),
|
||||
runId: identity.runId,
|
||||
creationOperationId: identity.id,
|
||||
workspaceMode: input.executionWorkspaceMode ?? undefined,
|
||||
agentId: input.agentId ?? undefined,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? undefined,
|
||||
@@ -3715,9 +3337,10 @@ function createPluginEnvironmentDriver(
|
||||
...(requestedExpiresAtParam(input.requestedExpiresAt) !== undefined
|
||||
? { requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt) }
|
||||
: {}),
|
||||
} as PluginEnvironmentAcquireLeaseParams, ...timeoutOverride);
|
||||
} as PluginEnvironmentAcquireLeaseParams, ...timeoutOverride));
|
||||
|
||||
return await environmentsSvc.acquireLease({
|
||||
creationOperationId: resumed ? undefined : providerLease.metadata?.creationOperationId as string | undefined,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
@@ -3964,14 +3587,13 @@ export function environmentRuntimeService(
|
||||
pluginWorkerManager?: PluginWorkerManager;
|
||||
pluginWorkerReadyTimeoutMs?: number;
|
||||
pluginWorkerReadyPollMs?: number;
|
||||
pendingCleanupWriteAttempts?: number;
|
||||
pendingCleanupWriteBackoffMs?: number;
|
||||
deferredOrphanCleanupBufferLimit?: number;
|
||||
orphanCleanupSpool?: SandboxOrphanCleanupSpool;
|
||||
orphanCleanupSpoolDir?: string;
|
||||
} = {},
|
||||
) {
|
||||
const environmentsSvc = environmentService(db);
|
||||
let orphanRecoveryFlush: Promise<{ recovered: number; pending: number }> | null = null;
|
||||
const drivers = new Map<string, EnvironmentRuntimeDriver>();
|
||||
|
||||
const defaultDrivers = [
|
||||
@@ -3981,8 +3603,6 @@ export function environmentRuntimeService(
|
||||
pluginWorkerManager: options.pluginWorkerManager,
|
||||
pluginWorkerReadyTimeoutMs: options.pluginWorkerReadyTimeoutMs,
|
||||
pluginWorkerReadyPollMs: options.pluginWorkerReadyPollMs,
|
||||
pendingCleanupWriteAttempts: options.pendingCleanupWriteAttempts,
|
||||
pendingCleanupWriteBackoffMs: options.pendingCleanupWriteBackoffMs,
|
||||
deferredOrphanCleanupBufferLimit: options.deferredOrphanCleanupBufferLimit,
|
||||
orphanCleanupSpool: options.orphanCleanupSpool,
|
||||
orphanCleanupSpoolDir: options.orphanCleanupSpoolDir,
|
||||
@@ -4439,14 +4059,53 @@ export function environmentRuntimeService(
|
||||
return driver.isPendingCleanupWorkerReady(input);
|
||||
},
|
||||
|
||||
// Flush the in-process orphan-cleanup buffers of every driver that keeps one.
|
||||
// The cleanup sweep calls this each tick, so a buffered orphan lands a durable
|
||||
// `pending_cleanup` row once the database recovers, and the same sweep tears
|
||||
// it down. The dispatcher sums the recovered and pending counts across all
|
||||
// drivers. A driver with no buffer contributes nothing.
|
||||
// Recover committed creation operations before replaying legacy cleanup inputs.
|
||||
// New creation failures retain their database operation and do not make files.
|
||||
// Legacy buffers and files remain until their cleanup rows are durable.
|
||||
async flushDeferredOrphanCleanups(): Promise<{ recovered: number; pending: number }> {
|
||||
return flushDeferredEnvironmentCleanup(db, Array.from(drivers.values())
|
||||
.flatMap(driver => driver.flushDeferredOrphanCleanups ? [driver.flushDeferredOrphanCleanups] : []));
|
||||
orphanRecoveryFlush ??= (async () => {
|
||||
await recoverSandboxCreations(db, async operation => {
|
||||
if (!operation.pluginId || operation.metadata.sandboxProviderPlugin !== true) return null;
|
||||
if (!options.pluginWorkerManager) throw new Error("Sandbox creation recovery worker is unavailable");
|
||||
const plugin = await pluginRegistryService(db).getById(operation.pluginId);
|
||||
const driver = plugin?.manifestJson.environmentDrivers?.find(value => value.driverKey === operation.driverKey);
|
||||
if (driver?.creationOperationVersion !== 1) return null;
|
||||
if (!options.pluginWorkerManager.isRunning(operation.pluginId) ||
|
||||
!options.pluginWorkerManager.getWorker(operation.pluginId)?.supportedMethods.includes("environmentInspectLease")) {
|
||||
throw new Error("Sandbox creation recovery worker is unavailable");
|
||||
}
|
||||
let resolved: SandboxEnvironmentConfig;
|
||||
try {
|
||||
resolved = await resolveSandboxCleanupConfigSecrets(db, operation.companyId,
|
||||
operation.config as SandboxEnvironmentConfig, { issueId: operation.issueId });
|
||||
} catch (error) {
|
||||
// Missing or explicitly inactive saved credentials need an operator.
|
||||
// Temporary database/provider failures still retain retry ownership.
|
||||
const code = error instanceof HttpError ? (error.details as { code?: string } | undefined)?.code : undefined;
|
||||
if (error instanceof HttpError && (
|
||||
(error.status === 404 && (code === "secret_deleted" || code === "version_missing" ||
|
||||
(error.details === undefined && error.message === "Secret not found"))) ||
|
||||
(error.status === 422 && (code === "secret_inactive" || code === "version_inactive" ||
|
||||
(error.details === undefined && (error.message === "Provider vault is disabled" ||
|
||||
error.message === "Provider vault is locked while coming soon"))))
|
||||
)) return null;
|
||||
throw error;
|
||||
}
|
||||
const credentialHashes = operation.metadata.creationCredentialHashes as Record<string, string> | undefined;
|
||||
for (const [path, expected] of Object.entries(credentialHashes ?? {})) {
|
||||
const actual = readConfigValueAtPath({ ...resolved }, path);
|
||||
if (typeof actual !== "string" || createHash("sha256").update(actual).digest("hex") !== expected) return null;
|
||||
}
|
||||
return options.pluginWorkerManager.call(operation.pluginId, "environmentInspectLease", {
|
||||
driverKey: operation.driverKey, companyId: operation.companyId, environmentId: operation.originalEnvironmentId,
|
||||
config: stripSandboxProviderEnvelope(resolved), providerLeaseId: operation.id,
|
||||
creationOperation: { id: operation.id, runId: operation.runId },
|
||||
});
|
||||
});
|
||||
return flushDeferredEnvironmentCleanup(db, Array.from(drivers.values())
|
||||
.flatMap(driver => driver.flushDeferredOrphanCleanups ? [driver.flushDeferredOrphanCleanups] : []));
|
||||
})().finally(() => { orphanRecoveryFlush = null; });
|
||||
return orphanRecoveryFlush;
|
||||
},
|
||||
|
||||
async destroyReusableSandboxLeases(input: {
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { beginIdleTrackedWork } from "./task-admission.js";
|
||||
import { registerDeferredEnvironmentCleanup, writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
import { environmentService } from "./environments.js";
|
||||
import { createSandboxOrphanCleanupSpool, type DeferredOrphanCleanupRecord } from "./sandbox-orphan-cleanup-spool.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { attachSandboxCreation, environmentService } from "./environments.js";
|
||||
import { createOwnedSandbox, recordSandboxCreationResource } from "./sandbox-creation-operations.js";
|
||||
import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js";
|
||||
import { readWorkspaceProviderBinding } from "@paperclipai/shared";
|
||||
import { settlePluginTaskCredentialHandoff } from "./plugin-task-credential-writeback.js";
|
||||
@@ -234,59 +232,38 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({runnerTaskExecutionStopped:true,
|
||||
runnerTaskSessionRetained:state.sessionStateAvailable === true})}::jsonb`,
|
||||
}).where(and(eq(environmentLeases.id, metadata.runnerTaskPreviousLeaseId), eq(environmentLeases.companyId, input.companyId)));
|
||||
const fresh = await workers.call(plugin.id, "environmentAcquireLease", {driverKey:metadata.driverKey,
|
||||
const replacementDriverKey = metadata.driverKey;
|
||||
const fresh = await createOwnedSandbox(db, {
|
||||
companyId: input.companyId, environmentId: environment!.id, heartbeatRunId: lease.heartbeatRunId,
|
||||
issueId: lease.issueId, executionWorkspaceId: lease.executionWorkspaceId, provider: lease.provider!,
|
||||
pluginId: plugin.id, driverKey: metadata.driverKey, config: {}, requestKey: taskId,
|
||||
metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: plugin.pluginKey, driverKey: metadata.driverKey, runnerTaskLease: true },
|
||||
}, identity => workers.call(plugin.id, "environmentAcquireLease", {driverKey:replacementDriverKey,
|
||||
companyId:input.companyId,environmentId:environment!.id,issueId:lease.issueId,config:driverConfig,
|
||||
runId:run!.id,agentId:run!.agentId,adapterType:"paperclip_runner",
|
||||
...(typeof metadata.runnerTaskRequestedExpiresAt === "string" ? {requestedExpiresAt:metadata.runnerTaskRequestedExpiresAt} : {})});
|
||||
runId:identity.runId,creationOperationId:identity.id,agentId:run!.agentId,adapterType:"paperclip_runner",
|
||||
...(typeof metadata.runnerTaskRequestedExpiresAt === "string" ? {requestedExpiresAt:metadata.runnerTaskRequestedExpiresAt} : {})}));
|
||||
if (!fresh.providerLeaseId || fresh.providerLeaseId === taskId) throw new Error("Plugin replacement task identity unavailable");
|
||||
await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId: input.companyId,
|
||||
environmentId: environment!.id, providerLeaseId: fresh.providerLeaseId });
|
||||
const {runnerTaskIdentity:_identity,runnerTaskPriorIdentity:_prior,runnerTaskCredentialHandoff:_handoff,
|
||||
runnerTaskResumeFrom:_resume,runnerTaskPreviousLeaseId:_previous,...rest}=metadata;
|
||||
const replacementMetadata={...rest,providerMetadata:fresh.metadata ?? {},runnerTaskReused:false,runnerTaskDispatched:false,
|
||||
runnerTaskExecutionStopped:false,runnerTaskSessionRetained:false};
|
||||
try {
|
||||
{
|
||||
const expiresAt=fresh.expiresAt ? new Date(fresh.expiresAt) : null;
|
||||
const deadline=typeof metadata.runnerTaskRequestedExpiresAt === "string" ? Date.parse(metadata.runnerTaskRequestedExpiresAt) : null;
|
||||
if ((expiresAt && (!Number.isFinite(expiresAt.getTime()) || expiresAt.getTime() <= Date.now())) ||
|
||||
(deadline !== null && (!Number.isFinite(deadline) || !expiresAt || expiresAt.getTime() > deadline)))
|
||||
throw new Error("Plugin replacement task expiry is not bounded");
|
||||
const replacement = await db.update(environmentLeases).set({providerLeaseId:fresh.providerLeaseId,expiresAt,
|
||||
metadata:replacementMetadata,
|
||||
await attachSandboxCreation(db, String(fresh.metadata!.creationOperationId), { companyId: input.companyId, environmentId: environment!.id, heartbeatRunId: lease.heartbeatRunId }, async tx => {
|
||||
const [updated] = await tx.update(environmentLeases).set({providerLeaseId:fresh.providerLeaseId,expiresAt,
|
||||
metadata:{...replacementMetadata,creationOperationId:fresh.metadata!.creationOperationId},
|
||||
}).where(and(eq(environmentLeases.id,lease.id),eq(environmentLeases.providerLeaseId,taskId),
|
||||
eq(environmentLeases.status,"active"))).returning({id:environmentLeases.id});
|
||||
if (replacement.length !== 1) throw new Error("Plugin task replacement ownership changed");
|
||||
} catch (error) {
|
||||
try {
|
||||
await workers.call(plugin.id,"environmentReleaseLease",{driverKey:metadata.driverKey,companyId:input.companyId,
|
||||
environmentId:environment!.id,issueId:lease.issueId,config:driverConfig,providerLeaseId:fresh.providerLeaseId,
|
||||
leaseMetadata:replacementMetadata});
|
||||
} catch {
|
||||
const orphan:DeferredOrphanCleanupRecord={companyId:input.companyId,environmentId:environment!.id,
|
||||
executionWorkspaceId:lease.executionWorkspaceId,issueId:lease.issueId,heartbeatRunId:lease.heartbeatRunId,
|
||||
provider:lease.provider!,providerLeaseId:fresh.providerLeaseId,metadata:replacementMetadata};
|
||||
try {
|
||||
await environmentService(db).insertPendingCleanupLease({...orphan,failureReason:"plugin_task_replacement_unowned"});
|
||||
} catch {
|
||||
// The existing orphan spool recovers cleanup even when the database is unavailable.
|
||||
const spool = createSandboxOrphanCleanupSpool();
|
||||
const persisted = await spool.append(orphan);
|
||||
const finish = beginIdleTrackedWork();
|
||||
let flushing: Promise<{ recovered: number; pending: number }> | null = null;
|
||||
const unregister = registerDeferredEnvironmentCleanup(db, () => {
|
||||
flushing ??= (async () => {
|
||||
await environmentService(db).insertPendingCleanupLease({...orphan,failureReason:"plugin_task_replacement_unowned"});
|
||||
await spool.remove(orphan);
|
||||
unregister();
|
||||
finish();
|
||||
return { recovered: 1, pending: 0 };
|
||||
})().catch(error => { flushing = null; throw error; });
|
||||
// Keep successful completion for a concurrent sweep that already
|
||||
// captured this callback before it was unregistered.
|
||||
return flushing;
|
||||
});
|
||||
logger.error({providerLeaseId:fresh.providerLeaseId,persisted},"Plugin replacement cleanup requires recovery");
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
if (!updated) throw new Error("Plugin task replacement ownership changed");
|
||||
return (await environmentService(tx).getLeaseById(updated.id))!;
|
||||
});
|
||||
|
||||
}
|
||||
return executeEnvironmentTask(db, workers, input);
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
companySecretBindings,
|
||||
environmentCustomImageSetupSessions,
|
||||
environmentLeases,
|
||||
sandboxCreationOperations,
|
||||
environments,
|
||||
executionWorkspaces,
|
||||
heartbeatRuns,
|
||||
@@ -48,6 +49,43 @@ import {
|
||||
type ManagedResourceStockStatus,
|
||||
} from "./managed-resource-drift.js";
|
||||
|
||||
export async function attachSandboxCreation(db: Db, id: string, scope: { companyId: string; environmentId: string; heartbeatRunId?: string | null },
|
||||
attach: (tx: Db) => Promise<EnvironmentLease>): Promise<EnvironmentLease> {
|
||||
return db.transaction(async tx => {
|
||||
await assertEnvironmentCompanyBinding(tx as unknown as Db, scope.environmentId, scope.companyId);
|
||||
const [operation] = await tx.select().from(sandboxCreationOperations)
|
||||
.where(and(eq(sandboxCreationOperations.id, id), eq(sandboxCreationOperations.companyId, scope.companyId),
|
||||
eq(sandboxCreationOperations.originalEnvironmentId, scope.environmentId))).for("update");
|
||||
if (scope.heartbeatRunId) {
|
||||
const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, scope.heartbeatRunId), eq(heartbeatRuns.companyId, scope.companyId), eq(heartbeatRuns.status, "running"),
|
||||
)).for("update");
|
||||
if (!run || operation?.runId !== scope.heartbeatRunId) throw new Error("Sandbox creation run authority changed");
|
||||
}
|
||||
if (operation?.state === "attached" && operation.leaseId) {
|
||||
const existing = await environmentService(tx as unknown as Db).getLeaseById(operation.leaseId);
|
||||
if (existing?.status === "active") return existing;
|
||||
}
|
||||
if (!operation || operation.state !== "pending") throw new Error("Sandbox creation ownership has already transferred");
|
||||
const lease = await attach(tx as unknown as Db);
|
||||
await notifyEnvironmentCleanup(tx, { companyId: scope.companyId });
|
||||
await tx.update(sandboxCreationOperations).set({ state: "attached", leaseId: lease.id, updatedAt: new Date() })
|
||||
.where(eq(sandboxCreationOperations.id, id));
|
||||
return lease;
|
||||
});
|
||||
}
|
||||
|
||||
export async function assertEnvironmentCompanyBinding(db: Db, environmentId: string, companyId: string): Promise<void> {
|
||||
const [environment] = await db.select({ id: environments.id }).from(environments)
|
||||
.where(eq(environments.id, environmentId)).for("update");
|
||||
if (!environment) throw new Error("Sandbox creation environment no longer exists");
|
||||
const bindings = await db.select({ companyId: builtInManagedResources.companyId }).from(builtInManagedResources)
|
||||
.where(and(eq(builtInManagedResources.resourceKind, "environment"), eq(builtInManagedResources.resourceId, environmentId)));
|
||||
if (bindings.length && !bindings.some(binding => binding.companyId === companyId)) {
|
||||
throw forbidden("The selected environment belongs to another company.", { code: "environment_company_mismatch" });
|
||||
}
|
||||
}
|
||||
|
||||
type EnvironmentRow = typeof environments.$inferSelect;
|
||||
type EnvironmentLeaseRow = typeof environmentLeases.$inferSelect;
|
||||
const DEFAULT_LOCAL_ENVIRONMENT_NAME = "Local";
|
||||
@@ -1428,6 +1466,7 @@ export function environmentService(db: Db) {
|
||||
},
|
||||
|
||||
acquireLease: async (input: {
|
||||
creationOperationId?: string;
|
||||
companyId: string;
|
||||
environmentId: string;
|
||||
executionWorkspaceId?: string | null;
|
||||
@@ -1465,6 +1504,11 @@ export function environmentService(db: Db) {
|
||||
*/
|
||||
assertCompanyBinding?: boolean;
|
||||
}): Promise<EnvironmentLease> => {
|
||||
if (input.creationOperationId) {
|
||||
const { creationOperationId, ...leaseInput } = input;
|
||||
return attachSandboxCreation(db, creationOperationId, { companyId: input.companyId, environmentId: input.environmentId, heartbeatRunId: input.heartbeatRunId },
|
||||
tx => environmentService(tx).acquireLease({ ...leaseInput, assertCompanyBinding: true }));
|
||||
}
|
||||
const now = new Date();
|
||||
const replacementLeaseId = input.replacesReusableLeaseId ? randomUUID() : null;
|
||||
const values = {
|
||||
@@ -1515,48 +1559,7 @@ export function environmentService(db: Db) {
|
||||
companyId: input.companyId, runId: input.heartbeatRunId,
|
||||
});
|
||||
if (input.assertCompanyBinding) {
|
||||
// Lock the environment row first. Managed reconciliation locks the
|
||||
// same sandbox environment rows with `for update` before it writes a
|
||||
// company binding, so this lock serializes the two transactions on
|
||||
// this row and closes the time-of-check to time-of-use window.
|
||||
await tx
|
||||
.select({ id: environments.id })
|
||||
.from(environments)
|
||||
.where(eq(environments.id, input.environmentId))
|
||||
.for("update");
|
||||
// Re-read the company binding inside the locked transaction. A
|
||||
// binding a reconciliation committed after the route guard now
|
||||
// appears here. Reject a foreign-company environment before the
|
||||
// insert, so the login holds no lease.
|
||||
const boundRows = await tx
|
||||
.select({ companyId: builtInManagedResources.companyId })
|
||||
.from(builtInManagedResources)
|
||||
.where(
|
||||
and(
|
||||
eq(
|
||||
builtInManagedResources.resourceKind,
|
||||
MANAGED_ENVIRONMENT_RESOURCE_KIND,
|
||||
),
|
||||
eq(
|
||||
builtInManagedResources.resourceId,
|
||||
input.environmentId,
|
||||
),
|
||||
),
|
||||
);
|
||||
const boundCompanyIds = Array.from(
|
||||
new Set(boundRows.map((boundRow) => boundRow.companyId)),
|
||||
);
|
||||
if (
|
||||
boundCompanyIds.length > 0 &&
|
||||
!boundCompanyIds.includes(input.companyId)
|
||||
) {
|
||||
throw forbidden(
|
||||
"The selected environment belongs to another company.",
|
||||
{
|
||||
code: "environment_company_mismatch",
|
||||
},
|
||||
);
|
||||
}
|
||||
await assertEnvironmentCompanyBinding(tx as unknown as Db, input.environmentId, input.companyId);
|
||||
}
|
||||
if (input.replacesReusableLeaseId) {
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId });
|
||||
@@ -1699,6 +1702,7 @@ export function environmentService(db: Db) {
|
||||
const [previous] = await tx.select({ companyId: environmentLeases.companyId, issueId: environmentLeases.issueId,
|
||||
heartbeatRunId: environmentLeases.heartbeatRunId, status: environmentLeases.status,
|
||||
cleanupStatus: environmentLeases.cleanupStatus, releasedAt: environmentLeases.releasedAt,
|
||||
creationOperationId: sql<string | null>`${environmentLeases.metadata}->>'creationOperationId'`,
|
||||
}).from(environmentLeases).where(condition).for("update");
|
||||
if (!previous) return null;
|
||||
await notifyEnvironmentCleanup(tx, { companyId: previous.companyId, runId: previous.heartbeatRunId ?? undefined });
|
||||
@@ -1715,6 +1719,11 @@ export function environmentService(db: Db) {
|
||||
companyId: previous.companyId, runId: previous.heartbeatRunId,
|
||||
});
|
||||
}
|
||||
if (previous.creationOperationId && previous.status === "pending_cleanup" && options?.cleanupStatus === "success" && status !== "pending_cleanup") {
|
||||
await tx.update(sandboxCreationOperations).set({ state: "resolved", updatedAt: now })
|
||||
.where(and(eq(sandboxCreationOperations.id, previous.creationOperationId), eq(sandboxCreationOperations.companyId, previous.companyId),
|
||||
eq(sandboxCreationOperations.leaseId, id), eq(sandboxCreationOperations.state, "cleanup_pending")));
|
||||
}
|
||||
return tx.update(environmentLeases)
|
||||
.set({
|
||||
status,
|
||||
|
||||
@@ -54,6 +54,7 @@ const WORK_CHECKS = [
|
||||
state NOT IN ('saved', 'unchanged', 'resolved', 'unavailable', 'conflict')
|
||||
OR process_stopped_at IS NULL OR next_attempt_at IS NOT NULL
|
||||
OR receipt ? 'baseline' OR receipt->>'cleanupPending' = 'true'`,
|
||||
`SELECT 1 FROM sandbox_creation_operations WHERE state = 'pending'`,
|
||||
`SELECT 1 FROM budget_reservations WHERE state NOT IN ('settled', 'released')`,
|
||||
`SELECT 1 FROM budget_policies WHERE enforcement_version > enforcement_delivered_version`,
|
||||
`SELECT 1 FROM agents WHERE lifecycle_state IN
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
import { collectSecretRefPaths, isUuidSecretRef, parseSecretRefBindingObject, readConfigValueAtPath } from "./json-schema-secret-refs.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { and, eq, lte, or, sql } from "drizzle-orm";
|
||||
import { companies, environments, sandboxCreationOperations, type Db } from "@paperclipai/db";
|
||||
import { readEnvironmentCreationCleanupError, type PluginEnvironmentLease, type PluginEnvironmentLeaseObservation } from "@paperclipai/plugin-sdk";
|
||||
import { assertEnvironmentCompanyBinding, environmentService } from "./environments.js";
|
||||
import { ENVIRONMENT_CLEANUP_RETRY_MS } from "./environment-cleanup-state.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { notifyEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
|
||||
export type SandboxCreationOperation = typeof sandboxCreationOperations.$inferSelect;
|
||||
|
||||
type CreationOwner = {
|
||||
companyId: string;
|
||||
environmentId: string;
|
||||
executionWorkspaceId?: string | null;
|
||||
issueId?: string | null;
|
||||
heartbeatRunId?: string | null;
|
||||
provider: string;
|
||||
pluginId?: string;
|
||||
driverKey: string;
|
||||
config: Record<string, unknown>;
|
||||
metadata: Record<string, unknown>;
|
||||
timeoutMs?: number;
|
||||
requestKey?: string;
|
||||
configSchema?: Record<string, unknown> | null;
|
||||
resolvedConfig?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export async function createOwnedSandbox(
|
||||
db: Db, owner: CreationOwner,
|
||||
create: (identity: { id: string; runId: string }) => Promise<PluginEnvironmentLease>,
|
||||
): Promise<PluginEnvironmentLease> {
|
||||
const credentialHashes: Record<string, string> = {};
|
||||
for (const path of collectSecretRefPaths(owner.configSchema)) {
|
||||
const value = readConfigValueAtPath(owner.config, path);
|
||||
if (value != null && value !== "" && !(typeof value === "string" && isUuidSecretRef(value)) && !parseSecretRefBindingObject(value)) {
|
||||
throw new Error("Sandbox creation requires saved secret references");
|
||||
}
|
||||
if (value != null && value !== "") {
|
||||
const resolved = readConfigValueAtPath(owner.resolvedConfig ?? {}, path);
|
||||
if (typeof resolved !== "string") throw new Error("Sandbox creation credential identity is unavailable");
|
||||
credentialHashes[path] = createHash("sha256").update(resolved).digest("hex");
|
||||
}
|
||||
}
|
||||
const id = randomUUID();
|
||||
const runId = owner.heartbeatRunId ?? randomUUID();
|
||||
const operation = await db.transaction(async tx => {
|
||||
await notifyEnvironmentCleanup(tx, { companyId: owner.companyId, runId: owner.heartbeatRunId ?? undefined });
|
||||
await assertEnvironmentCompanyBinding(tx as unknown as Db, owner.environmentId, owner.companyId);
|
||||
const [inserted] = await tx.insert(sandboxCreationOperations).values({
|
||||
id, companyId: owner.companyId, environmentId: owner.environmentId, originalEnvironmentId: owner.environmentId,
|
||||
runId, requestKey: owner.requestKey ?? "run", issueId: owner.issueId, executionWorkspaceId: owner.executionWorkspaceId,
|
||||
provider: owner.provider, pluginId: owner.pluginId, driverKey: owner.driverKey,
|
||||
config: owner.config, metadata: { ...owner.metadata, creationCredentialHashes: credentialHashes },
|
||||
nextAttemptAt: new Date(Date.now() + Math.max(30 * 60_000, (owner.timeoutMs ?? 0) + 60_000)),
|
||||
}).onConflictDoNothing().returning();
|
||||
if (inserted) return inserted;
|
||||
const [existing] = await tx.select().from(sandboxCreationOperations).where(and(
|
||||
eq(sandboxCreationOperations.companyId, owner.companyId), eq(sandboxCreationOperations.originalEnvironmentId, owner.environmentId),
|
||||
eq(sandboxCreationOperations.runId, runId), eq(sandboxCreationOperations.requestKey, owner.requestKey ?? "run"),
|
||||
));
|
||||
if (!existing) throw new Error("Sandbox creation ownership was not recorded");
|
||||
return existing;
|
||||
});
|
||||
if (operation.id !== id) {
|
||||
if (operation.state !== "attached" || !operation.leaseId) throw new Error("Sandbox creation requires reconciliation before another attempt");
|
||||
const existing = await environmentService(db).getLeaseById(operation.leaseId);
|
||||
if (!existing || existing.status !== "active" || !existing.providerLeaseId) throw new Error("Sandbox creation has already transferred ownership");
|
||||
return { providerLeaseId: existing.providerLeaseId, expiresAt: existing.expiresAt?.toISOString(),
|
||||
metadata: { ...existing.metadata, creationOperationId: operation.id } };
|
||||
}
|
||||
const lease = await create({ id, runId });
|
||||
return { ...lease, metadata: { ...lease.metadata, creationOperationId: id } };
|
||||
}
|
||||
|
||||
/** Keep an already returned resource identity even when task attachment fails.
|
||||
* This enriches the existing owner; it never transfers or reopens ownership. */
|
||||
export async function recordSandboxCreationResource(db: Db, input: {
|
||||
id: string; companyId: string; environmentId: string; providerLeaseId: string;
|
||||
}): Promise<void> {
|
||||
const [recorded] = await db.update(sandboxCreationOperations).set({
|
||||
metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || ${JSON.stringify({ creationProviderLeaseId: input.providerLeaseId })}::jsonb`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(sandboxCreationOperations.id, input.id), eq(sandboxCreationOperations.companyId, input.companyId),
|
||||
eq(sandboxCreationOperations.originalEnvironmentId, input.environmentId),
|
||||
or(sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' is null`,
|
||||
sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' = ${input.providerLeaseId}`),
|
||||
)).returning({ id: sandboxCreationOperations.id });
|
||||
if (!recorded) throw new Error("Sandbox creation resource identity changed");
|
||||
}
|
||||
|
||||
export async function recoverSandboxCreations(db: Db,
|
||||
inspect: (operation: SandboxCreationOperation) => Promise<PluginEnvironmentLeaseObservation | null>,
|
||||
): Promise<void> {
|
||||
const due = await db.select().from(sandboxCreationOperations).where(and(
|
||||
eq(sandboxCreationOperations.state, "pending"), lte(sandboxCreationOperations.nextAttemptAt, new Date()),
|
||||
)).limit(100);
|
||||
for (const candidate of due) {
|
||||
try {
|
||||
const [existingCompany] = await db.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).limit(1);
|
||||
const observation = existingCompany ? await inspect(candidate) : null;
|
||||
const cleanup = observation?.creationCleanup ? readEnvironmentCreationCleanupError({ data: {
|
||||
schema: "paperclip/environment-creation-cleanup/v1", cleanup: observation.creationCleanup,
|
||||
} }) : null;
|
||||
if (cleanup && (cleanup.companyId !== candidate.companyId || cleanup.environmentId !== candidate.originalEnvironmentId ||
|
||||
cleanup.runId !== candidate.runId || cleanup.attemptId !== candidate.id || cleanup.providerLeaseId !== observation!.providerLeaseId)) {
|
||||
throw new Error("Sandbox creation recovery returned foreign ownership");
|
||||
}
|
||||
await db.transaction(async tx => {
|
||||
const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).for("key share");
|
||||
await tx.select({ id: environments.id }).from(environments).where(eq(environments.id, candidate.originalEnvironmentId)).for("update");
|
||||
const [operation] = await tx.select().from(sandboxCreationOperations)
|
||||
.where(eq(sandboxCreationOperations.id, candidate.id)).for("update");
|
||||
if (!operation || operation.state !== "pending") return;
|
||||
await notifyEnvironmentCleanup(tx, { companyId: operation.companyId });
|
||||
if (!company || !cleanup) {
|
||||
logger.warn({ errorKind: "sandbox_creation_needs_reconciliation", operationId: operation.id, companyId: operation.companyId },
|
||||
"Sandbox creation could not be identified; operator reconciliation is required");
|
||||
await tx.update(sandboxCreationOperations).set({ state: "needs_reconciliation", updatedAt: new Date(),
|
||||
metadata: { ...operation.metadata, creationReconciliationReason: company ? "creation_identity_unavailable" : "company_deleted" },
|
||||
})
|
||||
.where(eq(sandboxCreationOperations.id, operation.id));
|
||||
return;
|
||||
}
|
||||
const lease = await environmentService(tx as unknown as Db).insertPendingCleanupLease({
|
||||
companyId: operation.companyId, environmentId: operation.originalEnvironmentId,
|
||||
provider: operation.provider, providerLeaseId: cleanup.providerLeaseId,
|
||||
metadata: { ...operation.metadata, creationOperationId: operation.id, failedCreateCleanup: cleanup },
|
||||
failureReason: "interrupted_sandbox_creation",
|
||||
});
|
||||
await tx.update(sandboxCreationOperations).set({ state: "cleanup_pending", leaseId: lease.id, updatedAt: new Date() })
|
||||
.where(eq(sandboxCreationOperations.id, operation.id));
|
||||
});
|
||||
} catch {
|
||||
const retry = await db.transaction(async tx => {
|
||||
const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).for("key share");
|
||||
await notifyEnvironmentCleanup(tx, { companyId: candidate.companyId });
|
||||
await tx.update(sandboxCreationOperations).set(company
|
||||
? { nextAttemptAt: new Date(Date.now() + ENVIRONMENT_CLEANUP_RETRY_MS), updatedAt: new Date() }
|
||||
: { state: "needs_reconciliation", updatedAt: new Date(),
|
||||
metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || '{"creationReconciliationReason":"company_deleted"}'::jsonb` })
|
||||
.where(and(eq(sandboxCreationOperations.id, candidate.id), eq(sandboxCreationOperations.state, "pending")));
|
||||
return Boolean(company);
|
||||
});
|
||||
logger.warn({ errorKind: retry ? "sandbox_creation_recovery_failed" : "sandbox_creation_needs_reconciliation", operationId: candidate.id, companyId: candidate.companyId },
|
||||
retry ? "Sandbox creation recovery will retry at its saved deadline" : "Sandbox creation company was deleted; operator reconciliation is required");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user