mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-20 18:26:40 +01:00
fix(environments): clean up an unconfirmed setup sandbox and answer a provider outage with 503 (#22377)
A provider outage during an interactive custom-image setup start left a provisional sandbox with no cleanup owner, and the raw RPC error reached the HTTP error handler as a 500 crash report. - Keep the creation-cleanup envelope and create status for environmentStartInteractiveSetup errors at the worker RPC boundary. - Record an in-scope unconfirmed setup sandbox as a pending-cleanup lease that the existing sweep destroys. - Pin failed-creation cleanup (probe and teardown) to the plugin that created the sandbox. - Answer a transient provider refusal with 503 environment_provider_unavailable instead of a crash report. Co-Authored-By: Paperclip <Paperclip-Paperclip@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
10 files changed
+309
-10
No files matched your search
@@ -1675,7 +1675,9 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost
|
||||
: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR;
|
||||
|
||||
sendMessage(createErrorResponse(id, errorCode, errorMessage,
|
||||
method === "environmentAcquireLease" ? environmentAcquireErrorData(err)
|
||||
// An interactive setup creates a sandbox the same way an acquisition does,
|
||||
// so a failed create keeps the same ownership envelope and create status.
|
||||
method === "environmentAcquireLease" || method === "environmentStartInteractiveSetup" ? environmentAcquireErrorData(err)
|
||||
: method === "environmentDestroyLease" ? environmentCreationCleanupErrorData(err)
|
||||
: method === "environmentSyncOut" ? environmentSyncErrorData(err)
|
||||
: method === "environmentRunnerIngressEndpoint" ? environmentRunnerIngressErrorData(err)
|
||||
|
||||
@@ -12,13 +12,14 @@ import {
|
||||
import { createRequest, isJsonRpcResponse, JsonRpcCallError, parseMessage, serializeMessage, type JsonRpcResponse } from "../src/protocol.js";
|
||||
import { startWorkerRpcHost } from "../src/worker-rpc-host.js";
|
||||
|
||||
async function acquire(error: unknown) {
|
||||
async function acquire(error: unknown, method: "environmentAcquireLease" | "environmentStartInteractiveSetup" = "environmentAcquireLease") {
|
||||
const stdin = new PassThrough();
|
||||
const stdout = new PassThrough();
|
||||
const lines = createInterface({ input: stdout });
|
||||
const worker = startWorkerRpcHost({ stdin, stdout, plugin: definePlugin({
|
||||
async setup() {},
|
||||
async onEnvironmentAcquireLease() { throw error; },
|
||||
async onEnvironmentStartInteractiveSetup() { throw error; },
|
||||
}) });
|
||||
try {
|
||||
const response = new Promise<JsonRpcResponse>((resolve) => {
|
||||
@@ -27,7 +28,7 @@ async function acquire(error: unknown) {
|
||||
if (isJsonRpcResponse(message)) resolve(message);
|
||||
});
|
||||
});
|
||||
stdin.write(serializeMessage(createRequest("environmentAcquireLease", {}, "acquire-test")));
|
||||
stdin.write(serializeMessage(createRequest(method, {}, "acquire-test")));
|
||||
const result = await response;
|
||||
if (!("error" in result) || !result.error) throw new Error("Expected RPC error");
|
||||
return new JsonRpcCallError(result.error);
|
||||
@@ -59,6 +60,16 @@ describe("environment acquisition refusal diagnostics", () => {
|
||||
expect(readEnvironmentCreationCleanupError(callError)).toMatchObject({ providerLeaseId: cleanup.providerLeaseId, attemptId: cleanup.attemptId });
|
||||
});
|
||||
|
||||
it("keeps the cleanup envelope and create status when an interactive setup create fails", async () => {
|
||||
const createError = Object.assign(new Error("Bad gateway at https://private.example/api"), { statusCode: 502 });
|
||||
const callError = await acquire(new PluginEnvironmentCreationCleanupError([createError, new Error("lookup failed")],
|
||||
"Daytona sandbox creation failed (HTTP 502); cleanup could not be confirmed for paperclip-create-05f81d77", cleanup),
|
||||
"environmentStartInteractiveSetup");
|
||||
expect(readEnvironmentAcquireErrorDiagnostic(callError)).toEqual({ httpStatus: 502 });
|
||||
expect(readEnvironmentCreationCleanupError(callError)).toMatchObject({ providerLeaseId: cleanup.providerLeaseId, attemptId: cleanup.attemptId });
|
||||
expect(JSON.stringify(callError.data)).not.toContain("private.example");
|
||||
});
|
||||
|
||||
it("carries a closed connection errno and omits unstructured failures", async () => {
|
||||
expect(readEnvironmentAcquireErrorDiagnostic(await acquire(Object.assign(new Error("reset"), { code: "ECONNRESET" }))))
|
||||
.toEqual({ errorCode: "ECONNRESET" });
|
||||
|
||||
@@ -11,9 +11,11 @@ import {
|
||||
createDb,
|
||||
environmentCustomImageSetupSessions,
|
||||
environmentCustomImageTemplates,
|
||||
environmentLeases,
|
||||
environments,
|
||||
plugins,
|
||||
} from "@paperclipai/db";
|
||||
import { JsonRpcCallError } from "@paperclipai/plugin-sdk";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import {
|
||||
@@ -27,6 +29,8 @@ import {
|
||||
import {
|
||||
resolveEnvironmentDriverConfigForRuntime,
|
||||
} from "../services/environment-config.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
@@ -168,6 +172,7 @@ describeEmbeddedPostgres("environmentCustomImageService", () => {
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(environmentLeases);
|
||||
await db.delete(environmentCustomImageSetupSessions);
|
||||
await db.delete(environmentCustomImageTemplates);
|
||||
await db.delete(plugins);
|
||||
@@ -608,6 +613,61 @@ describeEmbeddedPostgres("environmentCustomImageService", () => {
|
||||
sourceEnvironmentConfigFingerprint: expired.session.metadata!.sourceEnvironmentConfigFingerprint });
|
||||
});
|
||||
|
||||
it.each(["setup scope", "foreign company", "instance scope"])(
|
||||
"hands an unconfirmed setup sandbox to the pending-cleanup sweep: %s", async (scope) => {
|
||||
// PAPERCLIP-BACKEND-75: a Daytona 502 during setup creation left a
|
||||
// provisional sandbox whose cleanup was never recorded anywhere.
|
||||
const { companyId, environmentId } = await seed();
|
||||
const workerManager = createWorkerManager();
|
||||
const startCall = workerManager.call.getMockImplementation()!;
|
||||
const providerLeaseId = `paperclip-create-${randomUUID()}`;
|
||||
let cleanup: Record<string, unknown> | undefined;
|
||||
let destroyed: Record<string, unknown> | undefined;
|
||||
workerManager.call.mockImplementation(async (pluginId, method, params) => {
|
||||
if (method === "environmentStartInteractiveSetup") {
|
||||
cleanup = {
|
||||
providerLeaseId, attemptId: providerLeaseId.slice("paperclip-create-".length),
|
||||
companyId: scope === "foreign company" ? randomUUID() : params.companyId, environmentId,
|
||||
accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" },
|
||||
};
|
||||
throw new JsonRpcCallError({ code: -32000,
|
||||
message: `Sandbox creation failed (HTTP 502); cleanup could not be confirmed for ${providerLeaseId}`,
|
||||
data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup, providerDiagnostic: { httpStatus: 502 } } });
|
||||
}
|
||||
if (method === "environmentDestroyLease") {
|
||||
destroyed = params;
|
||||
return { providerLeaseId: params.providerLeaseId, state: "destroyed" };
|
||||
}
|
||||
return startCall(pluginId, method, params);
|
||||
});
|
||||
const service = environmentCustomImageService(db, { pluginWorkerManager: workerManager });
|
||||
|
||||
const attempt = service.startSetupSession({ environmentId, actor: { userId: "user-1" },
|
||||
...(scope === "instance scope" ? {} : { secretContextCompanyId: companyId }) });
|
||||
if (scope === "setup scope") await expect(attempt).rejects.toBeInstanceOf(JsonRpcCallError);
|
||||
else await expect(attempt).rejects.toThrow("Interactive setup sandbox cleanup could not be recorded");
|
||||
|
||||
const [session] = await db.select().from(environmentCustomImageSetupSessions)
|
||||
.where(eq(environmentCustomImageSetupSessions.environmentId, environmentId));
|
||||
expect(session.status).toBe("failed");
|
||||
const leases = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId));
|
||||
if (scope !== "setup scope") {
|
||||
expect(leases).toHaveLength(0);
|
||||
return;
|
||||
}
|
||||
expect(leases).toEqual([expect.objectContaining({ companyId, environmentId, status: "pending_cleanup",
|
||||
provider: "fake-plugin", failureReason: "interrupted_setup_creation" })]);
|
||||
expect(leases[0].metadata).toMatchObject({ provider: "fake-plugin", image: "fake:base", purpose: "interactive_setup",
|
||||
setupSessionId: session.id, failedCreateCleanup: cleanup });
|
||||
|
||||
// The existing sweep teardown accepts the recorded lease and destroys the allocation.
|
||||
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
||||
await expect(runtime.retryPendingSandboxTeardown({ environment: null, lease: (await environmentService(db).getLeaseById(leases[0].id))! }))
|
||||
.resolves.toEqual({ providerLeaseId, state: "destroyed" });
|
||||
expect(destroyed).toMatchObject({ providerLeaseId, companyId, environmentId, leaseMetadata: { failedCreateCleanup: cleanup } });
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects interactive setup when the provider declares it but the worker omits a setup method", async () => {
|
||||
const { environmentId } = await seed();
|
||||
// The manifest declares supportsInteractiveSetup, but the worker reports
|
||||
|
||||
@@ -2411,6 +2411,58 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it("probes the plugin that recorded a failed creation when two plugins share its provider key", async () => {
|
||||
// An interactive setup can create its sandbox through the second of two
|
||||
// plugins with the same provider key, while the first is disabled. The
|
||||
// cleanup must follow the recorded plugin, not the first one with the key.
|
||||
const disabledPluginId = randomUUID();
|
||||
const creatorPluginId = randomUUID();
|
||||
const { companyId, environment, runId } = await seedEnvironment({
|
||||
driver: "sandbox",
|
||||
name: "Shared Provider Key Cleanup",
|
||||
config: { provider: "fake-plugin-shared", image: "fake:test", reuseLease: false },
|
||||
});
|
||||
const sharedDriver = [{ driverKey: "fake-plugin-shared", kind: "sandbox_provider", displayName: "Fake Shared", configSchema: { type: "object" } }];
|
||||
await insertSandboxProviderPlugin(disabledPluginId, {
|
||||
pluginKey: "paperclip.fake-shared-disabled", packageName: "@paperclipai/plugin-fake-shared-disabled",
|
||||
environmentDrivers: sharedDriver, status: "disabled", installOrder: 1,
|
||||
});
|
||||
await insertSandboxProviderPlugin(creatorPluginId, {
|
||||
pluginKey: "paperclip.fake-shared-creator", packageName: "@paperclipai/plugin-fake-shared-creator",
|
||||
environmentDrivers: sharedDriver, status: "ready", installOrder: 2,
|
||||
});
|
||||
const workerManager = {
|
||||
isRunning: vi.fn((id: string) => id === creatorPluginId),
|
||||
call: vi.fn(async () => {
|
||||
throw new Error("call must not run during a readiness probe");
|
||||
}),
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
||||
const insertOrphan = async (providerLeaseId: string, metadata: Record<string, unknown>) => {
|
||||
const orphan = await environmentService(db).insertPendingCleanupLease({
|
||||
companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId,
|
||||
provider: "fake-plugin-shared", providerLeaseId,
|
||||
metadata: { provider: "fake-plugin-shared", driver: "sandbox", reuseLease: false, ...metadata },
|
||||
failureReason: "interrupted_setup_creation",
|
||||
});
|
||||
return (await environmentService(db).getLeaseById(orphan.id))!;
|
||||
};
|
||||
|
||||
const setupLease = await insertOrphan("paperclip-create-shared", {
|
||||
pluginId: creatorPluginId,
|
||||
failedCreateCleanup: {
|
||||
providerLeaseId: "paperclip-create-shared", companyId, environmentId: environment.id,
|
||||
attemptId: "attempt-shared", accountFingerprint: "account-shared", labels: {},
|
||||
},
|
||||
});
|
||||
await expect(runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease: setupLease })).resolves.toBe(true);
|
||||
|
||||
// The pinned plugin's worker is down: a transient window, not ready.
|
||||
(workerManager.isRunning as ReturnType<typeof vi.fn>).mockReturnValue(false);
|
||||
await expect(runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease: setupLease })).resolves.toBe(false);
|
||||
expect((workerManager.call as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports the plugin cleanup provider not ready while the plugin is not ready, then ready after it recovers", async () => {
|
||||
// A pending_cleanup orphan targets a plugin-backed provider. A plugin reload
|
||||
// or a plugin reinstall moves the plugin through a "not ready" status. The
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { JsonRpcCallError } from "@paperclipai/plugin-sdk";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
|
||||
const captureExceptionMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../sentry.js", () => ({ captureException: captureExceptionMock }));
|
||||
vi.mock("../telemetry.js", () => ({ getTelemetryClient: () => null }));
|
||||
|
||||
const cleanup = {
|
||||
providerLeaseId: "paperclip-create-7c3c865d", companyId: "company-1", environmentId: "environment-1",
|
||||
attemptId: "7c3c865d", accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "daytona" },
|
||||
};
|
||||
|
||||
function refusal(data: unknown) {
|
||||
return new JsonRpcCallError({ code: -32000,
|
||||
message: "Daytona sandbox creation failed (HTTP 502); cleanup could not be confirmed for paperclip-create-7c3c865d", data });
|
||||
}
|
||||
|
||||
function createApp(error: unknown) {
|
||||
const app = express();
|
||||
app.post("/start", async () => { throw error; });
|
||||
app.use(errorHandler);
|
||||
return app;
|
||||
}
|
||||
|
||||
describe("sandbox provider outages during a requested creation", () => {
|
||||
beforeEach(() => captureExceptionMock.mockReset());
|
||||
|
||||
it("answers a transient create refusal with 503 and no crash report", async () => {
|
||||
// PAPERCLIP-BACKEND-75: a Daytona 502 reached this handler as a raw RPC
|
||||
// error, answering 500 with a crash report.
|
||||
const res = await request(createApp(refusal({ schema: "paperclip/environment-creation-cleanup/v1", cleanup,
|
||||
providerDiagnostic: { httpStatus: 502 } })))
|
||||
.post("/start");
|
||||
|
||||
expect(res.status).toBe(503);
|
||||
expect(res.headers["retry-after"]).toBe("30");
|
||||
expect(res.body).toEqual({
|
||||
error: "The sandbox provider was temporarily unavailable (HTTP 502). Try again in a few minutes.",
|
||||
code: "environment_provider_unavailable",
|
||||
});
|
||||
expect(JSON.stringify(res.body)).not.toContain("paperclip-create-");
|
||||
expect(captureExceptionMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("answers a connection failure and an expired creation deadline the same way", async () => {
|
||||
const reset = await request(createApp(refusal({ schema: "paperclip/environment-acquire-error/v1",
|
||||
providerDiagnostic: { errorCode: "ECONNRESET" } }))).post("/start");
|
||||
expect(reset.status).toBe(503);
|
||||
expect(reset.body.error).toBe("The sandbox provider was temporarily unavailable (ECONNRESET). Try again in a few minutes.");
|
||||
|
||||
const deadline = await request(createApp(refusal({ schema: "paperclip/environment-creation-cleanup/v1", cleanup,
|
||||
acquisitionDiagnostic: { phase: "create", elapsedMs: 300_000, budgetMs: 300_000 } }))).post("/start");
|
||||
expect(deadline.status).toBe(503);
|
||||
expect(deadline.body.error).toBe("The sandbox provider did not finish starting the sandbox in time. Try again in a few minutes.");
|
||||
expect(captureExceptionMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a definite provider status", refusal({ schema: "paperclip/environment-acquire-error/v1", providerDiagnostic: { httpStatus: 401 } })],
|
||||
["no structured evidence", refusal(undefined)],
|
||||
["a forged envelope on another error", Object.assign(new Error("x"), { data: { schema: "paperclip/environment-acquire-error/v1",
|
||||
providerDiagnostic: { httpStatus: 502 } } })],
|
||||
["a later Paperclip failure", new Error("Interactive setup sandbox cleanup could not be recorded", {
|
||||
cause: refusal({ schema: "paperclip/environment-acquire-error/v1", providerDiagnostic: { httpStatus: 502 } }) })],
|
||||
])("still reports %s as a server fault", async (_label, error) => {
|
||||
const res = await request(createApp(error)).post("/start");
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
expect(captureExceptionMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -21,6 +21,7 @@ import { recordResponsibleUserDenialOnActiveRun } from "../services/responsible-
|
||||
import { isObjectJournalBusyError } from "../services/object-byte-journal.js";
|
||||
import { isExpectedMcpConnectionFailure } from "../services/mcp-connection-failure.js";
|
||||
import { isOpenRouterCatalogUnavailable } from "../services/openrouter-models.js";
|
||||
import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE, readTransientSandboxCreationRefusal } from "../services/environment-provider-unavailable-error.js";
|
||||
import { isEndedDbPoolError } from "../database-retry.js";
|
||||
import { ToolGatewayHttpError } from "../services/tool-gateway.js";
|
||||
|
||||
@@ -340,6 +341,20 @@ export function errorHandler(
|
||||
return;
|
||||
}
|
||||
|
||||
// A sandbox provider outage during a sandbox creation that this request
|
||||
// started is not a server fault. The answer is fixed text: provider text and
|
||||
// provisional sandbox names stay out, and cleanup stays with its own record.
|
||||
const providerRefusal = readTransientSandboxCreationRefusal(rootError);
|
||||
if (providerRefusal) {
|
||||
if (!res.headersSent) {
|
||||
res.set("Retry-After", "30");
|
||||
res.status(503).json({ error: providerRefusal.message, code: ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE });
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const reportableError = sanitizeSecretSensitiveError(req, rootError);
|
||||
attachErrorContext(
|
||||
req,
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
type SandboxEnvironmentConfig,
|
||||
redactEnvironmentCustomImageValue,
|
||||
} from "@paperclipai/shared";
|
||||
import { readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
|
||||
import type {
|
||||
PluginEnvironmentCancelInteractiveSetupResult,
|
||||
PluginEnvironmentCaptureTemplateResult,
|
||||
@@ -442,6 +443,7 @@ export function environmentCustomImageService(
|
||||
provider,
|
||||
rpcCompanyId: storedRpcCompanyId ?? secretContextCompanyId ?? "instance",
|
||||
pluginId: resolved.plugin.id,
|
||||
pluginKey: resolved.plugin.pluginKey,
|
||||
driver: resolved.driver,
|
||||
runtimeConfig: parsed.config,
|
||||
driverConfig: stripSandboxProviderEnvelope(parsed.config),
|
||||
@@ -645,6 +647,45 @@ export function environmentCustomImageService(
|
||||
return toSession(row);
|
||||
}
|
||||
|
||||
// A setup sandbox whose failed creation could not be cleaned up has no
|
||||
// session lease to cancel. Hand it to the pending-cleanup sweep, the same
|
||||
// owner as a failed run acquisition, so the provider allocation is not lost.
|
||||
async function recordUnconfirmedSetupCreation(input: {
|
||||
error: unknown;
|
||||
environment: Environment;
|
||||
provider: Awaited<ReturnType<typeof resolveSetupProvider>>;
|
||||
sessionId: string;
|
||||
}): Promise<void> {
|
||||
const cleanup = readEnvironmentCreationCleanupError(input.error);
|
||||
if (!cleanup) return;
|
||||
const stored = parseEnvironmentDriverConfig(input.environment);
|
||||
// Accept only an allocation in this setup's own scope. An instance-scoped
|
||||
// setup has no company to own the cleanup record, so it is reported instead.
|
||||
if (input.provider.rpcCompanyId === "instance" || cleanup.companyId !== input.provider.rpcCompanyId ||
|
||||
cleanup.environmentId !== input.environment.id || cleanup.runId !== undefined ||
|
||||
stored.driver !== "sandbox" || stored.config.provider !== input.provider.provider) {
|
||||
throw new Error("Interactive setup sandbox cleanup could not be recorded", { cause: input.error });
|
||||
}
|
||||
// The stored config keeps secret references, never resolved values.
|
||||
await environments.insertPendingCleanupLease({
|
||||
companyId: cleanup.companyId,
|
||||
environmentId: input.environment.id,
|
||||
provider: input.provider.provider,
|
||||
providerLeaseId: cleanup.providerLeaseId,
|
||||
metadata: {
|
||||
...stored.config,
|
||||
driver: "sandbox",
|
||||
pluginId: input.provider.pluginId,
|
||||
pluginKey: input.provider.pluginKey,
|
||||
sandboxProviderPlugin: true,
|
||||
purpose: "interactive_setup",
|
||||
setupSessionId: input.sessionId,
|
||||
failedCreateCleanup: cleanup,
|
||||
},
|
||||
failureReason: "interrupted_setup_creation",
|
||||
});
|
||||
}
|
||||
|
||||
async function cancelSession(
|
||||
session: EnvironmentCustomImageSetupSession,
|
||||
reason: string | null,
|
||||
@@ -873,11 +914,15 @@ export function environmentCustomImageService(
|
||||
connectionPayload: providerSession.connectionPayload ?? null,
|
||||
};
|
||||
} catch (error) {
|
||||
await markSessionStatus({
|
||||
sessionId,
|
||||
status: "failed",
|
||||
failureReason: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
try {
|
||||
await recordUnconfirmedSetupCreation({ error, environment, provider, sessionId });
|
||||
} finally {
|
||||
await markSessionStatus({
|
||||
sessionId,
|
||||
status: "failed",
|
||||
failureReason: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
@@ -53,6 +53,28 @@ export function readTransientEnvironmentProviderFailure(error: unknown): Transie
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A sandbox creation that a request started directly (an interactive setup or
|
||||
* a login sandbox), refused by the provider with a transient status. Only the
|
||||
* worker's validated RPC envelope on the error itself counts: a wrapper means
|
||||
* that Paperclip failed afterwards, for example while recording cleanup, and
|
||||
* must still be reported. Ownership of a provisional allocation stays with its
|
||||
* creation-cleanup record; this only decides the HTTP answer.
|
||||
*/
|
||||
export function readTransientSandboxCreationRefusal(error: unknown): { message: string } | null {
|
||||
if (!(error instanceof JsonRpcCallError)) return null;
|
||||
const acquisition = readEnvironmentAcquisitionDiagnostic(error);
|
||||
const envelope = readEnvironmentAcquireErrorDiagnostic(error);
|
||||
const timedOut = acquisition !== null && acquisition.elapsedMs >= acquisition.budgetMs;
|
||||
const httpStatus = envelope?.httpStatus !== undefined && isRetryableEnvironmentHttpStatus(envelope.httpStatus) ? envelope.httpStatus : undefined;
|
||||
const errorCode = envelope?.errorCode !== undefined && isRetryableEnvironmentErrorCode(envelope.errorCode) ? envelope.errorCode : undefined;
|
||||
if (!timedOut && httpStatus === undefined && errorCode === undefined) return null;
|
||||
const evidence = httpStatus !== undefined ? ` (HTTP ${httpStatus})` : errorCode !== undefined ? ` (${errorCode})` : "";
|
||||
return { message: evidence === ""
|
||||
? "The sandbox provider did not finish starting the sandbox in time. Try again in a few minutes."
|
||||
: `The sandbox provider was temporarily unavailable${evidence}. Try again in a few minutes.` };
|
||||
}
|
||||
|
||||
/** A definite non-transient HTTP status anywhere on the chain is authoritative: retrying cannot change it. */
|
||||
function definiteNonTransientStatus(error: unknown): boolean {
|
||||
for (let cause = error, depth = 0; cause && depth < 5; cause = own(cause, "cause"), depth++) {
|
||||
|
||||
@@ -2508,7 +2508,7 @@ function createSandboxEnvironmentDriver(
|
||||
`Sandbox provider "${recordedProvider}" needs a plugin worker manager for cleanup, but none is available.`,
|
||||
);
|
||||
}
|
||||
const pinnedPluginId = exportResume || input.lease.metadata?.creationOperationId ? readString(input.lease.metadata?.pluginId) : null;
|
||||
const pinnedPluginId = exportResume ? readString(input.lease.metadata?.pluginId) : pinnedCreationCleanupPluginId(input.lease);
|
||||
if (exportResume && !pinnedPluginId) throw new Error("Workspace export cleanup has no recorded provider plugin.");
|
||||
const pluginProvider = await resolveSandboxProviderPlugin({ provider: recordedProvider, ...(pinnedPluginId ? { pluginId: pinnedPluginId } : {}) });
|
||||
if (pluginProvider.state !== "running") {
|
||||
@@ -2627,6 +2627,13 @@ function createSandboxEnvironmentDriver(
|
||||
return Boolean(pinned?.plugin.status === "ready" && pluginWorkerManager.isRunning(pinned.plugin.id)
|
||||
&& pluginWorkerVerifiesLifecycleMethod(pinned.plugin.id, "environmentStopLease"));
|
||||
}
|
||||
// A failed creation is torn down by the plugin that created it, so probe
|
||||
// that plugin. A missing or not-ready pinned plugin is a transient window.
|
||||
const pinnedCreationPluginId = pinnedCreationCleanupPluginId(input.lease);
|
||||
if (pinnedCreationPluginId) {
|
||||
const pinned = await resolvePluginSandboxProviderDriverById({ db, pluginId: pinnedCreationPluginId, driverKey: recordedProvider });
|
||||
return Boolean(pinned?.plugin.status === "ready" && pluginWorkerManager.isRunning(pinned.plugin.id));
|
||||
}
|
||||
// Resolve the installed plugin without a wait. A plugin reload or a plugin
|
||||
// reinstall can remove the plugin row for a short window, so a missing
|
||||
// plugin is a transient condition, not a permanent one. Report not ready,
|
||||
@@ -3360,6 +3367,16 @@ function readString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A failed sandbox creation belongs to the plugin that started it. Two plugins
|
||||
* can declare the same provider key, so the cleanup readiness probe and the
|
||||
* teardown both use this pin instead of the first installed plugin with the key.
|
||||
*/
|
||||
function pinnedCreationCleanupPluginId(lease: { metadata: Record<string, unknown> | null }): string | null {
|
||||
const metadata = lease.metadata ?? {};
|
||||
return metadata.creationOperationId || metadata.failedCreateCleanup ? readString(metadata.pluginId) : null;
|
||||
}
|
||||
|
||||
// Keys the runtime stores in the lease metadata that are not part of the
|
||||
// provider driver config. Some are host-internal control fields. `remoteCwd` is
|
||||
// a per-lease runtime value. The host reads `remoteCwd` from the lease metadata
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
"packages/plugins/sdk/src/ui/index.ts": "f7d1cee64a8f0c924bd10a1bd20745a52fe2060e",
|
||||
"packages/plugins/sdk/src/ui/runtime.ts": "998428e9701eba3b762d3417236d32c2d0d5763d",
|
||||
"packages/plugins/sdk/src/ui/types.ts": "5a5644acf6554d2e5e1d000bfd03b97d57001eb2",
|
||||
"packages/plugins/sdk/src/worker-rpc-host.ts": "afb577313ed099e20446fae6640e4f921089bafd",
|
||||
"packages/plugins/sdk/src/worker-rpc-host.ts": "820c73c7d4c681296ef8c813a68ab2bd9901d339",
|
||||
"packages/plugins/sdk/src/workspace-operation-error.ts": "2cd247d002d83b163239229e631bca4e519e5075",
|
||||
"packages/plugins/sdk/src/workspace-provider.ts": "38e945b807f22150710c11b136af0a28aab2cb7c",
|
||||
"packages/plugins/sdk/tsconfig.json": "d62ebdff94bc4b75dabf1fcad187e1e7a90fc794",
|
||||
|
||||
Reference in new issue
Block a user