mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
7209 lines
287 KiB
TypeScript
7209 lines
287 KiB
TypeScript
import { JsonRpcCallError } from "@paperclipai/plugin-sdk";
|
|
import { collectRunFailureDiagnostics } from "../services/run-failure-diagnostics.js";
|
|
import { idleWorkSnapshot } from "../services/task-admission.js";
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import { eq } from "drizzle-orm";
|
|
import {
|
|
buildSshEnvLabFixtureConfig,
|
|
getSshEnvLabSupport,
|
|
startSshEnvLabFixture,
|
|
stopSshEnvLabFixture,
|
|
} from "@paperclipai/adapter-utils/ssh";
|
|
import {
|
|
activityLog,
|
|
costEvents,
|
|
agents,
|
|
builtInManagedResources,
|
|
companies,
|
|
companySecretVersions,
|
|
companySecrets,
|
|
createDb,
|
|
environmentLeases,
|
|
environments,
|
|
executionWorkspaces,
|
|
heartbeatRuns,
|
|
issues,
|
|
plugins,
|
|
projects,
|
|
} from "@paperclipai/db";
|
|
import {
|
|
getEmbeddedPostgresTestSupport,
|
|
startEmbeddedPostgresTestDatabase,
|
|
} from "./helpers/embedded-postgres.js";
|
|
import { resolveEnvironmentDriverConfigForRuntime } from "../services/environment-config.ts";
|
|
import {
|
|
SANDBOX_CAPABILITY_KEYS,
|
|
environmentRuntimeService,
|
|
findReusableSandboxLeaseId,
|
|
SandboxOrphanCleanupWriteError,
|
|
} from "../services/environment-runtime.ts";
|
|
import * as sandboxProviderRuntime from "../services/sandbox-provider-runtime.ts";
|
|
import * as environmentsModule from "../services/environments.ts";
|
|
import { logger } from "../middleware/logger.ts";
|
|
import { resolveRunnerEnvironmentForRun } from "../services/runner-environment-lifecycle.js";
|
|
import { environmentService } from "../services/environments.ts";
|
|
import { remoteExecutionHasStopped } from "../services/remote-execution-termination.ts";
|
|
import { heartbeatService } from "../services/heartbeat.ts";
|
|
import { secretService } from "../services/secrets.ts";
|
|
import type { PluginWorkerManager } from "../services/plugin-worker-manager.ts";
|
|
import {
|
|
getActiveStepContext,
|
|
runWithRuntimeParent,
|
|
type StartupSpanContext,
|
|
} from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
|
|
import { traceparentFromContextToken } from "../instrumentation.ts";
|
|
import { ROOT_CONTEXT, trace } from "@opentelemetry/api";
|
|
import { buildNativeHarnessBackupManifest } from "../services/native-runtime/native-session-executor.ts";
|
|
import { createNativeHarnessBackupStamp } from "../services/native-runtime/native-harness-backup-stamp.ts";
|
|
|
|
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
|
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
|
const sshFixtureSupport = await getSshEnvLabSupport();
|
|
|
|
if (!embeddedPostgresSupport.supported) {
|
|
console.warn(
|
|
`Skipping embedded Postgres environment runtime tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
|
);
|
|
}
|
|
|
|
function stableStringify(value: unknown): string {
|
|
if (Array.isArray(value)) {
|
|
return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
|
|
}
|
|
if (value && typeof value === "object") {
|
|
const record = value as Record<string, unknown>;
|
|
return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`;
|
|
}
|
|
return JSON.stringify(value) ?? "null";
|
|
}
|
|
|
|
function reusableRuntimeFingerprint(input: {
|
|
provider: string;
|
|
adapterType: string | null;
|
|
config: Record<string, unknown>;
|
|
}) {
|
|
return createHash("sha256").update(stableStringify(input)).digest("hex");
|
|
}
|
|
|
|
describe("findReusableSandboxLeaseId", () => {
|
|
it("matches reusable plugin-backed sandbox leases by provider", () => {
|
|
const selected = findReusableSandboxLeaseId({
|
|
config: {
|
|
provider: "fake-plugin",
|
|
image: "template-b",
|
|
timeoutMs: 300000,
|
|
reuseLease: true,
|
|
},
|
|
leases: [
|
|
{
|
|
providerLeaseId: "sandbox-template-a",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "template-a",
|
|
timeoutMs: 300000,
|
|
reuseLease: true,
|
|
},
|
|
},
|
|
{
|
|
providerLeaseId: "sandbox-template-b",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "template-b",
|
|
timeoutMs: 300000,
|
|
reuseLease: true,
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
expect(selected).toBe("sandbox-template-b");
|
|
});
|
|
|
|
it("ignores host-only log streaming when matching a reusable plugin lease", () => {
|
|
const selected = findReusableSandboxLeaseId({
|
|
config: {
|
|
provider: "fake-plugin",
|
|
image: "template-b",
|
|
timeoutMs: 300000,
|
|
reuseLease: true,
|
|
streamRunLogs: true,
|
|
runnerLifecycleMode: "warm",
|
|
target: null,
|
|
},
|
|
leases: [
|
|
{
|
|
providerLeaseId: "sandbox-template-b",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "template-b",
|
|
timeoutMs: 300000,
|
|
reuseLease: true,
|
|
runnerLifecycleMode: "warm",
|
|
target: "us",
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
expect(selected).toBe("sandbox-template-b");
|
|
});
|
|
|
|
it("requires image identity for reusable fake sandbox leases", () => {
|
|
const selected = findReusableSandboxLeaseId({
|
|
config: {
|
|
provider: "fake",
|
|
image: "ubuntu:24.04",
|
|
reuseLease: true,
|
|
},
|
|
leases: [
|
|
{
|
|
providerLeaseId: "sandbox-image-a",
|
|
metadata: {
|
|
provider: "fake",
|
|
image: "debian:12",
|
|
reuseLease: true,
|
|
},
|
|
},
|
|
{
|
|
providerLeaseId: "sandbox-image-b",
|
|
metadata: {
|
|
provider: "fake",
|
|
image: "ubuntu:24.04",
|
|
reuseLease: true,
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
expect(selected).toBe("sandbox-image-b");
|
|
});
|
|
});
|
|
|
|
describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|
let stopDb: (() => Promise<void>) | null = null;
|
|
let db!: ReturnType<typeof createDb>;
|
|
let runtime!: ReturnType<typeof environmentRuntimeService>;
|
|
const fixtureRoots: string[] = [];
|
|
// Give each test its own orphan-cleanup spool directory, so a spool file one
|
|
// test writes never leaks into another test's flush. The default runtime reads
|
|
// this env override when a test does not pass an explicit spool directory.
|
|
let orphanCleanupSpoolDir: string | null = null;
|
|
const previousOrphanCleanupSpoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
|
|
|
|
beforeAll(async () => {
|
|
const started = await startEmbeddedPostgresTestDatabase("environment-runtime");
|
|
stopDb = started.stop;
|
|
db = createDb(started.connectionString);
|
|
runtime = environmentRuntimeService(db);
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
orphanCleanupSpoolDir = await mkdtemp(path.join(os.tmpdir(), "orphan-cleanup-spool-"));
|
|
process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR = orphanCleanupSpoolDir;
|
|
});
|
|
|
|
afterEach(async () => {
|
|
if (orphanCleanupSpoolDir) {
|
|
await rm(orphanCleanupSpoolDir, { recursive: true, force: true }).catch(() => undefined);
|
|
orphanCleanupSpoolDir = null;
|
|
}
|
|
if (previousOrphanCleanupSpoolDir === undefined) {
|
|
delete process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
|
|
} else {
|
|
process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR = previousOrphanCleanupSpoolDir;
|
|
}
|
|
while (fixtureRoots.length > 0) {
|
|
const root = fixtureRoots.pop();
|
|
if (!root) continue;
|
|
await stopSshEnvLabFixture(path.join(root, "state.json")).catch(() => undefined);
|
|
await rm(root, { recursive: true, force: true }).catch(() => undefined);
|
|
}
|
|
await db.delete(activityLog);
|
|
await db.delete(environmentLeases);
|
|
await db.delete(issues);
|
|
await db.delete(costEvents);
|
|
await db.delete(heartbeatRuns);
|
|
await db.delete(agents);
|
|
await db.delete(environments);
|
|
await db.delete(executionWorkspaces);
|
|
await db.delete(plugins);
|
|
await db.delete(companySecretVersions);
|
|
await db.delete(companySecrets);
|
|
await db.delete(projects);
|
|
await db.delete(companies);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await stopDb?.();
|
|
});
|
|
|
|
function insertSandboxProviderPlugin(pluginId: string, input: {
|
|
pluginKey?: string;
|
|
packageName?: string;
|
|
displayName?: string;
|
|
description?: string;
|
|
author?: string;
|
|
environmentDrivers?: Array<Record<string, unknown>>;
|
|
status?: string;
|
|
installOrder?: number;
|
|
} = {}) {
|
|
const pluginKey = input.pluginKey ?? "paperclip.fake-plugin-sandbox-provider";
|
|
return db.insert(plugins).values({
|
|
id: pluginId,
|
|
pluginKey,
|
|
packageName: input.packageName ?? "@paperclipai/plugin-fake-sandbox",
|
|
version: "1.0.0",
|
|
apiVersion: 1,
|
|
categories: ["automation"],
|
|
manifestJson: {
|
|
id: pluginKey,
|
|
apiVersion: 1,
|
|
version: "1.0.0",
|
|
displayName: input.displayName ?? "Fake Plugin Sandbox Provider",
|
|
description: input.description ?? "Test fake plugin provider",
|
|
author: input.author ?? "Paperclip",
|
|
categories: ["automation"],
|
|
capabilities: ["environment.drivers.register"],
|
|
entrypoints: { worker: "dist/worker.js" },
|
|
environmentDrivers: input.environmentDrivers ?? [{ driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" } }],
|
|
},
|
|
status: input.status ?? "ready",
|
|
installOrder: input.installOrder ?? 1,
|
|
updatedAt: new Date(),
|
|
} as any);
|
|
}
|
|
|
|
function insertForeignCompanyEnvironmentBinding(otherCompanyId: string, environment: { id: string }) {
|
|
return db.insert(builtInManagedResources).values({
|
|
companyId: otherCompanyId,
|
|
bundleKey: "managed-environment",
|
|
resourceKind: "environment",
|
|
resourceKey: "managed-sandbox",
|
|
resourceId: environment.id,
|
|
stockVersion: "1",
|
|
stockHash: "hash",
|
|
});
|
|
}
|
|
|
|
function insertReusableProject(projectId: string, companyId: string) {
|
|
return db.insert(projects).values({
|
|
id: projectId,
|
|
companyId,
|
|
name: `Workspace ${projectId.slice(0, 8)}`,
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
}
|
|
|
|
function createFreshPluginLeaseWorker(pluginId: string) {
|
|
return {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "fresh-plugin-lease",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
}
|
|
|
|
function insertReusableExecutionWorkspace(executionWorkspaceId: string, companyId: string, projectId: string) {
|
|
return db.insert(executionWorkspaces).values({
|
|
id: executionWorkspaceId,
|
|
companyId,
|
|
projectId,
|
|
mode: "shared_workspace",
|
|
strategyType: "project_primary",
|
|
name: "Reusable workspace",
|
|
status: "active",
|
|
providerType: "local_fs",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
}
|
|
|
|
function insertStaleReusablePluginLease(companyId: string, environment: { id: string }, executionWorkspaceId: string, runId: string, agentId: string, pluginId: string, providerConfig: Record<string, unknown>) {
|
|
return environmentService(db).acquireLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
heartbeatRunId: runId,
|
|
leasePolicy: "reuse_by_environment",
|
|
provider: "fake-plugin",
|
|
providerLeaseId: "stale-plugin-lease",
|
|
metadata: {
|
|
agentId,
|
|
driver: "sandbox",
|
|
pluginId,
|
|
pluginKey: "acme.fake-sandbox-provider",
|
|
sandboxProviderPlugin: true,
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
reusableSandboxLease: {
|
|
version: 1,
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
agentId,
|
|
adapterType: null,
|
|
provider: "fake-plugin",
|
|
runtimeFingerprint: reusableRuntimeFingerprint({
|
|
provider: "fake-plugin",
|
|
adapterType: null,
|
|
config: providerConfig,
|
|
}),
|
|
},
|
|
},
|
|
});
|
|
}
|
|
|
|
function createReferencedWorkspaceRealizationRequest(companyId: string, environment: { id: string }, runId: string) {
|
|
return {
|
|
version: 1,
|
|
adapterType: "codex_local",
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId: null,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
requestedMode: "ephemeral",
|
|
source: {
|
|
kind: "project_primary",
|
|
localPath: "/tmp/anchor",
|
|
projectId: "anchor-project",
|
|
projectWorkspaceId: "anchor-workspace",
|
|
repoUrl: null,
|
|
repoRef: null,
|
|
strategy: "project_primary",
|
|
branchName: null,
|
|
worktreePath: null,
|
|
},
|
|
additionalSources: [
|
|
{
|
|
localPath: "/tmp/referenced-project",
|
|
projectId: "referenced-project-1",
|
|
projectWorkspaceId: "referenced-workspace-1",
|
|
repoUrl: null,
|
|
repoRef: null,
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
function insertOldReusablePluginLease(companyId: string, environment: { id: string }, executionWorkspaceId: string, runId: string, agentId: string, providerConfig: Record<string, unknown>) {
|
|
return environmentService(db).acquireLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
heartbeatRunId: runId,
|
|
leasePolicy: "reuse_by_environment",
|
|
provider: "fake-plugin",
|
|
providerLeaseId: "old-plugin-lease",
|
|
metadata: {
|
|
agentId,
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
reusableSandboxLease: {
|
|
version: 1,
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
agentId,
|
|
adapterType: null,
|
|
provider: "fake-plugin",
|
|
runtimeFingerprint: reusableRuntimeFingerprint({
|
|
provider: "fake-plugin",
|
|
adapterType: null,
|
|
config: providerConfig,
|
|
}),
|
|
},
|
|
},
|
|
});
|
|
}
|
|
|
|
function insertRunningManualRun(nextRunId: string, companyId: string, agentId: string) {
|
|
return db.insert(heartbeatRuns).values({
|
|
id: nextRunId,
|
|
companyId,
|
|
agentId,
|
|
invocationSource: "manual",
|
|
status: "running",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
}
|
|
|
|
async function seedEnvironment(input: {
|
|
driver?: string;
|
|
name?: string;
|
|
status?: "active" | "disabled";
|
|
config?: Record<string, unknown>;
|
|
} = {}) {
|
|
const companyId = randomUUID();
|
|
const agentId = randomUUID();
|
|
const environmentId = randomUUID();
|
|
const runId = randomUUID();
|
|
const driver = input.driver ?? "local";
|
|
const environmentName = input.name ?? `${driver}-${environmentId.slice(0, 8)}`;
|
|
let config = input.config ?? {};
|
|
|
|
await db.insert(companies).values({
|
|
id: companyId,
|
|
name: "Acme",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await db.insert(agents).values({
|
|
id: agentId,
|
|
companyId,
|
|
name: "CodexCoder",
|
|
role: "engineer",
|
|
status: "active",
|
|
adapterType: "codex_local",
|
|
adapterConfig: {},
|
|
runtimeConfig: {},
|
|
permissions: {},
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
if (typeof config.privateKey === "string" && config.privateKey.length > 0) {
|
|
const secret = await secretService(db).create(companyId, {
|
|
name: `environment-runtime-private-key-${randomUUID()}`,
|
|
provider: "local_encrypted",
|
|
value: config.privateKey,
|
|
});
|
|
await secretService(db).createBinding({
|
|
companyId,
|
|
secretId: secret.id,
|
|
targetType: "environment",
|
|
targetId: environmentId,
|
|
configPath: "privateKeySecretRef",
|
|
});
|
|
config = {
|
|
...config,
|
|
privateKey: null,
|
|
privateKeySecretRef: {
|
|
type: "secret_ref",
|
|
secretId: secret.id,
|
|
version: "latest",
|
|
},
|
|
};
|
|
}
|
|
const existingLocalEnvironment = driver === "local"
|
|
? await db
|
|
.select()
|
|
.from(environments)
|
|
.where(eq(environments.driver, "local"))
|
|
.then((rows) => rows[0] ?? null)
|
|
: null;
|
|
const environmentRecord = existingLocalEnvironment ?? {
|
|
id: environmentId,
|
|
name: environmentName,
|
|
description: null,
|
|
driver,
|
|
status: input.status ?? "active",
|
|
config,
|
|
metadata: null,
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
};
|
|
if (!existingLocalEnvironment) {
|
|
await db.insert(environments).values({
|
|
id: environmentRecord.id,
|
|
name: environmentRecord.name,
|
|
driver: environmentRecord.driver,
|
|
status: environmentRecord.status,
|
|
config: environmentRecord.config,
|
|
createdAt: environmentRecord.createdAt,
|
|
updatedAt: environmentRecord.updatedAt,
|
|
});
|
|
}
|
|
await db.insert(heartbeatRuns).values({
|
|
id: runId,
|
|
companyId,
|
|
agentId,
|
|
invocationSource: "manual",
|
|
status: "running",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
|
|
return {
|
|
companyId,
|
|
agentId,
|
|
environment: {
|
|
id: environmentRecord.id,
|
|
companyId,
|
|
name: environmentRecord.name,
|
|
description: environmentRecord.description,
|
|
driver: environmentRecord.driver,
|
|
status: environmentRecord.status,
|
|
config: environmentRecord.config,
|
|
metadata: environmentRecord.metadata,
|
|
createdAt: environmentRecord.createdAt,
|
|
updatedAt: environmentRecord.updatedAt,
|
|
} as const,
|
|
runId,
|
|
};
|
|
}
|
|
|
|
async function seedReusablePluginSandboxLease(adapterType: string | null = null) {
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Reusable Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.reusable-sandbox-provider",
|
|
packageName: "@acme/reusable-sandbox-provider",
|
|
displayName: "Reusable Sandbox Provider",
|
|
description: "Test provider with reusable lease support",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
supportsReusableLeases: true,
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
image: { type: "string" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await db.insert(projects).values({
|
|
id: projectId,
|
|
companyId,
|
|
name: `Workspace ${projectId.slice(0, 8)}`,
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await db.insert(executionWorkspaces).values({
|
|
id: executionWorkspaceId,
|
|
companyId,
|
|
projectId,
|
|
mode: "shared_workspace",
|
|
strategyType: "project_primary",
|
|
name: "Reusable workspace",
|
|
status: "active",
|
|
providerType: "local_fs",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
const reusableLease = await environmentService(db).acquireLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
heartbeatRunId: runId,
|
|
leasePolicy: "reuse_by_environment",
|
|
provider: "fake-plugin",
|
|
providerLeaseId: "reusable-plugin-lease",
|
|
metadata: {
|
|
agentId,
|
|
driver: "sandbox",
|
|
pluginId,
|
|
pluginKey: "acme.reusable-sandbox-provider",
|
|
sandboxProviderPlugin: true,
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
reusableSandboxLease: {
|
|
version: 1,
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
agentId,
|
|
adapterType,
|
|
provider: "fake-plugin",
|
|
runtimeFingerprint: reusableRuntimeFingerprint({
|
|
provider: "fake-plugin",
|
|
adapterType,
|
|
config: providerConfig,
|
|
}),
|
|
},
|
|
},
|
|
});
|
|
|
|
return { pluginId, companyId, agentId, environment, runId, executionWorkspaceId, reusableLease };
|
|
}
|
|
|
|
it("acquires and resumes a reusable lease after a task changes from per-turn to warm", async () => {
|
|
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
|
await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
|
|
const environment = { ...seeded.environment, config: { ...seeded.environment.config, reuseLease: false } };
|
|
await environmentService(db).update(environment.id, { config: environment.config });
|
|
const issueId = randomUUID();
|
|
await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Lifecycle switch", status: "in_progress", assigneeAgentId: seeded.agentId });
|
|
let acquisitions = 0;
|
|
let warmProviderLeaseId = "";
|
|
const call = vi.fn(async (_pluginId: string, method: string, input: any) => {
|
|
if (method === "environmentAcquireLease") return {
|
|
providerLeaseId: `lifecycle-${++acquisitions}`, metadata: { remoteCwd: "/workspace" },
|
|
};
|
|
if (method === "environmentResumeLease") return {
|
|
providerLeaseId: warmProviderLeaseId, metadata: { remoteCwd: "/workspace" },
|
|
};
|
|
if (method === "environmentDestroyLease" || method === "environmentStopLease" || method === "environmentReleaseLease") return {
|
|
providerLeaseId: input.providerLeaseId, state: method === "environmentDestroyLease" ? "destroyed" : "stopped",
|
|
};
|
|
throw new Error(`Unexpected lifecycle method: ${method}`);
|
|
});
|
|
const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
|
|
isRunning: () => true, call,
|
|
getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }),
|
|
} as unknown as PluginWorkerManager });
|
|
const input = {
|
|
companyId: seeded.companyId, agentId: seeded.agentId, issueId,
|
|
adapterType: "paperclip_runner", persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" as const },
|
|
};
|
|
const cold = await runtime.acquireRunLease({ ...input, environment, heartbeatRunId: seeded.runId });
|
|
expect(cold.lease.leasePolicy).toBe("ephemeral");
|
|
await runtime.releaseRunLeases(seeded.runId, "released");
|
|
const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" });
|
|
const warmRunId = randomUUID();
|
|
await db.insert(heartbeatRuns).values({ id: warmRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
|
|
const warm = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: warmRunId });
|
|
warmProviderLeaseId = warm.lease.providerLeaseId!;
|
|
expect(warm.lease.leasePolicy).toBe("reuse_by_environment");
|
|
expect(warm.lease.executionWorkspaceId).toBe(seeded.executionWorkspaceId);
|
|
expect((await runtime.resolveCapabilities({ environment: warmEnvironment, lease: warm.lease })).reusableLeases).toBe(true);
|
|
await runtime.releaseRunLeases(warmRunId, "released", undefined, "stop_and_retain");
|
|
const nextRunId = randomUUID();
|
|
await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
|
|
const next = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: nextRunId });
|
|
expect(next.lease.providerLeaseId).toBe(warmProviderLeaseId);
|
|
expect(acquisitions).toBe(2);
|
|
expect((await environmentService(db).getById(environment.id))?.config.reuseLease).toBe(false);
|
|
});
|
|
|
|
it.each(["stopped", "missing", "wrong-lease", "error"])(
|
|
"startup cancellation is run-scoped and needs a matching receipt: %s",
|
|
async (outcome) => {
|
|
const { pluginId, companyId, runId, reusableLease, environment } = await seedReusablePluginSandboxLease();
|
|
const other = await environmentService(db).acquireLease({
|
|
companyId, environmentId: environment.id, heartbeatRunId: null,
|
|
leasePolicy: "ephemeral", provider: "fake-plugin", providerLeaseId: "other-sandbox",
|
|
});
|
|
const call = vi.fn(async () => {
|
|
if (outcome === "error") throw new Error("provider unavailable");
|
|
if (outcome === "missing") return undefined;
|
|
return { providerLeaseId: outcome === "wrong-lease" ? "other-sandbox" : reusableLease.providerLeaseId, state: "stopped" };
|
|
});
|
|
const workerManager = {
|
|
isRunning: () => true, call,
|
|
getWorker: () => ({ supportedMethods: ["environmentStopLease"] }),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
await runtimeWithPlugin.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true);
|
|
expect(call).toHaveBeenCalledWith(pluginId, "environmentStopLease", expect.objectContaining({
|
|
companyId, providerLeaseId: reusableLease.providerLeaseId, cancelActiveWork: true,
|
|
}), expect.any(Number));
|
|
expect(call).toHaveBeenCalledOnce();
|
|
expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(outcome === "stopped");
|
|
await expect(environmentService(db).getLeaseById(other.id)).resolves.toMatchObject({ status: "active" });
|
|
},
|
|
);
|
|
|
|
it.each(["release_only", "stopped", "stop_failed", "unconfirmed", "restart", "competing_owner", "missing_pin", "late_receipt"])(
|
|
"never dispatches destructive release for an untagged stop-and-retain request: %s", async outcome => {
|
|
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
|
const lease = seeded.reusableLease;
|
|
await db.update(environmentLeases).set({ leasePolicy: "ephemeral", metadata: {
|
|
...lease.metadata, reuseLease: false, ...(outcome === "missing_pin" ? { pluginId: undefined } : {}),
|
|
} }).where(eq(environmentLeases.id, lease.id));
|
|
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
|
|
let supported = !["release_only", "restart"].includes(outcome);
|
|
let calls = 0;
|
|
const call = vi.fn(async (_id: string, method: string) => {
|
|
if (method !== "environmentStopLease") return { providerLeaseId: lease.providerLeaseId, state: "destroyed" };
|
|
if (++calls === 1 && outcome === "stop_failed") throw new Error("injected stop failure");
|
|
if (calls === 1 && outcome === "unconfirmed") return undefined;
|
|
if (outcome === "late_receipt") await db.update(environmentLeases).set({
|
|
status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true },
|
|
}).where(eq(environmentLeases.id, lease.id));
|
|
return { providerLeaseId: lease.providerLeaseId, state: "stopped" };
|
|
});
|
|
const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: {
|
|
isRunning: () => true, call,
|
|
getWorker: () => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", ...(supported ? ["environmentStopLease"] : [])] }),
|
|
} as unknown as PluginWorkerManager });
|
|
if (outcome === "competing_owner") await environmentService(db).acquireLease({
|
|
companyId: seeded.companyId, environmentId: seeded.environment.id, heartbeatRunId: null,
|
|
leasePolicy: "ephemeral", provider: lease.provider, providerLeaseId: lease.providerLeaseId,
|
|
});
|
|
await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain");
|
|
expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true);
|
|
if (outcome === "late_receipt") {
|
|
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "active", heartbeatRunId: null, metadata: { newOwner: true } });
|
|
return;
|
|
}
|
|
if (outcome === "missing_pin") {
|
|
expect(call).not.toHaveBeenCalled();
|
|
await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
expect(call).not.toHaveBeenCalled();
|
|
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup" });
|
|
return;
|
|
}
|
|
if (["release_only", "restart", "competing_owner"].includes(outcome)) {
|
|
expect(call).not.toHaveBeenCalled();
|
|
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup", metadata: {
|
|
sandboxStopAndRetain: { pluginId: seeded.pluginId, runId: seeded.runId, providerLeaseId: lease.providerLeaseId },
|
|
} });
|
|
await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
expect(call).not.toHaveBeenCalled();
|
|
if (outcome === "competing_owner") return;
|
|
supported = true;
|
|
}
|
|
if (["release_only", "stop_failed", "unconfirmed", "restart"].includes(outcome)) {
|
|
await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
}
|
|
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: null,
|
|
metadata: { remoteExecutionTermination: { state: "stopped", providerLeaseId: lease.providerLeaseId },
|
|
sandboxStopAndRetainReceipt: { method: "environmentStopLease", runId: seeded.runId, pluginId: seeded.pluginId } } });
|
|
expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true);
|
|
},
|
|
);
|
|
|
|
it("reports an unwired manager separately from a stopped sandbox worker", async () => {
|
|
const { companyId, environment, runId } = await seedReusablePluginSandboxLease();
|
|
const input = {
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
};
|
|
const runtimeWithoutManager = environmentRuntimeService(db);
|
|
await expect(runtimeWithoutManager.acquireRunLease(input))
|
|
.rejects.toThrow("sandbox plugin workers are unavailable in this server process");
|
|
|
|
const offlineManager = { isRunning: () => false, call: vi.fn() } as unknown as PluginWorkerManager;
|
|
const runtimeWithStoppedWorker = environmentRuntimeService(db, {
|
|
pluginWorkerManager: offlineManager,
|
|
pluginWorkerReadyTimeoutMs: 0,
|
|
});
|
|
await expect(runtimeWithStoppedWorker.acquireRunLease(input))
|
|
.rejects.toThrow("its worker is not running");
|
|
expect(offlineManager.call).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("retains a successful reusable sandbox lease without stopping the provider resource", async () => {
|
|
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
throw new Error(`Unexpected plugin method while retaining warm lease: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const released = await runtimeWithPlugin.releaseRunLeases(
|
|
runId,
|
|
"released",
|
|
undefined,
|
|
"keep_running",
|
|
);
|
|
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.lease).toMatchObject({
|
|
id: reusableLease.id,
|
|
status: "retained",
|
|
cleanupStatus: "success",
|
|
});
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "retained",
|
|
cleanupStatus: "success",
|
|
});
|
|
expect(workerManager.call).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("stops a reusable sandbox and keeps its lease resumable", async () => {
|
|
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentStopLease") return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" };
|
|
throw new Error(`Unexpected plugin method while stopping lease: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentStopLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const released = await runtimeWithPlugin.releaseRunLeases(
|
|
runId,
|
|
"failed",
|
|
undefined,
|
|
"stop_and_retain",
|
|
);
|
|
|
|
expect(released[0]?.lease).toMatchObject({
|
|
id: reusableLease.id,
|
|
status: "released",
|
|
});
|
|
expect(workerManager.call).toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentStopLease",
|
|
expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }),
|
|
expect.any(Number),
|
|
);
|
|
});
|
|
|
|
it.each(["same task", "different task", "different agent", "no task"])(
|
|
"scopes projectless reusable leases to their task and agent: %s", async (scenario) => {
|
|
const seeded = await seedReusablePluginSandboxLease();
|
|
await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
|
|
const issueId = randomUUID();
|
|
const otherIssueId = randomUUID();
|
|
await db.insert(issues).values([issueId, otherIssueId].map(id => ({
|
|
id, companyId: seeded.companyId, title: "Projectless chat", status: "in_progress",
|
|
assigneeAgentId: seeded.agentId,
|
|
})));
|
|
const nextAgentId = scenario === "different agent" ? randomUUID() : seeded.agentId;
|
|
if (nextAgentId !== seeded.agentId) await db.insert(agents).values({
|
|
id: nextAgentId, companyId: seeded.companyId, name: "Other agent", adapterType: "paperclip_runner",
|
|
});
|
|
let acquisitions = 0;
|
|
const call = vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") return {
|
|
providerLeaseId: `projectless-${++acquisitions}`, metadata: { remoteCwd: "/workspace" },
|
|
};
|
|
if (method === "environmentResumeLease") return {
|
|
providerLeaseId: "projectless-1", metadata: { remoteCwd: "/workspace" },
|
|
};
|
|
if (method === "environmentStopLease") return { providerLeaseId: "projectless-1", state: "stopped" };
|
|
throw new Error(`Unexpected projectless lease method: ${method}`);
|
|
});
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: {
|
|
isRunning: () => true, call,
|
|
getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }),
|
|
} as unknown as PluginWorkerManager });
|
|
const first = await runtimeWithPlugin.acquireRunLease({
|
|
companyId: seeded.companyId, environment: seeded.environment, issueId,
|
|
agentId: seeded.agentId, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null,
|
|
});
|
|
expect(first.lease.metadata?.reusableSandboxLease).toMatchObject({
|
|
projectlessIssueId: issueId, executionWorkspaceId: null, agentId: seeded.agentId,
|
|
});
|
|
await runtimeWithPlugin.releaseRunLeases(seeded.runId, "released", undefined, "stop_and_retain");
|
|
const nextRunId = randomUUID();
|
|
await db.insert(heartbeatRuns).values({
|
|
id: nextRunId, companyId: seeded.companyId, agentId: nextAgentId, status: "running",
|
|
});
|
|
if (scenario === "different task" || scenario === "different agent") {
|
|
await expect(environmentService(db).acquireLease({
|
|
companyId: seeded.companyId, environmentId: seeded.environment.id,
|
|
issueId: scenario === "different task" ? otherIssueId : issueId,
|
|
heartbeatRunId: nextRunId, executionWorkspaceId: null,
|
|
provider: "fake-plugin", providerLeaseId: first.lease.providerLeaseId,
|
|
leasePolicy: "reuse_by_environment", replacesReusableLeaseId: first.lease.id,
|
|
metadata: { agentId: nextAgentId },
|
|
})).rejects.toMatchObject({ status: 409 });
|
|
}
|
|
const next = await runtimeWithPlugin.acquireRunLease({
|
|
companyId: seeded.companyId, environment: seeded.environment,
|
|
issueId: scenario === "no task" ? null : scenario === "different task" ? otherIssueId : issueId,
|
|
agentId: nextAgentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: null,
|
|
});
|
|
const shouldResume = scenario === "same task";
|
|
expect(next.lease.providerLeaseId).toBe(shouldResume ? "projectless-1" : "projectless-2");
|
|
expect(acquisitions).toBe(shouldResume ? 1 : 2);
|
|
expect(call.mock.calls.filter((entry) => entry[1] === "environmentResumeLease")).toHaveLength(shouldResume ? 1 : 0);
|
|
});
|
|
|
|
it.each(["pending", "inline", "foreign scope", "foreign environment", "foreign run", "malformed", "deleted environment", "forged error", "invalid diagnostic", "old worker"])("retains uncertain plugin creation for durable cleanup: %s", async (mode) => {
|
|
const seeded = await seedReusablePluginSandboxLease();
|
|
await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
|
|
const attemptId = randomUUID();
|
|
const providerLeaseId = `paperclip-create-${attemptId}`;
|
|
const cleanup = {
|
|
providerLeaseId, attemptId, companyId: mode === "foreign scope" ? randomUUID() : seeded.companyId,
|
|
environmentId: mode === "foreign environment" ? randomUUID() : seeded.environment.id,
|
|
runId: mode === "foreign run" ? randomUUID() : seeded.runId,
|
|
accountFingerprint: mode === "malformed" ? "invalid" : "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" },
|
|
};
|
|
const acquisitionDiagnostic = { phase: "shell", elapsedMs: 300_000, budgetMs: 300_000 };
|
|
const data = { schema: "paperclip/environment-creation-cleanup/v1", cleanup,
|
|
...(mode === "old worker" ? {} : { acquisitionDiagnostic: mode === "invalid diagnostic"
|
|
? { ...acquisitionDiagnostic, phase: "private-command" } : acquisitionDiagnostic }),
|
|
};
|
|
const failure = mode === "forged error"
|
|
? Object.assign(new Error("Provider creation cleanup required"), { data })
|
|
: new JsonRpcCallError({ code: -32002, message: "Provider creation cleanup required", data });
|
|
let available = mode === "inline";
|
|
const call = vi.fn(async (_id: string, method: string, params: Record<string, unknown>) => {
|
|
if (method === "environmentAcquireLease") throw failure;
|
|
if (method === "environmentDestroyLease") {
|
|
const pending = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId));
|
|
expect(pending).toHaveLength(1);
|
|
expect(pending[0].status).toBe("pending_cleanup");
|
|
expect(params).toMatchObject({ providerLeaseId, companyId: seeded.companyId, environmentId: seeded.environment.id, leaseMetadata: { failedCreateCleanup: cleanup } });
|
|
if (!available) throw new Error("provider not visible yet");
|
|
return { providerLeaseId, state: "destroyed" };
|
|
}
|
|
throw new Error(`Unexpected method ${method}`);
|
|
});
|
|
const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager;
|
|
const candidate = environmentRuntimeService(db, { pluginWorkerManager: worker });
|
|
const acquisition = candidate.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment,
|
|
issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null });
|
|
const caught = await acquisition.catch(error => error);
|
|
const execution = collectRunFailureDiagnostics({ id: seeded.runId, companyId: seeded.companyId,
|
|
errorCode: "setup_failed", resultJson: null,
|
|
} as typeof heartbeatRuns.$inferSelect, { error: caught, phase: "setup" }).execution;
|
|
if (["pending", "inline", "deleted environment"].includes(mode)) {
|
|
expect(execution).toMatchObject({ environmentAcquisitionPhase: "shell",
|
|
environmentAcquisitionElapsedMs: 300_000, environmentAcquisitionBudgetMs: 300_000 });
|
|
} else expect(execution).not.toHaveProperty("environmentAcquisitionPhase");
|
|
if (mode === "inline") {
|
|
await expect(acquisition).rejects.toMatchObject({
|
|
message: "Sandbox creation failed; allocated sandbox cleanup was confirmed.", cause: failure,
|
|
});
|
|
} else {
|
|
await expect(acquisition).rejects.toBe(failure);
|
|
}
|
|
const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId));
|
|
if (["foreign scope", "foreign environment", "foreign run", "malformed"].includes(mode)) {
|
|
expect(rows).toHaveLength(0);
|
|
expect(call.mock.calls.filter((c) => c[1] === "environmentDestroyLease")).toHaveLength(0);
|
|
return;
|
|
}
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]).toMatchObject({ status: mode === "inline" ? "expired" : "pending_cleanup",
|
|
cleanupStatus: mode === "inline" ? "success" : "failed", companyId: seeded.companyId,
|
|
metadata: { pluginId: seeded.pluginId, sandboxProviderPlugin: true, failedCreateCleanup: cleanup } });
|
|
if (mode === "pending" || mode === "deleted environment") {
|
|
available = true;
|
|
if (mode === "deleted environment") {
|
|
await db.update(environmentLeases).set({ environmentId: null }).where(eq(environmentLeases.id, rows[0].id));
|
|
}
|
|
const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker });
|
|
const persisted = await environmentService(db).getLeaseById(rows[0].id);
|
|
await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: persisted! })).resolves.toEqual({ providerLeaseId, state: "destroyed" });
|
|
}
|
|
});
|
|
|
|
it.each(["deleted receipt lost", "observation response lost", "journal write failed", "foreign observation"])(
|
|
"journals observed creation identity before deletion and recovers: %s", async (fault) => {
|
|
const seeded = await seedReusablePluginSandboxLease();
|
|
await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
|
|
const cleanup = {
|
|
providerLeaseId: `paperclip-create-${randomUUID()}`, attemptId: randomUUID(),
|
|
companyId: seeded.companyId, environmentId: seeded.environment.id, runId: seeded.runId,
|
|
accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" },
|
|
};
|
|
const observed = { ...cleanup, observedProviderLeaseId: "verified-provider-id" };
|
|
const failure = Object.assign(new Error("Creation uncertain"), {
|
|
data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup },
|
|
});
|
|
let injected = false;
|
|
let deleted = false;
|
|
const call = vi.fn(async (_id: string, method: string, params: Record<string, unknown>) => {
|
|
if (method === "environmentAcquireLease") throw failure;
|
|
if (method !== "environmentDestroyLease") throw new Error(`Unexpected ${method}`);
|
|
const evidence = (params.leaseMetadata as { failedCreateCleanup: typeof observed }).failedCreateCleanup;
|
|
if (!evidence.observedProviderLeaseId) {
|
|
expect(deleted).toBe(false);
|
|
if (!injected && fault === "observation response lost") {
|
|
injected = true;
|
|
throw new Error("Lost observation response");
|
|
}
|
|
if (!injected && fault === "foreign observation") {
|
|
injected = true;
|
|
throw Object.assign(new Error("Foreign observation"), {
|
|
data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup: { ...observed, companyId: "another-company" } },
|
|
});
|
|
}
|
|
if (!injected && fault === "journal write failed") {
|
|
injected = true;
|
|
vi.spyOn(db, "update").mockImplementationOnce(() => { throw new Error("Database unavailable"); });
|
|
}
|
|
throw Object.assign(new Error("Persist observed sandbox identity before cleanup"), {
|
|
data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup: observed },
|
|
});
|
|
}
|
|
const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0].metadata).toMatchObject({ failedCreateCleanup: observed });
|
|
if (!deleted) {
|
|
deleted = true;
|
|
if (fault === "deleted receipt lost") throw new Error("Deletion succeeded but reply lost");
|
|
}
|
|
return { providerLeaseId: cleanup.providerLeaseId, state: "destroyed" };
|
|
});
|
|
const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager;
|
|
const runtime = environmentRuntimeService(db, { pluginWorkerManager: worker });
|
|
await expect(runtime.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment,
|
|
issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null })).rejects.toBe(failure);
|
|
const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0].status).toBe("pending_cleanup");
|
|
expect(deleted).toBe(fault === "deleted receipt lost");
|
|
const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker });
|
|
await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: rows[0]! })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" });
|
|
expect(deleted).toBe(true);
|
|
},
|
|
);
|
|
|
|
it("resumes the same provider lease on the next per-turn run", async () => {
|
|
const seeded = await seedReusablePluginSandboxLease();
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === seeded.pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-exact-resume",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentStopLease") return { providerLeaseId: "sandbox-exact-resume", state: "stopped" };
|
|
if (method === "environmentResumeLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-exact-resume",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method during exact resume: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentStopLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
|
|
const first = await runtimeWithPlugin.acquireRunLease({
|
|
companyId: seeded.companyId,
|
|
environment: seeded.environment,
|
|
issueId: null,
|
|
agentId: seeded.agentId,
|
|
heartbeatRunId: seeded.runId,
|
|
persistedExecutionWorkspace: {
|
|
id: seeded.executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
const workspaceSyncStamp = {
|
|
schema: "paperclip.native-workspace-stamp/v1",
|
|
workspaceId: seeded.executionWorkspaceId,
|
|
providerLeaseId: "sandbox-exact-resume",
|
|
remoteCwd: "/workspace",
|
|
hostSha256: "a".repeat(64),
|
|
finalizedRunId: seeded.runId,
|
|
};
|
|
await environmentService(db).updateLeaseMetadata(first.lease.id, {
|
|
...(first.lease.metadata ?? {}),
|
|
nativeWorkspaceSync: workspaceSyncStamp,
|
|
});
|
|
await runtimeWithPlugin.releaseRunLeases(
|
|
seeded.runId,
|
|
"released",
|
|
undefined,
|
|
"stop_and_retain",
|
|
);
|
|
|
|
const nextRunId = randomUUID();
|
|
await db.insert(heartbeatRuns).values({
|
|
id: nextRunId,
|
|
companyId: seeded.companyId,
|
|
agentId: seeded.agentId,
|
|
invocationSource: "manual",
|
|
status: "running",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId: seeded.companyId,
|
|
environment: seeded.environment,
|
|
issueId: null,
|
|
agentId: seeded.agentId,
|
|
heartbeatRunId: nextRunId,
|
|
persistedExecutionWorkspace: {
|
|
id: seeded.executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" });
|
|
expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume");
|
|
expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({
|
|
outcome: "resumed",
|
|
});
|
|
expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual(
|
|
workspaceSyncStamp,
|
|
);
|
|
await expect(
|
|
environmentService(db).getLeaseById(first.lease.id),
|
|
).resolves.toMatchObject({
|
|
status: "expired",
|
|
cleanupStatus: "success",
|
|
});
|
|
expect(
|
|
workerManager.call.mock.calls.filter(
|
|
(call) => call[1] === "environmentAcquireLease",
|
|
),
|
|
).toHaveLength(1);
|
|
});
|
|
|
|
it("reacquires one provider lease row idempotently during same-run recovery", async () => {
|
|
const seeded = await seedReusablePluginSandboxLease();
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === seeded.pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentResumeLease") {
|
|
return {
|
|
providerLeaseId: seeded.reusableLease.providerLeaseId,
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
throw new Error(
|
|
`Unexpected plugin method during same-run reacquisition: ${method}`,
|
|
);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: workerManager,
|
|
});
|
|
|
|
const reacquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId: seeded.companyId,
|
|
environment: seeded.environment,
|
|
issueId: null,
|
|
agentId: seeded.agentId,
|
|
heartbeatRunId: seeded.runId,
|
|
persistedExecutionWorkspace: {
|
|
id: seeded.executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(reacquired.lease).toMatchObject({
|
|
id: seeded.reusableLease.id,
|
|
heartbeatRunId: seeded.runId,
|
|
providerLeaseId: seeded.reusableLease.providerLeaseId,
|
|
status: "active",
|
|
cleanupStatus: null,
|
|
});
|
|
const rows = await environmentService(db).listLeases(seeded.environment.id);
|
|
expect(
|
|
rows.filter(
|
|
(lease) =>
|
|
lease.heartbeatRunId === seeded.runId &&
|
|
lease.providerLeaseId === seeded.reusableLease.providerLeaseId,
|
|
),
|
|
).toHaveLength(1);
|
|
expect(workerManager.call).toHaveBeenCalledWith(
|
|
seeded.pluginId,
|
|
"environmentResumeLease",
|
|
expect.objectContaining({
|
|
providerLeaseId: seeded.reusableLease.providerLeaseId,
|
|
}),
|
|
expect.any(Number),
|
|
);
|
|
});
|
|
|
|
it("destroys a disposable paperclip_runner sandbox after the turn", async () => {
|
|
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentDestroyLease") return undefined;
|
|
throw new Error(`Unexpected plugin method while destroying lease: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const released = await runtimeWithPlugin.releaseRunLeases(
|
|
runId,
|
|
"released",
|
|
undefined,
|
|
"destroy",
|
|
);
|
|
|
|
expect(released[0]?.lease).toMatchObject({
|
|
id: reusableLease.id,
|
|
status: "expired",
|
|
});
|
|
expect(workerManager.call).toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentDestroyLease",
|
|
expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }),
|
|
expect.any(Number),
|
|
);
|
|
});
|
|
|
|
it("does not destroy native sandbox state when its verified backup is missing", async () => {
|
|
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
|
|
await environmentService(db).updateLeaseMetadata(reusableLease.id, {
|
|
...(reusableLease.metadata ?? {}),
|
|
reusableSandboxLease: {
|
|
...((reusableLease.metadata?.reusableSandboxLease as Record<
|
|
string,
|
|
unknown
|
|
>) ?? {}),
|
|
adapterType: "paperclip_runner",
|
|
},
|
|
sandboxLeaseAcquisition: { outcome: "created" },
|
|
});
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
throw new Error(`Provider teardown must not run without a backup: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
const errors: unknown[] = [];
|
|
|
|
const released = await runtimeWithPlugin.releaseRunLeases(
|
|
runId,
|
|
"released",
|
|
(_leaseId, error) => errors.push(error),
|
|
"destroy",
|
|
);
|
|
|
|
expect(released).toEqual([]);
|
|
expect(errors).toHaveLength(1);
|
|
expect(String(errors[0])).toContain("runner_harness_backup_unavailable");
|
|
expect(workerManager.call).not.toHaveBeenCalled();
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "active",
|
|
});
|
|
});
|
|
|
|
it("acquires and releases a projectless local run lease through the runtime seam", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment();
|
|
|
|
const acquired = await runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
expect(acquired.lease.status).toBe("active");
|
|
expect(acquired.lease.executionWorkspaceId).toBeNull();
|
|
expect(acquired.lease.metadata).toMatchObject({
|
|
driver: "local",
|
|
executionWorkspaceMode: null,
|
|
});
|
|
expect(acquired.leaseContext).toEqual({
|
|
executionWorkspaceId: null,
|
|
executionWorkspaceMode: null,
|
|
});
|
|
|
|
const released = await runtime.releaseRunLeases(runId);
|
|
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.environment.driver).toBe("local");
|
|
expect(released[0]?.lease.status).toBe("released");
|
|
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.id, acquired.lease.id));
|
|
expect(rows[0]?.status).toBe("released");
|
|
});
|
|
|
|
it("rejects truly unsupported drivers before acquiring a lease", async () => {
|
|
const { companyId, agentId, environment, runId } = await seedEnvironment({
|
|
driver: "ssh",
|
|
name: "Fixture SSH",
|
|
config: {
|
|
host: "ssh.example.test",
|
|
port: 22,
|
|
username: "ssh-user",
|
|
remoteWorkspacePath: "/srv/paperclip/workspace",
|
|
privateKey: null,
|
|
knownHosts: null,
|
|
strictHostKeyChecking: true,
|
|
},
|
|
});
|
|
const runtimeWithoutSsh = environmentRuntimeService(db, {
|
|
drivers: [
|
|
{
|
|
driver: "local",
|
|
acquireRunLease: async () => {
|
|
throw new Error("should not acquire");
|
|
},
|
|
releaseRunLease: async () => null,
|
|
},
|
|
],
|
|
});
|
|
|
|
await expect(
|
|
runtimeWithoutSsh.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
}),
|
|
).rejects.toThrow('Environment driver "ssh" is not registered in the environment runtime yet.');
|
|
|
|
const rows = await db.select().from(environmentLeases);
|
|
expect(rows).toHaveLength(0);
|
|
});
|
|
|
|
it("acquires and releases an SSH run lease through the runtime seam", async () => {
|
|
if (!sshFixtureSupport.supported) {
|
|
console.warn(
|
|
`Skipping SSH runtime fixture test: ${sshFixtureSupport.reason ?? "unsupported environment"}`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
const fixtureRoot = await mkdtemp(path.join(os.tmpdir(), "paperclip-environment-runtime-ssh-"));
|
|
fixtureRoots.push(fixtureRoot);
|
|
const statePath = path.join(fixtureRoot, "state.json");
|
|
const fixture = await startSshEnvLabFixture({ statePath });
|
|
const sshConfig = await buildSshEnvLabFixtureConfig(fixture);
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "ssh",
|
|
name: "Fixture SSH",
|
|
config: sshConfig,
|
|
});
|
|
try {
|
|
const acquired = await runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
expect(acquired.lease.status).toBe("active");
|
|
expect(acquired.lease.providerLeaseId).toContain(`ssh://${sshConfig.username}@${sshConfig.host}:${sshConfig.port}`);
|
|
expect(acquired.lease.metadata).toMatchObject({
|
|
driver: "ssh",
|
|
host: sshConfig.host,
|
|
port: sshConfig.port,
|
|
username: sshConfig.username,
|
|
remoteWorkspacePath: sshConfig.remoteWorkspacePath,
|
|
remoteCwd: sshConfig.remoteWorkspacePath,
|
|
});
|
|
|
|
const released = await runtime.releaseRunLeases(runId);
|
|
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.environment.driver).toBe("ssh");
|
|
expect(released[0]?.lease.status).toBe("released");
|
|
} finally {
|
|
}
|
|
});
|
|
|
|
it("acquires and releases a fake sandbox run lease through the runtime seam", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Fake Sandbox",
|
|
config: {
|
|
provider: "fake",
|
|
image: "ubuntu:24.04",
|
|
reuseLease: true,
|
|
},
|
|
});
|
|
|
|
const acquired = await runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
expect(acquired.lease.status).toBe("active");
|
|
expect(acquired.lease.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`));
|
|
expect(acquired.lease.leasePolicy).toBe("ephemeral");
|
|
expect(acquired.lease.metadata).toMatchObject({
|
|
driver: "sandbox",
|
|
provider: "fake",
|
|
image: "ubuntu:24.04",
|
|
reuseLease: true,
|
|
});
|
|
|
|
const released = await runtime.releaseRunLeases(runId);
|
|
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.environment.driver).toBe("sandbox");
|
|
expect(released[0]?.lease.status).toBe("released");
|
|
});
|
|
|
|
it.each([false, true])("confirms a built-in fake stop without release or destroy fallback: cancel=%s", async cancel => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null,
|
|
heartbeatRunId: runId, persistedExecutionWorkspace: null });
|
|
const provider = sandboxProviderRuntime.requireSandboxProvider("fake");
|
|
const release = vi.spyOn(provider, "releaseLease"), destroy = vi.spyOn(provider, "destroyLease");
|
|
const released = await runtime.releaseRunLeases(runId, "released", undefined, "stop_and_retain", cancel);
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.lease).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: null,
|
|
metadata: { remoteExecutionTermination: { state: "stopped", providerLeaseId: acquired.lease.providerLeaseId },
|
|
sandboxStopAndRetainReceipt: { method: "builtin.stopLease", builtinProvider: "fake", runId } } });
|
|
expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true);
|
|
expect(release).not.toHaveBeenCalled();
|
|
expect(destroy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it.each(["stop_failed", "wrong_receipt", "missing_capability", "changed_binding"])(
|
|
"preserves a built-in stop through restart without destructive fallback: %s", async outcome => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null,
|
|
heartbeatRunId: runId, persistedExecutionWorkspace: null });
|
|
const provider = sandboxProviderRuntime.requireSandboxProvider("fake"), originalStop = provider.stopLease!;
|
|
const release = vi.spyOn(provider, "releaseLease"), destroy = vi.spyOn(provider, "destroyLease");
|
|
try {
|
|
provider.stopLease = outcome === "missing_capability" ? undefined : vi.fn(async () => {
|
|
if (outcome === "wrong_receipt") return { providerLeaseId: "sandbox://fake/other", state: "stopped" as const };
|
|
throw new Error("injected built-in stop failure");
|
|
});
|
|
await runtime.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true);
|
|
expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" });
|
|
expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(false);
|
|
if (outcome === "missing_capability") {
|
|
await heartbeatService(db, { environmentRuntime: environmentRuntimeService(db) }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" });
|
|
}
|
|
const stop = vi.fn(originalStop.bind(provider));
|
|
provider.stopLease = stop;
|
|
if (outcome === "changed_binding") await db.update(environmentLeases).set({ providerLeaseId: "sandbox://fake/replacement" })
|
|
.where(eq(environmentLeases.id, acquired.lease.id));
|
|
await heartbeatService(db, { environmentRuntime: environmentRuntimeService(db) }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
if (outcome === "changed_binding") {
|
|
expect(stop).not.toHaveBeenCalled();
|
|
expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" });
|
|
} else {
|
|
expect(stop).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ providerLeaseId: acquired.lease.providerLeaseId }));
|
|
expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "released", cleanupStatus: "success" });
|
|
expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true);
|
|
}
|
|
expect(release).not.toHaveBeenCalled();
|
|
expect(destroy).not.toHaveBeenCalled();
|
|
} finally { provider.stopLease = originalStop; }
|
|
},
|
|
);
|
|
|
|
it("releases the remote sandbox when the lease insert rejects a foreign-company binding", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox",
|
|
config: {
|
|
provider: "fake",
|
|
image: "ubuntu:24.04",
|
|
reuseLease: false,
|
|
},
|
|
});
|
|
|
|
// A managed reconciliation binds the environment to another company after the
|
|
// route guard read an empty binding list. The lease insert then rejects the
|
|
// foreign-company binding with the 403 `environment_company_mismatch`.
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co",
|
|
issuePrefix: "OTA",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
const destroySpy = vi.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease");
|
|
try {
|
|
await expect(
|
|
runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({
|
|
status: 403,
|
|
details: { code: "environment_company_mismatch" },
|
|
});
|
|
|
|
// The acquire records the durable pending-cleanup row before the teardown,
|
|
// so the row lands while the database is proven reachable. The successful
|
|
// teardown then releases the row to the terminal `expired` state, so no
|
|
// active or pending_cleanup row remains for the orphan.
|
|
const leaseRows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(leaseRows).toHaveLength(1);
|
|
expect(leaseRows[0]?.status).toBe("expired");
|
|
expect(leaseRows[0]?.cleanupStatus).toBe("success");
|
|
|
|
// The acquire already provisioned the remote sandbox, so it releases the
|
|
// sandbox on the rejection. Without this teardown the rejected insert leaks
|
|
// a live sandbox that no lease row tracks. The teardown carries the
|
|
// provisioned provider lease id.
|
|
expect(destroySpy).toHaveBeenCalledTimes(1);
|
|
expect(destroySpy.mock.calls[0]?.[0]?.providerLeaseId).toMatch(
|
|
new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`),
|
|
);
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("destroys the remote plugin sandbox when the lease insert rejects a foreign-company binding", async () => {
|
|
// The Claude login runs on a plugin-backed sandbox provider (Daytona), so this
|
|
// path is the one the login uses. It must also release the remote sandbox when
|
|
// the conditional lease insert rejects a foreign-company binding.
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false };
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Foreign-bound Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: pluginConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: pluginConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId);
|
|
|
|
// A managed reconciliation binds the environment to another company after the
|
|
// route guard read an empty binding list. The lease insert then rejects the
|
|
// foreign-company binding with the 403 `environment_company_mismatch`.
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co",
|
|
issuePrefix: "OTB",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "plugin-lease-1",
|
|
metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false },
|
|
};
|
|
}
|
|
if (method === "environmentDestroyLease") {
|
|
return { providerLeaseId: "plugin-lease-1", state: "destroyed" };
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
await expect(
|
|
runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({
|
|
status: 403,
|
|
details: { code: "environment_company_mismatch" },
|
|
});
|
|
|
|
// The acquire records the durable pending-cleanup row before the teardown,
|
|
// so the row lands while the database is proven reachable. The successful
|
|
// teardown then releases the row to the terminal `expired` state, so no
|
|
// active or pending_cleanup row remains for the orphan.
|
|
const leaseRows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(leaseRows).toHaveLength(1);
|
|
expect(leaseRows[0]?.status).toBe("expired");
|
|
expect(leaseRows[0]?.cleanupStatus).toBe("success");
|
|
expect(leaseRows[0]?.metadata?.remoteExecutionTermination).toMatchObject({
|
|
companyId, runId, leaseId: leaseRows[0]!.id, providerLeaseId: "plugin-lease-1", state: "destroyed",
|
|
});
|
|
|
|
// The acquire provisioned the remote plugin sandbox, so it destroys the
|
|
// sandbox on the rejection. Without this teardown the rejected insert leaks a
|
|
// live sandbox that no lease row tracks.
|
|
const destroyCalls = (workerManager.call as unknown as ReturnType<typeof vi.fn>).mock.calls.filter(
|
|
(callArgs) => callArgs[1] === "environmentDestroyLease",
|
|
);
|
|
expect(destroyCalls).toHaveLength(1);
|
|
expect(destroyCalls[0]?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-1" });
|
|
|
|
await environmentService(db).update(environment.id, { driver: "local", config: {} });
|
|
});
|
|
|
|
it("records a durable pending-cleanup lease when the built-in teardown fails after a foreign-company rejection", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Teardown Fail",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Built-in",
|
|
issuePrefix: "OTC",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// The remote teardown fails, so the acquire cannot release the sandbox
|
|
// directly. It must record a durable pending-cleanup row instead.
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValue(new Error("teardown failed"));
|
|
try {
|
|
await expect(
|
|
runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
|
|
|
|
expect(destroySpy).toHaveBeenCalledTimes(1);
|
|
// The failed teardown left a durable pending-cleanup row that carries the
|
|
// provider lease id for a later sweep.
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]?.status).toBe("pending_cleanup");
|
|
expect(rows[0]?.cleanupStatus).toBe("failed");
|
|
expect(rows[0]?.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`));
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("records the durable pending-cleanup row before it runs the inline teardown", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Write First",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Write First",
|
|
issuePrefix: "OWF",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// The teardown reads the lease table when it runs. A durable pending_cleanup
|
|
// row must already exist, which proves the acquire records the row before the
|
|
// teardown. This ordering closes the window the finding describes: the durable
|
|
// write lands while the database is proven reachable, before the teardown RPC
|
|
// that a database outage could otherwise interrupt.
|
|
let pendingCleanupRowsAtTeardown = -1;
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockImplementation(async () => {
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
pendingCleanupRowsAtTeardown = rows.filter((row) => row.status === "pending_cleanup").length;
|
|
});
|
|
try {
|
|
await expect(
|
|
runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
|
|
|
|
// The durable row already tracked the orphan when the teardown ran.
|
|
expect(pendingCleanupRowsAtTeardown).toBe(1);
|
|
expect(destroySpy).toHaveBeenCalledTimes(1);
|
|
|
|
// The teardown succeeded, so the acquire released the row to the terminal
|
|
// `expired` state and left no pending_cleanup row.
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]?.status).toBe("expired");
|
|
expect(rows[0]?.cleanupStatus).toBe("success");
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("leaves no orphan and no error when the durable write fails but the teardown succeeds", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Write Fail Teardown Ok",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Write Fail",
|
|
issuePrefix: "OWK",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// Force every durable pending-cleanup write to fail, but let the teardown
|
|
// succeed. A successful teardown removes the orphan, so the acquire needs no
|
|
// durable row and raises no orphan-cleanup write error. The write-first order
|
|
// never turns a clean teardown into a failure.
|
|
const realEnvironmentService = environmentsModule.environmentService;
|
|
const factorySpy = vi
|
|
.spyOn(environmentsModule, "environmentService")
|
|
.mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
|
|
const real = realEnvironmentService(database);
|
|
return {
|
|
...real,
|
|
insertPendingCleanupLease: () =>
|
|
Promise.reject(new Error("pending-cleanup write failed; database down")),
|
|
};
|
|
});
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockResolvedValue(undefined as never);
|
|
try {
|
|
// Set the retry backoff to zero, so the failed write retries never slow the
|
|
// test.
|
|
const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
|
|
const rejection = await runtime
|
|
.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
})
|
|
.then(
|
|
() => {
|
|
throw new Error("acquireRunLease resolved but must reject");
|
|
},
|
|
(error: unknown) => error,
|
|
);
|
|
|
|
// The rejection is the original company mismatch, not an orphan-cleanup
|
|
// write error, because the teardown removed the orphan.
|
|
expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError);
|
|
expect(rejection).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
|
|
expect(destroySpy).toHaveBeenCalledTimes(1);
|
|
|
|
// No orphan remains and no durable row was needed, so the lease table is
|
|
// empty for the environment.
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(0);
|
|
|
|
// The buffer holds no orphan, so a flush recovers nothing.
|
|
const flushed = await runtime.flushDeferredOrphanCleanups();
|
|
expect(flushed).toEqual({ recovered: 0, pending: 0 });
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
factorySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("records a durable pending-cleanup lease when the plugin teardown fails after a foreign-company rejection", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false };
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Foreign-bound Plugin Sandbox Teardown Fail",
|
|
driver: "sandbox",
|
|
config: pluginConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: pluginConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId);
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Plugin",
|
|
issuePrefix: "OTD",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "plugin-lease-2",
|
|
metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false },
|
|
};
|
|
}
|
|
if (method === "environmentDestroyLease") {
|
|
throw new Error("destroy failed");
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
await expect(
|
|
runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
|
|
|
|
// The failed plugin teardown left a durable pending-cleanup row that carries
|
|
// the provider lease id for a later sweep.
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]?.status).toBe("pending_cleanup");
|
|
expect(rows[0]?.cleanupStatus).toBe("failed");
|
|
expect(rows[0]?.providerLeaseId).toBe("plugin-lease-2");
|
|
|
|
await environmentService(db).update(environment.id, { driver: "local", config: {} });
|
|
});
|
|
|
|
it("throws a SandboxOrphanCleanupWriteError when the built-in durable pending-cleanup write also fails", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Cleanup Write Fail",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Cleanup Write",
|
|
issuePrefix: "OCW",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// The remote teardown fails, so the acquire tries to record a durable
|
|
// pending-cleanup row instead.
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValue(new Error("teardown failed"));
|
|
// Force the durable pending-cleanup write to fail on every attempt. The
|
|
// write retries a few times, so reject the insert each time and let the
|
|
// primary company-bound acquire reject for real.
|
|
const failingInsert = vi
|
|
.fn<Parameters<ReturnType<typeof environmentService>["insertPendingCleanupLease"]>, Promise<never>>()
|
|
.mockRejectedValue(new Error("pending-cleanup write failed"));
|
|
const realEnvironmentService = environmentsModule.environmentService;
|
|
const factorySpy = vi
|
|
.spyOn(environmentsModule, "environmentService")
|
|
.mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
|
|
const real = realEnvironmentService(database);
|
|
return {
|
|
...real,
|
|
insertPendingCleanupLease: failingInsert,
|
|
};
|
|
});
|
|
// The durable database write fails, so the only durable handle left is the
|
|
// error log. Capture it to prove the leaked sandbox stays discoverable.
|
|
const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
|
|
try {
|
|
// Set the retry backoff to zero, so the retries never slow the test.
|
|
const runtimeWithFailingCleanup = environmentRuntimeService(db, {
|
|
pendingCleanupWriteBackoffMs: 0,
|
|
});
|
|
const rejection = await runtimeWithFailingCleanup
|
|
.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
})
|
|
.then(
|
|
() => {
|
|
throw new Error("acquireRunLease resolved but must reject");
|
|
},
|
|
(error: unknown) => error,
|
|
);
|
|
|
|
// The failed durable write is not swallowed: the acquire throws a
|
|
// SandboxOrphanCleanupWriteError that keeps the original 403 rejection as
|
|
// its cause.
|
|
expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError);
|
|
expect(rejection).toMatchObject({ provider: "fake" });
|
|
expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({
|
|
status: 403,
|
|
details: { code: "environment_company_mismatch" },
|
|
});
|
|
expect(destroySpy).toHaveBeenCalledTimes(1);
|
|
// The write retried the durable insert before it gave up, so more than one
|
|
// attempt ran.
|
|
expect(failingInsert.mock.calls.length).toBeGreaterThan(1);
|
|
// The durable write failed before it created a row, so no lease row exists.
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(0);
|
|
// The durable log preserves the provider identifiers, so an operator keeps
|
|
// a handle to find and tear the leaked sandbox down by hand.
|
|
const cleanupLog = logSpy.mock.calls.find(
|
|
([fields]) =>
|
|
(fields as { errorKind?: string } | undefined)?.errorKind ===
|
|
"sandbox_orphan_cleanup_write_failed",
|
|
);
|
|
expect(cleanupLog).toBeDefined();
|
|
expect(cleanupLog?.[0]).toMatchObject({
|
|
provider: "fake",
|
|
companyId,
|
|
environmentId: environment.id,
|
|
});
|
|
// The log never carries the caught exception, so no credential leaks.
|
|
expect(cleanupLog?.[0]).not.toHaveProperty("cause");
|
|
expect(cleanupLog?.[0]).not.toHaveProperty("cleanupWriteError");
|
|
} finally {
|
|
logSpy.mockRestore();
|
|
factorySpy.mockRestore();
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("recovers cleanup state when a retry of the durable pending-cleanup write succeeds", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Cleanup Write Retry",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Cleanup Retry",
|
|
issuePrefix: "OCR",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// The remote teardown fails, so the acquire tries to record a durable
|
|
// pending-cleanup row instead.
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValue(new Error("teardown failed"));
|
|
// The first durable write attempt fails, so the acquire retries. The second
|
|
// attempt delegates to the real insert, so a durable pending_cleanup row
|
|
// lands and a later sweep can find the orphan.
|
|
let insertAttempts = 0;
|
|
const realEnvironmentService = environmentsModule.environmentService;
|
|
const factorySpy = vi
|
|
.spyOn(environmentsModule, "environmentService")
|
|
.mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
|
|
const real = realEnvironmentService(database);
|
|
return {
|
|
...real,
|
|
insertPendingCleanupLease: (
|
|
input: Parameters<typeof real.insertPendingCleanupLease>[0],
|
|
) => {
|
|
insertAttempts += 1;
|
|
if (insertAttempts === 1) {
|
|
return Promise.reject(new Error("pending-cleanup write failed once"));
|
|
}
|
|
return real.insertPendingCleanupLease(input);
|
|
},
|
|
};
|
|
});
|
|
try {
|
|
// Set the retry backoff to zero, so the retry never slows the test.
|
|
const runtimeWithRetry = environmentRuntimeService(db, {
|
|
pendingCleanupWriteBackoffMs: 0,
|
|
});
|
|
const rejection = await runtimeWithRetry
|
|
.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
})
|
|
.then(
|
|
() => {
|
|
throw new Error("acquireRunLease resolved but must reject");
|
|
},
|
|
(error: unknown) => error,
|
|
);
|
|
|
|
// The retry landed the durable row, so the acquire rejects with the
|
|
// original company-mismatch rejection, not a SandboxOrphanCleanupWriteError.
|
|
expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError);
|
|
expect(rejection).toMatchObject({
|
|
status: 403,
|
|
details: { code: "environment_company_mismatch" },
|
|
});
|
|
expect(destroySpy).toHaveBeenCalledTimes(1);
|
|
// The first attempt failed and the second succeeded, so the write ran twice.
|
|
expect(insertAttempts).toBe(2);
|
|
// A durable pending_cleanup lease row now tracks the orphan, so a sweep can
|
|
// find and release the leaked sandbox.
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]?.status).toBe("pending_cleanup");
|
|
expect(rows[0]?.cleanupStatus).toBe("failed");
|
|
expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed");
|
|
} finally {
|
|
factorySpy.mockRestore();
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("buffers the orphan in-process and a later sweep flush lands the durable row after the database recovers", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Cleanup Buffer Flush",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Cleanup Buffer",
|
|
issuePrefix: "OCB",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// The remote teardown fails, so the acquire tries to record a durable
|
|
// pending-cleanup row instead.
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValue(new Error("teardown failed"));
|
|
// The database is down, so every synchronous pending-cleanup write rejects.
|
|
// The flag flips to false when the database recovers, so a later flush lands
|
|
// the durable row.
|
|
let databaseDown = true;
|
|
const realEnvironmentService = environmentsModule.environmentService;
|
|
const factorySpy = vi
|
|
.spyOn(environmentsModule, "environmentService")
|
|
.mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
|
|
const real = realEnvironmentService(database);
|
|
return {
|
|
...real,
|
|
insertPendingCleanupLease: (
|
|
input: Parameters<typeof real.insertPendingCleanupLease>[0],
|
|
) => {
|
|
if (databaseDown) {
|
|
return Promise.reject(new Error("pending-cleanup write failed; database down"));
|
|
}
|
|
return real.insertPendingCleanupLease(input);
|
|
},
|
|
};
|
|
});
|
|
const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
|
|
try {
|
|
// Set the retry backoff to zero, so the retries never slow the test.
|
|
const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
|
|
const rejection = await runtime
|
|
.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
})
|
|
.then(
|
|
() => {
|
|
throw new Error("acquireRunLease resolved but must reject");
|
|
},
|
|
(error: unknown) => error,
|
|
);
|
|
|
|
// The synchronous write failed on every attempt, so the acquire still
|
|
// throws the orphan-cleanup write error that keeps the original rejection.
|
|
expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError);
|
|
// The database was down, so no durable row exists yet.
|
|
let rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(0);
|
|
// The acquire buffered the orphan in-process, so the log records the buffer.
|
|
const bufferedLog = logSpy.mock.calls.find(
|
|
([fields]) =>
|
|
(fields as { errorKind?: string } | undefined)?.errorKind ===
|
|
"sandbox_orphan_cleanup_write_failed",
|
|
);
|
|
expect(bufferedLog?.[0]).toMatchObject({ buffered: true });
|
|
|
|
// The database recovers, so the next cleanup-sweep flush lands the row.
|
|
databaseDown = false;
|
|
const flushed = await runtime.flushDeferredOrphanCleanups();
|
|
expect(flushed).toEqual({ recovered: 1, pending: 0 });
|
|
|
|
// A durable pending_cleanup row now tracks the orphan, so a sweep finds and
|
|
// releases the leaked sandbox.
|
|
rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]?.status).toBe("pending_cleanup");
|
|
expect(rows[0]?.cleanupStatus).toBe("failed");
|
|
expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed");
|
|
expect(rows[0]?.providerLeaseId).toBeTruthy();
|
|
|
|
// The buffer is empty now, so a second flush inserts nothing.
|
|
const second = await runtime.flushDeferredOrphanCleanups();
|
|
expect(second).toEqual({ recovered: 0, pending: 0 });
|
|
const rowsAfter = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rowsAfter).toHaveLength(1);
|
|
} finally {
|
|
logSpy.mockRestore();
|
|
factorySpy.mockRestore();
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("persists the orphan to the durable spool so a restart recovers and lands the durable row", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Cleanup Spool Restart",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Cleanup Spool Restart",
|
|
issuePrefix: "OCS",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// The remote teardown fails, so the acquire tries to record a durable
|
|
// pending-cleanup row instead.
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValue(new Error("teardown failed"));
|
|
// The database is down, so every synchronous pending-cleanup write rejects.
|
|
// The flag flips to false to simulate the database recovering after a
|
|
// restart.
|
|
let databaseDown = true;
|
|
const realEnvironmentService = environmentsModule.environmentService;
|
|
const factorySpy = vi
|
|
.spyOn(environmentsModule, "environmentService")
|
|
.mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
|
|
const real = realEnvironmentService(database);
|
|
return {
|
|
...real,
|
|
insertPendingCleanupLease: (
|
|
input: Parameters<typeof real.insertPendingCleanupLease>[0],
|
|
) => {
|
|
if (databaseDown) {
|
|
return Promise.reject(new Error("pending-cleanup write failed; database down"));
|
|
}
|
|
return real.insertPendingCleanupLease(input);
|
|
},
|
|
};
|
|
});
|
|
const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
|
|
const spoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR!;
|
|
try {
|
|
// The first process acquires, fails every synchronous write, and persists
|
|
// the orphan to the durable spool. The retry backoff is zero, so the test
|
|
// never waits.
|
|
const firstProcessRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
|
|
const rejection = await firstProcessRuntime
|
|
.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
})
|
|
.then(
|
|
() => {
|
|
throw new Error("acquireRunLease resolved but must reject");
|
|
},
|
|
(error: unknown) => error,
|
|
);
|
|
expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError);
|
|
|
|
// The database was down, so no durable row exists yet.
|
|
let rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(0);
|
|
|
|
// The acquire persisted the orphan to the durable spool, so a file waits on
|
|
// the disk and the log records the durable persist.
|
|
const spoolFiles = (await readdir(spoolDir)).filter((name) => name.endsWith(".json"));
|
|
expect(spoolFiles).toHaveLength(1);
|
|
const persistedLog = logSpy.mock.calls.find(
|
|
([fields]) =>
|
|
(fields as { errorKind?: string } | undefined)?.errorKind ===
|
|
"sandbox_orphan_cleanup_write_failed",
|
|
);
|
|
expect(persistedLog?.[0]).toMatchObject({ persisted: true });
|
|
|
|
// The process restarts. A fresh runtime keeps no in-process buffer, so only
|
|
// the durable spool carries the orphan. The database recovers, so the first
|
|
// cleanup-sweep flush loads the spool and lands the durable row.
|
|
databaseDown = false;
|
|
const restartedRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
|
|
const flushed = await restartedRuntime.flushDeferredOrphanCleanups();
|
|
expect(flushed).toEqual({ recovered: 1, pending: 0 });
|
|
|
|
rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]?.status).toBe("pending_cleanup");
|
|
expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed");
|
|
expect(rows[0]?.providerLeaseId).toBeTruthy();
|
|
|
|
// The flush removed the spooled copy after the row landed, so a second
|
|
// restart never re-inserts a duplicate.
|
|
const spoolFilesAfter = (await readdir(spoolDir)).filter((name) => name.endsWith(".json"));
|
|
expect(spoolFilesAfter).toHaveLength(0);
|
|
const secondRestartRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
|
|
const secondFlush = await secondRestartRuntime.flushDeferredOrphanCleanups();
|
|
expect(secondFlush).toEqual({ recovered: 0, pending: 0 });
|
|
const rowsAfter = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rowsAfter).toHaveLength(1);
|
|
} finally {
|
|
logSpy.mockRestore();
|
|
factorySpy.mockRestore();
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("buffers the orphan in-process when the durable spool write also fails", async () => {
|
|
const idleBefore = idleWorkSnapshot().active;
|
|
let releaseFlush!: () => void;
|
|
let enteredFlush!: () => void;
|
|
const flushGate = new Promise<void>(resolve => { releaseFlush = resolve; });
|
|
const flushEntered = new Promise<void>(resolve => { enteredFlush = resolve; });
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Cleanup Spool Unwritable",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Cleanup Spool Unwritable",
|
|
issuePrefix: "OCU",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValue(new Error("teardown failed"));
|
|
let databaseDown = true;
|
|
const realEnvironmentService = environmentsModule.environmentService;
|
|
const factorySpy = vi
|
|
.spyOn(environmentsModule, "environmentService")
|
|
.mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
|
|
const real = realEnvironmentService(database);
|
|
return {
|
|
...real,
|
|
insertPendingCleanupLease: (
|
|
input: Parameters<typeof real.insertPendingCleanupLease>[0],
|
|
) => {
|
|
if (databaseDown) {
|
|
return Promise.reject(new Error("pending-cleanup write failed; database down"));
|
|
}
|
|
enteredFlush();
|
|
return flushGate.then(() => real.insertPendingCleanupLease(input));
|
|
},
|
|
};
|
|
});
|
|
const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
|
|
try {
|
|
// Point the spool at an unwritable path, so the durable spool write fails
|
|
// like a full or read-only disk. The acquire must fall back to the
|
|
// in-process buffer and never lose the orphan silently.
|
|
const unwritableSpool = {
|
|
append: () => Promise.resolve(false),
|
|
load: () => Promise.resolve([]),
|
|
remove: () => Promise.resolve(),
|
|
};
|
|
const runtime = environmentRuntimeService(db, {
|
|
pendingCleanupWriteBackoffMs: 0,
|
|
orphanCleanupSpool: unwritableSpool,
|
|
});
|
|
await runtime
|
|
.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
})
|
|
.then(
|
|
() => {
|
|
throw new Error("acquireRunLease resolved but must reject");
|
|
},
|
|
() => undefined,
|
|
);
|
|
|
|
// The spool write failed, so the log records the in-process-only fallback.
|
|
const fallbackLog = logSpy.mock.calls.find(
|
|
([fields]) =>
|
|
(fields as { errorKind?: string } | undefined)?.errorKind ===
|
|
"sandbox_orphan_cleanup_write_failed",
|
|
);
|
|
expect(fallbackLog?.[0]).toMatchObject({ persisted: false, buffered: true });
|
|
expect(idleWorkSnapshot().active).toBe(idleBefore + 1);
|
|
|
|
// The same runtime still keeps the orphan in-process, so a flush after the
|
|
// database recovers lands the durable row.
|
|
databaseDown = false;
|
|
const flushing = runtime.flushDeferredOrphanCleanups();
|
|
await flushEntered;
|
|
// The batch was spliced out of the queue, but the DB does not own it yet.
|
|
expect(idleWorkSnapshot().active).toBe(idleBefore + 1);
|
|
releaseFlush();
|
|
const flushed = await flushing;
|
|
expect(idleWorkSnapshot().active).toBe(idleBefore);
|
|
expect(flushed).toEqual({ recovered: 1, pending: 0 });
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]?.status).toBe("pending_cleanup");
|
|
} finally {
|
|
releaseFlush();
|
|
logSpy.mockRestore();
|
|
factorySpy.mockRestore();
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("keeps the buffered orphan when the flush write still fails, then recovers on a later flush", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Foreign-bound Fake Sandbox Cleanup Buffer Requeue",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Cleanup Requeue",
|
|
issuePrefix: "OCQ",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValue(new Error("teardown failed"));
|
|
let databaseDown = true;
|
|
const realEnvironmentService = environmentsModule.environmentService;
|
|
const factorySpy = vi
|
|
.spyOn(environmentsModule, "environmentService")
|
|
.mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
|
|
const real = realEnvironmentService(database);
|
|
return {
|
|
...real,
|
|
insertPendingCleanupLease: (
|
|
input: Parameters<typeof real.insertPendingCleanupLease>[0],
|
|
) => {
|
|
if (databaseDown) {
|
|
return Promise.reject(new Error("pending-cleanup write failed; database down"));
|
|
}
|
|
return real.insertPendingCleanupLease(input);
|
|
},
|
|
};
|
|
});
|
|
const errorLogSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
|
|
const warnLogSpy = vi.spyOn(logger, "warn").mockImplementation(() => logger);
|
|
try {
|
|
const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
|
|
await runtime
|
|
.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
})
|
|
.then(
|
|
() => {
|
|
throw new Error("acquireRunLease resolved but must reject");
|
|
},
|
|
() => undefined,
|
|
);
|
|
|
|
// The database is still down, so the flush re-queues the orphan instead of
|
|
// losing it. The buffer keeps the record for a later tick.
|
|
const firstFlush = await runtime.flushDeferredOrphanCleanups();
|
|
expect(firstFlush).toEqual({ recovered: 0, pending: 1 });
|
|
const rowsWhileDown = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rowsWhileDown).toHaveLength(0);
|
|
// The flush logs the re-queue and never carries the caught write exception.
|
|
// The sync-retry path also warns with the same error kind, so match on the
|
|
// `requeued` field that only the flush warn carries.
|
|
const requeueLog = warnLogSpy.mock.calls.find(
|
|
([fields]) => (fields as { requeued?: boolean } | undefined)?.requeued === true,
|
|
);
|
|
expect(requeueLog).toBeDefined();
|
|
expect(requeueLog?.[0]).toMatchObject({
|
|
errorKind: "sandbox_orphan_cleanup_write_failed",
|
|
requeued: true,
|
|
});
|
|
expect(requeueLog?.[0]).not.toHaveProperty("cause");
|
|
expect(requeueLog?.[0]).not.toHaveProperty("cleanupWriteError");
|
|
|
|
// The database recovers, so the next flush lands the durable row exactly
|
|
// once from the still-buffered record.
|
|
databaseDown = false;
|
|
const secondFlush = await runtime.flushDeferredOrphanCleanups();
|
|
expect(secondFlush).toEqual({ recovered: 1, pending: 0 });
|
|
const rowsAfter = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rowsAfter).toHaveLength(1);
|
|
expect(rowsAfter[0]?.status).toBe("pending_cleanup");
|
|
} finally {
|
|
warnLogSpy.mockRestore();
|
|
errorLogSpy.mockRestore();
|
|
factorySpy.mockRestore();
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("throws a SandboxOrphanCleanupWriteError when the plugin durable pending-cleanup write also fails", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false };
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Foreign-bound Plugin Sandbox Cleanup Write Fail",
|
|
driver: "sandbox",
|
|
config: pluginConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: pluginConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId);
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Plugin Cleanup Write",
|
|
issuePrefix: "OPW",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "plugin-lease-write-fail",
|
|
metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false },
|
|
};
|
|
}
|
|
if (method === "environmentDestroyLease") {
|
|
throw new Error("destroy failed");
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
|
|
const realEnvironmentService = environmentsModule.environmentService;
|
|
const factorySpy = vi
|
|
.spyOn(environmentsModule, "environmentService")
|
|
.mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
|
|
const real = realEnvironmentService(database);
|
|
return {
|
|
...real,
|
|
insertPendingCleanupLease: () =>
|
|
Promise.reject(new Error("pending-cleanup write failed")),
|
|
};
|
|
});
|
|
try {
|
|
const runtimeWithPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: workerManager,
|
|
pendingCleanupWriteBackoffMs: 0,
|
|
});
|
|
const rejection = await runtimeWithPlugin
|
|
.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
})
|
|
.then(
|
|
() => {
|
|
throw new Error("acquireRunLease resolved but must reject");
|
|
},
|
|
(error: unknown) => error,
|
|
);
|
|
|
|
expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError);
|
|
expect(rejection).toMatchObject({ provider: "fake-plugin", providerLeaseId: "plugin-lease-write-fail" });
|
|
expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({
|
|
status: 403,
|
|
details: { code: "environment_company_mismatch" },
|
|
});
|
|
// The durable write failed before it created a row, so no lease row exists.
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(0);
|
|
} finally {
|
|
factorySpy.mockRestore();
|
|
}
|
|
|
|
await environmentService(db).update(environment.id, { driver: "local", config: {} });
|
|
});
|
|
|
|
it("records the orphan directly in the pending_cleanup state with one atomic insert", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Atomic Pending Cleanup Insert",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
|
|
const lease = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId: null,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
provider: "fake",
|
|
providerLeaseId: "sandbox://fake/atomic-orphan",
|
|
metadata: { provider: "fake", reuseLease: false },
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
|
|
// The returned lease is already in the terminal recovery state. It never
|
|
// passes through the `active` state, so a crash cannot strand the orphan.
|
|
expect(lease.status).toBe("pending_cleanup");
|
|
expect(lease.cleanupStatus).toBe("failed");
|
|
expect(lease.leasePolicy).toBe("ephemeral");
|
|
expect(lease.failureReason).toBe("acquire_rejected_teardown_failed");
|
|
expect(lease.providerLeaseId).toBe("sandbox://fake/atomic-orphan");
|
|
expect(lease.releasedAt).not.toBeNull();
|
|
|
|
// One insert created exactly one row, and the sweep reads that row directly.
|
|
const rows = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id));
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0]?.status).toBe("pending_cleanup");
|
|
expect(rows[0]?.cleanupStatus).toBe("failed");
|
|
});
|
|
|
|
it("tears the recorded built-in provider lease down for a pending-cleanup orphan", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Cleanup Retry Built-in",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Retry Built-in",
|
|
issuePrefix: "OTE",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// The first teardown fails, so the acquire records a pending-cleanup orphan.
|
|
// The sweep then retries the teardown, which succeeds this time.
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValueOnce(new Error("teardown failed"))
|
|
.mockResolvedValue(undefined);
|
|
try {
|
|
await expect(
|
|
runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
|
|
|
|
const orphan = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id))
|
|
.then((r) => r[0]!);
|
|
expect(orphan.status).toBe("pending_cleanup");
|
|
const providerLeaseId = orphan.providerLeaseId;
|
|
|
|
// The teardown accepts a null environment and reads the recorded provider
|
|
// lease, so a delete or a provider change never strands it.
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! });
|
|
// The retry called the provider teardown a second time with the orphan
|
|
// provider lease id.
|
|
expect(destroySpy).toHaveBeenCalledTimes(2);
|
|
expect(destroySpy).toHaveBeenLastCalledWith(expect.objectContaining({ providerLeaseId }));
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("throws and keeps the pending-cleanup orphan when the teardown retry fails", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Cleanup Retry Scope",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Retry Scope",
|
|
issuePrefix: "OTF",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValue(new Error("teardown failed"));
|
|
try {
|
|
await expect(
|
|
runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
|
|
|
|
const orphan = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id))
|
|
.then((r) => r[0]!);
|
|
expect(orphan.status).toBe("pending_cleanup");
|
|
|
|
// The teardown retries, but the provider destroy still fails, so the
|
|
// teardown throws. The caller keeps the orphan pending for a later sweep.
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
await expect(
|
|
runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }),
|
|
).rejects.toThrow();
|
|
|
|
const still = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.id, orphan.id))
|
|
.then((r) => r[0]!);
|
|
expect(still.status).toBe("pending_cleanup");
|
|
expect(still.cleanupStatus).toBe("failed");
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("tears the recorded provider down after the environment provider changes", async () => {
|
|
// Ordering: an acquire provisions a sandbox with provider A, the lease
|
|
// insert rejects a foreign-company binding, and the compensating teardown
|
|
// fails, so the acquire records an orphan for provider A. A provider change
|
|
// then re-points the environment before the sweep runs. The sweep must tear
|
|
// the orphan down from the recorded provider metadata, not the current
|
|
// environment provider, so the change cannot strand the teardown.
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Provider Change Vs Orphan",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Provider Change",
|
|
issuePrefix: "OTP",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockRejectedValueOnce(new Error("teardown failed"))
|
|
.mockResolvedValue(undefined);
|
|
try {
|
|
await expect(
|
|
runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
|
|
|
|
const orphan = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id))
|
|
.then((r) => r[0]!);
|
|
expect(orphan.status).toBe("pending_cleanup");
|
|
expect(orphan.provider).toBe("fake");
|
|
const providerLeaseId = orphan.providerLeaseId;
|
|
|
|
// The provider-target mutation re-points the environment to a different
|
|
// driver. The orphan teardown must not depend on this current config.
|
|
await environmentService(db).update(environment.id, { driver: "local", config: {} });
|
|
|
|
// The teardown tears the recorded provider lease down, so it never reads
|
|
// the re-pointed environment provider.
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! });
|
|
expect(destroySpy).toHaveBeenLastCalledWith(expect.objectContaining({ providerLeaseId }));
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("reports the plugin cleanup worker not ready while its worker is down, then ready after it recovers", async () => {
|
|
// A pending_cleanup orphan targets a plugin-backed provider. The sweep must
|
|
// not consume a finite retry attempt while the plugin worker is briefly down.
|
|
// So the readiness probe reports "not ready" for a down worker and "ready"
|
|
// after the worker recovers. A built-in provider has no worker, so it is
|
|
// always ready.
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Cleanup Worker Readiness",
|
|
config: { provider: "fake-plugin-ready", image: "fake:test", reuseLease: false },
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "paperclip.fake-plugin-ready-sandbox-provider",
|
|
packageName: "@paperclipai/plugin-fake-ready-sandbox",
|
|
displayName: "Fake Plugin Ready Sandbox Provider",
|
|
description: "Test fake plugin provider readiness",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin-ready",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin Ready",
|
|
configSchema: { type: "object" },
|
|
},
|
|
],
|
|
});
|
|
|
|
// The worker is down at first, then recovers on the second probe.
|
|
let workerRunning = false;
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId && workerRunning),
|
|
call: vi.fn(async () => {
|
|
throw new Error("call must not run during a readiness probe");
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId: null,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
provider: "fake-plugin-ready",
|
|
providerLeaseId: "plugin-lease-readiness",
|
|
metadata: { provider: "fake-plugin-ready", driver: "sandbox", reuseLease: false },
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
const lease = (await environmentService(db).getLeaseById(orphan.id))!;
|
|
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
// A down plugin worker is the transient restart window, so the probe reports
|
|
// not ready. The sweep skips the lease without a claim, so no attempt burns.
|
|
await expect(
|
|
runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
|
|
).resolves.toBe(false);
|
|
// The probe never calls the worker; it only checks the live worker state.
|
|
expect((workerManager.call as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();
|
|
|
|
// The worker recovers, so the probe now reports ready and the sweep proceeds.
|
|
workerRunning = true;
|
|
await expect(
|
|
runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
|
|
).resolves.toBe(true);
|
|
|
|
// A built-in provider has no plugin worker, so its orphan is always ready.
|
|
const builtinOrphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId: null,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
provider: "fake",
|
|
providerLeaseId: "builtin-lease-readiness",
|
|
metadata: { provider: "fake", driver: "sandbox", reuseLease: false },
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
const builtinLease = (await environmentService(db).getLeaseById(builtinOrphan.id))!;
|
|
await expect(
|
|
runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease: builtinLease }),
|
|
).resolves.toBe(true);
|
|
});
|
|
|
|
it("reports the plugin cleanup provider not ready while the plugin is not ready, then ready after it recovers", async () => {
|
|
// A pending_cleanup orphan targets a plugin-backed provider. A plugin reload
|
|
// or a plugin reinstall moves the plugin through a "not ready" status. The
|
|
// sweep must not consume a finite retry attempt in that transient window. So
|
|
// the readiness probe reports "not ready" while the plugin status is not
|
|
// "ready", even when its worker runs, and "ready" after the plugin recovers.
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Cleanup Plugin Readiness",
|
|
config: { provider: "fake-plugin-reload", image: "fake:test", reuseLease: false },
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "paperclip.fake-plugin-reload-sandbox-provider",
|
|
packageName: "@paperclipai/plugin-fake-reload-sandbox",
|
|
displayName: "Fake Plugin Reload Sandbox Provider",
|
|
description: "Test fake plugin provider reload readiness",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin-reload",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin Reload",
|
|
configSchema: { type: "object" },
|
|
},
|
|
],
|
|
status: "installing",
|
|
});
|
|
|
|
// The worker runs the whole time, so the probe result depends on the plugin
|
|
// status alone, not on a down worker.
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async () => {
|
|
throw new Error("call must not run during a readiness probe");
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId: null,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
provider: "fake-plugin-reload",
|
|
providerLeaseId: "plugin-lease-reload",
|
|
metadata: { provider: "fake-plugin-reload", driver: "sandbox", reuseLease: false },
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
const lease = (await environmentService(db).getLeaseById(orphan.id))!;
|
|
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
// A not-ready plugin is the transient reload window, so the probe reports
|
|
// not ready. The sweep skips the lease without a claim, so no attempt burns.
|
|
await expect(
|
|
runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
|
|
).resolves.toBe(false);
|
|
expect((workerManager.call as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();
|
|
|
|
// The plugin becomes ready, so the probe now reports ready and the sweep
|
|
// proceeds.
|
|
await db.update(plugins).set({ status: "ready" }).where(eq(plugins.id, pluginId));
|
|
await expect(
|
|
runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
|
|
).resolves.toBe(true);
|
|
});
|
|
|
|
it("reports the plugin cleanup provider not ready while the plugin is missing, then ready after it returns", async () => {
|
|
// A pending_cleanup orphan targets a plugin-backed provider whose plugin row
|
|
// is gone. A plugin reinstall removes and re-adds the plugin row, so the row
|
|
// is missing for a short window. The sweep must not consume a finite retry
|
|
// attempt in that window. So the readiness probe reports "not ready" while no
|
|
// plugin row resolves the provider, and "ready" after the plugin returns.
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Cleanup Plugin Missing",
|
|
config: { provider: "fake-plugin-missing", image: "fake:test", reuseLease: false },
|
|
});
|
|
|
|
// The worker manager reports the plugin worker as running, so the probe
|
|
// result depends on the missing plugin row alone, not on a down worker.
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async () => {
|
|
throw new Error("call must not run during a readiness probe");
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId: null,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
provider: "fake-plugin-missing",
|
|
providerLeaseId: "plugin-lease-missing",
|
|
metadata: { provider: "fake-plugin-missing", driver: "sandbox", reuseLease: false },
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
const lease = (await environmentService(db).getLeaseById(orphan.id))!;
|
|
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
// No plugin row resolves the provider, so the probe reports not ready. The
|
|
// sweep skips the lease without a claim, so no attempt burns while the plugin
|
|
// reinstall is in flight.
|
|
await expect(
|
|
runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
|
|
).resolves.toBe(false);
|
|
expect((workerManager.call as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();
|
|
|
|
// The plugin reinstall completes, so a ready plugin row now resolves the
|
|
// provider and the probe reports ready.
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "paperclip.fake-plugin-missing-sandbox-provider",
|
|
packageName: "@paperclipai/plugin-fake-missing-sandbox",
|
|
displayName: "Fake Plugin Missing Sandbox Provider",
|
|
description: "Test fake plugin provider missing readiness",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin-missing",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin Missing",
|
|
configSchema: { type: "object" },
|
|
},
|
|
],
|
|
});
|
|
await expect(
|
|
runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
|
|
).resolves.toBe(true);
|
|
});
|
|
|
|
it("records an orphan with a null reference when a delete already removed the environment", async () => {
|
|
// Ordering: a delete removes the environment before the acquire records the
|
|
// orphan. The environment foreign key must not fail the insert. The record
|
|
// persists with a null reference and its immutable provider metadata, so a
|
|
// later sweep still tears the orphan down.
|
|
const { companyId, environment } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Delete Before Orphan Record",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
|
|
// Remove the environment row first, so the record happens after the delete.
|
|
await environmentService(db).remove(environment.id);
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
provider: "fake",
|
|
providerLeaseId: "sandbox://fake/delete-before-record",
|
|
metadata: { provider: "fake", image: "ubuntu:24.04", driver: "sandbox" },
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
// The insert kept no environment reference, so the delete did not fail it.
|
|
expect(orphan.environmentId).toBeNull();
|
|
expect(orphan.status).toBe("pending_cleanup");
|
|
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockResolvedValue(undefined);
|
|
try {
|
|
// The recorded metadata drives the teardown, so a null environment tears
|
|
// the orphan down.
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! });
|
|
expect(destroySpy).toHaveBeenCalledWith(
|
|
expect.objectContaining({ providerLeaseId: "sandbox://fake/delete-before-record" }),
|
|
);
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("keeps the orphan and tears it down after the environment is deleted", async () => {
|
|
// Ordering: an orphan is recorded, then the environment is deleted. The
|
|
// foreign key sets the reference null instead of cascading the row away, so
|
|
// the orphan survives. The sweep tears it down from the recorded metadata.
|
|
const { companyId, environment } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Delete After Orphan Record",
|
|
config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
|
|
});
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
provider: "fake",
|
|
providerLeaseId: "sandbox://fake/delete-after-record",
|
|
metadata: { provider: "fake", image: "ubuntu:24.04", driver: "sandbox" },
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
expect(orphan.environmentId).toBe(environment.id);
|
|
|
|
// Delete the environment row directly. The foreign key sets the lease
|
|
// reference null, so the orphan survives instead of being cascaded away.
|
|
await environmentService(db).remove(environment.id);
|
|
const survived = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.id, orphan.id))
|
|
.then((r) => r[0]!);
|
|
expect(survived.environmentId).toBeNull();
|
|
expect(survived.status).toBe("pending_cleanup");
|
|
|
|
const destroySpy = vi
|
|
.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
|
|
.mockResolvedValue(undefined);
|
|
try {
|
|
// The recorded metadata drives the teardown, so a null environment tears
|
|
// the orphan down after the delete set the reference null.
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! });
|
|
expect(destroySpy).toHaveBeenCalledWith(
|
|
expect.objectContaining({ providerLeaseId: "sandbox://fake/delete-after-record" }),
|
|
);
|
|
} finally {
|
|
destroySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
// A plugin provider whose config declares an `apiKey` secret-ref field. The
|
|
// orphan teardown must resolve that recorded ref even when the environment
|
|
// binding is gone, so these tests register a provider with a real secret-ref
|
|
// field, unlike the plain `fake-plugin` tests above.
|
|
const SECRET_REF_PLUGIN_KEY = "paperclip.secret-plugin-sandbox-provider";
|
|
const SECRET_REF_PROVIDER = "secret-plugin";
|
|
|
|
async function registerSecretRefPluginProvider(): Promise<string> {
|
|
const pluginId = randomUUID();
|
|
await db.insert(plugins).values({
|
|
id: pluginId,
|
|
pluginKey: SECRET_REF_PLUGIN_KEY,
|
|
packageName: "@paperclipai/plugin-secret-sandbox",
|
|
version: "1.0.0",
|
|
apiVersion: 1,
|
|
categories: ["automation"],
|
|
manifestJson: {
|
|
id: SECRET_REF_PLUGIN_KEY,
|
|
apiVersion: 1,
|
|
version: "1.0.0",
|
|
displayName: "Secret Plugin Sandbox Provider",
|
|
description: "Test plugin provider with a secret-ref config field",
|
|
author: "Paperclip",
|
|
categories: ["automation"],
|
|
capabilities: ["environment.drivers.register"],
|
|
entrypoints: { worker: "dist/worker.js" },
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: SECRET_REF_PROVIDER,
|
|
kind: "sandbox_provider",
|
|
displayName: "Secret Plugin",
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
apiKey: { type: "string", format: "secret-ref" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
status: "ready",
|
|
installOrder: 1,
|
|
updatedAt: new Date(),
|
|
} as any);
|
|
return pluginId;
|
|
}
|
|
|
|
// A worker manager mock that records the config of each destroy call, so a
|
|
// test can assert the resolved API key reached the provider teardown.
|
|
function createDestroyRecordingWorkerManager(pluginId: string): {
|
|
workerManager: PluginWorkerManager;
|
|
destroyConfigs: Array<Record<string, unknown> | undefined>;
|
|
} {
|
|
const destroyConfigs: Array<Record<string, unknown> | undefined> = [];
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string, args: any) => {
|
|
if (method === "environmentDestroyLease") {
|
|
destroyConfigs.push(args?.config as Record<string, unknown> | undefined);
|
|
return {};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
return { workerManager, destroyConfigs };
|
|
}
|
|
|
|
async function createApiKeySecret(companyId: string, value: string): Promise<string> {
|
|
const secret = await secretService(db).create(companyId, {
|
|
name: `sandbox-cleanup-api-key-${randomUUID()}`,
|
|
provider: "local_encrypted",
|
|
value,
|
|
});
|
|
return secret.id;
|
|
}
|
|
|
|
it("tears down a secret-backed plugin orphan when a delete already removed the environment", async () => {
|
|
// Ordering: a delete removed the environment (and its secret binding) before
|
|
// the acquire recorded the orphan. The recorded config keeps the API-key
|
|
// secret ref, so the teardown must resolve it without the environment
|
|
// binding and send the resolved value to the provider destroy call.
|
|
const pluginId = await registerSecretRefPluginProvider();
|
|
const { companyId, environment } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Secret Plugin Delete Before Record",
|
|
config: { provider: SECRET_REF_PROVIDER, reuseLease: false },
|
|
});
|
|
const secretId = await createApiKeySecret(companyId, "old-provider-api-key");
|
|
await secretService(db).createBinding({
|
|
companyId,
|
|
secretId,
|
|
targetType: "environment",
|
|
targetId: environment.id,
|
|
configPath: "apiKey",
|
|
});
|
|
|
|
// Remove the environment first, so the record happens after the delete.
|
|
await environmentService(db).remove(environment.id);
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
provider: SECRET_REF_PROVIDER,
|
|
providerLeaseId: "secret-plugin-lease-delete-before",
|
|
metadata: {
|
|
provider: SECRET_REF_PROVIDER,
|
|
driver: "sandbox",
|
|
sandboxProviderPlugin: true,
|
|
pluginId,
|
|
pluginKey: SECRET_REF_PLUGIN_KEY,
|
|
apiKey: secretId,
|
|
reuseLease: false,
|
|
},
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
expect(orphan.environmentId).toBeNull();
|
|
expect(orphan.status).toBe("pending_cleanup");
|
|
|
|
const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId);
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
vi.mocked(workerManager.call).mockImplementationOnce(async (_pluginId, _method, args: any) => {
|
|
destroyConfigs.push(args.config);
|
|
return { providerLeaseId: args.providerLeaseId, state: "destroyed" };
|
|
});
|
|
await expect(runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }))
|
|
.resolves.toEqual({ providerLeaseId: orphan.providerLeaseId, state: "destroyed" });
|
|
expect(destroyConfigs).toHaveLength(1);
|
|
// The recorded secret ref resolved to the old credential, and the resolved
|
|
// value reached the provider teardown instead of the secret ref.
|
|
expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" });
|
|
expect(destroyConfigs[0]?.apiKey).not.toBe(secretId);
|
|
});
|
|
|
|
it("tears down a secret-backed plugin orphan after the environment is deleted", async () => {
|
|
// Ordering: the orphan is recorded, then a delete removes the environment.
|
|
// The foreign key sets the lease reference null, so the orphan survives. The
|
|
// teardown resolves the recorded API-key ref without the environment.
|
|
const pluginId = await registerSecretRefPluginProvider();
|
|
const { companyId, environment } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Secret Plugin Delete After Record",
|
|
config: { provider: SECRET_REF_PROVIDER, reuseLease: false },
|
|
});
|
|
const secretId = await createApiKeySecret(companyId, "old-provider-api-key");
|
|
await secretService(db).createBinding({
|
|
companyId,
|
|
secretId,
|
|
targetType: "environment",
|
|
targetId: environment.id,
|
|
configPath: "apiKey",
|
|
});
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
provider: SECRET_REF_PROVIDER,
|
|
providerLeaseId: "secret-plugin-lease-delete-after",
|
|
metadata: {
|
|
provider: SECRET_REF_PROVIDER,
|
|
driver: "sandbox",
|
|
sandboxProviderPlugin: true,
|
|
pluginId,
|
|
pluginKey: SECRET_REF_PLUGIN_KEY,
|
|
apiKey: secretId,
|
|
reuseLease: false,
|
|
},
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
expect(orphan.environmentId).toBe(environment.id);
|
|
|
|
// Delete the environment. The foreign key sets the lease reference null, so
|
|
// the orphan survives and the current binding is gone.
|
|
await environmentService(db).remove(environment.id);
|
|
const survived = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.id, orphan.id))
|
|
.then((r) => r[0]!);
|
|
expect(survived.environmentId).toBeNull();
|
|
|
|
const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId);
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! });
|
|
expect(destroyConfigs).toHaveLength(1);
|
|
expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" });
|
|
});
|
|
|
|
it("tears down a secret-backed plugin orphan with the recorded credential after the provider is reconfigured", async () => {
|
|
// Ordering: the orphan is recorded for the old credential, then the
|
|
// environment is reconfigured to a new credential. The environment binding
|
|
// now points to a different secret at the same path. The teardown must
|
|
// resolve the OLD recorded ref, not the new binding, so it destroys the
|
|
// sandbox the old credential provisioned.
|
|
const pluginId = await registerSecretRefPluginProvider();
|
|
const { companyId, environment } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Secret Plugin Reconfigured",
|
|
config: { provider: SECRET_REF_PROVIDER, reuseLease: false },
|
|
});
|
|
const oldSecretId = await createApiKeySecret(companyId, "old-provider-api-key");
|
|
|
|
const orphan = await environmentService(db).insertPendingCleanupLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
provider: SECRET_REF_PROVIDER,
|
|
providerLeaseId: "secret-plugin-lease-reconfigured",
|
|
metadata: {
|
|
provider: SECRET_REF_PROVIDER,
|
|
driver: "sandbox",
|
|
sandboxProviderPlugin: true,
|
|
pluginId,
|
|
pluginKey: SECRET_REF_PLUGIN_KEY,
|
|
apiKey: oldSecretId,
|
|
reuseLease: false,
|
|
},
|
|
failureReason: "acquire_rejected_teardown_failed",
|
|
});
|
|
expect(orphan.environmentId).toBe(environment.id);
|
|
|
|
// Reconfigure the environment: a new secret replaces the binding at the same
|
|
// path. The old recorded ref is no longer bound to the environment.
|
|
const newSecretId = await createApiKeySecret(companyId, "new-provider-api-key");
|
|
await secretService(db).createBinding({
|
|
companyId,
|
|
secretId: newSecretId,
|
|
targetType: "environment",
|
|
targetId: environment.id,
|
|
configPath: "apiKey",
|
|
});
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
config: { provider: SECRET_REF_PROVIDER, apiKey: newSecretId, reuseLease: false },
|
|
});
|
|
|
|
const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId);
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! });
|
|
expect(destroyConfigs).toHaveLength(1);
|
|
// The teardown resolved the OLD recorded credential, not the new binding.
|
|
expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" });
|
|
expect(destroyConfigs[0]?.apiKey).not.toBe("new-provider-api-key");
|
|
|
|
await environmentService(db).update(environment.id, { driver: "local", config: {} });
|
|
});
|
|
|
|
it("tears the recorded plugin provider lease down for a pending-cleanup orphan", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false };
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Cleanup Retry Plugin",
|
|
driver: "sandbox",
|
|
config: pluginConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: pluginConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId);
|
|
const otherCompanyId = randomUUID();
|
|
await db.insert(companies).values({
|
|
id: otherCompanyId,
|
|
name: "Other Co Retry Plugin",
|
|
issuePrefix: "OTG",
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment);
|
|
|
|
// The first destroy fails, so the acquire records a pending-cleanup orphan.
|
|
// The sweep retries the destroy, which succeeds this time.
|
|
let destroyAttempts = 0;
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "plugin-lease-3",
|
|
metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false },
|
|
};
|
|
}
|
|
if (method === "environmentDestroyLease") {
|
|
destroyAttempts += 1;
|
|
if (destroyAttempts === 1) {
|
|
throw new Error("destroy failed");
|
|
}
|
|
return {};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
await expect(
|
|
runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
assertCompanyBinding: true,
|
|
}),
|
|
).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
|
|
|
|
const orphan = await db
|
|
.select()
|
|
.from(environmentLeases)
|
|
.where(eq(environmentLeases.environmentId, environment.id))
|
|
.then((r) => r[0]!);
|
|
expect(orphan.status).toBe("pending_cleanup");
|
|
expect(orphan.providerLeaseId).toBe("plugin-lease-3");
|
|
|
|
const lease = await environmentService(db).getLeaseById(orphan.id);
|
|
await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! });
|
|
expect(destroyAttempts).toBe(2);
|
|
const destroyCall = (workerManager.call as unknown as ReturnType<typeof vi.fn>).mock.calls
|
|
.filter((callArgs) => callArgs[1] === "environmentDestroyLease")
|
|
.at(-1);
|
|
expect(destroyCall?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-3" });
|
|
|
|
await environmentService(db).update(environment.id, { driver: "local", config: {} });
|
|
});
|
|
|
|
it("uses plugin-backed sandbox config for execute and release", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const fakePluginConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Fake Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: fakePluginConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: fakePluginConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId);
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string, params: any) => {
|
|
expect(params.config).toEqual(expect.objectContaining({
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
}));
|
|
expect(params.config).not.toHaveProperty("provider");
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-1",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentExecute") {
|
|
return {
|
|
exitCode: 0,
|
|
signal: null,
|
|
timedOut: false,
|
|
stdout: "ok\n",
|
|
stderr: "",
|
|
};
|
|
}
|
|
if (method === "environmentReleaseLease") {
|
|
expect(params.config).toEqual({
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
});
|
|
expect(params.config).not.toHaveProperty("driver");
|
|
expect(params.config).not.toHaveProperty("executionWorkspaceMode");
|
|
expect(params.config).not.toHaveProperty("pluginId");
|
|
expect(params.config).not.toHaveProperty("pluginKey");
|
|
expect(params.config).not.toHaveProperty("providerMetadata");
|
|
expect(params.config).not.toHaveProperty("provider");
|
|
expect(params.config).not.toHaveProperty("sandboxProviderPlugin");
|
|
return undefined;
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
const executed = await runtimeWithPlugin.execute({
|
|
environment,
|
|
lease: acquired.lease,
|
|
command: "printf",
|
|
args: ["ok"],
|
|
cwd: "/workspace",
|
|
env: {},
|
|
timeoutMs: 1000,
|
|
commandTimeoutMs: 900,
|
|
forceSession: true,
|
|
});
|
|
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "local",
|
|
config: {},
|
|
});
|
|
const released = await runtimeWithPlugin.releaseRunLeases(runId);
|
|
|
|
expect(executed.stdout).toBe("ok\n");
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.lease.status).toBe("released");
|
|
// The execute call carries the optional log sink as the fifth argument; it
|
|
// is undefined when the caller passes no sink.
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({
|
|
timeoutMs: 1000,
|
|
commandTimeoutMs: 900,
|
|
bypassSession: false,
|
|
}), 31000, undefined);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.anything(), 31234);
|
|
});
|
|
|
|
// Build the fake plugin fixture for the run-parent release tests below. A
|
|
// plugin sandbox provider can open a persistent session on the first command
|
|
// and delete it on lease release; the delete emits a provider
|
|
// `session.close` span. The host mints that span's parent from the active
|
|
// step context at the release RPC. So the release must run under the run
|
|
// parent, or the span loses its traceparent and the backend drops it.
|
|
async function seedFakePluginSandbox() {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const fakePluginConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Fake Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: fakePluginConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: fakePluginConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId);
|
|
return { companyId, environment, runId, pluginId };
|
|
}
|
|
|
|
it("runs a plugin-backed lease release under the run-time exec parent, so the teardown span keeps a valid traceparent", async () => {
|
|
const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox();
|
|
|
|
// The active step context observed at the moment the host issues the release
|
|
// RPC. The real worker manager mints the provider-span traceparent from
|
|
// exactly this value, so a non-null context with a valid parent proves the
|
|
// teardown span keeps a host-minted parent.
|
|
let releaseStepContext: ReturnType<typeof getActiveStepContext> = null;
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-1",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentExecute") {
|
|
return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" };
|
|
}
|
|
if (method === "environmentReleaseLease") {
|
|
releaseStepContext = getActiveStepContext();
|
|
return undefined;
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
// Build a real host run-parent context and run the first command under it,
|
|
// exactly as the run drives an exec. The driver records this context for the
|
|
// lease and replays it around the later release RPC.
|
|
const runParent: StartupSpanContext = trace.setSpanContext(ROOT_CONTEXT, {
|
|
traceId: "0af7651916cd43dd8448eb211c80319c",
|
|
spanId: "b7ad6b7169203331",
|
|
traceFlags: 1,
|
|
isRemote: false,
|
|
});
|
|
await runWithRuntimeParent(runParent, () =>
|
|
runtimeWithPlugin.execute({
|
|
environment,
|
|
lease: acquired.lease,
|
|
command: "printf",
|
|
args: ["ok"],
|
|
cwd: "/workspace",
|
|
env: {},
|
|
timeoutMs: 1000,
|
|
}),
|
|
);
|
|
|
|
await environmentService(db).update(environment.id, { driver: "local", config: {} });
|
|
const released = await runtimeWithPlugin.releaseRunLeases(runId);
|
|
|
|
expect(released).toHaveLength(1);
|
|
// The release RPC ran under the recorded run parent, so the host mints a
|
|
// valid W3C traceparent from it. A dropped span would show a null context.
|
|
expect(releaseStepContext).not.toBeNull();
|
|
expect(releaseStepContext?.parentContext).toBe(runParent);
|
|
expect(traceparentFromContextToken(releaseStepContext?.parentContext)).toBe(
|
|
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
|
|
);
|
|
});
|
|
|
|
it("releases a plugin-backed lease with no active step context when the run drove no exec", async () => {
|
|
const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox();
|
|
|
|
let releaseStepContext: ReturnType<typeof getActiveStepContext> = null;
|
|
let releaseCalled = false;
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-1",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentReleaseLease") {
|
|
releaseCalled = true;
|
|
releaseStepContext = getActiveStepContext();
|
|
return undefined;
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
await environmentService(db).update(environment.id, { driver: "local", config: {} });
|
|
const released = await runtimeWithPlugin.releaseRunLeases(runId);
|
|
|
|
expect(released).toHaveLength(1);
|
|
// No command ran, so no session opened and no teardown span is emitted. The
|
|
// release must not invent a parent from an unrelated ambient context; it
|
|
// runs unwrapped, exactly as before the fix.
|
|
expect(releaseCalled).toBe(true);
|
|
expect(releaseStepContext).toBeNull();
|
|
});
|
|
|
|
it("forwards the bypassSession flag to the plugin execute RPC so a pre-run command skips the session", async () => {
|
|
const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox();
|
|
|
|
// Capture the params of each environmentExecute RPC, so the test can assert
|
|
// the host forwards `bypassSession` to the provider unchanged.
|
|
const executeParams: Array<Record<string, unknown>> = [];
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string, params: Record<string, unknown>) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-1",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentExecute") {
|
|
executeParams.push(params);
|
|
return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" };
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
// A pre-run command (the provision command) sets bypassSession explicitly.
|
|
await runtimeWithPlugin.execute({
|
|
environment,
|
|
lease: acquired.lease,
|
|
command: "bash",
|
|
args: ["-lc", "true"],
|
|
cwd: "/workspace",
|
|
env: {},
|
|
timeoutMs: 1000,
|
|
bypassSession: true,
|
|
});
|
|
// An in-run command runs under the run-parent context, so the host does not
|
|
// bypass the session and the provider opens/uses the session.
|
|
const runParent: StartupSpanContext = trace.setSpanContext(ROOT_CONTEXT, {
|
|
traceId: "0af7651916cd43dd8448eb211c80319c",
|
|
spanId: "b7ad6b7169203331",
|
|
traceFlags: 1,
|
|
isRemote: false,
|
|
});
|
|
await runWithRuntimeParent(runParent, () =>
|
|
runtimeWithPlugin.execute({
|
|
environment,
|
|
lease: acquired.lease,
|
|
command: "printf",
|
|
args: ["ok"],
|
|
cwd: "/workspace",
|
|
env: {},
|
|
timeoutMs: 1000,
|
|
}),
|
|
);
|
|
|
|
expect(executeParams).toHaveLength(2);
|
|
expect(executeParams[0]?.bypassSession).toBe(true);
|
|
// An in-run command carries a run parent, so it does not bypass the session.
|
|
expect(executeParams[1]?.bypassSession).toBe(false);
|
|
});
|
|
|
|
it("bypasses the session for a context-less command so the setup span keeps a run parent", async () => {
|
|
const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox();
|
|
|
|
// Capture the params of each environmentExecute RPC, so the test can assert
|
|
// the host derives `bypassSession` from the active run-parent context.
|
|
const executeParams: Array<Record<string, unknown>> = [];
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string, params: Record<string, unknown>) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-1",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentExecute") {
|
|
executeParams.push(params);
|
|
return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" };
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
// A command with no active run-parent context (a pre-run install/probe or
|
|
// the agent process launch) runs before the run trace is active. The host
|
|
// mints no plugin RPC traceparent for it, so a session opened here would drop
|
|
// its setup span. The host bypasses the session for such a command.
|
|
await runtimeWithPlugin.execute({
|
|
environment,
|
|
lease: acquired.lease,
|
|
command: "sh",
|
|
args: ["-c", "command -v claude"],
|
|
cwd: "/workspace",
|
|
env: {},
|
|
timeoutMs: 1000,
|
|
});
|
|
|
|
expect(executeParams).toHaveLength(1);
|
|
expect(executeParams[0]?.bypassSession).toBe(true);
|
|
});
|
|
|
|
it("builds the workspace-realization record with referenced sources for a plugin-backed sandbox realize", async () => {
|
|
// A provider plugin realize handler returns only its realized cwd and provider metadata; it does
|
|
// not build the workspace-realization record. The server must build that record from the run
|
|
// request, so the referenced (mentioned) project sources reach the adapter through
|
|
// `realization.additional`. Without the record the sandbox agent never receives the mentioned
|
|
// projects. This test drives the plugin-backed sandbox realize path and asserts the referenced
|
|
// source survives into the returned record.
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const fakePluginConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Fake Plugin Sandbox Realize",
|
|
driver: "sandbox",
|
|
config: fakePluginConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: fakePluginConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId);
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-realize-1",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentRealizeWorkspace") {
|
|
// Mimic a real provider (for example Daytona): return only the realized cwd and provider
|
|
// metadata, never a `workspaceRealization` record.
|
|
return {
|
|
cwd: "/workspace/project",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
remoteCwd: "/workspace/project",
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
const workspaceRealizationRequest = createReferencedWorkspaceRealizationRequest(companyId, environment, runId);
|
|
const realized = await runtimeWithPlugin.realizeWorkspace({
|
|
environment,
|
|
lease: acquired.lease,
|
|
workspace: {
|
|
localPath: "/tmp/anchor",
|
|
mode: "ephemeral",
|
|
metadata: { workspaceRealizationRequest },
|
|
},
|
|
});
|
|
|
|
// The provider realized cwd and provider metadata survive.
|
|
expect(realized.cwd).toBe("/workspace/project");
|
|
expect(realized.metadata?.provider).toBe("fake-plugin");
|
|
// The server-built record carries the referenced source through `additional`, so the adapter can
|
|
// stage the mentioned project into the sandbox.
|
|
const realization = realized.metadata?.workspaceRealization as Record<string, unknown> | undefined;
|
|
expect(realization).toBeDefined();
|
|
expect(realization?.additional).toEqual([
|
|
expect.objectContaining({
|
|
path: "/tmp/referenced-project",
|
|
projectId: "referenced-project-1",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("builds the workspace-realization record with referenced sources for a built-in sandbox realize", async () => {
|
|
// The sandbox driver `realizeWorkspace` has two exits: a plugin-backed provider and a
|
|
// built-in provider. Both must build the same workspace-realization record, so the
|
|
// referenced (mentioned) project sources reach the adapter through `realization.additional`.
|
|
// The test above covers the plugin exit. This test covers the built-in exit (no provider
|
|
// plugin), so a regression on either exit fails a test.
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Fake Sandbox Realize",
|
|
config: {
|
|
provider: "fake",
|
|
image: "ubuntu:24.04",
|
|
reuseLease: true,
|
|
},
|
|
});
|
|
|
|
const acquired = await runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
const workspaceRealizationRequest = createReferencedWorkspaceRealizationRequest(companyId, environment, runId);
|
|
const realized = await runtime.realizeWorkspace({
|
|
environment,
|
|
lease: acquired.lease,
|
|
workspace: {
|
|
localPath: "/tmp/anchor",
|
|
mode: "ephemeral",
|
|
metadata: { workspaceRealizationRequest },
|
|
},
|
|
});
|
|
|
|
// The built-in exit builds the record and carries the referenced source through `additional`,
|
|
// so the adapter can stage the mentioned project into the sandbox.
|
|
const realization = realized.metadata?.workspaceRealization as Record<string, unknown> | undefined;
|
|
expect(realization).toBeDefined();
|
|
expect(realization?.additional).toEqual([
|
|
expect.objectContaining({
|
|
path: "/tmp/referenced-project",
|
|
projectId: "referenced-project-1",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("uses resolved secret-ref config for plugin-backed sandbox execute and release", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const apiSecret = await secretService(db).create(companyId, {
|
|
name: `secure-plugin-api-key-${randomUUID()}`,
|
|
provider: "local_encrypted",
|
|
value: "resolved-provider-key",
|
|
});
|
|
const providerConfig = {
|
|
provider: "secure-plugin",
|
|
template: "base",
|
|
apiKey: apiSecret.id,
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Secure Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await secretService(db).createBinding({
|
|
companyId,
|
|
secretId: apiSecret.id,
|
|
targetType: "environment",
|
|
targetId: environment.id,
|
|
configPath: "apiKey",
|
|
});
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.secure-sandbox-provider",
|
|
packageName: "@acme/secure-sandbox-provider",
|
|
displayName: "Secure Sandbox Provider",
|
|
description: "Test schema-driven provider",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "secure-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Secure Sandbox",
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
template: { type: "string" },
|
|
apiKey: { type: "string", format: "secret-ref" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string, params: any) => {
|
|
expect(params.config.apiKey).toBe("resolved-provider-key");
|
|
expect(params.config).not.toHaveProperty("provider");
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-1",
|
|
metadata: {
|
|
provider: "secure-plugin",
|
|
template: "base",
|
|
apiKey: "resolved-provider-key",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
sandboxId: "sandbox-1",
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentExecute") {
|
|
return {
|
|
exitCode: 0,
|
|
signal: null,
|
|
timedOut: false,
|
|
stdout: "ok\n",
|
|
stderr: "",
|
|
};
|
|
}
|
|
if (method === "environmentReleaseLease") {
|
|
return undefined;
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
expect(acquired.lease.metadata).toMatchObject({
|
|
provider: "secure-plugin",
|
|
template: "base",
|
|
apiKey: apiSecret.id,
|
|
timeoutMs: 1234,
|
|
sandboxId: "sandbox-1",
|
|
});
|
|
const executed = await runtimeWithPlugin.execute({
|
|
environment,
|
|
lease: acquired.lease,
|
|
command: "printf",
|
|
args: ["ok"],
|
|
cwd: "/workspace",
|
|
env: {},
|
|
});
|
|
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "local",
|
|
config: {},
|
|
});
|
|
const released = await runtimeWithPlugin.releaseRunLeases(runId);
|
|
|
|
expect(executed.stdout).toBe("ok\n");
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.lease.status).toBe("released");
|
|
// The execute call carries the optional log sink as the fifth argument; it
|
|
// is undefined when the caller passes no sink.
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({
|
|
config: expect.objectContaining({
|
|
apiKey: "resolved-provider-key",
|
|
}),
|
|
}), 31234, undefined);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.objectContaining({
|
|
config: expect.objectContaining({
|
|
apiKey: "resolved-provider-key",
|
|
}),
|
|
}), 31234);
|
|
});
|
|
|
|
it("resolves persisted secret_ref binding objects in sandbox provider config at runtime", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment } = await seedEnvironment();
|
|
const apiSecret = await secretService(db).create(companyId, {
|
|
name: `secure-plugin-api-key-${randomUUID()}`,
|
|
provider: "local_encrypted",
|
|
value: "resolved-provider-key",
|
|
});
|
|
const providerConfig = {
|
|
provider: "secure-plugin",
|
|
template: "base",
|
|
apiKey: { type: "secret_ref", secretId: apiSecret.id, version: "latest" },
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Secure Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await secretService(db).createBinding({
|
|
companyId,
|
|
secretId: apiSecret.id,
|
|
targetType: "environment",
|
|
targetId: environment.id,
|
|
configPath: "apiKey",
|
|
});
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.secure-sandbox-provider",
|
|
packageName: "@acme/secure-sandbox-provider",
|
|
displayName: "Secure Sandbox Provider",
|
|
description: "Test schema-driven provider",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "secure-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Secure Sandbox",
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
template: { type: "string" },
|
|
apiKey: { type: "string", format: "secret-ref" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
const resolved = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment);
|
|
|
|
expect(resolved.driver).toBe("sandbox");
|
|
expect(resolved.config).toMatchObject({
|
|
provider: "secure-plugin",
|
|
template: "base",
|
|
apiKey: "resolved-provider-key",
|
|
});
|
|
});
|
|
|
|
it("waits briefly for a ready sandbox provider plugin worker to come online", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Eventually Running Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.eventually-running-sandbox-provider",
|
|
packageName: "@acme/eventually-running-sandbox-provider",
|
|
displayName: "Eventually Running Sandbox Provider",
|
|
description: "Test plugin worker startup grace period",
|
|
author: "Acme",
|
|
});
|
|
|
|
let runningChecks = 0;
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => {
|
|
if (id !== pluginId) return false;
|
|
runningChecks += 1;
|
|
return runningChecks >= 3;
|
|
}),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-1",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: workerManager,
|
|
pluginWorkerReadyTimeoutMs: 25,
|
|
pluginWorkerReadyPollMs: 1,
|
|
});
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("sandbox-1");
|
|
expect(workerManager.isRunning).toHaveBeenCalledTimes(3);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
|
|
});
|
|
|
|
it("throws a worker-not-running error once the readiness deadline is exhausted", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Never Running Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.never-running-sandbox-provider",
|
|
packageName: "@acme/never-running-sandbox-provider",
|
|
displayName: "Never Running Sandbox Provider",
|
|
description: "Test plugin worker that never comes online before the readiness deadline",
|
|
author: "Acme",
|
|
});
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn(() => false),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: workerManager,
|
|
pluginWorkerReadyTimeoutMs: 25,
|
|
pluginWorkerReadyPollMs: 1,
|
|
});
|
|
|
|
await expect(
|
|
runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
}),
|
|
).rejects.toThrow(/worker is not running/);
|
|
// Confirms the loop actually polled repeatedly across the deadline window
|
|
// instead of giving up after a single check.
|
|
expect(workerManager.isRunning.mock.calls.length).toBeGreaterThanOrEqual(2);
|
|
});
|
|
|
|
it("keeps polling across a longer worker restart window and succeeds once the handle registers late", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Registered Late Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.registered-late-sandbox-provider",
|
|
packageName: "@acme/registered-late-sandbox-provider",
|
|
displayName: "Registered Late Sandbox Provider",
|
|
description: "Test plugin worker handle that stays unregistered/starting for most of the readiness window",
|
|
author: "Acme",
|
|
});
|
|
|
|
// Simulates a worker process restart: the handle is absent (or "starting")
|
|
// for most of the readiness window and only reports running with several
|
|
// checks left before the deadline. A larger attempt count than the
|
|
// existing "waits briefly" sanity test, to prove the loop is bound by the
|
|
// deadline rather than by a small fixed number of attempts.
|
|
const readyAfterChecks = 8;
|
|
let checks = 0;
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => {
|
|
if (id !== pluginId) return false;
|
|
checks += 1;
|
|
return checks >= readyAfterChecks;
|
|
}),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-registered-late",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: false,
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: workerManager,
|
|
pluginWorkerReadyTimeoutMs: 200,
|
|
pluginWorkerReadyPollMs: 1,
|
|
});
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("sandbox-registered-late");
|
|
expect(checks).toBe(readyAfterChecks);
|
|
});
|
|
|
|
it.each([
|
|
{ label: "explicit provider and bridge budgets", config: { timeoutMs: 1_234, bridgeRequestTimeoutMs: 40_000 }, declared: 300_000, expected: 70_000 },
|
|
{ label: "declared acquisition default", config: {}, declared: 300_000, expected: 330_000 },
|
|
{ label: "explicit shorter provider budget", config: { timeoutMs: 5_000 }, declared: 300_000, expected: 35_000 },
|
|
{ label: "bridge budget does not shorten acquisition default", config: { bridgeRequestTimeoutMs: 40_000 }, declared: 300_000, expected: 330_000 },
|
|
{ label: "bridge extends the acquisition default", config: { bridgeRequestTimeoutMs: 400_000 }, declared: 300_000, expected: 430_000 },
|
|
{ label: "undeclared provider retains worker default", config: {}, declared: undefined, expected: undefined },
|
|
])("uses $label for plugin-backed sandbox acquisition", async ({ config, declared, expected }) => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
...config,
|
|
reuseLease: false,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Long Lease Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.long-lease-sandbox-provider",
|
|
packageName: "@acme/long-lease-sandbox-provider",
|
|
displayName: "Long Lease Sandbox Provider",
|
|
description: "Test plugin worker acquire timeout",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
defaultAcquireTimeoutMs: declared,
|
|
// A timeoutMs schema default can mean lease lifetime (for example
|
|
// E2B). Only the dedicated declaration sets the host RPC budget.
|
|
configSchema: { type: "object", properties: { timeoutMs: { type: "number", default: 3_600_000 } } },
|
|
},
|
|
],
|
|
});
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "sandbox-1",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
...config,
|
|
reuseLease: false,
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("sandbox-1");
|
|
expect(workerManager.call).toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentAcquireLease",
|
|
expect.objectContaining({
|
|
driverKey: "fake-plugin",
|
|
config: {
|
|
image: "fake:test",
|
|
...config,
|
|
reuseLease: false,
|
|
},
|
|
}),
|
|
expected,
|
|
);
|
|
});
|
|
|
|
it("preserves the exact lease when plugin-backed sandbox resume throws", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Reusable Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.fake-sandbox-provider",
|
|
packageName: "@acme/fake-sandbox-provider",
|
|
displayName: "Fake Sandbox Provider",
|
|
description: "Test schema-driven provider",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
supportsReusableLeases: true,
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
image: { type: "string" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await insertReusableProject(projectId, companyId);
|
|
await insertReusableExecutionWorkspace(executionWorkspaceId, companyId, projectId);
|
|
const staleLease = await insertStaleReusablePluginLease(companyId, environment, executionWorkspaceId, runId, agentId, pluginId, providerConfig);
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentResumeLease") {
|
|
throw new Error("stale sandbox");
|
|
}
|
|
if (method === "environmentDestroyLease") {
|
|
return undefined;
|
|
}
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "fresh-plugin-lease",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
await expect(runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
})).rejects.toThrow("the lease was preserved and no replacement was created");
|
|
|
|
expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentResumeLease", expect.objectContaining({
|
|
driverKey: "fake-plugin",
|
|
providerLeaseId: "stale-plugin-lease",
|
|
}), 31234);
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentDestroyLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentAcquireLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({
|
|
status: "active",
|
|
});
|
|
});
|
|
|
|
it("resumes only the exact stopped sandbox lifecycle without acquiring or reseeding", async () => {
|
|
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
|
await environmentService(db).releaseLease(seeded.reusableLease.id, "released", { cleanupStatus: "success" });
|
|
const lease = (await environmentService(db).getLeaseById(seeded.reusableLease.id))!;
|
|
const call = vi.fn(async (_id: string, method: string) => {
|
|
if (method !== "environmentResumeLease") throw new Error("Only exact resume is permitted");
|
|
return { providerLeaseId: lease.providerLeaseId, metadata: { remoteCwd: "/workspace" } };
|
|
});
|
|
const workerManager = { isRunning: () => true, call,
|
|
getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
await expect(runtime.resumeRunLease({ environment: seeded.environment, lease })).resolves.toMatchObject({ providerLeaseId: lease.providerLeaseId });
|
|
expect(call).toHaveBeenCalledOnce();
|
|
expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentResumeLease", expect.objectContaining({
|
|
companyId: seeded.companyId, environmentId: seeded.environment.id, providerLeaseId: lease.providerLeaseId,
|
|
leaseMetadata: lease.metadata,
|
|
}), expect.any(Number));
|
|
expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("released");
|
|
expect(await db.select().from(environmentLeases)).toHaveLength(1);
|
|
});
|
|
|
|
it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "retained_owner", "no_stop_capability", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent", "foreign_plugin_pin", "v2_missing_pin"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => {
|
|
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
|
const lease = seeded.reusableLease;
|
|
const requestId = randomUUID();
|
|
if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent"].includes(outcome)) {
|
|
const [owner] = await db.select().from(plugins).where(eq(plugins.id, seeded.pluginId));
|
|
await db.insert(plugins).values({ ...owner, id: randomUUID(), pluginKey: "other.same-driver", packageName: "@other/same-driver",
|
|
installOrder: -1, manifestJson: { ...owner.manifestJson, id: "other.same-driver" } });
|
|
if (outcome === "owner_driver_changed") await db.update(plugins).set({ manifestJson: { ...owner.manifestJson, environmentDrivers: [] } }).where(eq(plugins.id, seeded.pluginId));
|
|
if (outcome === "owner_missing") await db.delete(plugins).where(eq(plugins.id, seeded.pluginId));
|
|
}
|
|
await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", metadata: {
|
|
...lease.metadata,
|
|
nativeWorkspaceExportResume: { schema: outcome === "v2_missing_pin" ? "paperclip.workspace-export-resume.v2" : "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId,
|
|
...(["legacy_intent", "v2_missing_pin"].includes(outcome) ? {} : { pluginId: outcome === "foreign_plugin_pin" ? randomUUID() : seeded.pluginId }),
|
|
runId: outcome === "foreign_marker" ? randomUUID() : lease.heartbeatRunId, leaseId: lease.id,
|
|
provider: lease.provider, providerLeaseId: lease.providerLeaseId, resultId: randomUUID() },
|
|
pendingCleanupAttemptId: requestId, pendingCleanupInFlight: true,
|
|
pendingCleanupLeaseExpiresAtMs: Date.now() + (outcome === "live_request" ? 60_000 : -1),
|
|
} }).where(eq(environmentLeases.id, lease.id));
|
|
if (["competing_owner", "retained_owner"].includes(outcome)) await db.insert(environmentLeases).values({ companyId: seeded.companyId,
|
|
environmentId: seeded.environment.id, status: outcome === "retained_owner" ? "retained" : "active", provider: lease.provider, providerLeaseId: lease.providerLeaseId });
|
|
const call = vi.fn(async (_id: string, method: string) => {
|
|
if (method !== "environmentStopLease") throw new Error("Saved workspace must never be destroyed");
|
|
if (outcome === "late_receipt") await db.update(environmentLeases).set({ status: "active", heartbeatRunId: null,
|
|
cleanupStatus: null, metadata: { newOwner: true } }).where(eq(environmentLeases.id, lease.id));
|
|
return outcome === "unconfirmed" ? undefined : { providerLeaseId: lease.providerLeaseId, state: "stopped" };
|
|
});
|
|
const workerManager = { isRunning: (id: string) => !(outcome === "owner_stopped" && id === seeded.pluginId), call,
|
|
getWorker: () => ({ supportedMethods: outcome === "no_stop_capability" ? ["environmentReleaseLease", "environmentDestroyLease"] : ["environmentStopLease", "environmentDestroyLease"] }),
|
|
} as unknown as PluginWorkerManager;
|
|
const restarted = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
await heartbeatService(db, { environmentRuntime: restarted }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
expect(call.mock.calls.some((entry) => entry[1] === "environmentDestroyLease")).toBe(false);
|
|
const persisted = (await environmentService(db).getLeaseById(lease.id))!;
|
|
if (["stopped", "duplicate_key", "legacy_intent"].includes(outcome)) {
|
|
expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentStopLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId, cancelActiveWork: true }), expect.any(Number));
|
|
expect(persisted).toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped", runId: lease.heartbeatRunId } } });
|
|
expect(persisted.metadata?.nativeWorkspaceExportResume).toBeUndefined();
|
|
} else if (outcome === "late_receipt") {
|
|
expect(persisted).toMatchObject({ status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true } });
|
|
} else {
|
|
expect(persisted).toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed" });
|
|
expect(persisted.metadata?.remoteExecutionTermination).toBeUndefined();
|
|
if (["foreign_marker", "foreign_plugin_pin", "v2_missing_pin", "competing_owner", "retained_owner", "no_stop_capability", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled();
|
|
if (["owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) {
|
|
expect(persisted.metadata?.pendingCleanupAttemptId).toBe(requestId);
|
|
}
|
|
}
|
|
});
|
|
|
|
it.each(["stopped", "stop_failed", "unconfirmed", "restart", "committed_release", "inflight"])("preserves terminal ephemeral export work through release and restart: %s", async outcome => {
|
|
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
|
const lease = seeded.reusableLease, requestId = randomUUID();
|
|
await db.update(heartbeatRuns).set({ status: outcome === "committed_release" ? "succeeded" : "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
|
|
await db.update(environmentLeases).set({ status: outcome === "committed_release" ? "active" : "pending_cleanup", leasePolicy: "ephemeral", cleanupStatus: "failed", metadata: {
|
|
...lease.metadata, reuseLease: false,
|
|
nativeWorkspaceExportResume: { schema: "paperclip.workspace-export-resume.v2", purpose: "terminal_export", requestId,
|
|
companyId: seeded.companyId, pluginId: seeded.pluginId, runId: seeded.runId, resultId: randomUUID(),
|
|
leaseId: lease.id, provider: lease.provider, providerLeaseId: lease.providerLeaseId },
|
|
pendingCleanupAttemptId: requestId, pendingCleanupInFlight: outcome === "inflight",
|
|
pendingCleanupLeaseExpiresAtMs: outcome === "inflight" ? Date.now() + 60_000 : 0,
|
|
} }).where(eq(environmentLeases.id, lease.id));
|
|
let calls = 0;
|
|
const call = vi.fn(async (_id: string, method: string, params: any) => {
|
|
expect(method).toBe("environmentStopLease");
|
|
expect(params).toMatchObject({ providerLeaseId: lease.providerLeaseId, resourceDisposition: "stop_and_retain", config: { reuseLease: false } });
|
|
if (++calls === 1 && outcome === "stop_failed") throw new Error("injected stop failure");
|
|
if (calls === 1 && outcome === "unconfirmed") return undefined;
|
|
return { providerLeaseId: lease.providerLeaseId, state: "stopped" };
|
|
});
|
|
const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call,
|
|
getWorker: () => ({ supportedMethods: ["environmentStopLease", "environmentDestroyLease"] }),
|
|
} as unknown as PluginWorkerManager });
|
|
if (outcome === "restart") await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
else await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain");
|
|
if (outcome === "inflight") { expect(call).not.toHaveBeenCalled(); return; }
|
|
if (["stop_failed", "unconfirmed"].includes(outcome)) {
|
|
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup", metadata: { pendingCleanupInFlight: false } });
|
|
await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
}
|
|
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped" } } });
|
|
expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true);
|
|
});
|
|
|
|
it("does not allocate a native-runner replacement without a verified backup", async () => {
|
|
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === seeded.pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentResumeLease") {
|
|
return { providerLeaseId: null, metadata: { expired: true } };
|
|
}
|
|
if (method === "environmentDestroyLease") return undefined;
|
|
if (method === "environmentAcquireLease") {
|
|
throw new Error("replacement must not be allocated");
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
await expect(runtimeWithPlugin.acquireRunLease({
|
|
companyId: seeded.companyId,
|
|
environment: seeded.environment,
|
|
issueId: null,
|
|
agentId: seeded.agentId,
|
|
adapterType: "paperclip_runner",
|
|
heartbeatRunId: seeded.runId,
|
|
persistedExecutionWorkspace: {
|
|
id: seeded.executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
})).rejects.toThrow("runner_harness_backup_unavailable");
|
|
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
seeded.pluginId,
|
|
"environmentAcquireLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
seeded.pluginId,
|
|
"environmentDestroyLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
});
|
|
|
|
it("permits native-runner replacement only after verifying the stamped backup", async () => {
|
|
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
|
const backupBase = await mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-replacement-"));
|
|
const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR;
|
|
process.env.PAPERCLIP_RUNNER_STATE_DIR = backupBase;
|
|
try {
|
|
const normalizedSessionId = "native-replacement-session";
|
|
const runnerInstanceId = "native-replacement-runner";
|
|
const sessionScopeId = "native-replacement-session-scope-v2";
|
|
const sessionRoot = path.join(
|
|
backupBase,
|
|
createHash("sha256").update(sessionScopeId).digest("hex"),
|
|
);
|
|
const current = path.join(sessionRoot, "failover-backups", "current");
|
|
await mkdir(path.join(current, "runner"), { recursive: true });
|
|
await mkdir(path.join(current, "codex-home", "sessions"), { recursive: true });
|
|
await writeFile(path.join(current, "runner", "runner-state.json"), "runner-state");
|
|
await writeFile(path.join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state");
|
|
const execution = {
|
|
provider: { kind: "codex", model: null, approvalPolicy: "never" },
|
|
binding: {
|
|
companyId: seeded.companyId,
|
|
runId: seeded.runId,
|
|
issueId: "issue",
|
|
agentId: seeded.agentId,
|
|
executionWorkspaceId: seeded.executionWorkspaceId,
|
|
},
|
|
workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
|
session: {
|
|
normalizedSessionId,
|
|
driverKind: "codex_app_server",
|
|
protocolVersion: 1,
|
|
lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null },
|
|
},
|
|
} as never;
|
|
const manifest = buildNativeHarnessBackupManifest({
|
|
backupRoot: current,
|
|
execution,
|
|
runnerInstanceId,
|
|
providerSessionIdentity: {
|
|
providerSessionId: "thread-1",
|
|
providerBackendSessionId: "session-1",
|
|
providerSessionIdentity: null,
|
|
},
|
|
sourceProviderLeaseId: seeded.reusableLease.providerLeaseId!,
|
|
});
|
|
const manifestPath = path.join(current, "manifest.json");
|
|
await writeFile(manifestPath, JSON.stringify(manifest));
|
|
const stamp = createNativeHarnessBackupStamp({
|
|
manifestPath,
|
|
sessionScopeId,
|
|
authorizedProviderLeaseId: seeded.reusableLease.providerLeaseId!,
|
|
normalizedSessionId,
|
|
runnerInstanceId,
|
|
completedAt: manifest.completedAt,
|
|
});
|
|
await environmentService(db).updateLeaseMetadata(seeded.reusableLease.id, {
|
|
...(seeded.reusableLease.metadata ?? {}),
|
|
nativeHarnessBackup: stamp,
|
|
});
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === seeded.pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentResumeLease") {
|
|
return { providerLeaseId: null, metadata: { expired: true } };
|
|
}
|
|
if (method === "environmentDestroyLease") return undefined;
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "replacement-plugin-lease",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId: seeded.companyId,
|
|
environment: seeded.environment,
|
|
issueId: null,
|
|
agentId: seeded.agentId,
|
|
adapterType: "paperclip_runner",
|
|
heartbeatRunId: seeded.runId,
|
|
persistedExecutionWorkspace: {
|
|
id: seeded.executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("replacement-plugin-lease");
|
|
expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({
|
|
outcome: "replacement",
|
|
reason: "not_found",
|
|
});
|
|
} finally {
|
|
if (previousStateDirectory === undefined) {
|
|
delete process.env.PAPERCLIP_RUNNER_STATE_DIR;
|
|
} else {
|
|
process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory;
|
|
}
|
|
await rm(backupBase, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("fails closed and does not resume when a worker restart drops the resume method after the capability snapshot", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Reusable Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.fake-sandbox-provider",
|
|
packageName: "@acme/fake-sandbox-provider",
|
|
displayName: "Fake Sandbox Provider",
|
|
description: "Test schema-driven provider",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
supportsReusableLeases: true,
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
image: { type: "string" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await insertReusableProject(projectId, companyId);
|
|
await insertReusableExecutionWorkspace(executionWorkspaceId, companyId, projectId);
|
|
const staleLease = await insertStaleReusablePluginLease(companyId, environment, executionWorkspaceId, runId, agentId, pluginId, providerConfig);
|
|
|
|
// The runtime reads the worker methods once to decide reuse, then does
|
|
// asynchronous database work before the resume dispatch. A worker restart
|
|
// in that window drops `environmentResumeLease`. The first read returns the
|
|
// reuse verbs, so the runtime treats the lease as resumable. Every later
|
|
// read returns the restarted worker's methods, which no longer include
|
|
// `environmentResumeLease`.
|
|
let getWorkerReads = 0;
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentResumeLease") {
|
|
// A regression that reads the stale snapshot dispatches the resume
|
|
// RPC and fails here. The live worker cannot serve the method.
|
|
throw new Error("worker no longer advertises environmentResumeLease");
|
|
}
|
|
if (method === "environmentDestroyLease") {
|
|
return undefined;
|
|
}
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "fresh-plugin-lease",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => {
|
|
getWorkerReads += 1;
|
|
return {
|
|
supportedMethods:
|
|
getWorkerReads === 1
|
|
? ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"]
|
|
: ["environmentReleaseLease", "environmentDestroyLease"],
|
|
};
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
await expect(runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
})).rejects.toThrow("the lease was preserved and no replacement was created");
|
|
|
|
// The runtime re-checks the live worker before the resume dispatch. The
|
|
// restarted worker no longer advertises `environmentResumeLease`, so the
|
|
// runtime must not dispatch the resume RPC.
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentResumeLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
// It preserves the stale reusable lease and does not allocate a replacement.
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentDestroyLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentAcquireLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({
|
|
status: "active",
|
|
});
|
|
});
|
|
|
|
// Seed a reusable plugin sandbox lease that a worker created under an earlier
|
|
// capability set. The worker restarts and no longer advertises the lifecycle
|
|
// methods. The lease lifecycle paths must verify the live worker before they
|
|
// dispatch a lifecycle RPC, so the runtime fails closed instead of a doomed
|
|
// dispatch.
|
|
async function seedStaleLifecycleReusableLease(
|
|
leasePolicy: "reuse_by_environment" | "retain_on_failure" = "reuse_by_environment",
|
|
) {
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Reusable Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.fake-sandbox-provider",
|
|
packageName: "@acme/fake-sandbox-provider",
|
|
displayName: "Fake Sandbox Provider",
|
|
description: "Test schema-driven provider",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
supportsReusableLeases: true,
|
|
configSchema: { type: "object" },
|
|
},
|
|
],
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await db.insert(projects).values({
|
|
id: projectId,
|
|
companyId,
|
|
name: `Workspace ${projectId.slice(0, 8)}`,
|
|
status: "active",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await db.insert(executionWorkspaces).values({
|
|
id: executionWorkspaceId,
|
|
companyId,
|
|
projectId,
|
|
mode: "shared_workspace",
|
|
strategyType: "project_primary",
|
|
name: "Reusable workspace",
|
|
status: "active",
|
|
providerType: "local_fs",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
const lease = await environmentService(db).acquireLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
heartbeatRunId: runId,
|
|
leasePolicy,
|
|
provider: "fake-plugin",
|
|
providerLeaseId: "stale-lifecycle-lease",
|
|
metadata: {
|
|
agentId,
|
|
driver: "sandbox",
|
|
pluginId,
|
|
pluginKey: "acme.fake-sandbox-provider",
|
|
sandboxProviderPlugin: true,
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
},
|
|
});
|
|
|
|
// The worker is running, but its discovery list dropped the reusable-lease
|
|
// lifecycle methods. `call` throws on any lifecycle RPC so a regression that
|
|
// dispatches one fails the test through the `not.toHaveBeenCalledWith` check.
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentAcquireLease", "environmentExecute"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
return { pluginId, environment, runId, lease, workerManager, runtimeWithPlugin };
|
|
}
|
|
|
|
it("routes release to pending_cleanup when the worker no longer advertises the release lifecycle method", async () => {
|
|
const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease();
|
|
|
|
const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!);
|
|
|
|
expect(released).toHaveLength(1);
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentReleaseLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
// The failed release verification must enter the pending-cleanup retry flow.
|
|
// The reaper sweeps only `pending_cleanup` leases, so a `released` status
|
|
// here would strand the still-active provider resource.
|
|
await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({
|
|
status: "pending_cleanup",
|
|
cleanupStatus: "failed",
|
|
failureReason: "release_cleanup_failed",
|
|
});
|
|
});
|
|
|
|
it("retains a retain_on_failure lease on failed release instead of routing to pending_cleanup", async () => {
|
|
const { lease, runtimeWithPlugin } = await seedStaleLifecycleReusableLease("retain_on_failure");
|
|
|
|
const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!, "failed");
|
|
|
|
expect(released).toHaveLength(1);
|
|
// A retain_on_failure lease keeps the provider resource for reuse. The
|
|
// reaper destroys `pending_cleanup` leases, so the retained lease must not
|
|
// enter that flow even when the release verification fails.
|
|
await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({
|
|
status: "retained",
|
|
cleanupStatus: "failed",
|
|
});
|
|
});
|
|
|
|
it("destroys a failed retained lease when the agent stops", async () => {
|
|
const { lease, workerManager, runtimeWithPlugin, runId } = await seedStaleLifecycleReusableLease("retain_on_failure");
|
|
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId));
|
|
await runtimeWithPlugin.releaseRunLeases(runId, "failed");
|
|
await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "failed" });
|
|
vi.mocked(workerManager.getWorker).mockReturnValue({ supportedMethods: ["environmentDestroyLease"] } as never);
|
|
vi.mocked(workerManager.call).mockResolvedValue({ providerLeaseId: lease.providerLeaseId, state: "destroyed" });
|
|
const { heartbeatService } = await import("../services/heartbeat.js");
|
|
const stopped = await heartbeatService(db, { pluginWorkerManager: workerManager })
|
|
.stopInvocationsForAgents([lease.metadata!.agentId as string], "Agent lifecycle stop requested");
|
|
expect(stopped).toBe(true);
|
|
expect(workerManager.call).toHaveBeenCalledWith(expect.any(String), "environmentDestroyLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId }), expect.any(Number));
|
|
await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success" });
|
|
});
|
|
|
|
it("fails closed on expiry destruction when the worker no longer advertises the destroy lifecycle method", async () => {
|
|
const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease();
|
|
|
|
await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!, "expired");
|
|
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentDestroyLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({
|
|
status: "pending_cleanup",
|
|
cleanupStatus: "failed",
|
|
});
|
|
});
|
|
|
|
it("does not resume released reusable plugin sandbox leases after provider config drift", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "template-a",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Reusable Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.fake-sandbox-provider",
|
|
packageName: "@acme/fake-sandbox-provider",
|
|
displayName: "Fake Sandbox Provider",
|
|
description: "Test schema-driven provider",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
supportsReusableLeases: true,
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
image: { type: "string" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await insertReusableProject(projectId, companyId);
|
|
await insertReusableExecutionWorkspace(executionWorkspaceId, companyId, projectId);
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string, params: any) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: `lease-${params.config.image}`,
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: params.config.image,
|
|
timeoutMs: params.config.timeoutMs,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentReleaseLease" || method === "environmentDestroyLease") {
|
|
return undefined;
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const first = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
expect(first.lease.metadata?.reusableSandboxLease).toMatchObject({
|
|
provider: "fake-plugin",
|
|
leaseFingerprint: expect.objectContaining({
|
|
category: "lease",
|
|
fingerprint: expect.stringMatching(/^v1:sha256:[a-f0-9]{64}$/),
|
|
}),
|
|
});
|
|
await runtimeWithPlugin.releaseRunLeases(runId);
|
|
|
|
const nextRunId = randomUUID();
|
|
await insertRunningManualRun(nextRunId, companyId, agentId);
|
|
const updatedEnvironment = {
|
|
...environment,
|
|
config: {
|
|
...providerConfig,
|
|
image: "template-b",
|
|
},
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
config: updatedEnvironment.config,
|
|
});
|
|
|
|
const second = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment: updatedEnvironment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: nextRunId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(second.lease.providerLeaseId).toBe("lease-template-b");
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentResumeLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({
|
|
providerLeaseId: "lease-template-a",
|
|
}), 31234);
|
|
await expect(environmentService(db).getLeaseById(first.lease.id)).resolves.toMatchObject({
|
|
status: "expired",
|
|
cleanupStatus: "success",
|
|
failureReason: "lease_fingerprint_mismatch",
|
|
});
|
|
});
|
|
|
|
it("does not resume released reusable plugin sandbox leases after secret version drift", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const apiSecret = await secretService(db).create(companyId, {
|
|
name: `secure-plugin-api-key-${randomUUID()}`,
|
|
provider: "local_encrypted",
|
|
value: "resolved-provider-key",
|
|
});
|
|
const providerConfig = {
|
|
provider: "secure-plugin",
|
|
template: "base",
|
|
apiKey: apiSecret.id,
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Secure Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await secretService(db).createBinding({
|
|
companyId,
|
|
secretId: apiSecret.id,
|
|
targetType: "environment",
|
|
targetId: environment.id,
|
|
configPath: "apiKey",
|
|
});
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.secure-sandbox-provider",
|
|
packageName: "@acme/secure-sandbox-provider",
|
|
displayName: "Secure Sandbox Provider",
|
|
description: "Test schema-driven provider",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "secure-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Secure Sandbox",
|
|
supportsReusableLeases: true,
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
template: { type: "string" },
|
|
apiKey: { type: "string", format: "secret-ref" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await insertReusableProject(projectId, companyId);
|
|
await insertReusableExecutionWorkspace(executionWorkspaceId, companyId, projectId);
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string, params: any) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: `lease-${params.config.apiKey}`,
|
|
metadata: {
|
|
provider: "secure-plugin",
|
|
template: params.config.template,
|
|
apiKey: params.config.apiKey,
|
|
timeoutMs: params.config.timeoutMs,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentReleaseLease" || method === "environmentDestroyLease") {
|
|
return undefined;
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const first = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
await runtimeWithPlugin.releaseRunLeases(runId);
|
|
await secretService(db).rotate(apiSecret.id, { value: "rotated-provider-key" });
|
|
|
|
const nextRunId = randomUUID();
|
|
await insertRunningManualRun(nextRunId, companyId, agentId);
|
|
const second = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: nextRunId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(second.lease.providerLeaseId).toBe("lease-rotated-provider-key");
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentResumeLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({
|
|
providerLeaseId: "lease-resolved-provider-key",
|
|
}), 31234);
|
|
await expect(environmentService(db).getLeaseById(first.lease.id)).resolves.toMatchObject({
|
|
status: "expired",
|
|
cleanupStatus: "success",
|
|
failureReason: "lease_fingerprint_mismatch",
|
|
});
|
|
const firstMetadata = JSON.stringify(first.lease.metadata);
|
|
expect(firstMetadata).not.toContain("resolved-provider-key");
|
|
expect(firstMetadata).not.toContain("rotated-provider-key");
|
|
});
|
|
|
|
it("preserves active reusable sandbox leases held by another running run", async () => {
|
|
const { pluginId, companyId, agentId, environment, executionWorkspaceId, reusableLease } =
|
|
await seedReusablePluginSandboxLease();
|
|
const nextRunId = randomUUID();
|
|
await insertRunningManualRun(nextRunId, companyId, agentId);
|
|
const workerManager = createFreshPluginLeaseWorker(pluginId);
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: nextRunId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
|
|
expect(workerManager.call).toHaveBeenCalledOnce();
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({
|
|
agentId,
|
|
executionWorkspaceId,
|
|
runId: nextRunId,
|
|
}), 31234);
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "active",
|
|
cleanupStatus: null,
|
|
});
|
|
});
|
|
|
|
it("does not retain or resume plugin-backed sandbox leases unless the provider opts in", async () => {
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Non-reusable Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.non-reusable-sandbox-provider",
|
|
packageName: "@acme/non-reusable-sandbox-provider",
|
|
displayName: "Non-reusable Sandbox Provider",
|
|
description: "Test provider without reusable lease support",
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await insertReusableProject(projectId, companyId);
|
|
await db.insert(executionWorkspaces).values({
|
|
id: executionWorkspaceId,
|
|
companyId,
|
|
projectId,
|
|
mode: "shared_workspace",
|
|
strategyType: "project_primary",
|
|
name: "Non-reusable workspace",
|
|
status: "active",
|
|
providerType: "local_fs",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertOldReusablePluginLease(companyId, environment, executionWorkspaceId, runId, agentId, providerConfig);
|
|
|
|
const workerManager = createFreshPluginLeaseWorker(pluginId);
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
|
|
expect(acquired.lease.leasePolicy).toBe("ephemeral");
|
|
expect(workerManager.call).toHaveBeenCalledTimes(1);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
|
|
});
|
|
|
|
it("does not resume a lease when the nested capability disables reusable leases", async () => {
|
|
// The provider declares the legacy `supportsReusableLeases: true` flag but
|
|
// the nested `sandboxCapabilities.reusableLeases: false`. The nested value
|
|
// wins through the capability contract, so acquisition must acquire a fresh
|
|
// lease and must never resume the existing reusable lease.
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Nested-disabled Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.nested-disabled-sandbox-provider",
|
|
packageName: "@acme/nested-disabled-sandbox-provider",
|
|
displayName: "Nested-disabled Sandbox Provider",
|
|
description: "Test provider with a legacy flag and a disabled nested capability",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
supportsReusableLeases: true,
|
|
sandboxCapabilities: { reusableLeases: false },
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
image: { type: "string" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await insertReusableProject(projectId, companyId);
|
|
await db.insert(executionWorkspaces).values({
|
|
id: executionWorkspaceId,
|
|
companyId,
|
|
projectId,
|
|
mode: "shared_workspace",
|
|
strategyType: "project_primary",
|
|
name: "Nested-disabled workspace",
|
|
status: "active",
|
|
providerType: "local_fs",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
await insertOldReusablePluginLease(companyId, environment, executionWorkspaceId, runId, agentId, providerConfig);
|
|
|
|
const workerManager = createFreshPluginLeaseWorker(pluginId);
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
|
|
expect(acquired.lease.leasePolicy).toBe("ephemeral");
|
|
expect(workerManager.call).toHaveBeenCalledTimes(1);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentResumeLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
});
|
|
|
|
it("fails closed and does not resume when the worker does not verify the reuse methods", async () => {
|
|
// The provider declares `reusableLeases: true`, but its worker advertises
|
|
// neither `environmentResumeLease` nor `environmentReleaseLease`. The runtime
|
|
// must not resume or reuse the lease, because it cannot dispatch a resume or
|
|
// a release the worker does not serve. It acquires a fresh ephemeral lease
|
|
// and leaves the old reusable lease untouched.
|
|
const pluginId = randomUUID();
|
|
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
|
const providerConfig = {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
};
|
|
const environment = {
|
|
...baseEnvironment,
|
|
name: "Unverified-worker Plugin Sandbox",
|
|
driver: "sandbox",
|
|
config: providerConfig,
|
|
};
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "sandbox",
|
|
name: environment.name,
|
|
config: providerConfig,
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.unverified-worker-sandbox-provider",
|
|
packageName: "@acme/unverified-worker-sandbox-provider",
|
|
displayName: "Unverified-worker Sandbox Provider",
|
|
description: "Test provider that declares reusable leases but whose worker lacks the reuse methods",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
sandboxCapabilities: { reusableLeases: true },
|
|
configSchema: {
|
|
type: "object",
|
|
properties: {
|
|
image: { type: "string" },
|
|
timeoutMs: { type: "number" },
|
|
reuseLease: { type: "boolean" },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
});
|
|
const executionWorkspaceId = randomUUID();
|
|
const projectId = randomUUID();
|
|
await insertReusableProject(projectId, companyId);
|
|
await db.insert(executionWorkspaces).values({
|
|
id: executionWorkspaceId,
|
|
companyId,
|
|
projectId,
|
|
mode: "shared_workspace",
|
|
strategyType: "project_primary",
|
|
name: "Unverified-worker workspace",
|
|
status: "active",
|
|
providerType: "local_fs",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
const existingLease = await environmentService(db).acquireLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
heartbeatRunId: runId,
|
|
leasePolicy: "reuse_by_environment",
|
|
provider: "fake-plugin",
|
|
providerLeaseId: "old-plugin-lease",
|
|
metadata: {
|
|
agentId,
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
reusableSandboxLease: {
|
|
version: 1,
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
agentId,
|
|
adapterType: null,
|
|
provider: "fake-plugin",
|
|
runtimeFingerprint: reusableRuntimeFingerprint({
|
|
provider: "fake-plugin",
|
|
adapterType: null,
|
|
config: providerConfig,
|
|
}),
|
|
},
|
|
},
|
|
});
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "fresh-plugin-lease",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
// The worker verifies only `environmentExecute`. It advertises neither
|
|
// reuse verb, so the reusable-lease capability fails closed.
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentExecute"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
|
|
expect(acquired.lease.leasePolicy).toBe("ephemeral");
|
|
expect(workerManager.call).toHaveBeenCalledTimes(1);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
|
|
expect(workerManager.call).not.toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentResumeLease",
|
|
expect.anything(),
|
|
expect.anything(),
|
|
);
|
|
// The runtime leaves the old reusable lease untouched: it neither resumes
|
|
// nor destroys a lease it cannot serve.
|
|
await expect(environmentService(db).getLeaseById(existingLease.id)).resolves.toMatchObject({
|
|
status: "active",
|
|
leasePolicy: "reuse_by_environment",
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
{ status: "active", scope: "workspace" },
|
|
{ status: "failed", scope: "workspace" },
|
|
{ status: "failed", scope: "issue" },
|
|
] as const)("destroys $status reusable plugin-backed sandbox leases scoped to an $scope", async ({ status, scope }) => {
|
|
const { pluginId, companyId, runId, executionWorkspaceId, reusableLease } =
|
|
await seedReusablePluginSandboxLease();
|
|
await db
|
|
.update(heartbeatRuns)
|
|
.set({ status: status === "failed" ? "failed" : "succeeded" })
|
|
.where(eq(heartbeatRuns.id, runId));
|
|
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
|
|
const issueId = randomUUID();
|
|
if (scope === "issue") {
|
|
await db.insert(issues).values({ id: issueId, companyId, title: "Failed reusable lease cleanup", status: "done", priority: "medium" });
|
|
await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, reusableLease.id));
|
|
}
|
|
const selection = scope === "issue" ? { issueId } : { executionWorkspaceId };
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentDestroyLease") {
|
|
return undefined;
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
// A known issue/workspace id does not authorize another company's teardown.
|
|
expect(await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId: randomUUID(), ...selection })).toEqual([]);
|
|
expect(workerManager.call).not.toHaveBeenCalled();
|
|
const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
|
|
companyId,
|
|
...selection,
|
|
failureReason: "execution_workspace_closed",
|
|
});
|
|
|
|
expect(destroyed).toHaveLength(1);
|
|
expect(destroyed[0]?.lease.id).toBe(reusableLease.id);
|
|
expect(destroyed[0]?.lease.status).toBe("expired");
|
|
expect(workerManager.call).toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentDestroyLease",
|
|
expect.objectContaining({
|
|
driverKey: "fake-plugin",
|
|
providerLeaseId: "reusable-plugin-lease",
|
|
}),
|
|
31234,
|
|
);
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "expired",
|
|
failureReason: "execution_workspace_closed",
|
|
cleanupStatus: "success",
|
|
});
|
|
});
|
|
|
|
it.each(["active", "failed"] as const)("does not destroy a %s scoped reusable lease while its run is active", async (status) => {
|
|
const { pluginId, companyId, executionWorkspaceId, reusableLease } =
|
|
await seedReusablePluginSandboxLease();
|
|
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async () => undefined),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentDestroyLease",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: workerManager,
|
|
});
|
|
|
|
const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
|
|
companyId,
|
|
executionWorkspaceId,
|
|
failureReason: "issue_terminal_done",
|
|
});
|
|
|
|
expect(destroyed).toEqual([]);
|
|
expect(workerManager.call).not.toHaveBeenCalled();
|
|
await expect(
|
|
environmentService(db).getLeaseById(reusableLease.id),
|
|
).resolves.toMatchObject({ status });
|
|
});
|
|
|
|
it.each(["issue", "workspace", "environment"] as const)(
|
|
"durably claims failed reusable cleanup before %s teardown and recovers a provider failure",
|
|
async (scope) => {
|
|
const seeded = await seedReusablePluginSandboxLease();
|
|
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
|
|
await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
|
|
const issueId = randomUUID();
|
|
if (scope === "issue") {
|
|
await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Cleanup crash window", status: "done" });
|
|
await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, seeded.reusableLease.id));
|
|
}
|
|
let enterProvider!: () => void;
|
|
let finishProvider!: () => void;
|
|
const entered = new Promise<void>((resolve) => { enterProvider = resolve; });
|
|
const finish = new Promise<void>((resolve) => { finishProvider = resolve; });
|
|
let unavailable = true;
|
|
const call = vi.fn(async (_id: string, method: string) => {
|
|
if (method !== "environmentDestroyLease") throw new Error(`Unexpected method ${method}`);
|
|
if (unavailable) {
|
|
enterProvider();
|
|
await finish;
|
|
throw new Error("Provider response lost");
|
|
}
|
|
return { providerLeaseId: seeded.reusableLease.providerLeaseId, state: "destroyed" };
|
|
});
|
|
const makeRuntime = () => environmentRuntimeService(db, { pluginWorkerManager: {
|
|
isRunning: () => true, call,
|
|
getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
|
|
} as unknown as PluginWorkerManager });
|
|
const runtime = makeRuntime();
|
|
const close = (service: ReturnType<typeof makeRuntime>) => scope === "environment"
|
|
? service.destroyReusableSandboxLeasesForEnvironment({ environmentId: seeded.environment.id })
|
|
: service.destroyReusableSandboxLeases({ companyId: seeded.companyId,
|
|
...(scope === "issue" ? { issueId } : { executionWorkspaceId: seeded.executionWorkspaceId }) });
|
|
const closing = close(runtime);
|
|
try {
|
|
await entered;
|
|
// This independently persisted row is what survives controller loss
|
|
// during the provider call. No provider receipt has arrived yet.
|
|
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
|
|
status: "pending_cleanup", cleanupStatus: "failed", companyId: seeded.companyId,
|
|
providerLeaseId: seeded.reusableLease.providerLeaseId,
|
|
metadata: { pendingCleanupAttemptId: expect.any(String), pendingCleanupInFlight: true,
|
|
pendingCleanupLeaseExpiresAtMs: expect.any(Number) },
|
|
});
|
|
await close(makeRuntime());
|
|
expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
|
|
.toEqual({ swept: 0, destroyed: 0, capped: 0 });
|
|
expect(call).toHaveBeenCalledOnce();
|
|
} finally {
|
|
finishProvider();
|
|
await closing;
|
|
}
|
|
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
|
|
status: "pending_cleanup", metadata: { pendingCleanupInFlight: false },
|
|
});
|
|
unavailable = false;
|
|
expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
|
|
.toEqual({ swept: 1, destroyed: 1, capped: 0 });
|
|
expect(call).toHaveBeenCalledTimes(2);
|
|
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
|
|
status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "destroyed" } },
|
|
});
|
|
},
|
|
);
|
|
|
|
it.each(["queued", "scheduled_retry", "running"] as const)(
|
|
"fences scoped cleanup when its holding run becomes %s after selection", async (status) => {
|
|
const seeded = await seedReusablePluginSandboxLease();
|
|
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
|
|
await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
|
|
const originalService = environmentsModule.environmentService;
|
|
const serviceSpy = vi.spyOn(environmentsModule, "environmentService").mockImplementation((client) => {
|
|
const service = originalService(client);
|
|
return { ...service, getById: async (id: string) => {
|
|
const environment = await service.getById(id);
|
|
if (id === seeded.environment.id) {
|
|
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, seeded.runId));
|
|
}
|
|
return environment;
|
|
} };
|
|
});
|
|
const call = vi.fn(async () => undefined);
|
|
const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
|
|
isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
|
|
} as unknown as PluginWorkerManager });
|
|
try {
|
|
expect(await runtime.destroyReusableSandboxLeases({ companyId: seeded.companyId,
|
|
executionWorkspaceId: seeded.executionWorkspaceId })).toEqual([]);
|
|
expect(call).not.toHaveBeenCalled();
|
|
expect(await originalService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "failed" });
|
|
} finally {
|
|
serviceSpy.mockRestore();
|
|
}
|
|
},
|
|
);
|
|
|
|
it("destroys reusable plugin-backed sandbox leases scoped to an environment", async () => {
|
|
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
|
|
// The holding run is finished, so the reservation is stale and destroyable.
|
|
await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId));
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentDestroyLease") {
|
|
return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" };
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
|
|
environmentId: reusableLease.environmentId!,
|
|
failureReason: "environment_deleted",
|
|
});
|
|
|
|
expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 });
|
|
expect(workerManager.call).toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentDestroyLease",
|
|
expect.objectContaining({
|
|
driverKey: "fake-plugin",
|
|
providerLeaseId: "reusable-plugin-lease",
|
|
}),
|
|
31234,
|
|
);
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "expired",
|
|
failureReason: "environment_deleted",
|
|
cleanupStatus: "success",
|
|
metadata: { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1",
|
|
leaseId: reusableLease.id, runId, providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" } },
|
|
});
|
|
});
|
|
|
|
it("destroys a failed reusable sandbox after successful release before deleting its environment", async () => {
|
|
const { pluginId, companyId, runId, environment, reusableLease } = await seedReusablePluginSandboxLease();
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentReleaseLease") {
|
|
return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" };
|
|
}
|
|
if (method === "environmentDestroyLease") {
|
|
// The provider call must already have a durable cleanup reference.
|
|
expect(await environmentService(db).getLeaseById(reusableLease.id)).toMatchObject({
|
|
status: "pending_cleanup", environmentId: environment.id,
|
|
});
|
|
return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" };
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId));
|
|
|
|
const released = await runtimeWithPlugin.releaseRunLeases(runId, "failed", undefined, undefined, true);
|
|
expect(released[0]?.lease).toMatchObject({
|
|
id: reusableLease.id, status: "failed", cleanupStatus: "success",
|
|
metadata: { remoteExecutionTermination: { state: "stopped" } },
|
|
});
|
|
const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
|
|
environmentId: environment.id, failureReason: "environment_deleted",
|
|
});
|
|
expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 });
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease",
|
|
expect.objectContaining({ companyId, environmentId: environment.id, providerLeaseId: reusableLease.providerLeaseId }),
|
|
31234);
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "expired", cleanupStatus: "success",
|
|
metadata: { remoteExecutionTermination: { state: "destroyed" } },
|
|
});
|
|
expect((await environmentService(db).removeIfDeletable(environment.id))?.id).toBe(environment.id);
|
|
});
|
|
|
|
it.each(["active", "failed"] as const)("keeps a %s reusable lease held by an in-flight run out of the environment-scoped destroy", async (status) => {
|
|
const { pluginId, reusableLease } = await seedReusablePluginSandboxLease();
|
|
// seedEnvironment leaves the holding run in `running` status.
|
|
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async () => undefined),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
|
|
environmentId: reusableLease.environmentId!,
|
|
failureReason: "environment_deleted",
|
|
});
|
|
|
|
expect(result).toEqual({ destroyed: 0, failed: 0, skippedLiveRun: 1 });
|
|
expect(workerManager.call).not.toHaveBeenCalled();
|
|
// The lease keeps its reusable status, so the delete guard still blocks.
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status,
|
|
leasePolicy: "reuse_by_environment",
|
|
});
|
|
});
|
|
|
|
it("sweeps reusable cleanup with the configuration recorded on the lease", async () => {
|
|
const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
|
|
const environmentsSvc = environmentService(db);
|
|
await environmentsSvc.releaseLease(reusableLease.id, "pending_cleanup", {
|
|
failureReason: "release_cleanup_failed",
|
|
cleanupStatus: "failed",
|
|
});
|
|
|
|
// Re-point the environment after the reusable sandbox was provisioned.
|
|
// The pending-cleanup sweep still enters destroyRunLease because the
|
|
// environment exists, but destroy must target the provider and config
|
|
// captured on the lease rather than this current configuration.
|
|
await environmentsSvc.update(environment.id, {
|
|
config: {
|
|
provider: "replacement-provider",
|
|
image: "replacement:test",
|
|
timeoutMs: 9999,
|
|
reuseLease: true,
|
|
},
|
|
});
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentDestroyLease") return undefined;
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
const heartbeat = heartbeatService(db, { environmentRuntime: runtimeWithPlugin });
|
|
|
|
const result = await heartbeat.sweepPendingCleanupLeases({ backoffMs: 0 });
|
|
|
|
expect(result).toEqual({ swept: 1, destroyed: 1, capped: 0 });
|
|
expect(workerManager.call).toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentDestroyLease",
|
|
expect.objectContaining({
|
|
driverKey: "fake-plugin",
|
|
environmentId: environment.id,
|
|
providerLeaseId: "reusable-plugin-lease",
|
|
config: expect.objectContaining({
|
|
image: "fake:test",
|
|
timeoutMs: 1234,
|
|
reuseLease: true,
|
|
}),
|
|
}),
|
|
31234,
|
|
);
|
|
const destroyInput = vi.mocked(workerManager.call).mock.calls[0]?.[2] as {
|
|
config?: Record<string, unknown>;
|
|
};
|
|
expect(destroyInput.config).not.toMatchObject({
|
|
image: "replacement:test",
|
|
timeoutMs: 9999,
|
|
});
|
|
await expect(environmentsSvc.getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "expired",
|
|
cleanupStatus: "success",
|
|
});
|
|
});
|
|
|
|
it("retries reusable plugin-backed sandbox destroy when the worker is unavailable", async () => {
|
|
const { pluginId, companyId, executionWorkspaceId, runId, reusableLease } =
|
|
await seedReusablePluginSandboxLease();
|
|
await db
|
|
.update(heartbeatRuns)
|
|
.set({ status: "succeeded", finishedAt: new Date() })
|
|
.where(eq(heartbeatRuns.id, runId));
|
|
|
|
const offlineWorkerManager = {
|
|
isRunning: vi.fn(() => false),
|
|
call: vi.fn(),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithOfflinePlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: offlineWorkerManager,
|
|
});
|
|
|
|
const pending = await runtimeWithOfflinePlugin.destroyReusableSandboxLeases({
|
|
companyId,
|
|
executionWorkspaceId,
|
|
failureReason: "execution_workspace_closed",
|
|
});
|
|
|
|
expect(pending).toHaveLength(1);
|
|
expect(pending[0]?.lease.id).toBe(reusableLease.id);
|
|
expect(pending[0]?.lease.status).toBe("pending_cleanup");
|
|
expect(offlineWorkerManager.call).not.toHaveBeenCalled();
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "pending_cleanup",
|
|
failureReason: "execution_workspace_closed",
|
|
cleanupStatus: "failed",
|
|
});
|
|
|
|
const recoveredWorkerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentDestroyLease") {
|
|
return undefined;
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithRecoveredPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: recoveredWorkerManager,
|
|
});
|
|
|
|
const retried = await runtimeWithRecoveredPlugin.destroyReusableSandboxLeases({
|
|
companyId,
|
|
executionWorkspaceId,
|
|
failureReason: "cleanup_retry",
|
|
});
|
|
|
|
expect(retried).toHaveLength(1);
|
|
expect(retried[0]?.lease.id).toBe(reusableLease.id);
|
|
expect(retried[0]?.lease.status).toBe("expired");
|
|
expect(recoveredWorkerManager.call).toHaveBeenCalledWith(
|
|
pluginId,
|
|
"environmentDestroyLease",
|
|
expect.objectContaining({
|
|
driverKey: "fake-plugin",
|
|
providerLeaseId: "reusable-plugin-lease",
|
|
}),
|
|
31234,
|
|
);
|
|
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
|
status: "expired",
|
|
failureReason: "cleanup_retry",
|
|
cleanupStatus: "success",
|
|
});
|
|
});
|
|
|
|
it("resolves effective capabilities from the lease's exact plugin, not an earlier plugin that shares the driver key", async () => {
|
|
// The helper seeds the plugin that owns the lease. Pin it to `pluginId`
|
|
// through the lease metadata and give it a lower-priority sibling.
|
|
const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
|
|
|
|
// Rewrite the owner plugin so it DENIES reusable leases through the nested
|
|
// capability declaration.
|
|
await db
|
|
.update(plugins)
|
|
.set({
|
|
manifestJson: {
|
|
id: "acme.reusable-sandbox-provider",
|
|
apiVersion: 1,
|
|
version: "1.0.0",
|
|
displayName: "Reusable Sandbox Provider",
|
|
description: "Owner plugin that denies reusable leases",
|
|
author: "Paperclip",
|
|
categories: ["automation"],
|
|
capabilities: ["environment.drivers.register"],
|
|
entrypoints: { worker: "dist/worker.js" },
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
sandboxCapabilities: { reusableLeases: false },
|
|
configSchema: { type: "object", properties: {} },
|
|
},
|
|
],
|
|
},
|
|
updatedAt: new Date(),
|
|
} as any)
|
|
.where(eq(plugins.id, pluginId));
|
|
|
|
// Install an EARLIER plugin that shares the driver key and grants reusable
|
|
// leases. It is not ready and never acquired this lease. A resolver keyed by
|
|
// driver key alone would read this declaration and grant a capability the
|
|
// owner plugin denied.
|
|
const collidingPluginId = randomUUID();
|
|
await insertSandboxProviderPlugin(collidingPluginId, {
|
|
pluginKey: "acme.colliding-sandbox-provider",
|
|
packageName: "@acme/colliding-sandbox-provider",
|
|
displayName: "Colliding Sandbox Provider",
|
|
description: "Earlier plugin that shares the driver key",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
supportsReusableLeases: true,
|
|
configSchema: { type: "object", properties: {} },
|
|
},
|
|
],
|
|
status: "installed",
|
|
installOrder: 0,
|
|
});
|
|
|
|
// The owner worker verifies the reuse verbs and the sync verbs, so every
|
|
// capability is verified. Only the owner's declaration can narrow one.
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(),
|
|
getWorker: vi.fn((id: string) =>
|
|
id === pluginId
|
|
? {
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentSyncIn",
|
|
"environmentSyncOut",
|
|
],
|
|
}
|
|
: undefined,
|
|
),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
expect(reusableLease.metadata?.pluginId).toBe(pluginId);
|
|
|
|
const effective = await runtimeWithPlugin.resolveCapabilities({
|
|
environment,
|
|
lease: reusableLease,
|
|
});
|
|
|
|
// The runtime read the owner plugin's declaration, so it denies reusable
|
|
// leases even though the earlier plugin grants them.
|
|
expect(effective?.reusableLeases).toBe(false);
|
|
// The owner plugin does not restrict native sync, and its worker verifies
|
|
// the sync verbs, so those stay granted. This proves the resolver read the
|
|
// owner declaration and did not fail every capability closed.
|
|
expect(effective?.nativeSyncIn).toBe(true);
|
|
expect(effective?.nativeSyncOut).toBe(true);
|
|
});
|
|
|
|
it("fails every effective capability closed when the pinned plugin id is absent from the registry", async () => {
|
|
// The lease pins a plugin id, but that plugin record is gone. A stale worker
|
|
// entry still advertises every method. The runtime must not read the stale
|
|
// methods; it must fail closed because the exact-plugin identity is gone.
|
|
const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
|
|
await db.delete(plugins).where(eq(plugins.id, pluginId));
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn(() => true),
|
|
call: vi.fn(),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentExecute",
|
|
"environmentSyncIn",
|
|
"environmentSyncOut",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const effective = await runtimeWithPlugin.resolveCapabilities({
|
|
environment,
|
|
lease: reusableLease,
|
|
});
|
|
|
|
for (const key of SANDBOX_CAPABILITY_KEYS) {
|
|
expect(effective?.[key]).toBe(false);
|
|
}
|
|
});
|
|
|
|
it("fails every effective capability closed when the pinned plugin no longer declares this provider key", async () => {
|
|
// The pinned plugin still exists, but it no longer declares a
|
|
// `sandbox_provider` driver with this key (here it changed the driver kind).
|
|
// A running worker still advertises every method. The runtime must fail
|
|
// closed because the exact-plugin declaration is gone.
|
|
const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
|
|
await db
|
|
.update(plugins)
|
|
.set({
|
|
manifestJson: {
|
|
id: "acme.reusable-sandbox-provider",
|
|
apiVersion: 1,
|
|
version: "1.0.0",
|
|
displayName: "Reusable Sandbox Provider",
|
|
description: "Owner plugin that no longer declares the provider key",
|
|
author: "Paperclip",
|
|
categories: ["automation"],
|
|
capabilities: ["environment.drivers.register"],
|
|
entrypoints: { worker: "dist/worker.js" },
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
// The key exists, but the kind is now a plain environment driver,
|
|
// not a sandbox provider. The by-id resolver fails closed.
|
|
kind: "environment_driver",
|
|
displayName: "Fake Plugin",
|
|
configSchema: { type: "object", properties: {} },
|
|
},
|
|
],
|
|
},
|
|
updatedAt: new Date(),
|
|
} as any)
|
|
.where(eq(plugins.id, pluginId));
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(),
|
|
getWorker: vi.fn(() => ({
|
|
supportedMethods: [
|
|
"environmentResumeLease",
|
|
"environmentReleaseLease",
|
|
"environmentExecute",
|
|
"environmentSyncIn",
|
|
"environmentSyncOut",
|
|
],
|
|
})),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const effective = await runtimeWithPlugin.resolveCapabilities({
|
|
environment,
|
|
lease: reusableLease,
|
|
});
|
|
|
|
for (const key of SANDBOX_CAPABILITY_KEYS) {
|
|
expect(effective?.[key]).toBe(false);
|
|
}
|
|
});
|
|
|
|
it("defers to verified worker discovery for a valid pinned plugin that omits sandboxCapabilities", async () => {
|
|
// A valid, identified plugin whose manifest declares no `sandboxCapabilities`
|
|
// and no legacy reuse flag. Its worker verifies the sync verbs. An omitted
|
|
// declaration is NOT an identity failure: the runtime defers to the verified
|
|
// baseline, so native sync stays granted while unverified capabilities stay
|
|
// false.
|
|
const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
|
|
await db
|
|
.update(plugins)
|
|
.set({
|
|
manifestJson: {
|
|
id: "acme.reusable-sandbox-provider",
|
|
apiVersion: 1,
|
|
version: "1.0.0",
|
|
displayName: "Reusable Sandbox Provider",
|
|
description: "Owner plugin that omits the capability declaration",
|
|
author: "Paperclip",
|
|
categories: ["automation"],
|
|
capabilities: ["environment.drivers.register"],
|
|
entrypoints: { worker: "dist/worker.js" },
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
kind: "sandbox_provider",
|
|
displayName: "Fake Plugin",
|
|
configSchema: { type: "object", properties: {} },
|
|
},
|
|
],
|
|
},
|
|
updatedAt: new Date(),
|
|
} as any)
|
|
.where(eq(plugins.id, pluginId));
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(),
|
|
getWorker: vi.fn((id: string) =>
|
|
id === pluginId
|
|
? { supportedMethods: ["environmentSyncIn", "environmentSyncOut"] }
|
|
: undefined,
|
|
),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const effective = await runtimeWithPlugin.resolveCapabilities({
|
|
environment,
|
|
lease: reusableLease,
|
|
});
|
|
|
|
// The worker verified the sync verbs and the omitted declaration adds no
|
|
// restriction, so native sync stays granted.
|
|
expect(effective?.nativeSyncIn).toBe(true);
|
|
expect(effective?.nativeSyncOut).toBe(true);
|
|
// The worker did not verify the reuse verbs, so reusable leases stay false.
|
|
expect(effective?.reusableLeases).toBe(false);
|
|
});
|
|
|
|
it("releases a sandbox run lease from metadata after the environment config changes", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "sandbox",
|
|
name: "Fake Sandbox",
|
|
config: {
|
|
provider: "fake",
|
|
image: "ubuntu:24.04",
|
|
reuseLease: true,
|
|
},
|
|
});
|
|
|
|
const acquired = await runtime.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "local",
|
|
config: {},
|
|
});
|
|
|
|
const released = await runtime.releaseRunLeases(runId);
|
|
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.lease.id).toBe(acquired.lease.id);
|
|
expect(released[0]?.lease.status).toBe("released");
|
|
});
|
|
|
|
it("does not reuse a sandbox lease owned by a different agent for the same execution workspace", async () => {
|
|
const { companyId, agentId, environment, runId } = await seedEnvironment({
|
|
driver: "plugin",
|
|
name: "Plugin Fake plugin",
|
|
config: {
|
|
pluginKey: "acme.environments",
|
|
driverKey: "fake-plugin",
|
|
driverConfig: {
|
|
template: "base",
|
|
},
|
|
},
|
|
});
|
|
const otherAgentId = randomUUID();
|
|
const executionWorkspaceId = randomUUID();
|
|
const pluginId = randomUUID();
|
|
const projectId = randomUUID();
|
|
|
|
await insertReusableProject(projectId, companyId);
|
|
await db.insert(executionWorkspaces).values({
|
|
id: executionWorkspaceId,
|
|
companyId,
|
|
projectId,
|
|
mode: "shared_workspace",
|
|
strategyType: "project_primary",
|
|
name: "Existing workspace",
|
|
status: "active",
|
|
providerType: "local_fs",
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
});
|
|
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.environments",
|
|
packageName: "@acme/paperclip-environments",
|
|
displayName: "Acme Environments",
|
|
description: "Test plugin environment driver",
|
|
author: "Acme",
|
|
});
|
|
|
|
await environmentService(db).acquireLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId,
|
|
heartbeatRunId: runId,
|
|
leasePolicy: "reuse_by_environment",
|
|
provider: "fake-plugin",
|
|
providerLeaseId: "other-agent-lease",
|
|
metadata: {
|
|
agentId,
|
|
provider: "fake-plugin",
|
|
template: "base",
|
|
reuseLease: true,
|
|
},
|
|
});
|
|
|
|
const workerManager = {
|
|
isRunning: vi.fn((id: string) => id === pluginId),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "fresh-agent-lease",
|
|
metadata: {
|
|
provider: "fake-plugin",
|
|
template: "base",
|
|
reuseLease: true,
|
|
},
|
|
};
|
|
}
|
|
throw new Error(`Unexpected plugin method: ${method}`);
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
agentId: otherAgentId,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: {
|
|
id: executionWorkspaceId,
|
|
mode: "shared_workspace",
|
|
},
|
|
});
|
|
|
|
expect(acquired.lease.providerLeaseId).toBe("fresh-agent-lease");
|
|
expect(workerManager.call).toHaveBeenCalledTimes(1);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({
|
|
agentId: otherAgentId,
|
|
executionWorkspaceId,
|
|
}));
|
|
});
|
|
|
|
it.each([undefined, 300_000])("delegates plugin environment leases with acquisition budget %s", async (defaultAcquireTimeoutMs) => {
|
|
const pluginId = randomUUID();
|
|
const expiresAt = new Date(Date.now() + 60_000).toISOString();
|
|
const workerManager = {
|
|
isRunning: vi.fn(() => true),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "plugin-lease-1",
|
|
expiresAt,
|
|
metadata: {
|
|
driver: "local",
|
|
pluginId: "provider-plugin-id",
|
|
pluginKey: "provider.plugin",
|
|
driverKey: "provider-driver",
|
|
executionWorkspaceMode: "provider-mode",
|
|
provider: "test-provider",
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
return undefined;
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: workerManager,
|
|
});
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "plugin",
|
|
name: "Plugin Fake plugin",
|
|
config: {
|
|
pluginKey: "acme.environments",
|
|
driverKey: "fake-plugin",
|
|
driverConfig: {
|
|
template: "base",
|
|
},
|
|
},
|
|
});
|
|
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.environments",
|
|
packageName: "@acme/paperclip-environments",
|
|
displayName: "Acme Environments",
|
|
description: "Test plugin environment driver",
|
|
author: "Acme",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
displayName: "Fake plugin",
|
|
defaultAcquireTimeoutMs,
|
|
configSchema: { type: "object" },
|
|
},
|
|
],
|
|
});
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", {
|
|
driverKey: "fake-plugin",
|
|
companyId,
|
|
environmentId: environment.id,
|
|
executionWorkspaceId: undefined,
|
|
executionWorkspaceSettings: null,
|
|
issueId: null,
|
|
config: { template: "base" },
|
|
agentId: undefined,
|
|
adapterType: undefined,
|
|
runId,
|
|
workspaceMode: undefined,
|
|
}, ...(defaultAcquireTimeoutMs === undefined ? [] : [330_000]));
|
|
expect(acquired.lease.providerLeaseId).toBe("plugin-lease-1");
|
|
expect(acquired.lease.expiresAt?.toISOString()).toBe(expiresAt);
|
|
expect(acquired.lease.metadata).toMatchObject({
|
|
driver: "plugin",
|
|
pluginId,
|
|
pluginKey: "acme.environments",
|
|
driverKey: "fake-plugin",
|
|
executionWorkspaceMode: null,
|
|
providerMetadata: {
|
|
driver: "local",
|
|
pluginId: "provider-plugin-id",
|
|
pluginKey: "provider.plugin",
|
|
driverKey: "provider-driver",
|
|
executionWorkspaceMode: "provider-mode",
|
|
provider: "test-provider",
|
|
remoteCwd: "/workspace",
|
|
},
|
|
});
|
|
|
|
await environmentService(db).update(environment.id, {
|
|
driver: "local",
|
|
config: {},
|
|
});
|
|
|
|
const released = await runtimeWithPlugin.releaseRunLeases(runId);
|
|
|
|
expect(released).toHaveLength(1);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", {
|
|
driverKey: "fake-plugin",
|
|
companyId,
|
|
environmentId: environment.id,
|
|
issueId: null,
|
|
config: {},
|
|
providerLeaseId: "plugin-lease-1",
|
|
leaseMetadata: expect.objectContaining({
|
|
driver: "plugin",
|
|
pluginId,
|
|
providerMetadata: expect.objectContaining({
|
|
driver: "local",
|
|
}),
|
|
}),
|
|
});
|
|
expect(released[0]?.lease.status).toBe("released");
|
|
});
|
|
|
|
async function seedPluginDriverEnvironment(pluginId: string) {
|
|
const seeded = await seedEnvironment({
|
|
driver: "plugin",
|
|
name: "Plugin Fake plugin",
|
|
config: {
|
|
pluginKey: "acme.environments",
|
|
driverKey: "fake-plugin",
|
|
driverConfig: { template: "base" },
|
|
},
|
|
});
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.environments",
|
|
packageName: "@acme/paperclip-environments",
|
|
displayName: "Acme Environments",
|
|
description: "Test plugin environment driver",
|
|
author: "Acme",
|
|
environmentDrivers: [
|
|
{ driverKey: "fake-plugin", displayName: "Fake plugin", configSchema: { type: "object" } },
|
|
],
|
|
});
|
|
return seeded;
|
|
}
|
|
|
|
it("forwards a requested expiry to the acquire RPC and records only a provider-attested expiry", async () => {
|
|
const pluginId = randomUUID();
|
|
let receivedRequestedExpiresAt: unknown;
|
|
const workerManager = {
|
|
isRunning: vi.fn(() => true),
|
|
call: vi.fn(async (_pluginId: string, method: string, params: Record<string, unknown>) => {
|
|
if (method === "environmentAcquireLease") {
|
|
receivedRequestedExpiresAt = params.requestedExpiresAt;
|
|
// The provider grants no expiry.
|
|
return { providerLeaseId: "plugin-lease-no-expiry", metadata: { remoteCwd: "/workspace" } };
|
|
}
|
|
return undefined;
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
const { companyId, environment, runId } = await seedPluginDriverEnvironment(pluginId);
|
|
|
|
const requestedExpiresAt = new Date(Date.now() + 60_000);
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
requestedExpiresAt,
|
|
});
|
|
|
|
// The acquire RPC receives the requested expiry as an ISO 8601 string.
|
|
expect(receivedRequestedExpiresAt).toBe(requestedExpiresAt.toISOString());
|
|
// The provider gave no expiry, so the lease row records no expiry. The runtime
|
|
// never synthesizes the requested deadline as evidence of provider enforcement.
|
|
expect(acquired.lease.expiresAt ?? null).toBeNull();
|
|
});
|
|
|
|
it("records the real provider expiry when the provider grants one later than the requested deadline", async () => {
|
|
const pluginId = randomUUID();
|
|
const providerExpiresAt = new Date(Date.now() + 120_000).toISOString();
|
|
const workerManager = {
|
|
isRunning: vi.fn(() => true),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "plugin-lease-later",
|
|
expiresAt: providerExpiresAt,
|
|
metadata: { remoteCwd: "/workspace" },
|
|
};
|
|
}
|
|
return undefined;
|
|
}),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
|
const { companyId, environment, runId } = await seedPluginDriverEnvironment(pluginId);
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
requestedExpiresAt: new Date(Date.now() + 60_000),
|
|
});
|
|
|
|
// The lease row records the real provider expiry, not the requested deadline.
|
|
// The caller then verifies the expiry bounds its deadline and fails closed.
|
|
expect(acquired.lease.expiresAt?.toISOString()).toBe(providerExpiresAt);
|
|
});
|
|
|
|
it("delegates the full plugin environment lifecycle through the worker manager", async () => {
|
|
const pluginId = randomUUID();
|
|
const workerManager = {
|
|
isRunning: vi.fn(() => true),
|
|
call: vi.fn(async (_pluginId: string, method: string) => {
|
|
if (method === "environmentAcquireLease") {
|
|
return {
|
|
providerLeaseId: "plugin-lease-full",
|
|
metadata: {
|
|
remoteCwd: "/workspace",
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentResumeLease") {
|
|
return {
|
|
providerLeaseId: "plugin-lease-full",
|
|
metadata: {
|
|
resumed: true,
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentRealizeWorkspace") {
|
|
return {
|
|
cwd: "/workspace/project",
|
|
metadata: {
|
|
realized: true,
|
|
},
|
|
};
|
|
}
|
|
if (method === "environmentExecute") {
|
|
return {
|
|
exitCode: 0,
|
|
signal: null,
|
|
timedOut: false,
|
|
stdout: "ok\n",
|
|
stderr: "",
|
|
metadata: {
|
|
commandId: "cmd-1",
|
|
},
|
|
};
|
|
}
|
|
return undefined;
|
|
}),
|
|
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
|
|
} as unknown as PluginWorkerManager;
|
|
const runtimeWithPlugin = environmentRuntimeService(db, {
|
|
pluginWorkerManager: workerManager,
|
|
});
|
|
const { companyId, environment, runId } = await seedEnvironment({
|
|
driver: "plugin",
|
|
name: "Plugin Full Lifecycle",
|
|
config: {
|
|
pluginKey: "acme.environments",
|
|
driverKey: "fake-plugin",
|
|
driverConfig: {
|
|
template: "base",
|
|
},
|
|
},
|
|
});
|
|
|
|
await insertSandboxProviderPlugin(pluginId, {
|
|
pluginKey: "acme.environments",
|
|
packageName: "@acme/paperclip-environments",
|
|
displayName: "Acme Environments",
|
|
description: "Test plugin environment driver",
|
|
author: "Acme",
|
|
environmentDrivers: [
|
|
{
|
|
driverKey: "fake-plugin",
|
|
displayName: "Fake plugin",
|
|
configSchema: { type: "object" },
|
|
},
|
|
],
|
|
});
|
|
|
|
const acquired = await runtimeWithPlugin.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
const resumed = await runtimeWithPlugin.resumeRunLease({
|
|
environment,
|
|
lease: acquired.lease,
|
|
});
|
|
const realized = await runtimeWithPlugin.realizeWorkspace({
|
|
environment,
|
|
lease: acquired.lease,
|
|
workspace: {
|
|
localPath: "/tmp/project",
|
|
mode: "ephemeral",
|
|
},
|
|
});
|
|
const executed = await runtimeWithPlugin.execute({
|
|
environment,
|
|
lease: acquired.lease,
|
|
command: "echo",
|
|
args: ["ok"],
|
|
cwd: realized.cwd,
|
|
env: { FOO: "bar" },
|
|
stdin: "",
|
|
timeoutMs: 1000,
|
|
});
|
|
const destroyed = await runtimeWithPlugin.destroyRunLease({
|
|
environment,
|
|
lease: acquired.lease,
|
|
});
|
|
|
|
expect(resumed).toMatchObject({
|
|
providerLeaseId: "plugin-lease-full",
|
|
metadata: {
|
|
resumed: true,
|
|
},
|
|
});
|
|
expect(realized).toEqual({
|
|
cwd: "/workspace/project",
|
|
metadata: {
|
|
realized: true,
|
|
},
|
|
});
|
|
expect(executed).toMatchObject({
|
|
exitCode: 0,
|
|
timedOut: false,
|
|
stdout: "ok\n",
|
|
});
|
|
expect(destroyed?.status).toBe("failed");
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentResumeLease", {
|
|
driverKey: "fake-plugin",
|
|
companyId,
|
|
environmentId: environment.id,
|
|
issueId: null,
|
|
config: { template: "base" },
|
|
providerLeaseId: "plugin-lease-full",
|
|
leaseMetadata: expect.objectContaining({
|
|
driver: "plugin",
|
|
pluginId,
|
|
}),
|
|
});
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentRealizeWorkspace", expect.objectContaining({
|
|
driverKey: "fake-plugin",
|
|
companyId,
|
|
environmentId: environment.id,
|
|
config: { template: "base" },
|
|
workspace: {
|
|
localPath: "/tmp/project",
|
|
mode: "ephemeral",
|
|
},
|
|
}));
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({
|
|
driverKey: "fake-plugin",
|
|
companyId,
|
|
environmentId: environment.id,
|
|
command: "echo",
|
|
args: ["ok"],
|
|
cwd: "/workspace/project",
|
|
env: { FOO: "bar" },
|
|
}), 31000);
|
|
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", {
|
|
driverKey: "fake-plugin",
|
|
companyId,
|
|
environmentId: environment.id,
|
|
issueId: null,
|
|
config: { template: "base" },
|
|
providerLeaseId: "plugin-lease-full",
|
|
leaseMetadata: expect.objectContaining({
|
|
driver: "plugin",
|
|
pluginId,
|
|
}),
|
|
});
|
|
});
|
|
|
|
it("releases with the driver captured on the lease even if the environment driver changes later", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment();
|
|
const environmentsSvc = environmentService(db);
|
|
const localRelease = vi.fn(async ({ lease, status }: { lease: { id: string }; status: "released" | "expired" | "failed" }) =>
|
|
await environmentsSvc.releaseLease(lease.id, status)
|
|
);
|
|
const sshRelease = vi.fn(async () => {
|
|
throw new Error("ssh release should not be called");
|
|
});
|
|
const runtimeWithSpies = environmentRuntimeService(db, {
|
|
drivers: [
|
|
{
|
|
driver: "local",
|
|
acquireRunLease: async (input) => await environmentsSvc.acquireLease({
|
|
companyId: input.companyId,
|
|
environmentId: input.environment.id,
|
|
executionWorkspaceId: input.executionWorkspaceId,
|
|
issueId: input.issueId,
|
|
heartbeatRunId: input.heartbeatRunId,
|
|
metadata: {
|
|
driver: input.environment.driver,
|
|
executionWorkspaceMode: input.executionWorkspaceMode,
|
|
},
|
|
}),
|
|
releaseRunLease: localRelease,
|
|
},
|
|
{
|
|
driver: "ssh",
|
|
acquireRunLease: async () => {
|
|
throw new Error("ssh acquire should not be called");
|
|
},
|
|
releaseRunLease: sshRelease,
|
|
},
|
|
],
|
|
});
|
|
|
|
const acquired = await runtimeWithSpies.acquireRunLease({
|
|
companyId,
|
|
environment,
|
|
issueId: null,
|
|
heartbeatRunId: runId,
|
|
persistedExecutionWorkspace: null,
|
|
});
|
|
|
|
await environmentsSvc.update(environment.id, { driver: "ssh" });
|
|
|
|
const released = await runtimeWithSpies.releaseRunLeases(runId);
|
|
|
|
expect(released).toHaveLength(1);
|
|
expect(localRelease).toHaveBeenCalledTimes(1);
|
|
expect(sshRelease).not.toHaveBeenCalled();
|
|
expect(acquired.lease.metadata?.driver).toBe("local");
|
|
});
|
|
|
|
it("test_release_run_leases_continues_after_first_release_fails", async () => {
|
|
const { companyId, environment, runId } = await seedEnvironment();
|
|
const environmentsSvc = environmentService(db);
|
|
|
|
// Seed two active leases for one run. The first driver release throws; the
|
|
// second must still release.
|
|
const failingLease = await environmentsSvc.acquireLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
heartbeatRunId: runId,
|
|
provider: "local",
|
|
providerLeaseId: "fail-release",
|
|
metadata: { driver: "local" },
|
|
});
|
|
const healthyLease = await environmentsSvc.acquireLease({
|
|
companyId,
|
|
environmentId: environment.id,
|
|
heartbeatRunId: runId,
|
|
provider: "local",
|
|
providerLeaseId: "healthy-release",
|
|
metadata: { driver: "local" },
|
|
});
|
|
|
|
const runtimeWithFailingDriver = environmentRuntimeService(db, {
|
|
drivers: [
|
|
{
|
|
driver: "local",
|
|
acquireRunLease: async () => {
|
|
throw new Error("acquire should not be called");
|
|
},
|
|
releaseRunLease: async ({ lease, status }) => {
|
|
if (lease.providerLeaseId === "fail-release") {
|
|
throw new Error("driver release failed");
|
|
}
|
|
return await environmentsSvc.releaseLease(lease.id, status);
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
const errors: Array<{ leaseId: string; error: unknown }> = [];
|
|
const released = await runtimeWithFailingDriver.releaseRunLeases(
|
|
runId,
|
|
"released",
|
|
(leaseId, error) => errors.push({ leaseId, error }),
|
|
);
|
|
|
|
// The healthy lease released even though the first release failed.
|
|
expect(released).toHaveLength(1);
|
|
expect(released[0]?.lease.id).toBe(healthyLease.id);
|
|
expect(released[0]?.lease.status).toBe("released");
|
|
|
|
// The failed release reported one lease-specific error.
|
|
expect(errors).toHaveLength(1);
|
|
expect(errors[0]?.leaseId).toBe(failingLease.id);
|
|
expect((errors[0]?.error as Error).message).toBe("driver release failed");
|
|
|
|
// The database confirms the isolation. The healthy lease released; the
|
|
// failing lease stayed active.
|
|
const rows = await db.select().from(environmentLeases);
|
|
expect(rows.find((row) => row.id === healthyLease.id)?.status).toBe("released");
|
|
expect(rows.find((row) => row.id === failingLease.id)?.status).toBe("active");
|
|
});
|
|
});
|