Files
PaperClipAI/server/src/__tests__/plugin-runner-task.integration.test.ts
T
Nicky LeachandPaperclip 54ac0032dc Verify plugin workspaces on run leases and release settled tasks
Acquire the logical placement before verifying a retained checkout and reuse it through preparation. Persist definite refusal before release and preserve the existing workspace and session choices on fresh fallback and restart.

Always invoke provider release after task settlement. Reuse saved stop proof during cleanup retries and preserve completed run outcomes. Add database regressions for verification placement, restart, refusal, transport failure, and release replay.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-11-12 20:06:46 -08:00

434 lines
37 KiB
TypeScript

import * as pluginProbe from "../services/native-runtime/plugin-runner-probe.js";
import { randomUUID } from "node:crypto";
import { PassThrough } from "node:stream";
import { setTimeout as delay } from "node:timers/promises";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, heartbeatRuns, plugins, projects } from "@paperclipai/db";
import { definePlugin } from "../../../packages/plugins/sdk/src/define-plugin.js";
import { startWorkerRpcHost } from "../../../packages/plugins/sdk/src/worker-rpc-host.js";
import { createRequest, parseMessage, serializeMessage, isJsonRpcResponse, isJsonRpcSuccessResponse, JsonRpcCallError } from "../../../packages/plugins/sdk/src/protocol.js";
import { environmentTaskOperationSchema, environmentTaskResultSchema, readEnvironmentTaskErrorDiagnostic, type PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { agentEnvironmentTestService } from "../services/agent-environment-test.js";
import { heartbeatService } from "../services/heartbeat.js";
import { remoteExecutionHasStopped, remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { resolvePluginTaskResume } from "../services/environment-task-runtime.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import { environmentRunOrchestrator } from "../services/environment-run-orchestrator.js";
import { createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type RunnerProviderLaunchRequirements } from "@paperclipai/paperclip-runner";
import { createPluginRunnerTaskOwner } from "../services/native-runtime/plugin-runner-task.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
const ingress = vi.hoisted(() => ({ connect: vi.fn() }));
vi.mock("../realtime/runner-prp-outbound.js", () => ({ connectRunnerPrpIngress: ingress.connect }));
const databaseSuite = support.supported ? describe : describe.skip;
/** Real SDK serialization and dispatch with synthetic provider resources. */
function workerHarness(plugin: ReturnType<typeof definePlugin>, manifest: PaperclipPluginManifestV1) {
const stdin = new PassThrough(), stdout = new PassThrough();
const host = startWorkerRpcHost({ plugin, stdin, stdout });
const pending = new Map<number, { resolve: (value: any) => void; reject: (error: Error) => void }>();
let nextId = 0, buffer = "";
stdout.on("data", chunk => {
buffer += String(chunk);
let end: number;
while ((end = buffer.indexOf("\n")) >= 0) {
const message = parseMessage(buffer.slice(0, end));
buffer = buffer.slice(end + 1);
if (!isJsonRpcResponse(message)) continue;
const waiter = pending.get(Number(message.id));
pending.delete(Number(message.id));
if (isJsonRpcSuccessResponse(message)) waiter?.resolve(message.result);
else waiter?.reject(new JsonRpcCallError(message.error));
}
});
const call = (method: string, params: unknown) => new Promise<any>((resolve, reject) => {
const id = ++nextId;
pending.set(id, { resolve, reject });
stdin.write(serializeMessage(createRequest(method as never, params as never, id)));
});
return { call, stop: () => host.stop(), initialize: () => call("initialize", {
manifest, config: {}, instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" }, apiVersion: 1,
}) };
}
databaseSuite("Core plugin Runner task lifecycle", () => {
let db!: ReturnType<typeof createDb>;
let stopDb: (() => Promise<void>) | undefined;
beforeAll(async () => {
const started = await startRecoveryPostgres("plugin-runner-task");
stopDb = started.cleanup; db = createDb(started.connectionString);
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
afterAll(async () => { await stopDb?.(); });
it.each(["unresolved_operation", "source_state_unverified", "malformed_rpc", "extra_rpc_fields"] as const)("relays only the closed source refusal through Core (%s)", async mode => {
const reason = mode === "source_state_unverified" ? mode : "unresolved_operation";
const companyId = randomUUID(), agentId = randomUUID(), pluginId = randomUUID(), environmentId = randomUUID();
const sourceRunId = randomUUID(), runId = randomUUID(), sourceLeaseId = randomUUID(), leaseId = randomUUID();
const manifest: PaperclipPluginManifestV1 = { id: pluginId, apiVersion: 1, version: "1.0.0", displayName: "Task diagnostics",
description: "Synthetic task provider", author: "Paperclip", categories: ["automation"],
capabilities: ["environment.drivers.register"], entrypoints: { worker: "fixture" },
environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" } }] };
await db.insert(companies).values({ id: companyId, name: "Diagnostic fixture", issuePrefix: `D${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
await db.insert(environments).values({ id: environmentId, name: `Diagnostic ${environmentId}`, driver: "plugin", config: {} });
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture", version: "1.0.0", status: "ready", manifestJson: manifest });
await db.insert(heartbeatRuns).values([
{ id: sourceRunId, companyId, agentId, invocationSource: "manual", status: "succeeded", resultJson: { outcome: "preserved" } },
{ id: runId, companyId, agentId, invocationSource: "manual", status: "running" },
]);
const pin = { driver: "plugin", pluginId, driverKey: "tasks", runnerTaskLease: true,
runnerTaskProjectAuthValidation: { configurationId: "fixture", provider: "openai", projectIds: [] } };
const source = { id: sourceLeaseId, companyId, environmentId, heartbeatRunId: sourceRunId, provider: "plugin:fixture:tasks", providerLeaseId: "source-task" };
await db.insert(environmentLeases).values([
{ ...source, status: "released", leasePolicy: "ephemeral", cleanupStatus: "success", releasedAt: new Date(), metadata: { ...pin,
runnerTaskSessionRetained: true, runnerTaskIdentity: { runnerId: "runner", leaseId: sourceLeaseId, runId: sourceRunId, sessionId: "session", turnId: "old-turn", itemId: "old-item" },
remoteExecutionTermination: remoteTerminationReceipt(source, { providerLeaseId: "source-task", state: "stopped" }) } },
{ id: leaseId, companyId, environmentId, heartbeatRunId: runId, provider: source.provider, providerLeaseId: "current-task", status: "active", metadata: pin },
]);
const dispatched: string[] = [];
const rpc = workerHarness(definePlugin({ async setup() {}, async onEnvironmentTask(params) {
dispatched.push(params.operation.kind);
throw Object.assign(new Error("synthetic-private-provider-text"), { status: 409, code: "SESSION_STATE_UNAVAILABLE", reason,
operationId: "forged-operation", callId: "forged-call", taskId: "foreign-task" });
} }), manifest);
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask"] }),
call: (_id: string, method: string, params: unknown) => {
if (mode === "malformed_rpc" || mode === "extra_rpc_fields") throw new JsonRpcCallError({ code: -32000,
message: "synthetic-private-provider-text", data: { schema: "paperclip/environment-task-error/v1", diagnostic: {
httpStatus: 409, errorCode: "SESSION_STATE_UNAVAILABLE", reason: mode === "malformed_rpc" ? "provider text" : reason,
operationId: "forged-operation", callId: "forged-call", taskId: "foreign-task",
} } });
return rpc.call(method, params);
} } as unknown as PluginWorkerManager;
try {
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers });
const error = await runtime.task({ companyId, leaseId, operation: { kind: "submit", resumeFrom: "source-task", preparedLaunchId: "fixture",
projectIds: [], bootstrapTicket: "fixture-ticket", runner: { runnerId: "runner", leaseId: sourceLeaseId, runId, sessionId: "session",
turnId: "turn", itemId: "item", harness: "codex", protocolMin: 1, protocolMax: 3 } } }).catch(error => error);
expect(readEnvironmentTaskErrorDiagnostic(error)).toEqual(mode === "malformed_rpc" ? null : { httpStatus: 409, errorCode: "SESSION_STATE_UNAVAILABLE", reason });
expect(error.message).toBe(mode === "malformed_rpc"
? "Plugin-provided Runner execution operation unavailable; reconcile the same task before retrying" : "Plugin Runner source state unavailable");
expect(JSON.stringify(error)).not.toMatch(/synthetic-private|forged|foreign-task/);
expect(dispatched).toEqual(mode === "malformed_rpc" || mode === "extra_rpc_fields" ? [] : ["submit"]);
const [savedSource] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, sourceRunId));
expect(savedSource).toMatchObject({ status: "succeeded", resultJson: { outcome: "preserved" } });
const [current] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
expect(current.metadata).toMatchObject({ runnerTaskDispatched: true, runnerTaskExecutionStopped: false });
expect(JSON.stringify(current.metadata)).not.toMatch(/synthetic-private|forged|unresolved_operation|source_state_unverified/);
} finally { rpc.stop(); }
});
it("does not allocate agent-dependent setup leases before the agent is prepared", async () => {
const companyId = randomUUID(), pluginId = randomUUID(), environmentId = randomUUID(), agentId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Setup lease fixture", issuePrefix: "SETUP" });
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "@test/agent-owned-tasks", version: "1.0.0", status: "ready",
manifestJson: { agentLifecycle: true, capabilities: ["agents.lifecycle.manage", "environment.drivers.register"],
environmentDrivers: [{ driverKey: "tasks", supportsTasks: true, requiresAgent: true }] } as never });
await db.insert(environments).values({ id: environmentId, name: "Agent computer", driver: "plugin", status: "active",
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
const call = vi.fn();
const setup = agentEnvironmentTestService(db, { call } as never);
const input = { companyId, environmentId, adapterType: "paperclip_runner" };
expect((await setup.resolveAdapterTestExecutionContext(input)).fallbackChecks).toMatchObject([{ code: "environment_agent_required", level: "warn" }]);
await db.insert(agents).values({ id: agentId, companyId, name: "Draft", lifecycleState: "preparing", lifecycleHolds: ["setup"], status: "paused" });
expect((await setup.resolveAdapterTestExecutionContext({ ...input, agentId })).fallbackChecks).toMatchObject([{ code: "environment_agent_preparing", level: "warn" }]);
await db.update(companies).set({ requireBoardApprovalForNewAgents: true }).where(eq(companies.id, companyId));
await db.update(agents).set({ lifecycleState: "paused", pauseReason: "setup" }).where(eq(agents.id, agentId));
expect((await setup.resolveAdapterTestExecutionContext({ ...input, agentId })).fallbackChecks).toMatchObject([{ code: "environment_agent_approval_required", level: "warn" }]);
const foreign = randomUUID();
await db.insert(companies).values({ id: foreign, name: "Other company", issuePrefix: "OTHER" });
expect((await setup.resolveAdapterTestExecutionContext({ ...input, companyId: foreign, agentId })).fallbackChecks).toMatchObject([{ code: "environment_agent_preparing" }]);
expect(call).not.toHaveBeenCalled();
expect(await db.select().from(environmentLeases).where(eq(environmentLeases.environmentId, environmentId))).toEqual([]);
});
it("uses an acquired lease through SDK RPC, native dispatch, connection, stop proof and release", async () => {
const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID();
const environmentId = randomUUID(), pluginId = randomUUID(), projectId = randomUUID();
const manifest: PaperclipPluginManifestV1 = {
id: "test.runner-tasks", apiVersion: 1, version: "1.0.0", displayName: "Runner tasks",
description: "Synthetic task provider", author: "Paperclip", categories: ["automation"],
capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" },
environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" } }],
};
await db.insert(companies).values({ id: companyId, name: "Fixture", status: "active" });
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", role: "engineer", status: "active",
adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {} });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
await db.insert(projects).values({ id: projectId, companyId, name: "Fixture", status: "active" });
const [environment] = await db.insert(environments).values({ id: environmentId, name: "Fixture", driver: "plugin", status: "active",
config: { pluginKey: manifest.id, driverKey: "tasks", driverConfig: {} } }).returning();
await db.insert(plugins).values({ id: pluginId, pluginKey: manifest.id, packageName: "@test/runner-tasks",
version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: manifest, status: "ready" });
const calls: PluginEnvironmentTaskOperation[] = [];
let executionStopped = false, stopRequested = false, acquisition = 0, releaseFailure = false;
let projectAuthAllowed = true;
const released = vi.fn(), complete = vi.fn();
const plugin = definePlugin({
async setup() {},
async onEnvironmentAcquireLease() { return { providerLeaseId: `attempt:fixture.${++acquisition}`, metadata: {} }; },
async onEnvironmentReleaseLease() { if (releaseFailure) throw new Error("synthetic release failure"); released(); },
async onEnvironmentTask(params) {
calls.push(params.operation);
switch (params.operation.kind) {
case "describe": return { kind: "description", launchPreparationVersion: 1, projectAuthCheckVersion: 1, taskId: params.taskId,
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } };
case "prepare": return { kind: "prepared", taskId: params.taskId, launch: {
configurationId: `config-${params.taskId}`, requirementsDigest: runnerLaunchRequirementsDigest(params.operation.requirements as RunnerProviderLaunchRequirements),
provider: createRunnerProviderLaunch(params.operation.requirements as RunnerProviderLaunchRequirements, {
options: { codexArgs: [] }, root: "/runner", cwd: "/workspace", codexHome: "/account",
codexCommand: "/bin/codex", providerNodeCommand: "/bin/node", opencodeProxyPath: "", acpxSidecarPath: "",
}),
} };
case "check_project_auth": return { kind: "project_auth", taskId: params.taskId,
configurationId: params.operation.configurationId, provider: params.operation.provider, allowed: projectAuthAllowed };
case "read_state": return { kind: "state", taskId: params.taskId, state: {
schema: "paperclip.runner.durable.state.v1", ...params.operation.identity, lifecycle: "suspended",
} };
case "submit": stopRequested = false; executionStopped = false; return { kind: "accepted", taskId: params.taskId };
case "status": return { kind: "status", taskId: params.taskId, phase: stopRequested ? "cancelled" : "running", executionStopped, sessionStateAvailable: executionStopped };
case "connection": return { kind: "connection", taskId: params.taskId, endpoint: {
kind: "authenticated_websocket", websocketUrl: "wss://runner.example.test/connect", generation: "fixture",
secretHeaders: [{ name: "Authorization", value: "Bearer synthetic-fixture" }],
} };
case "stop": stopRequested = true; return { kind: "accepted", taskId: params.taskId };
case "complete": complete(); return { kind: "accepted", taskId: params.taskId };
default: throw new Error(`Unexpected runner task operation: ${params.operation.kind}`);
}
},
});
const harness = workerHarness(plugin, manifest);
const { supportedMethods } = await harness.initialize();
const workers = { isRunning: () => true, getWorker: () => ({ supportedMethods }),
call: (_id: string, method: string, params: unknown) => harness.call(method, params) } as unknown as PluginWorkerManager;
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers });
const setup = agentEnvironmentTestService(db, workers);
await setup.assertAdapterTestEnvironmentForCompany(companyId, environmentId, "paperclip_runner");
const probe = await setup.resolveAdapterTestExecutionContext({ companyId, agentId, adapterType: "paperclip_runner", environmentId });
expect(probe.fallbackChecks).toEqual([]);
expect(probe.executionTarget).toMatchObject({ runnerTaskProbe: { runId: expect.any(String), sessionId: expect.any(String) } });
if (probe.executionTarget?.kind !== "remote" || probe.executionTarget.transport !== "sandbox" || !probe.executionTarget.runnerTaskProbe) throw new Error("Probe task target unavailable");
await expect(probe.executionTarget.runnerTask!({ kind: "describe" }))
.resolves.toMatchObject({ kind: "description" });
const [probeLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, probe.executionTarget.leaseId!));
expect(probeLease.heartbeatRunId).toBeNull();
const selected = { identity: "core-selected-revision", sessionIdentity: "core-selected-account", home: "/core-owned-home" };
const verify = vi.spyOn(pluginProbe, "probePluginRunnerEnvironment").mockResolvedValue({ adapterType: "paperclip_runner", status: "pass", testedAt: new Date().toISOString(), checks: [] });
try {
await setup.testManagedEnvironment("paperclip_runner", { companyId, adapterType: "paperclip_runner", config: {}, executionTarget: probe.executionTarget },
{ provider: "openai" } as any, selected as any);
expect(verify).toHaveBeenCalledWith(expect.anything(), { identity: selected.identity, accountIdentity: selected.sessionIdentity, home: "/core-owned-home/provider" });
verify.mockClear();
// A preconfigured Runner home does not use the selected connection, so the probe refuses it.
const operationsBefore = calls.length;
await expect(setup.testManagedEnvironment("paperclip_runner", { companyId, adapterType: "paperclip_runner", config: { homeMode: "preconfigured" }, executionTarget: probe.executionTarget },
{ provider: "openai" } as any, selected as any)).resolves.toMatchObject({ status: "fail",
checks: [{ code: "ai_connection_required", message: expect.stringContaining("isolated Runner home") }] });
expect(verify).not.toHaveBeenCalled();
expect(calls).toHaveLength(operationsBefore);
} finally { verify.mockRestore(); }
await probe.executionTarget.runnerTask!({ kind: "check_project_auth", configurationId: "probe-config", provider: "openai", projectIds: [] });
await probe.executionTarget.runnerTask!({ kind: "submit", preparedLaunchId: "probe-config", bootstrapTicket: "synthetic-probe-ticket", projectIds: [],
runner: { protocolMin: 1, protocolMax: 3, harness: "codex", runnerId: "probe-runner", leaseId: probeLease.id,
runId: probe.executionTarget.runnerTaskProbe.runId, sessionId: probe.executionTarget.runnerTaskProbe.sessionId, turnId: "probe-turn", itemId: "probe-item" },
launch: { runnerVersion: "fixture", runnerDigest: `sha256:${"a".repeat(64)}`,
maxOutboxBytes: 262144, p0ReserveBytes: 65536, maxRuntimeMs: 0,
lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
});
const probeSubmit = calls.find((op): op is Extract<PluginEnvironmentTaskOperation, { kind: "submit" }> => op.kind === "submit")!;
expect(probeSubmit.launch?.maxRuntimeMs).toBeGreaterThan(0);
expect(probeSubmit.launch?.maxRuntimeMs).toBeLessThanOrEqual(120_000);
executionStopped = true;
await probe.release(); released.mockClear(); complete.mockClear(); calls.length = 0;
const orchestrator = environmentRunOrchestrator(db, { environmentRuntime: runtime });
const acquired = await orchestrator.acquireForRun({ companyId, selectedEnvironmentId: environmentId, localEnvironmentId: randomUUID(),
agentId, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null,
executionWorkspaceSettings: null, adapterType: "paperclip_runner" });
const realization = await orchestrator.realizeForRun({ companyId, environment: acquired.environment, lease: acquired.lease,
adapterType: "paperclip_runner", issueId: null, heartbeatRunId: runId,
executionWorkspace: { baseCwd: "/local/project", cwd: "/local/project", source: "project_primary", projectId,
workspaceId: null, repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null,
warnings: [], created: false, branchCreatedByRuntime: false },
effectiveExecutionWorkspaceMode: null, persistedExecutionWorkspace: null });
const target = realization.executionTarget;
expect(target).toMatchObject({ kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: acquired.lease.id });
if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask) throw new Error("Missing plugin target");
expect(target.runner).toBeUndefined();
const description = environmentTaskResultSchema.parse(await target.runnerTask({ kind: "describe" }));
expect(description).toMatchObject({ kind: "description", runner: { version: "fixture" } });
const onDispatched = vi.fn(), checkpoint = vi.fn(async () => {}), close = vi.fn(async () => {});
ingress.connect.mockReturnValue({ ready: Promise.resolve(), close });
const owner = createPluginRunnerTaskOwner({ taskId: description.taskId, task: target.runnerTask, projectAuthCheckVersion: 1,
credentialIdentity: "revision", accountIdentity: "account", credential: { type: "codex-auth-json", authJson: '{"OPENAI_API_KEY":"synthetic-account-key"}' }, projectIds: target.runnerTaskProjectIds ?? [], harness: "codex",
onSessionRetained: checkpoint, onDispatched, pollMs: 1 });
const identity = { runnerInstanceId: randomUUID(), environmentLeaseId: acquired.lease.id, runId,
normalizedSessionId: randomUUID(), turnId: "turn", itemId: "item" };
const requirements: RunnerProviderLaunchRequirements = { schema: "paperclip.runner.launch-requirements.v1", identity,
provider: "codex", model: "fixture", instructions: "Core policy", runtimeContext: null, settings: {},
approvalPolicy: "never", readOnly: false, preparedConversation: false, includeCollaborationModeInstructions: false };
const prepared = await owner.driver.prepareLaunch!(requirements);
expect(calls).toContainEqual(expect.objectContaining({kind:"prepare",accountIdentity:"account"}));
// Re-preparation invalidates the old credential validation before the worker reply.
await runtime.task({ companyId, leaseId: acquired.lease.id, operation: environmentTaskOperationSchema.parse({
kind: "prepare", requirements, projectIds: [projectId],
}) });
await expect(runtime.task({ companyId, leaseId: acquired.lease.id, operation: {
kind: "submit", projectIds: [projectId], preparedLaunchId: prepared.configurationId, bootstrapTicket: "synthetic-ticket",
accountIdentity: "account", credential: { type: "codex-auth-json", authJson: '{"OPENAI_API_KEY":"never-transfer-stale-proof"}' },
runner: { protocolMin: 1, protocolMax: 3, harness: "codex", runnerId: identity.runnerInstanceId,
leaseId: identity.environmentLeaseId, runId: identity.runId, sessionId: identity.normalizedSessionId,
turnId: identity.turnId, itemId: identity.itemId },
} })).rejects.toThrow("validation unavailable");
expect(calls.some(op => op.kind === "submit")).toBe(false);
await owner.driver.prepareLaunch!(requirements);
const registration = await owner.registration({} as never, runId);
const handle = owner.launcher({ command: "ignored", args: [], cwd: "/ignored", environment: { OPENAI_API_KEY: "synthetic-launch-key" },
nativeRunner: { identity, ticket: "synthetic-bootstrap-ticket",
connection: registration.connection, stateDirectory: "/runner/state", runnerVersion: "fixture", runnerDigest: `sha256:${"a".repeat(64)}`,
maxOutboxBytes: 262144, p0ReserveBytes: 65536, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } } });
let settled = false;
void handle.completion.then(() => { settled = true; });
try {
registration.activate(); await registration.ready();
expect(onDispatched).toHaveBeenCalledOnce();
expect(ingress.connect).toHaveBeenCalledWith(expect.objectContaining({ endpoint: expect.objectContaining({ websocketUrl: "wss://runner.example.test/connect" }) }));
const submit = calls.find(operation => operation.kind === "submit");
expect(submit).toMatchObject({ kind: "submit", projectIds: [projectId], runner: { leaseId: acquired.lease.id, runId },
launch: { runnerVersion: "fixture" } });
const [saved] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, acquired.lease.id));
expect(saved.metadata?.runnerTaskDispatched).toBe(true);
expect(JSON.stringify(saved.metadata)).not.toMatch(/synthetic-account-key|synthetic-launch-key|synthetic-bootstrap-ticket|synthetic-fixture/);
const acquisitionBefore = acquisition;
const callsBefore = calls.length;
const recovered = await environmentRunOrchestrator(db, { environmentRuntime: environmentRuntimeService(db, { pluginWorkerManager: workers }) }).acquireForRun({
companyId, selectedEnvironmentId: environmentId, localEnvironmentId: randomUUID(), agentId,
issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null,
executionWorkspaceSettings: null, adapterType: "paperclip_runner",
reattachRemoteLease: { kind: "plugin_task", leaseId: saved.id },
});
expect(recovered.lease.id).toBe(saved.id);
const adopted = createPluginRunnerTaskOwner({ taskId: description.taskId, task: target.runnerTask, projectIds: [projectId], harness: "codex",
projectAuthCheckVersion: 1, adoptIdentity: identity });
await adopted.driver.authorizeRunnerAdoption!(prepared.configurationId, identity);
const adoptedRegistration = await adopted.registration({} as never, runId);
adoptedRegistration.activate(); await adoptedRegistration.ready();
await adoptedRegistration.release("detach");
expect(acquisition).toBe(acquisitionBefore);
expect(calls.slice(callsBefore).some(op => ["prepare", "submit", "stop", "complete"].includes(op.kind))).toBe(false);
expect(executionStopped).toBe(false);
projectAuthAllowed = false;
const revoked = createPluginRunnerTaskOwner({ taskId: description.taskId, task: target.runnerTask, projectIds: [projectId], harness: "codex",
projectAuthCheckVersion: 1, adoptIdentity: identity });
const connectionsBeforeRevocation = calls.filter(op => op.kind === "connection").length;
await expect(revoked.driver.authorizeRunnerAdoption!(prepared.configurationId, identity)).rejects.toThrow("project authentication conflicts");
expect(calls.filter(op => op.kind === "connection")).toHaveLength(connectionsBeforeRevocation);
expect(calls.slice(callsBefore).filter(op => op.kind === "check_project_auth")).toEqual(Array(2).fill({
kind: "check_project_auth", configurationId: prepared.configurationId, provider: "openai", projectIds: [projectId],
}));
projectAuthAllowed = true;
close.mockClear();
await expect(runtime.task({ companyId: randomUUID(), leaseId: acquired.lease.id, operation: { kind: "stop" } })).rejects.toThrow("lease unavailable");
const before = calls.filter(value => value.kind === "submit").length;
await expect(runtime.task({ companyId, leaseId: acquired.lease.id, operation: { ...submit!, projectIds: [randomUUID()] } as never })).rejects.toThrow("project unavailable");
expect(calls.filter(value => value.kind === "submit")).toHaveLength(before);
handle.child.kill("SIGTERM");
await vi.waitFor(() => expect(stopRequested).toBe(true));
await delay(10); expect(settled).toBe(false);
executionStopped = true; await handle.completion;
await registration.release(); expect(complete).not.toHaveBeenCalled();
await registration.checkpoint("settled");
expect(checkpoint).toHaveBeenCalledOnce(); expect(complete).toHaveBeenCalledOnce();
await runtime.releaseRunLeases(runId, "released");
expect(released).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce();
expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true);
// A controller that dies after dispatch leaves only the lease marker.
const second = await runtime.acquireRunLease({ companyId, environment: environment as never, agentId,
issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, adapterType: "paperclip_runner" });
if (submit?.kind !== "submit") throw new Error("Missing submit fixture");
await runtime.task({ companyId, leaseId: second.lease.id, operation: { kind: "check_project_auth", configurationId: "second-config", provider: "openai", projectIds: [projectId] } });
await runtime.task({ companyId, leaseId: second.lease.id,
operation: { ...submit, credential: undefined, accountIdentity: undefined, credentialIdentity: undefined,
preparedLaunchId: "second-config", runner: { ...submit.runner, leaseId: second.lease.id } } });
await runtime.task({ companyId, leaseId: second.lease.id, operation: { kind: "complete" } });
const errors = vi.fn();
await runtime.releaseRunLeases(runId, "released", errors);
expect(errors).toHaveBeenCalledOnce();
expect(released).toHaveBeenCalledOnce(); // No release while termination is unknown.
const [pending] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, second.lease.id));
expect(pending).toMatchObject({ status: "pending_cleanup", metadata: { runnerTaskDispatched: true } });
await db.delete(environments).where(eq(environments.id, environmentId));
executionStopped = true;
releaseFailure = true;
const recovery = heartbeatService(db, { environmentRuntime: runtime, pluginWorkerManager: workers });
expect((await recovery.sweepPendingCleanupLeases()).destroyed).toBe(0);
const [retryable] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, second.lease.id));
expect(retryable.status).toBe("pending_cleanup");
releaseFailure = false;
await db.update(environmentLeases).set({ metadata: { ...retryable.metadata, pendingCleanupRetryAfterMs: 0 } }).where(eq(environmentLeases.id, second.lease.id));
const sweep = await recovery.sweepPendingCleanupLeases();
expect(released).toHaveBeenCalledTimes(2);
expect(sweep.destroyed).toBe(1);
const [cleaned] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, second.lease.id));
expect(cleaned).toMatchObject({ status: "expired", cleanupStatus: "success", environmentId: null,
metadata: { runnerTaskDispatched: false, remoteExecutionTermination: { state: "stopped" } } });
} finally { executionStopped = true; handle.child.kill(); await handle.completion.catch(() => {}); harness.stop(); }
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
it.each([false, true])("replaces a same-run task only after the provider confirms termination (%s)", async stopped => {
const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), pluginId = randomUUID(), environmentId = randomUUID(), oldLeaseId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Task recovery", issuePrefix: `R${companyId.slice(0, 7)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
await db.insert(environments).values({ id: environmentId, name: `Recovery ${environmentId}`, driver: "plugin", status: "active", config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-recovery", version: "1.0.0", status: "ready", manifestJson: {
capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }],
} as never });
const priorIdentity = { runnerInstanceId: "runner", environmentLeaseId: oldLeaseId, runId, normalizedSessionId: "session" };
const runner = { runnerId: "runner", leaseId: oldLeaseId, runId, sessionId: "session", turnId: "turn", itemId: "item", harness: "codex", protocolMin: 1, protocolMax: 3 };
await db.insert(environmentLeases).values({ id: oldLeaseId, companyId, environmentId, heartbeatRunId: runId, status: "active", provider: `plugin:${pluginId}:tasks`, providerLeaseId: "old-task",
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", agentId, runnerTaskLease: true, runnerTaskDispatched: true, runnerTaskIdentity: runner } });
const operations: string[] = [];
const workers = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentTask", "environmentReleaseLease", "environmentAcquireLease"] }),
call: async (_id: string, method: string, params: any) => {
operations.push(method === "environmentTask" ? params.operation.kind : method);
if (method === "environmentAcquireLease") {
expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true);
return { providerLeaseId: "new-task", metadata: {} };
}
if (method === "environmentReleaseLease") return;
if (params.operation.kind === "check_project_auth") return { kind: "project_auth", taskId: params.taskId, configurationId: params.operation.configurationId, provider: params.operation.provider, allowed: true };
if (params.operation.kind === "status") return { kind: "status", taskId: "old-task", phase: "cancelled", executionStopped: stopped, sessionStateAvailable: stopped };
return { kind: "accepted", taskId: params.taskId };
} } as unknown as PluginWorkerManager;
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers });
const acquire = environmentRunOrchestrator(db, { environmentRuntime: runtime }).acquireForRun({
companyId, selectedEnvironmentId: environmentId, localEnvironmentId: randomUUID(), adapterType: "paperclip_runner", agentId,
issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, executionWorkspaceSettings: null, recoverPluginTask: true,
});
if (!stopped) {
await expect(acquire).rejects.toThrow("termination_unconfirmed");
expect(operations).not.toContain("environmentAcquireLease");
expect(operations).not.toContain("complete");
return;
}
const next = await acquire;
expect(operations).toEqual(["stop", "status", "complete", "environmentReleaseLease", "environmentAcquireLease"]);
expect(next.lease.id).not.toBe(oldLeaseId);
await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).resolves.toBe("old-task");
await expect(runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "submit", projectIds: [], bootstrapTicket: "fixture", runner } })).rejects.toThrow("identity mismatch");
await runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "check_project_auth", configurationId: "recovery-config", provider: "openai", projectIds: [] } });
await expect(runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "submit", preparedLaunchId: "recovery-config", projectIds: [], bootstrapTicket: "fixture", runner, resumeFrom: "old-task" } })).resolves.toMatchObject({ kind: "accepted" });
const competingId = randomUUID();
await db.insert(environmentLeases).values({ id: competingId, companyId, environmentId, heartbeatRunId: runId, status: "active", provider: `plugin:${pluginId}:tasks`, providerLeaseId: "competing-task" });
await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).rejects.toThrow("not authorized");
await db.delete(environmentLeases).where(eq(environmentLeases.id, competingId));
await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, oldLeaseId));
await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).rejects.toThrow("not authorized");
});
});