Preserve plugin session continuity through warm execution and checkpoint recovery (#17820)

This commit is contained in:
Nicky Leach authored and GitHub committed 2026-10-26 20:25:10 -07:00
1 parent dcfc2acfa8
commit ea727dc3c2
23 files changed
+433 -82

No files matched your search

@@ -764,6 +764,20 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
describe("native runner lifecycle changes before lease acquisition", () => {
it.each([undefined, "account:generation"])("uses a warm plugin lease only with a managed credential identity: %s", async managedAiCredentialIdentity => {
const environment = { ...makeEnvironment("plugin"), config: { pluginKey: "fixture", driverKey: "tasks", driverConfig: {} } };
mockGetEnvironment.mockResolvedValue(environment);
const acquireRunLease = vi.fn(async input => ({ environment: input.environment,
lease: makeLease({ leasePolicy: input.environment.config.runnerLifecycleMode === "warm" ? "reuse_by_environment" : "ephemeral" }), leaseContext: {} }));
const orchestrator = environmentRunOrchestrator({} as never, { environmentRuntime: makeMockRuntime({ acquireRunLease }) });
const result = await orchestrator.acquireForRun({ companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local",
adapterType: "paperclip_runner", adapterConfig: { lifecycleMode: "warm" }, managedAiCredentialIdentity,
issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1", persistedExecutionWorkspace: null, executionWorkspaceSettings: null });
expect(result.environment.config.runnerLifecycleMode).toBe(managedAiCredentialIdentity ? "warm" : "per_turn");
expect(result.lease.leasePolicy).toBe(managedAiCredentialIdentity ? "reuse_by_environment" : "ephemeral");
expect(environment.config).not.toHaveProperty("runnerLifecycleMode");
});
it("requests a reusable lease when an existing task switches from per-turn to warm", async () => {
const environment = { ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } };
mockGetEnvironment.mockResolvedValue(environment);
@@ -824,7 +824,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
const cold = await runtime.acquireRunLease({ ...input, environment, heartbeatRunId: seeded.runId });
expect(cold.lease.leasePolicy).toBe("ephemeral");
await runtime.releaseRunLeases(seeded.runId, "released");
const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" });
const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" }, undefined, "selected-account:generation");
const warmRunId = randomUUID();
await db.insert(heartbeatRuns).values({ id: warmRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
const warm = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: warmRunId });
@@ -59,16 +59,41 @@ databaseSuite("environment task lease authority", () => {
await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, oldLeaseId));
await expect(authorize()).rejects.toThrow("not authorized");
await db.update(environmentLeases).set({ cleanupStatus: "success" }).where(eq(environmentLeases.id, oldLeaseId));
let sessionStateAvailable = false;
const operations: string[] = [];
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask"] }),
call: async (_id: string, _method: string, params: any) => ["read_state", "archive_state"].includes(params.operation.kind)
? { kind: "state", taskId: params.taskId, state: { schema: "paperclip.runner.durable.state.v1", lifecycle: "suspended",
...(params.operation.identity ?? params.operation.priorIdentity) } }
: params.operation.kind === "status"
? (operations.push("status"), { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true, sessionStateAvailable })
: { kind: "accepted", taskId: params.taskId },
} as unknown as PluginWorkerManager;
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers });
const runner = { runnerId: "runner", leaseId: oldLeaseId, sessionId: "session", runId: nextRunId,
turnId: "turn", itemId: "item", harness: "codex", protocolMin: 1, protocolMax: 3 };
const checkpointIdentity = { ...priorIdentity, turnId: "prior-turn", itemId: "prior-item" };
const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, oldLeaseId));
await db.update(environmentLeases).set({ metadata: { ...source.metadata, runnerTaskSessionRetained: false } }).where(eq(environmentLeases.id, oldLeaseId));
await expect(authorize()).rejects.toThrow("not authorized");
await expect(resolvePluginTaskResume(db, { companyId, leaseId: nextLeaseId, priorIdentity, inspectCheckpoint: true })).resolves.toBe("old-task");
const inspect = () => runtime.task({ companyId, leaseId: nextLeaseId, operation: {
kind: "read_state", identity: checkpointIdentity, resumeFrom: "old-task",
} });
await expect(inspect()).rejects.toThrow("not authorized");
await expect(authorize()).rejects.toThrow("not authorized");
// Discovery cannot launch, and cannot bypass the original termination receipt.
await expect(runtime.task({ companyId, leaseId: nextLeaseId, operation: {
kind: "submit", preparedLaunchId: "fixture", projectIds: [], bootstrapTicket: "fixture", runner, resumeFrom: "old-task",
} })).rejects.toThrow("resume identity mismatch");
await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, oldLeaseId));
await expect(inspect()).rejects.toThrow("not authorized");
expect(operations).toEqual(["status"]);
await db.update(environmentLeases).set({ cleanupStatus: "success" }).where(eq(environmentLeases.id, oldLeaseId));
sessionStateAvailable = true;
await expect(inspect()).resolves.toMatchObject({ kind: "state" });
expect(operations).toEqual(["status", "status"]);
await expect(authorize()).resolves.toBe("old-task");
for (const operation of [
{ kind: "read_state" as const, identity: checkpointIdentity, resumeFrom: "old-task" },
{ kind: "archive_state" as const, priorIdentity: checkpointIdentity, archiveKey: "a".repeat(24), resumeFrom: "old-task" },
@@ -93,7 +93,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => {
await stopDb?.();
});
it("dispatches a plugin-only project without Core workspace files through the native heartbeat boundary", async () => {
it("dispatches a plugin-owned account per-turn without Core workspace files through the native heartbeat boundary", async () => {
const companyId = randomUUID(), projectId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sourceId = randomUUID();
const environmentId = randomUUID(), pluginId = randomUUID(), pluginKey = `test.tasks.${pluginId}`;
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-task-heartbeat-")); tempRoots.push(root);
@@ -112,11 +112,11 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => {
capabilities: ["environment.drivers.register", "workspaces.manage", "events.subscribe"], resourceLifecycle: ["project"],
workspaceProvider: { modes: ["shared_workspace", "isolated_workspace", "operator_branch"], strategies: ["project_primary", "git_worktree"] },
entrypoints: { worker: "worker.js" },
environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" } }] } });
environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, warmTaskVersion: 1, configSchema: { type: "object" } }] } });
await db.insert(environments).values({ id: environmentId, companyId, name: "Tasks", driver: "plugin", status: "active",
config: { pluginKey, driverKey: "tasks", driverConfig: {} } });
await db.insert(agents).values({ id: agentId, companyId, name: "Task agent", role: "engineer", status: "idle",
adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol" }, runtimeConfig: {},
adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol", lifecycleMode: "warm" }, runtimeConfig: {},
defaultEnvironmentId: environmentId, permissions: {} });
await db.insert(issues).values({ id: issueId, companyId, projectId, projectWorkspaceId: sourceId, title: "Task-only dispatch", status: "in_progress",
assigneeAgentId: agentId, issueNumber: 1, identifier: "TASK-1" });
@@ -129,14 +129,15 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => {
if (method === "environmentReleaseLease" || method === "environmentDestroyLease") return undefined;
throw new Error(`Unexpected task provider call: ${method}`);
});
const workerManager = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["workspaceOperation", "environmentTask", "environmentAcquireLease", "environmentReleaseLease", "environmentDestroyLease"] }), call } as unknown as PluginWorkerManager;
const workerManager = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["workspaceOperation", "environmentTask", "environmentAcquireLease", "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), call } as unknown as PluginWorkerManager;
const entered = vi.fn(() => { throw new Error("synthetic_native_boundary_reached"); });
const heartbeat = heartbeatService(db, { pluginWorkerManager: workerManager, nativeSessionBackendFactory: entered });
heartbeats.push(heartbeat);
const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", contextSnapshot: { issueId, projectId } });
expect(run).not.toBeNull();
await vi.waitFor(async () => { expect((await heartbeat.getRun(run!.id))?.status).toBe("failed"); }, { timeout: 10_000 });
expect(entered).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({ runId: run!.id, companyId, issueId }) }));
expect(entered).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({ runId: run!.id, companyId, issueId }),
session: expect.objectContaining({ lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }) }));
expect((await heartbeat.getRun(run!.id))?.error).toContain("synthetic_native_boundary_reached");
await heartbeat.drainActiveRunExecutions();
expect(call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: "heartbeat-task-attempt" }));
@@ -48,6 +48,24 @@ const support = await getEmbeddedPostgresTestSupport();
async function hire(status = "idle") { return createAgentLifecycle(db).requestHire(companyId, { name: "Agent", status, adapterType: "process" }); }
async function current(id: string) { return (await db.select().from(agents).where(eq(agents.id, id)))[0]!; }
it.each((["codex", "claude"] as const).flatMap(provider => ["direct", "setup", "approval"].map(path => [provider, path] as const)))("defaults a newly hired %s agent on a warm task environment to warm execution through %s", async (provider, path) => {
const pluginId = randomUUID(), environmentId = randomUUID();
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "warm-hire-fixture", version: "1.0.0", status: "ready",
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true, warmTaskVersion: 1 }] } as never });
await db.insert(environments).values({ id: environmentId, name: `Warm computer ${environmentId}`, driver: "plugin",
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
const adapterConfig = provider === "claude" ? { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6", lifecycleMode: "per_turn" }
: { provider: "codex", model: "gpt-5.6-luna", lifecycleMode: "per_turn" };
const lifecycle = createAgentLifecycle(db);
const input = { name: "Warm hire", adapterType: "paperclip_runner", adapterConfig, defaultEnvironmentId: environmentId };
const draft = path === "setup" ? await lifecycle.requestHire(companyId, input, { setupOnly: true }) : null;
const hired = await lifecycle.requestHire(companyId, { ...input, ...(path === "approval" ? { status: "pending_approval" } : {}) }, draft ? { setupAgentId: draft.id } : undefined);
if (path === "approval") await lifecycle.approveHire(hired.id, "board");
expect((await current(hired.id)).adapterConfig).toMatchObject({ ...adapterConfig, lifecycleMode: "warm" });
await createAgentLifecycle(db).updateConfiguration(hired.id, { adapterConfig: { ...(await current(hired.id)).adapterConfig, lifecycleMode: "per_turn" } });
expect((await current(hired.id)).adapterConfig.lifecycleMode).toBe("per_turn");
});
it("rejects an outdated configuration snapshot under the write lock", async () => {
const agent = await hire();
const snapshot = await current(agent.id);
+5 -1
View File
@@ -12,7 +12,7 @@ import { secretService, assertClaudeOAuthBindingInvariant } from "./secrets.js";
import { agentCredentialService } from "./agent-credentials.js";
import { agentIdentityService } from "./agent-identity.js";
import { initializePrimaryAgent } from "./primary-agent.js";
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData> {
const { runner, ...data } = input;
@@ -44,6 +44,10 @@ export async function prepareAgentHire(db: Db, companyId: string, input: CreateA
? await secretService(db).normalizeAdapterConfigForPersistence(companyId, data.adapterConfig, { adapterType })
: {};
const adapterConfig = normalizePaperclipRunnerAdapterConfig(adapterType, rawAdapterConfig);
if (adapterType === "paperclip_runner" && !["openai_dot", "grok_bot", "muse"].includes(String(adapterConfig.provider)) &&
(await resolveAgentRunnerTaskDriverForCompany(db, companyId, data.defaultEnvironmentId))?.warmTaskVersion === 1) {
adapterConfig.lifecycleMode = "warm";
}
await agentCredentialService(db).enforceClaudeOAuthBindingClaim(db, {
companyId, consume: true, environmentId: data.defaultEnvironmentId ?? null,
childAdapterConfig: adapterConfig, claudeLogin: options?.claudeLogin,
@@ -12,7 +12,7 @@ vi.mock("./environment-config.js", async () => ({
}));
vi.mock("./plugin-registry.js", () => ({ pluginRegistryService: () => ({ getByKey: state.pluginByKey }) }));
vi.mock("@paperclipai/adapter-utils/ssh", () => ({ runSshCommand: state.ssh }));
import { AGENT_RUNNER_SSH_PLATFORM_COMMAND, agentRunnerAvailability, resolveAgentRunnerTargetForCompany, resolveNewAgentRunner, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
import { AGENT_RUNNER_SSH_PLATFORM_COMMAND, agentRunnerAvailability, resolveAgentRunnerTaskDriverForCompany, resolveAgentRunnerTargetForCompany, resolveNewAgentRunner, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
import { buildCodexAcpConfig, resolveCodexExecutionEngine } from "@paperclipai/adapter-codex-local/server";
import { secretService } from "./secrets.js";
import * as localPlatform from "../lib/agent-runner-platform.js";
@@ -522,6 +522,24 @@ describe("Runner-only plugin environments", () => {
expect(agentRunnerAvailability("gemini_local", target)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
expect(state.ssh).not.toHaveBeenCalled();
});
it("carries warm task capability from the selected driver without provider gates", async () => {
state.pluginByKey.mockResolvedValue(plugin({ supportsTasks: true, warmTaskVersion: 1 }));
expect(await resolveAgentRunnerTargetForCompany({} as never, "company", "computer"))
.toEqual({ driver: "plugin", runnerTasks: true, warmTaskVersion: 1 });
expect(await resolveAgentRunnerTaskDriverForCompany({} as never, "company", "computer"))
.toMatchObject({ driverKey: "tasks", supportsTasks: true, warmTaskVersion: 1 });
});
it.each(["ssh", "sandbox"])("does not qualify a %s runner while inspecting hire defaults", async driver => {
state.getEnvironment.mockResolvedValue({ ...pluginEnvironment, driver, config: { provider: "fake" } });
expect(await resolveAgentRunnerTaskDriverForCompany({} as never, "company", "computer")).toBeUndefined();
expect(state.ssh).not.toHaveBeenCalled();
expect(state.sandboxQualification).not.toHaveBeenCalled();
expect(state.pluginByKey).not.toHaveBeenCalled();
if (driver === "sandbox") await expect(resolveAgentRunnerTargetForCompany({} as never, "company", "computer"))
.rejects.toThrow("The fake sandbox provider cannot execute runs.");
});
it("saves a Claude agent on the plugin environment as Paperclip Runner", async () => {
const saved = await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "claude_local", defaultEnvironmentId: "computer" });
expect(saved).toMatchObject({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "claude" } });
+22 -7
View File
@@ -15,6 +15,7 @@ export type RunnerTarget = {
driver: string; platform?: string; architecture?: string; libc?: "glibc" | "musl" | "unknown"; glibcVersion?: string;
/** A plugin environment whose driver declares Runner task execution. It runs agents only through Paperclip Runner. */
runnerTasks?: boolean;
warmTaskVersion?: 1;
};
export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi";
export interface AgentRunnerSelection {
@@ -293,7 +294,10 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId);
let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget()
: { driver };
if (driver === "plugin" && environment && await pluginEnvironmentRunsRunnerTasks(db, environment)) target = { driver, runnerTasks: true };
if (driver === "plugin" && environment) {
const taskDriver = await pluginEnvironmentRunnerTaskDriver(db, environment);
if (taskDriver) target = { driver, runnerTasks: true, ...(taskDriver.warmTaskVersion === 1 ? { warmTaskVersion: 1 as const } : {}) };
}
if (driver === "sandbox" && environment) {
const { readSandboxPlatformQualification } = await import("./environment-probe.js");
const qualified = await readSandboxPlatformQualification(db, environment, companyId);
@@ -312,6 +316,18 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
}
export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) {
const resolved = await resolveAgentEnvironmentForCompany(db, companyId, defaultEnvironmentId);
if (resolved.environment?.driver === "sandbox" && resolved.environment.config?.provider === "fake") throw unprocessable("The fake sandbox provider cannot execute runs.");
return resolved;
}
// Hiring defaults inspect capabilities without probing or qualifying an already selected runner.
export async function resolveAgentRunnerTaskDriverForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) {
const { environment, driver } = await resolveAgentEnvironmentForCompany(db, companyId, defaultEnvironmentId);
return driver === "plugin" && environment ? pluginEnvironmentRunnerTaskDriver(db, environment) : undefined;
}
async function resolveAgentEnvironmentForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) {
const { instanceSettingsService } = await import("./instance-settings.js");
const { environmentService } = await import("./environments.js");
const settings = await instanceSettingsService(db).get();
@@ -331,7 +347,6 @@ export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId:
const boundCompanyIds = await envs.listBoundCompanyIds(environment.id);
if (boundCompanyIds.length > 0 && !boundCompanyIds.includes(companyId)) throw forbidden("The selected environment belongs to another company.", { code: "environment_company_mismatch" });
if (environment.status === "archived") throw unprocessable("Environment is archived.");
if (environment.driver === "sandbox" && environment.config?.provider === "fake") throw unprocessable("The fake sandbox provider cannot execute runs.");
}
return { environment, driver: environment?.driver ?? "local" };
}
@@ -339,17 +354,17 @@ export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId:
// The plugin's manifest declaration decides whether its environment runs
// Runner tasks. Run admission separately verifies a ready plugin and a live
// worker, so a plugin that is down still selects the Runner and fails its run.
async function pluginEnvironmentRunsRunnerTasks(db: Db, environment: Parameters<typeof parseEnvironmentDriverConfig>[0]): Promise<boolean> {
async function pluginEnvironmentRunnerTaskDriver(db: Db, environment: Parameters<typeof parseEnvironmentDriverConfig>[0]) {
let config: { pluginKey: string; driverKey: string };
try {
const parsed = parseEnvironmentDriverConfig(environment);
if (parsed.driver !== "plugin") return false;
if (parsed.driver !== "plugin") return undefined;
config = parsed.config;
} catch {
return false;
return undefined;
}
const { pluginRegistryService } = await import("./plugin-registry.js");
const plugin = await pluginRegistryService(db).getByKey(config.pluginKey);
return Boolean(plugin?.manifestJson.capabilities.includes("environment.drivers.register")
&& plugin.manifestJson.environmentDrivers?.some(driver => driver.driverKey === config.driverKey && driver.supportsTasks === true));
return plugin?.manifestJson.capabilities.includes("environment.drivers.register")
? plugin.manifestJson.environmentDrivers?.find(driver => driver.driverKey === config.driverKey && driver.supportsTasks === true) : undefined;
}
@@ -273,6 +273,7 @@ export function environmentRunOrchestrator(
adapterType: string;
adapterConfig?: Record<string, unknown>;
admittedLifecycleMode?: "warm" | "per_turn";
managedAiCredentialIdentity?: string;
issueId: string | null;
heartbeatRunId: string;
agentId: string;
@@ -292,7 +293,7 @@ export function environmentRunOrchestrator(
});
const environment = resolveRunnerEnvironmentForRun(
selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode,
selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, input.managedAiCredentialIdentity,
);
if (input.recoverPluginTask && environment.driver === "plugin" && input.adapterType === "paperclip_runner") {
@@ -85,10 +85,11 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
throw new Error("Plugin-provided Runner execution identity mismatch");
}
if (runner && resumeFrom) {
const sources = await db.select({ runId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }).from(environmentLeases).where(and(
const inspectCheckpoint = operation.kind === "read_state";
const sources = await db.select({ id: environmentLeases.id, runId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.provider, lease.provider!),
eq(environmentLeases.providerLeaseId, resumeFrom),
sql`${environmentLeases.metadata}->>'runnerTaskSessionRetained' = 'true'`,
inspectCheckpoint ? undefined : sql`${environmentLeases.metadata}->>'runnerTaskSessionRetained' = 'true'`,
checkpointIdentity ? eq(environmentLeases.heartbeatRunId, checkpointIdentity.runId) : undefined,
)).limit(2);
const priorRunId = sources.length === 1 ? sources[0]!.runId : null;
@@ -98,7 +99,20 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
companyId: input.companyId, leaseId: lease.id,
priorIdentity: { runnerInstanceId: runner.runnerId, environmentLeaseId: runner.leaseId,
runId: priorRunId, normalizedSessionId: runner.sessionId },
inspectCheckpoint,
})) throw new Error("Plugin Runner resume identity mismatch");
// Older providers misclassified suspended checkpoints. A fresh provider
// attestation may repair availability, but never supplies stop/owner authority.
if (inspectCheckpoint && sources[0]!.metadata?.runnerTaskSessionRetained !== true) {
const status = await executeEnvironmentTask(db, workers, { companyId: input.companyId,
leaseId: sources[0]!.id, operation: { kind: "status" } });
if (status.kind !== "status" || status.executionStopped !== true || status.sessionStateAvailable !== true ||
resumeFrom !== await resolvePluginTaskResume(db, { companyId: input.companyId, leaseId: lease.id,
priorIdentity: { runnerInstanceId: runner.runnerId, environmentLeaseId: runner.leaseId,
runId: priorRunId, normalizedSessionId: runner.sessionId } })) {
throw new Error("Plugin Runner retained session is not authorized");
}
}
}
if (metadata.driver !== "plugin" || typeof metadata.pluginId !== "string" || typeof metadata.driverKey !== "string") {
throw new Error("Plugin-provided Runner execution lease has no pinned plugin driver");
@@ -380,6 +394,8 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor
export async function resolvePluginTaskResume(db: Db, input: {
companyId: string; leaseId: string;
priorIdentity: { runnerInstanceId: string; environmentLeaseId: string; runId: string; normalizedSessionId: string };
/** Read-only discovery; launching still requires an attested retained checkpoint. */
inspectCheckpoint?: boolean;
}): Promise<string> {
const rows = await db.select({ lease: environmentLeases, run: heartbeatRuns }).from(environmentLeases)
.innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId), eq(heartbeatRuns.companyId, input.companyId)))
@@ -389,7 +405,8 @@ export async function resolvePluginTaskResume(db: Db, input: {
// A retained PRP lease is logical; later provider tasks have different physical lease IDs.
const candidates = rows.filter(row => {
const identity = row.lease.metadata?.runnerTaskIdentity as Record<string, unknown> | undefined;
return row.lease.id !== input.leaseId && row.run.id === input.priorIdentity.runId && row.lease.metadata?.runnerTaskSessionRetained === true &&
return row.lease.id !== input.leaseId && row.run.id === input.priorIdentity.runId &&
(input.inspectCheckpoint || row.lease.metadata?.runnerTaskSessionRetained === true) &&
identity?.runnerId === input.priorIdentity.runnerInstanceId && identity.leaseId === input.priorIdentity.environmentLeaseId &&
identity.sessionId === input.priorIdentity.normalizedSessionId && identity.runId === input.priorIdentity.runId &&
!rows.some(successor => successor.lease.id !== input.leaseId &&
+1
View File
@@ -2713,6 +2713,7 @@ export function heartbeatService(
mergedConfig,
configSnapshot,
secretManifest,
managedAiCredentialIdentity: managedAiRuntime?.identity,
},
reuse: {
requestedShouldReuseExisting,
@@ -311,6 +311,26 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () =
expect(await coordinators(run.id)).toHaveLength(1);
});
it.each(["api_key", "subscription"] as const)("keeps managed Claude %s task execution warm", async method => {
const { input, run, issue } = await fixture();
const adapterConfig = { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6", lifecycleMode: "warm" };
input.agent = { ...input.agent, adapterConfig };
await db.update(agents).set({ adapterConfig }).where(eq(agents.id, input.agent.id));
input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: input.run, enabled: true, runtimeConfig: {}, adapterConfig,
agent: input.agent, issue, target: null, workspaceId: null });
input.workspace.executionTarget = { ...sandbox, runnerTask: vi.fn(), reusableLeaseConfigured: true,
effectiveCapabilities: { reusableLeases: true } as never, runnerLifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 } };
const attribution = { connectionId: randomUUID(), grantId: randomUUID(), provider: "anthropic" as const, method,
mode: "responsible_user" as const, responsibleUserId: "task-owner" };
input.config.managedAiRuntime = { sessionIdentity: "account:epoch-1", identity: "account:generation-1",
config: { env: {}, managedAiConnection: { ...attribution, identity: "account:generation-1", sessionIdentity: "account:epoch-1" } },
attribution, accountName: "Saved account", accountOwnerUserId: "task-owner", home: instructionRoot, cleanup: async () => {} };
const selected = await selectHeartbeatRuntime(db, input);
if (!selected.selected || !selected.nativeExecution) throw new Error("Expected native Claude selection");
expect(selected.nativeExecution.session.lifecyclePolicy).toEqual({ mode: "warm", idleTimeoutMs: 300_000 });
expect((await read(run.id)).runnerProfileJson).toMatchObject({ nativeExecutionInput: { session: { lifecyclePolicy: { mode: "warm" } } } });
});
it("preserves only server-owned admission and dispatch evidence when recording legacy selection", async () => {
const { input, run } = await fixture("process");
const admission = { version: 1, kind: "fixture" }, dispatch = { entered: true };
@@ -506,11 +506,10 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
executionTarget.transport === "sandbox"
? (executionTarget.runnerLifecyclePolicy ?? null)
: null;
// Codex refreshes its durable auth home; Copilot owns a fenced home
// and an explicitly bound token. Both retire on credential generation
// changes. Other managed harnesses require per-turn cleanup.
// Task providers own credential lifetime; process-backed managed homes retain their provider-specific cleanup policy.
const supportsManagedWarmSession = agent.adapterType === "paperclip_runner" &&
supportsManagedNativeWarmSession({ ...nativeRuntimeResolution.profile, acpxAgent: parseObject(agent.adapterConfig).acpxAgent }, managedAiRuntime?.attribution);
((executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" && !!executionTarget.runnerTask) ||
supportsManagedNativeWarmSession({ ...nativeRuntimeResolution.profile, acpxAgent: parseObject(agent.adapterConfig).acpxAgent }, managedAiRuntime?.attribution));
const effectiveLifecyclePolicy = persistedNativeExecutionInput?.session.lifecyclePolicy ??
(["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession
? { mode: "per_turn" as const, idleTimeoutMs: null }
@@ -456,10 +456,12 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary",
it("persists the workspace and issue binding before acquiring its environment", async () => {
const { input, source, issue, envOrchestrator } = await fixture();
input.config.managedAiCredentialIdentity = "account:generation";
const acquire = envOrchestrator.acquireForRun;
vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => {
expect((await readIssue(issue.id)).executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id);
expect((await readRun(input.run.id)).contextSnapshot?.executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id);
expect(args.managedAiCredentialIdentity).toBe("account:generation");
return acquire(args);
});
const result = await prepareHeartbeatWorkspace(db, input);
@@ -129,6 +129,7 @@ export interface HeartbeatWorkspacePreparationInput {
mergedConfig: Record<string, unknown>;
configSnapshot: WorkspaceMetadataInput["configSnapshot"];
secretManifest: WorkspaceMetadataInput["secretManifest"];
managedAiCredentialIdentity?: string;
};
reuse: {
requestedShouldReuseExisting: boolean;
@@ -1121,6 +1122,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
adapterType: agent.adapterType,
adapterConfig: parseObject(agent.adapterConfig),
admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode,
managedAiCredentialIdentity: input.config.managedAiCredentialIdentity,
runnerTaskProjectIds:[...new Set([executionWorkspace.projectId,...resolvedWorkspace.runnerTaskProjectIds ?? [],
...(executionWorkspace.additionalWorkspaces ?? []).map(workspace=>workspace.projectId)]
.filter((id):id is string=>Boolean(id)))],
@@ -11128,13 +11128,15 @@ describe("runnerd provider runtime wiring", () => {
await rm(isolatedStateDirectory, { recursive: true, force: true });
});
it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => {
it.each((["codex", "claude"] as const).flatMap(provider => (["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const).filter(scenario => provider === "codex" || scenario.startsWith("plugin-")).map(scenario => [provider, scenario] as const)))("rotates an owned warm %s remote runner before checking changed configuration: %s", async (provider, scenario) => {
const plugin = scenario === "plugin-replaced" || scenario === "plugin-managed-github";
const reusable = scenario === "unchanged" || scenario === "plugin-managed-github";
const first = { ...scopedNativeExecution(`warm-admission-${scenario}`),
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
const first = { ...scopedNativeExecution(`warm-admission-${provider}-${scenario}`),
schema: "paperclip.native-execution-input.v5" as const,
provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
provider: provider === "claude" ? { kind: "acpx" as const, agent: "claude" as const, model: "claude-sonnet-4-6", permissionMode: "approve-all" as const, profile: claudeProfile } : { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() };
if (provider === "claude") first.session.driverKind = "acpx_runtime";
first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 };
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` },
session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner", "plugin-replaced", "plugin-managed-github"].includes(scenario) ? 60_000 : 90_000 } } };
@@ -11161,7 +11163,7 @@ describe("runnerd provider runtime wiring", () => {
if(plugin) {
await mkdir(home,{recursive:true});await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600});
}
const pluginOptions=plugin ? {runnerIngressAuthorized:true,managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home,
const pluginOptions=plugin ? {runnerIngressAuthorized:true,runnerEnvironment:provider === "claude" ? {ANTHROPIC_API_KEY:"synthetic-fixture"} : {},managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home,
managedAiCredentialIdentity:"revision",managedAiAccountIdentity:"account"} : {};
const resume=scenario === "plugin-replaced" ? vi.spyOn(environmentTaskRuntime,"resolvePluginTaskResume").mockResolvedValue("task-first") : undefined;
const createOwner = pluginTaskRuntime.createPluginRunnerTaskOwner;
@@ -11862,6 +11864,103 @@ describe("runnerd provider runtime wiring", () => {
}
});
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))(
"recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => {
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`),
schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, runtimeContext: nativeRuntimeContextFixture(),
...(provider === "claude" ? { provider: { kind: "acpx", agent: "claude", model: "claude-sonnet-4-6", permissionMode: "approve-all", profile: claudeProfile } } : { provider: { kind: "codex", model: null, approvalPolicy: "never" } }),
session: { ...execution.session, driverKind: provider === "claude" ? "acpx_runtime" : "codex_app_server",
normalizedSessionId: `plugin-${provider}-${policy}-${mode}`, lifecyclePolicy: policy === "warm" ? { mode: "warm", idleTimeoutMs: 60_000 } : { mode: "per_turn", idleTimeoutMs: null } },
} as NativeExecutionInput;
parseNativeExecutionInput(first);
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-reopened` } };
const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId!,
runnerInstanceId: "plugin-runner", environmentLeaseId: "old-lease", turnId: "old-turn", itemId: "old-item" };
const scoped = await initializeScopedRunnerRoot(first, identity.runnerInstanceId);
await mkdir(join(scoped, "control-plane"), { recursive: true });
const journal = JSON.stringify({ ...durableControlPlaneState(identity), commands: [{ status: mode.endsWith("pending-command") ? "pending" : "completed" }],
committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }] });
await writeFile(join(scoped, "control-plane/control-plane-state.json"), journal);
if (mode.startsWith("empty-runner") || mode === "nonempty-runner") {
await mkdir(join(scoped, "runner"));
if (mode === "nonempty-runner") await writeFile(join(scoped, "runner", "unknown.json"), "{}");
}
if (mode.startsWith("local-receipt")) await writeFile(join(scoped, "plugin-runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended")));
let root = scoped;
if (["quarantined", "empty-runner-quarantined", "ambiguous"].includes(mode)) {
await mkdir(join(isolatedStateDirectory, "quarantine"), { recursive: true });
root = join(isolatedStateDirectory, "quarantine", `${scoped.split("/").at(-1)}.identity_indeterminate.1`);
await rename(scoped, root);
if (mode === "ambiguous") await cp(root, `${root}-second`, { recursive: true });
}
if (mode === "contradictory") {
await mkdir(join(root, "runner"));
await writeFile(join(root, "runner/runner-state.json"), "{");
}
const db = leaseDb(next);
const ordinarySelect = db.select.bind(db);
db.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
? priorNativeRunDb(mode === "foreign-scope" ? { ...first, binding: { ...first.binding, agentId: "foreign" } } : first,
mode === "active" ? "running" : "succeeded").select(fields as never)
: ordinarySelect(fields as never)) as Db["select"];
const throughExecutor = policy === "warm" && ["canonical", "local-receipt", "pending-outbox"].includes(mode);
let inspected = false;
const resume = vi.spyOn(environmentTaskRuntime, "resolvePluginTaskResume").mockImplementation(async (_db, request) => {
if (request.inspectCheckpoint) inspected = true;
else if (throughExecutor && !inspected) throw new Error("stale negative checkpoint availability");
if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized");
return "old-task";
});
const task = vi.fn(async (operation: any) => {
environmentTaskOperationSchema.parse(operation);
if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: {
...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"),
outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null,
...(mode === "wrong-identity" ? { itemId: "foreign" } : {}),
} };
return { kind: "description", taskId: "replacement-task", launchPreparationVersion: 1, projectAuthCheckVersion: 1, warmTaskVersion: 1,
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } };
});
const owner = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner");
state.createBackend.mockClear();
const home = join(isolatedStateDirectory, "checkpoint-account");
await mkdir(home, { recursive: true });
await writeFile(join(home, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "synthetic-fixture" }), { mode: 0o600 });
try {
const launch = { db, execution: next, runnerInstanceId: "replacement-runner", runnerIngressAuthorized: true,
managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account",
runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" },
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task,
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never };
if (throughExecutor) {
const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done",
completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] },
evidence: [], verification: [], artifacts: [], attentionRequests: [] });
if (!validated.ok) throw new Error("Invalid completion fixture");
state.execute.mockResolvedValueOnce({ result: validated.result, terminal: { runTerminalState: "succeeded" }, turnId: "turn",
normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind,
driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null });
}
const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch);
if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
await expect(backend).resolves.toBeDefined();
expect(state.createBackend).toHaveBeenCalledOnce();
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity }));
expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" });
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
} else {
await expect(backend).rejects.toThrow();
expect(state.createBackend).not.toHaveBeenCalled();
if (["local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) {
expect(task).not.toHaveBeenCalled();
}
if (mode === "ambiguous") expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
}
} finally { owner.mockRestore(); resume.mockRestore(); }
});
it.each(["verified", "wrong-state", "no-stop-proof", "disposition-retry", "disposition-wrong-state", "disposition-no-stop-proof"] as const)("admits only an authorized same-run plugin checkpoint: %s", async mode => {
const current = scopedNativeExecution(`plugin-crash-${mode}`);
const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!,
@@ -11969,10 +12068,11 @@ describe("runnerd provider runtime wiring", () => {
} finally { owner.mockRestore(); }
});
it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx"] as const)("negotiates task-only warm execution: %s", async mode => {
it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx", "installed-codex", "installed-claude"] as const)("negotiates task-only execution: %s", async mode => {
const current = scopedNativeExecution(`plugin-warm-${mode}`);
current.session.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000};
if(mode==="acpx") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
const installed = mode.startsWith("installed-");
current.session.lifecyclePolicy = installed ? {mode:"per_turn",idleTimeoutMs:null} : {mode:"warm",idleTimeoutMs:300000};
if(mode==="acpx" || mode==="installed-claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
const home=await mkdtemp(join(tmpdir(),"plugin-warm-account-"));
await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600});
const owner=vi.spyOn(pluginTaskRuntime,"createPluginRunnerTaskOwner");
@@ -11983,20 +12083,22 @@ describe("runnerd provider runtime wiring", () => {
state.createBackend.mockClear();state.createTransport.mockClear();
try {
const backend=createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"runner",runnerIngressAuthorized:true,
managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" ? undefined : "revision",managedAiAccountIdentity:"account",
runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" || installed ? undefined : "revision",managedAiAccountIdentity:installed ? undefined : "account",
runnerExecutionTarget:{kind:"remote",transport:"sandbox",leaseId:"lease",remoteCwd:".",runnerTask:task,
...(mode==="fresh" ? {sandboxLeaseAcquisition:{outcome:"resumed",providerLeaseId:"old-task"}} : {})}});
if(mode!=="codex" && mode!=="fresh") {await expect(backend).rejects.toThrow(mode==="acpx" ? "Codex workspace" : "capability or account");return;}
if(mode==="missing-capability" || mode==="missing-account") {await expect(backend).rejects.toThrow("capability or account");return;}
await backend;
if(mode==="fresh") expect(task.mock.calls.map(([operation])=>operation.kind)).toEqual(["stop","status","complete","describe"]);
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:{mode:"warm",idleTimeoutMs:300000}});
expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,accountIdentity:"account",onCredentials:undefined});
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:current.session.lifecyclePolicy});
expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,onCredentials:undefined});
expect(owner.mock.calls.at(-1)![0].accountIdentity).toBe(installed ? undefined : "account");
} finally {owner.mockRestore();await rm(home,{recursive:true,force:true});}
});
it.each([true,false])("reconnects task-only warm recovery under the saved identity before reuse or retirement: %s", async reuseProcess => {
const current=scopedNativeExecution(`plugin-restart-${reuseProcess}`);
it.each((["codex", "claude"] as const).flatMap(provider => [true, false].map(reuseProcess => [provider, reuseProcess] as const)))("reconnects task-only %s warm recovery under the saved identity before reuse or retirement: %s", async (provider, reuseProcess) => {
const current=scopedNativeExecution(`plugin-restart-${provider}-${reuseProcess}`);
if(provider === "claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
current.session.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000};
const identity={runnerInstanceId:"old-runner",environmentLeaseId:"old-lease",runId:"old-run",
normalizedSessionId:current.session.normalizedSessionId!,turnId:"old-turn",itemId:"old-item"};
@@ -12008,7 +12110,7 @@ describe("runnerd provider runtime wiring", () => {
state.createBackend.mockClear();state.createTransport.mockClear();
try {
await createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"new-runner",runnerIngressAuthorized:true,
managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account",
runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account",
managedAiCredentialWriteback:{} as never,
remoteWarmRecovery:{identity,reuseProcess,previousExecution:current,checkpoint:{} as never,
remote:{providerLeaseId:"saved-task",remoteCwd:"."},revalidate:async()=>{}},
@@ -2062,29 +2062,34 @@ function savePluginRunnerCheckpoint(root: string, state: Record<string, unknown>
renameSync(temporary, path);
}
function pluginRunnerdAuthorityLifecycle(
root: string,
identity: RunnerdDurableIdentity,
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
try {
const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"),
NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8")));
if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle))
|| !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate";
return state.lifecycle === "suspended" ? "suspended" : "not_suspended";
} catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; }
}
function runnerdAuthorityLifecycle(
root: string,
identity: RunnerdDurableIdentity,
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
const runnerRoot = resolve(root, "runner");
if (!existsSync(runnerRoot)) {
try {
const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"),
NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8")));
if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle))
|| !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate";
return state.lifecycle === "suspended" ? "suspended" : "not_suspended";
} catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; }
}
if (!existsSync(runnerRoot)) return pluginRunnerdAuthorityLifecycle(root, identity);
if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate";
const statePath = resolve(runnerRoot, "runner-state.json");
if (!existsSync(statePath)) {
try {
// Older remote transports created this controller-side placeholder even
// though runner state was owned by the sandbox. It carries no authority,
// so a verified failover backup may be consulted. Any non-empty direct
// so a verified plugin receipt or failover backup may be consulted. Any non-empty direct
// directory remains indeterminate and therefore blocks fallback.
return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate";
return readdirSync(runnerRoot).length === 0 ? pluginRunnerdAuthorityLifecycle(root, identity) : "indeterminate";
} catch {
return "indeterminate";
}
@@ -4894,6 +4899,72 @@ async function verifyWarmTransitionRestart(input: {
return verified;
}
async function recoverPluginRunnerCheckpoint(input: {
db: Db;
execution: NativeExecutionInput;
runnerExecutionTarget?: AdapterExecutionTarget | null;
}, scoped: string): Promise<void> {
const target = input.runnerExecutionTarget;
if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask || !target.leaseId ||
target.sandboxLeaseAcquisition?.outcome === "resumed") return;
let root = scoped;
const quarantined = !existsSync(scoped);
const deny = (): never => { throw new Error("runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven"); };
if (quarantined) {
const quarantine = resolve(runnerdStateBase(), "quarantine");
if (!existsSync(quarantine)) return;
if (!isSafeNativeStateDirectory(quarantine)) return deny();
const candidates = readdirSync(quarantine).filter(name => name.startsWith(`${basename(scoped)}.identity_indeterminate.`));
if (candidates.length === 0) return;
if (candidates.length !== 1) return deny();
root = resolve(quarantine, candidates[0]!);
}
if (!existsSync(root)) return;
if (!isSafeNativeStateDirectory(root)) return deny();
if (hasRetainedWarmTransitionEvidence(root)) {
if (quarantined) return deny();
return;
}
const identity = readRunnerdDurableIdentity(root);
if (!durableIdentityMatchesSession(identity, input.execution) || identity.runId === input.execution.binding.runId) {
if (quarantined) return deny();
return;
}
const direct = runnerdAuthorityLifecycle(root, identity);
const verifiedPluginReceipt = direct === "suspended" &&
!existsSync(resolve(root, "runner", "runner-state.json")) && pluginRunnerdAuthorityLifecycle(root, identity) === "suspended";
// Never hide contradictory direct authority behind a remote checkpoint.
if (direct !== "absent" && !(quarantined && direct === "suspended") && !verifiedPluginReceipt) {
if (quarantined) return deny();
return;
}
await assertCleanupActivationCommitted(input.db, root, input.execution);
const prior = await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true });
if (prior !== "terminal_state_indeterminate" && !((quarantined || verifiedPluginReceipt) && prior === "verified")) {
if (quarantined) return deny();
return;
}
const control = record(JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"),
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8")));
const terminal = record(Array.isArray(control.committedEvents) ? control.committedEvents.at(-1) : null);
if (!Array.isArray(control.commands) || control.commands.some(command => record(command).status !== "completed") ||
terminal.eventType !== "run.terminal" || record(terminal.envelope).runId !== identity.runId ||
record(terminal.envelope).normalizedSessionId !== identity.normalizedSessionId) return deny();
const resumeFrom = await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId,
leaseId: target.leaseId, priorIdentity: identity, inspectCheckpoint: true });
const checkpoint = environmentTaskResultSchema.parse(await target.runnerTask({ kind: "read_state", identity, resumeFrom }));
if (checkpoint.kind !== "state" || checkpoint.state.lifecycle !== "suspended" ||
!Array.isArray(checkpoint.state.outbox) || checkpoint.state.outbox.length !== 0 ||
checkpoint.state.pendingTerminalDelivery != null || checkpoint.state.pendingProviderCleanup != null ||
checkpoint.state.warmTransition != null) return deny();
savePluginRunnerCheckpoint(root, checkpoint.state, identity);
if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny();
if (quarantined) {
if (existsSync(scoped)) return deny();
renameSync(root, scoped);
}
}
async function migrateRunnerdStateRootForExecution(input: {
db: Db;
execution: NativeExecutionInput;
@@ -4913,6 +4984,9 @@ async function migrateRunnerdStateRootForExecution(input: {
issueId: input.execution.binding.issueId,
stateKey: basename(scoped),
});
if (!input.restartRecovery && !input.retainedWarmOwnerRunId) {
await recoverPluginRunnerCheckpoint(input, scoped);
}
if (
input.allowLocalRecovery &&
!input.retainedWarmOwnerRunId &&
@@ -8000,10 +8074,7 @@ async function executePaperclipNativeSessionWithinScope(
// Inspection can replace a stopped retained task before deciding between adoption and checkpoint recovery.
await taskTarget.runnerTask({kind:"describe"});
if (taskTarget.sandboxLeaseAcquisition?.outcome !== "resumed") {
const resumeFrom=await resolvePluginTaskResume(input.db,{companyId:input.execution.binding.companyId,leaseId:taskTarget.leaseId,priorIdentity:identity});
const checkpoint=environmentTaskResultSchema.parse(await taskTarget.runnerTask({kind:"read_state",identity,resumeFrom}));
if(checkpoint.kind!=="state") throw new Error("Plugin Runner checkpoint unavailable");
savePluginRunnerCheckpoint(root,checkpoint.state,identity);
await recoverPluginRunnerCheckpoint(input, root);
}
}
if (durableIdentityMatchesSession(identity, input.execution) && identity.runId !== input.execution.binding.runId &&
@@ -11802,8 +11873,8 @@ async function createRunnerdBackendWithinSessionClaim(
const remoteTarget = target.kind === "remote" ? target : null;
const pluginTask = remoteTarget?.transport === "sandbox" ? remoteTarget.runnerTask : undefined;
if (pluginTask && input.runnerIngressAuthorized !== true) throw new Error("runner_ingress_unavailable");
if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8" || (input.execution.session.lifecyclePolicy.mode === "warm" && input.execution.provider.kind !== "codex"))) {
throw new Error("Plugin warm Runner tasks require a Codex workspace session");
if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8")) {
throw new Error("Plugin Runner tasks require a workspace session");
}
const remoteCommandRunner = remoteTarget
? remoteTarget.transport === "ssh"
@@ -11992,7 +12063,7 @@ async function createRunnerdBackendWithinSessionClaim(
.where(and(eq(environmentLeases.id, remoteTarget.leaseId!), eq(environmentLeases.companyId, input.execution.binding.companyId)));
}, onDispatched: input.onTaskDispatched,
};
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin Codex warm task capability or account unavailable");
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin warm task capability or account unavailable");
const owner = retainedOwner ?? createPluginRunnerTaskOwner(ownerInput);
if (retainedOwner) retainedOwner.rebind(ownerInput);
if (description.workspaceFileReadVersion === 1) {
@@ -23,8 +23,11 @@ const requirements: RunnerProviderLaunchRequirements = {
settings: {}, approvalPolicy: "never", readOnly: false, preparedConversation: false,
includeCollaborationModeInstructions: false,
};
async function prepare(owner: ReturnType<typeof createPluginRunnerTaskOwner>) {
await owner.driver.prepareLaunch!(requirements);
function launchRequirements(harness: string): RunnerProviderLaunchRequirements {
return harness === "claude" ? { ...requirements, provider: "acpx", settings: { acpxAgent: "claude" } } : requirements;
}
async function prepare(owner: ReturnType<typeof createPluginRunnerTaskOwner>, current = requirements) {
await owner.driver.prepareLaunch!(current);
}
function provider() {
let stopped = false;
@@ -323,14 +326,14 @@ describe("plugin task workspace file transfer", () => {
});
});
it("reuses a warm Codex process across run authorities without idle status polling", async () => {
it.each(["codex", "claude"])("reuses a warm %s process across run authorities without idle status polling", async harness => {
const {task} = provider();
ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}});
const retained = vi.fn(async()=>{});
const input = {task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
const input = {task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
accountIdentity:"account",credentialIdentity:"revision",pollMs:1,onSessionRetained:retained};
const owner = createPluginRunnerTaskOwner(input);
await prepare(owner);
await prepare(owner, launchRequirements(harness));
const registration = await owner.registration({} as never,"run");
const spec = launchSpec(); spec.nativeRunner!.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000};
const handle = owner.launcher(spec); registration.activate(); await registration.ready();
@@ -345,7 +348,7 @@ it("reuses a warm Codex process across run authorities without idle status polli
owner.rebind({...input,task:nextTask});
const nextRegistration=await owner.registration({} as never,"next-run");
const nextIdentity={...requirements.identity,runId:"next-run",turnId:"next-turn",itemId:"next-item"};
await owner.driver.prepareLaunch!({...requirements,identity:nextIdentity});
await owner.driver.prepareLaunch!({...launchRequirements(harness),identity:nextIdentity});
expect(nextTask.mock.calls[0][0]).toMatchObject({kind:"prepare",priorIdentity:requirements.identity,accountIdentity:"account"});
expect(nextTask.mock.calls.map(([op])=>op.kind)).toEqual(["prepare","check_project_auth"]);
await registration.release(); nextRegistration.activate(); await nextRegistration.ready();
@@ -355,13 +358,13 @@ it("reuses a warm Codex process across run authorities without idle status polli
expect(nextTask.mock.calls.filter(([op])=>op.kind==="stop")).toHaveLength(1);
});
it("adopts a verified retained Codex task without submitting a replacement", async () => {
it.each(["codex", "claude"])("adopts a verified retained %s task without submitting a replacement", async harness => {
const {task}=provider();
ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}});
const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness:"codex",projectAuthCheckVersion:1,
const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness,projectAuthCheckVersion:1,
warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity,pollMs:1});
const recovered = {configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}};
await expect(owner.driver.prepareLaunch!(requirements,recovered)).resolves.toEqual(recovered);
await expect(owner.driver.prepareLaunch!(launchRequirements(harness),recovered)).resolves.toEqual(recovered);
expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]);
expect(task.mock.calls[0][0]).toMatchObject({configurationId:"saved",projectIds:[]});
expect(await owner.driver.adoptExistingRunner!.isAlive()).toBe(true);
@@ -372,28 +375,30 @@ it("adopts a verified retained Codex task without submitting a replacement", asy
expect(task.mock.calls.some(([op])=>op.kind==="complete")).toBe(true);
});
it("denies retained process adoption when current project authentication is revoked",async()=>{
it.each(["codex", "claude"])("denies retained %s process adoption when current project authentication is revoked",async harness=>{
const {task}=provider();
task.mockImplementation(async op=>op.kind === "check_project_auth"
? {kind:"project_auth",taskId:"task",configurationId:op.configurationId,provider:op.provider,allowed:false} as never
: {kind:"accepted",taskId:"task"} as never);
const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness:"codex",projectAuthCheckVersion:1,
const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness,projectAuthCheckVersion:1,
warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity});
await expect(owner.driver.prepareLaunch!(requirements,{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}))
await expect(owner.driver.prepareLaunch!(launchRequirements(harness),{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}))
.rejects.toThrow("project authentication conflicts");
expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]);
});
it("refuses warm account or credential generation changes", async()=>{
it.each(["codex", "claude"])("refuses warm %s account, harness, project, or credential generation changes", async harness=>{
const {task}=provider();
const input={task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
const input={task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
accountIdentity:"account",credentialIdentity:"revision",pollMs:1};
const owner=createPluginRunnerTaskOwner(input);
await prepare(owner);
await prepare(owner, launchRequirements(harness));
const spec=launchSpec();spec.nativeRunner!.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000};
const process=owner.launcher(spec);
expect(()=>owner.rebind({...input,accountIdentity:"other"})).toThrow("cannot be rebound");
expect(()=>owner.rebind({...input,credentialIdentity:"rotated"})).toThrow("cannot be rebound");
expect(()=>owner.rebind({...input,harness:harness === "codex" ? "claude" : "codex"})).toThrow("cannot be rebound");
expect(()=>owner.rebind({...input,projectIds:["foreign"]})).toThrow("cannot be rebound");
const registration=await owner.registration({} as never,"run");
await registration.checkpoint("unsettled"); await process.completion;
});
@@ -128,7 +128,7 @@ export function createPluginRunnerTaskOwner(input: {
if (launched || released) throw new Error("Plugin Runner replacement requires a new task lease");
const native = spec.nativeRunner;
if (!preparedLaunchId) throw new Error("Plugin Runner launch requires preparation");
if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.harness === "codex" && input.accountIdentity && input.warmTaskVersion === 1))
if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.accountIdentity && input.warmTaskVersion === 1))
|| !preparedIdentity || Object.entries(preparedIdentity).some(([key, value]) => native.identity[key as keyof DurableRecoveryIdentity] !== value)) {
throw new Error("Plugin Runner requires a structured per-turn native launch");
}
@@ -295,7 +295,7 @@ export function createPluginRunnerTaskOwner(input: {
};
return { launcher, registration, driver, readWorkspaceFile,
rebind(next: typeof input) {
if (!launched || stopRequested || stoppedState || detached || next.harness !== "codex" ||
if (!launched || stopRequested || stoppedState || detached || next.harness !== input.harness ||
input.accountIdentity !== next.accountIdentity || input.credentialIdentity !== next.credentialIdentity ||
JSON.stringify(input.projectIds) !== JSON.stringify(next.projectIds)) throw new Error("Plugin warm task cannot be rebound");
input = next;
@@ -1,8 +1,20 @@
import {expect,it} from "vitest";
import {resolveRunnerEnvironmentForRun} from "./runner-environment-lifecycle.js";
it("opts Codex plugin tasks into warm retention without opting ACPX in",()=>{
it.each([undefined,"codex","acpx","opencode"])("defaults plugin Runner execution to warm without provider gates: %s",provider=>{
const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{}}};
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider:"codex",lifecycleMode:"warm",idleTimeoutMs:1234})).toMatchObject({config:{runnerLifecycleMode:"warm",runnerIdleTimeoutMs:1234}});
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider:"acpx",lifecycleMode:"warm"})).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider,idleTimeoutMs:1234},undefined,"selected-account:generation")).toMatchObject({config:{runnerLifecycleMode:"warm",runnerIdleTimeoutMs:1234}});
expect(environment.config).not.toHaveProperty("runnerLifecycleMode");
});
it("keeps plugin-owned credentials per-turn without changing admitted recovery",()=>{
const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{},runnerLifecycleMode:"warm"}};
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{lifecycleMode:"warm"})).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{},"warm")).toMatchObject({config:{runnerLifecycleMode:"warm"}});
});
it("preserves explicit per-turn settings and the admitted recovery policy",()=>{
const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{}}};
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{lifecycleMode:"per_turn"},undefined,"account:generation")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{},"per_turn")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
expect(resolveRunnerEnvironmentForRun({...environment,config:{...environment.config,runnerLifecycleMode:"per_turn"}},"paperclip_runner",{lifecycleMode:"warm"},undefined,"account:generation")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
expect(resolveRunnerEnvironmentForRun(environment,"claude_local",{})).toBe(environment);
});
@@ -10,14 +10,16 @@ export function resolveRunnerEnvironmentForRun<T extends Pick<Environment, "driv
adapterType: string,
adapterConfig: Record<string, unknown> = {},
admittedLifecycleMode?: "warm" | "per_turn",
managedAiCredentialIdentity?: string,
): T {
if (adapterType !== "paperclip_runner" || !["sandbox", "plugin"].includes(environment.driver)) return environment;
const config = environment.config ?? {};
const lifecycleMode = admittedLifecycleMode ?? (config.runnerLifecycleMode === "warm" || config.runnerLifecycleMode === "per_turn"
? config.runnerLifecycleMode
: adapterConfig.lifecycleMode);
// Warm tasks need a managed credential identity; recovery keeps its admitted policy.
if (environment.driver === "plugin") return { ...environment, config: { ...config,
runnerLifecycleMode: lifecycleMode === "warm" && (adapterConfig.provider ?? "codex") === "codex" ? "warm" : "per_turn",
runnerLifecycleMode: admittedLifecycleMode ?? (managedAiCredentialIdentity && lifecycleMode !== "per_turn" ? "warm" : "per_turn"),
runnerIdleTimeoutMs: resolvePaperclipRunnerIdleTimeoutMs(config.runnerIdleTimeoutMs ?? adapterConfig.idleTimeoutMs),
} };
if (lifecycleMode !== "warm") return environment;