mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 19:35:13 +01:00
Preserve plugin session continuity through warm execution and checkpoint recovery (#17820)
This commit is contained in:
1 parent
dcfc2acfa8
commit
ea727dc3c2
23 files changed
+433
-82
No files matched your search
@@ -23,7 +23,7 @@ bounded `read_file` operation for attachments and deliverables.
|
||||
|
||||
Core supports per-turn workspace sessions with Codex, OpenCode, and ACPX.
|
||||
OpenCode and ACPX require the existing qualified provider profiles.
|
||||
Workspace-free providers are not supported here. Codex warm retention is opt-in through `warmTaskVersion: 1`; ACPX and OpenCode remain per-turn.
|
||||
Workspace-free providers are not supported here. Drivers opt into warm retention through `warmTaskVersion: 1`. Core prefers warm execution for new task runs on capable drivers, including ACPX; existing provider admission still determines which harnesses the driver supports.
|
||||
The plugin owns project workspace preparation. Core does not prepare or reuse
|
||||
a Core Git worktree for a task-only environment.
|
||||
Selecting a plain task directory does not add redundant default configuration.
|
||||
@@ -164,13 +164,20 @@ The existing agent-independent `inspect` and `remove` operations continue to
|
||||
protect storage maintenance. Providers that do not declare finalization version 1
|
||||
retain the inspection-based completion path. No stored binding migration is required.
|
||||
|
||||
## Codex warm task retention
|
||||
## Warm task retention
|
||||
|
||||
A driver declares `warmTaskVersion: 1`, implements `onEnvironmentResumeLease`,
|
||||
and returns `warmTaskVersion: 1` from `describe` only when its execution host
|
||||
supports authorized run attachment. Select the agent's warm lifecycle or set
|
||||
`runnerLifecycleMode: "warm"` and `runnerIdleTimeoutMs` on its plugin environment.
|
||||
Core requires an explicitly selected account. Setup probes remain ephemeral.
|
||||
supports authorized run attachment. Newly hired agents targeting such a driver
|
||||
are configured for warm execution. Existing agents keep their explicit policy;
|
||||
plugin runs without an explicit policy prefer warm execution. There is no
|
||||
provider-specific warm gate or additional setting. Existing runs retain their
|
||||
admitted lifecycle policy during restart recovery. The existing
|
||||
`runnerIdleTimeoutMs` controls bounded retention. Fresh warm admission requires
|
||||
a Core-managed credential identity before lease acquisition. Plugin-owned installed
|
||||
accounts continue per-turn until such an identity is available; the saved warm
|
||||
preference does not bypass account ownership checks. Drivers without warm
|
||||
capability and setup probes remain ephemeral.
|
||||
|
||||
Plugin task execution does not create a Core-managed GitHub broker. Warm reuse
|
||||
and restart recovery use the same broker-ownership rule as initial startup.
|
||||
@@ -208,4 +215,5 @@ Changes to account, credential generation, workspace, runtime or process policy
|
||||
retire the old owner before recovery. A new authority epoch archives the stopped
|
||||
checkpoint under its stable archive key and preserves provider session files.
|
||||
Warm status monitoring relies on PRP after startup and does not poll an idle
|
||||
Launcher. General ACPX warm support is deliberately excluded.
|
||||
Launcher. ACPX uses this task-provider contract; other execution transports retain
|
||||
their provider-specific lifecycle policies.
|
||||
@@ -16,6 +16,20 @@ Each recovery case includes another healthy turn after the recovered turn and
|
||||
asserts that it does not start another runner or provider process.
|
||||
It does not qualify a live model or a sandbox provider API.
|
||||
|
||||
Per-turn plugin continuation uses a stopped checkpoint on a replacement lease.
|
||||
The executor tests cover Claude and Codex, including reopening blocked work.
|
||||
The lease-authority integration test uses PostgreSQL to check that checkpoint
|
||||
discovery cannot launch work or bypass a termination receipt. A stale negative
|
||||
availability flag requires a fresh provider status before the checkpoint can
|
||||
be read for continuation.
|
||||
|
||||
The executor can restore one quarantined journal in the same session scope.
|
||||
It requires a terminal prior run, completed commands, a terminal event, and an
|
||||
identity-matched suspended plugin checkpoint with no pending delivery or cleanup.
|
||||
Multiple candidates, contradictory direct state, and unfinished work still fail
|
||||
closed. Recovery preserves the journal and the provider's existing state owner.
|
||||
Do not delete a quarantine or change database ownership to force continuation.
|
||||
|
||||
Use a disposable Linux container with Node.js, `sh`, `tar`, `base64`, `ps`, and
|
||||
`setsid`. Build `paperclip-runnerd` and `fake-codex-app-server` from the checkout's
|
||||
`packages/paperclip-runner/runner/Cargo.toml` for that container's architecture.
|
||||
|
||||
@@ -764,6 +764,20 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
|
||||
|
||||
|
||||
describe("native runner lifecycle changes before lease acquisition", () => {
|
||||
it.each([undefined, "account:generation"])("uses a warm plugin lease only with a managed credential identity: %s", async managedAiCredentialIdentity => {
|
||||
const environment = { ...makeEnvironment("plugin"), config: { pluginKey: "fixture", driverKey: "tasks", driverConfig: {} } };
|
||||
mockGetEnvironment.mockResolvedValue(environment);
|
||||
const acquireRunLease = vi.fn(async input => ({ environment: input.environment,
|
||||
lease: makeLease({ leasePolicy: input.environment.config.runnerLifecycleMode === "warm" ? "reuse_by_environment" : "ephemeral" }), leaseContext: {} }));
|
||||
const orchestrator = environmentRunOrchestrator({} as never, { environmentRuntime: makeMockRuntime({ acquireRunLease }) });
|
||||
const result = await orchestrator.acquireForRun({ companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local",
|
||||
adapterType: "paperclip_runner", adapterConfig: { lifecycleMode: "warm" }, managedAiCredentialIdentity,
|
||||
issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1", persistedExecutionWorkspace: null, executionWorkspaceSettings: null });
|
||||
expect(result.environment.config.runnerLifecycleMode).toBe(managedAiCredentialIdentity ? "warm" : "per_turn");
|
||||
expect(result.lease.leasePolicy).toBe(managedAiCredentialIdentity ? "reuse_by_environment" : "ephemeral");
|
||||
expect(environment.config).not.toHaveProperty("runnerLifecycleMode");
|
||||
});
|
||||
|
||||
it("requests a reusable lease when an existing task switches from per-turn to warm", async () => {
|
||||
const environment = { ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } };
|
||||
mockGetEnvironment.mockResolvedValue(environment);
|
||||
|
||||
@@ -824,7 +824,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
const cold = await runtime.acquireRunLease({ ...input, environment, heartbeatRunId: seeded.runId });
|
||||
expect(cold.lease.leasePolicy).toBe("ephemeral");
|
||||
await runtime.releaseRunLeases(seeded.runId, "released");
|
||||
const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" });
|
||||
const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" }, undefined, "selected-account:generation");
|
||||
const warmRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: warmRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
|
||||
const warm = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: warmRunId });
|
||||
|
||||
@@ -59,16 +59,41 @@ databaseSuite("environment task lease authority", () => {
|
||||
await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, oldLeaseId));
|
||||
await expect(authorize()).rejects.toThrow("not authorized");
|
||||
await db.update(environmentLeases).set({ cleanupStatus: "success" }).where(eq(environmentLeases.id, oldLeaseId));
|
||||
let sessionStateAvailable = false;
|
||||
const operations: string[] = [];
|
||||
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask"] }),
|
||||
call: async (_id: string, _method: string, params: any) => ["read_state", "archive_state"].includes(params.operation.kind)
|
||||
? { kind: "state", taskId: params.taskId, state: { schema: "paperclip.runner.durable.state.v1", lifecycle: "suspended",
|
||||
...(params.operation.identity ?? params.operation.priorIdentity) } }
|
||||
: params.operation.kind === "status"
|
||||
? (operations.push("status"), { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true, sessionStateAvailable })
|
||||
: { kind: "accepted", taskId: params.taskId },
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers });
|
||||
const runner = { runnerId: "runner", leaseId: oldLeaseId, sessionId: "session", runId: nextRunId,
|
||||
turnId: "turn", itemId: "item", harness: "codex", protocolMin: 1, protocolMax: 3 };
|
||||
const checkpointIdentity = { ...priorIdentity, turnId: "prior-turn", itemId: "prior-item" };
|
||||
const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, oldLeaseId));
|
||||
await db.update(environmentLeases).set({ metadata: { ...source.metadata, runnerTaskSessionRetained: false } }).where(eq(environmentLeases.id, oldLeaseId));
|
||||
await expect(authorize()).rejects.toThrow("not authorized");
|
||||
await expect(resolvePluginTaskResume(db, { companyId, leaseId: nextLeaseId, priorIdentity, inspectCheckpoint: true })).resolves.toBe("old-task");
|
||||
const inspect = () => runtime.task({ companyId, leaseId: nextLeaseId, operation: {
|
||||
kind: "read_state", identity: checkpointIdentity, resumeFrom: "old-task",
|
||||
} });
|
||||
await expect(inspect()).rejects.toThrow("not authorized");
|
||||
await expect(authorize()).rejects.toThrow("not authorized");
|
||||
// Discovery cannot launch, and cannot bypass the original termination receipt.
|
||||
await expect(runtime.task({ companyId, leaseId: nextLeaseId, operation: {
|
||||
kind: "submit", preparedLaunchId: "fixture", projectIds: [], bootstrapTicket: "fixture", runner, resumeFrom: "old-task",
|
||||
} })).rejects.toThrow("resume identity mismatch");
|
||||
await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, oldLeaseId));
|
||||
await expect(inspect()).rejects.toThrow("not authorized");
|
||||
expect(operations).toEqual(["status"]);
|
||||
await db.update(environmentLeases).set({ cleanupStatus: "success" }).where(eq(environmentLeases.id, oldLeaseId));
|
||||
sessionStateAvailable = true;
|
||||
await expect(inspect()).resolves.toMatchObject({ kind: "state" });
|
||||
expect(operations).toEqual(["status", "status"]);
|
||||
await expect(authorize()).resolves.toBe("old-task");
|
||||
for (const operation of [
|
||||
{ kind: "read_state" as const, identity: checkpointIdentity, resumeFrom: "old-task" },
|
||||
{ kind: "archive_state" as const, priorIdentity: checkpointIdentity, archiveKey: "a".repeat(24), resumeFrom: "old-task" },
|
||||
|
||||
@@ -93,7 +93,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => {
|
||||
await stopDb?.();
|
||||
});
|
||||
|
||||
it("dispatches a plugin-only project without Core workspace files through the native heartbeat boundary", async () => {
|
||||
it("dispatches a plugin-owned account per-turn without Core workspace files through the native heartbeat boundary", async () => {
|
||||
const companyId = randomUUID(), projectId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sourceId = randomUUID();
|
||||
const environmentId = randomUUID(), pluginId = randomUUID(), pluginKey = `test.tasks.${pluginId}`;
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-task-heartbeat-")); tempRoots.push(root);
|
||||
@@ -112,11 +112,11 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => {
|
||||
capabilities: ["environment.drivers.register", "workspaces.manage", "events.subscribe"], resourceLifecycle: ["project"],
|
||||
workspaceProvider: { modes: ["shared_workspace", "isolated_workspace", "operator_branch"], strategies: ["project_primary", "git_worktree"] },
|
||||
entrypoints: { worker: "worker.js" },
|
||||
environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" } }] } });
|
||||
environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, warmTaskVersion: 1, configSchema: { type: "object" } }] } });
|
||||
await db.insert(environments).values({ id: environmentId, companyId, name: "Tasks", driver: "plugin", status: "active",
|
||||
config: { pluginKey, driverKey: "tasks", driverConfig: {} } });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Task agent", role: "engineer", status: "idle",
|
||||
adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol" }, runtimeConfig: {},
|
||||
adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol", lifecycleMode: "warm" }, runtimeConfig: {},
|
||||
defaultEnvironmentId: environmentId, permissions: {} });
|
||||
await db.insert(issues).values({ id: issueId, companyId, projectId, projectWorkspaceId: sourceId, title: "Task-only dispatch", status: "in_progress",
|
||||
assigneeAgentId: agentId, issueNumber: 1, identifier: "TASK-1" });
|
||||
@@ -129,14 +129,15 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => {
|
||||
if (method === "environmentReleaseLease" || method === "environmentDestroyLease") return undefined;
|
||||
throw new Error(`Unexpected task provider call: ${method}`);
|
||||
});
|
||||
const workerManager = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["workspaceOperation", "environmentTask", "environmentAcquireLease", "environmentReleaseLease", "environmentDestroyLease"] }), call } as unknown as PluginWorkerManager;
|
||||
const workerManager = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["workspaceOperation", "environmentTask", "environmentAcquireLease", "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), call } as unknown as PluginWorkerManager;
|
||||
const entered = vi.fn(() => { throw new Error("synthetic_native_boundary_reached"); });
|
||||
const heartbeat = heartbeatService(db, { pluginWorkerManager: workerManager, nativeSessionBackendFactory: entered });
|
||||
heartbeats.push(heartbeat);
|
||||
const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", contextSnapshot: { issueId, projectId } });
|
||||
expect(run).not.toBeNull();
|
||||
await vi.waitFor(async () => { expect((await heartbeat.getRun(run!.id))?.status).toBe("failed"); }, { timeout: 10_000 });
|
||||
expect(entered).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({ runId: run!.id, companyId, issueId }) }));
|
||||
expect(entered).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({ runId: run!.id, companyId, issueId }),
|
||||
session: expect.objectContaining({ lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }) }));
|
||||
expect((await heartbeat.getRun(run!.id))?.error).toContain("synthetic_native_boundary_reached");
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: "heartbeat-task-attempt" }));
|
||||
|
||||
@@ -48,6 +48,24 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
async function hire(status = "idle") { return createAgentLifecycle(db).requestHire(companyId, { name: "Agent", status, adapterType: "process" }); }
|
||||
async function current(id: string) { return (await db.select().from(agents).where(eq(agents.id, id)))[0]!; }
|
||||
|
||||
it.each((["codex", "claude"] as const).flatMap(provider => ["direct", "setup", "approval"].map(path => [provider, path] as const)))("defaults a newly hired %s agent on a warm task environment to warm execution through %s", async (provider, path) => {
|
||||
const pluginId = randomUUID(), environmentId = randomUUID();
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "warm-hire-fixture", version: "1.0.0", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true, warmTaskVersion: 1 }] } as never });
|
||||
await db.insert(environments).values({ id: environmentId, name: `Warm computer ${environmentId}`, driver: "plugin",
|
||||
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
|
||||
const adapterConfig = provider === "claude" ? { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6", lifecycleMode: "per_turn" }
|
||||
: { provider: "codex", model: "gpt-5.6-luna", lifecycleMode: "per_turn" };
|
||||
const lifecycle = createAgentLifecycle(db);
|
||||
const input = { name: "Warm hire", adapterType: "paperclip_runner", adapterConfig, defaultEnvironmentId: environmentId };
|
||||
const draft = path === "setup" ? await lifecycle.requestHire(companyId, input, { setupOnly: true }) : null;
|
||||
const hired = await lifecycle.requestHire(companyId, { ...input, ...(path === "approval" ? { status: "pending_approval" } : {}) }, draft ? { setupAgentId: draft.id } : undefined);
|
||||
if (path === "approval") await lifecycle.approveHire(hired.id, "board");
|
||||
expect((await current(hired.id)).adapterConfig).toMatchObject({ ...adapterConfig, lifecycleMode: "warm" });
|
||||
await createAgentLifecycle(db).updateConfiguration(hired.id, { adapterConfig: { ...(await current(hired.id)).adapterConfig, lifecycleMode: "per_turn" } });
|
||||
expect((await current(hired.id)).adapterConfig.lifecycleMode).toBe("per_turn");
|
||||
});
|
||||
|
||||
it("rejects an outdated configuration snapshot under the write lock", async () => {
|
||||
const agent = await hire();
|
||||
const snapshot = await current(agent.id);
|
||||
|
||||
@@ -12,7 +12,7 @@ import { secretService, assertClaudeOAuthBindingInvariant } from "./secrets.js";
|
||||
import { agentCredentialService } from "./agent-credentials.js";
|
||||
import { agentIdentityService } from "./agent-identity.js";
|
||||
import { initializePrimaryAgent } from "./primary-agent.js";
|
||||
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
|
||||
export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData> {
|
||||
const { runner, ...data } = input;
|
||||
@@ -44,6 +44,10 @@ export async function prepareAgentHire(db: Db, companyId: string, input: CreateA
|
||||
? await secretService(db).normalizeAdapterConfigForPersistence(companyId, data.adapterConfig, { adapterType })
|
||||
: {};
|
||||
const adapterConfig = normalizePaperclipRunnerAdapterConfig(adapterType, rawAdapterConfig);
|
||||
if (adapterType === "paperclip_runner" && !["openai_dot", "grok_bot", "muse"].includes(String(adapterConfig.provider)) &&
|
||||
(await resolveAgentRunnerTaskDriverForCompany(db, companyId, data.defaultEnvironmentId))?.warmTaskVersion === 1) {
|
||||
adapterConfig.lifecycleMode = "warm";
|
||||
}
|
||||
await agentCredentialService(db).enforceClaudeOAuthBindingClaim(db, {
|
||||
companyId, consume: true, environmentId: data.defaultEnvironmentId ?? null,
|
||||
childAdapterConfig: adapterConfig, claudeLogin: options?.claudeLogin,
|
||||
|
||||
@@ -12,7 +12,7 @@ vi.mock("./environment-config.js", async () => ({
|
||||
}));
|
||||
vi.mock("./plugin-registry.js", () => ({ pluginRegistryService: () => ({ getByKey: state.pluginByKey }) }));
|
||||
vi.mock("@paperclipai/adapter-utils/ssh", () => ({ runSshCommand: state.ssh }));
|
||||
import { AGENT_RUNNER_SSH_PLATFORM_COMMAND, agentRunnerAvailability, resolveAgentRunnerTargetForCompany, resolveNewAgentRunner, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
import { AGENT_RUNNER_SSH_PLATFORM_COMMAND, agentRunnerAvailability, resolveAgentRunnerTaskDriverForCompany, resolveAgentRunnerTargetForCompany, resolveNewAgentRunner, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
import { buildCodexAcpConfig, resolveCodexExecutionEngine } from "@paperclipai/adapter-codex-local/server";
|
||||
import { secretService } from "./secrets.js";
|
||||
import * as localPlatform from "../lib/agent-runner-platform.js";
|
||||
@@ -522,6 +522,24 @@ describe("Runner-only plugin environments", () => {
|
||||
expect(agentRunnerAvailability("gemini_local", target)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
|
||||
expect(state.ssh).not.toHaveBeenCalled();
|
||||
});
|
||||
it("carries warm task capability from the selected driver without provider gates", async () => {
|
||||
state.pluginByKey.mockResolvedValue(plugin({ supportsTasks: true, warmTaskVersion: 1 }));
|
||||
expect(await resolveAgentRunnerTargetForCompany({} as never, "company", "computer"))
|
||||
.toEqual({ driver: "plugin", runnerTasks: true, warmTaskVersion: 1 });
|
||||
expect(await resolveAgentRunnerTaskDriverForCompany({} as never, "company", "computer"))
|
||||
.toMatchObject({ driverKey: "tasks", supportsTasks: true, warmTaskVersion: 1 });
|
||||
});
|
||||
|
||||
it.each(["ssh", "sandbox"])("does not qualify a %s runner while inspecting hire defaults", async driver => {
|
||||
state.getEnvironment.mockResolvedValue({ ...pluginEnvironment, driver, config: { provider: "fake" } });
|
||||
expect(await resolveAgentRunnerTaskDriverForCompany({} as never, "company", "computer")).toBeUndefined();
|
||||
expect(state.ssh).not.toHaveBeenCalled();
|
||||
expect(state.sandboxQualification).not.toHaveBeenCalled();
|
||||
expect(state.pluginByKey).not.toHaveBeenCalled();
|
||||
if (driver === "sandbox") await expect(resolveAgentRunnerTargetForCompany({} as never, "company", "computer"))
|
||||
.rejects.toThrow("The fake sandbox provider cannot execute runs.");
|
||||
});
|
||||
|
||||
it("saves a Claude agent on the plugin environment as Paperclip Runner", async () => {
|
||||
const saved = await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "claude_local", defaultEnvironmentId: "computer" });
|
||||
expect(saved).toMatchObject({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "claude" } });
|
||||
|
||||
@@ -15,6 +15,7 @@ export type RunnerTarget = {
|
||||
driver: string; platform?: string; architecture?: string; libc?: "glibc" | "musl" | "unknown"; glibcVersion?: string;
|
||||
/** A plugin environment whose driver declares Runner task execution. It runs agents only through Paperclip Runner. */
|
||||
runnerTasks?: boolean;
|
||||
warmTaskVersion?: 1;
|
||||
};
|
||||
export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi";
|
||||
export interface AgentRunnerSelection {
|
||||
@@ -293,7 +294,10 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
|
||||
const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId);
|
||||
let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget()
|
||||
: { driver };
|
||||
if (driver === "plugin" && environment && await pluginEnvironmentRunsRunnerTasks(db, environment)) target = { driver, runnerTasks: true };
|
||||
if (driver === "plugin" && environment) {
|
||||
const taskDriver = await pluginEnvironmentRunnerTaskDriver(db, environment);
|
||||
if (taskDriver) target = { driver, runnerTasks: true, ...(taskDriver.warmTaskVersion === 1 ? { warmTaskVersion: 1 as const } : {}) };
|
||||
}
|
||||
if (driver === "sandbox" && environment) {
|
||||
const { readSandboxPlatformQualification } = await import("./environment-probe.js");
|
||||
const qualified = await readSandboxPlatformQualification(db, environment, companyId);
|
||||
@@ -312,6 +316,18 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
|
||||
}
|
||||
|
||||
export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) {
|
||||
const resolved = await resolveAgentEnvironmentForCompany(db, companyId, defaultEnvironmentId);
|
||||
if (resolved.environment?.driver === "sandbox" && resolved.environment.config?.provider === "fake") throw unprocessable("The fake sandbox provider cannot execute runs.");
|
||||
return resolved;
|
||||
}
|
||||
|
||||
// Hiring defaults inspect capabilities without probing or qualifying an already selected runner.
|
||||
export async function resolveAgentRunnerTaskDriverForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) {
|
||||
const { environment, driver } = await resolveAgentEnvironmentForCompany(db, companyId, defaultEnvironmentId);
|
||||
return driver === "plugin" && environment ? pluginEnvironmentRunnerTaskDriver(db, environment) : undefined;
|
||||
}
|
||||
|
||||
async function resolveAgentEnvironmentForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) {
|
||||
const { instanceSettingsService } = await import("./instance-settings.js");
|
||||
const { environmentService } = await import("./environments.js");
|
||||
const settings = await instanceSettingsService(db).get();
|
||||
@@ -331,7 +347,6 @@ export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId:
|
||||
const boundCompanyIds = await envs.listBoundCompanyIds(environment.id);
|
||||
if (boundCompanyIds.length > 0 && !boundCompanyIds.includes(companyId)) throw forbidden("The selected environment belongs to another company.", { code: "environment_company_mismatch" });
|
||||
if (environment.status === "archived") throw unprocessable("Environment is archived.");
|
||||
if (environment.driver === "sandbox" && environment.config?.provider === "fake") throw unprocessable("The fake sandbox provider cannot execute runs.");
|
||||
}
|
||||
return { environment, driver: environment?.driver ?? "local" };
|
||||
}
|
||||
@@ -339,17 +354,17 @@ export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId:
|
||||
// The plugin's manifest declaration decides whether its environment runs
|
||||
// Runner tasks. Run admission separately verifies a ready plugin and a live
|
||||
// worker, so a plugin that is down still selects the Runner and fails its run.
|
||||
async function pluginEnvironmentRunsRunnerTasks(db: Db, environment: Parameters<typeof parseEnvironmentDriverConfig>[0]): Promise<boolean> {
|
||||
async function pluginEnvironmentRunnerTaskDriver(db: Db, environment: Parameters<typeof parseEnvironmentDriverConfig>[0]) {
|
||||
let config: { pluginKey: string; driverKey: string };
|
||||
try {
|
||||
const parsed = parseEnvironmentDriverConfig(environment);
|
||||
if (parsed.driver !== "plugin") return false;
|
||||
if (parsed.driver !== "plugin") return undefined;
|
||||
config = parsed.config;
|
||||
} catch {
|
||||
return false;
|
||||
return undefined;
|
||||
}
|
||||
const { pluginRegistryService } = await import("./plugin-registry.js");
|
||||
const plugin = await pluginRegistryService(db).getByKey(config.pluginKey);
|
||||
return Boolean(plugin?.manifestJson.capabilities.includes("environment.drivers.register")
|
||||
&& plugin.manifestJson.environmentDrivers?.some(driver => driver.driverKey === config.driverKey && driver.supportsTasks === true));
|
||||
return plugin?.manifestJson.capabilities.includes("environment.drivers.register")
|
||||
? plugin.manifestJson.environmentDrivers?.find(driver => driver.driverKey === config.driverKey && driver.supportsTasks === true) : undefined;
|
||||
}
|
||||
@@ -273,6 +273,7 @@ export function environmentRunOrchestrator(
|
||||
adapterType: string;
|
||||
adapterConfig?: Record<string, unknown>;
|
||||
admittedLifecycleMode?: "warm" | "per_turn";
|
||||
managedAiCredentialIdentity?: string;
|
||||
issueId: string | null;
|
||||
heartbeatRunId: string;
|
||||
agentId: string;
|
||||
@@ -292,7 +293,7 @@ export function environmentRunOrchestrator(
|
||||
});
|
||||
|
||||
const environment = resolveRunnerEnvironmentForRun(
|
||||
selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode,
|
||||
selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, input.managedAiCredentialIdentity,
|
||||
);
|
||||
|
||||
if (input.recoverPluginTask && environment.driver === "plugin" && input.adapterType === "paperclip_runner") {
|
||||
|
||||
@@ -85,10 +85,11 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
throw new Error("Plugin-provided Runner execution identity mismatch");
|
||||
}
|
||||
if (runner && resumeFrom) {
|
||||
const sources = await db.select({ runId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }).from(environmentLeases).where(and(
|
||||
const inspectCheckpoint = operation.kind === "read_state";
|
||||
const sources = await db.select({ id: environmentLeases.id, runId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.provider, lease.provider!),
|
||||
eq(environmentLeases.providerLeaseId, resumeFrom),
|
||||
sql`${environmentLeases.metadata}->>'runnerTaskSessionRetained' = 'true'`,
|
||||
inspectCheckpoint ? undefined : sql`${environmentLeases.metadata}->>'runnerTaskSessionRetained' = 'true'`,
|
||||
checkpointIdentity ? eq(environmentLeases.heartbeatRunId, checkpointIdentity.runId) : undefined,
|
||||
)).limit(2);
|
||||
const priorRunId = sources.length === 1 ? sources[0]!.runId : null;
|
||||
@@ -98,7 +99,20 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage
|
||||
companyId: input.companyId, leaseId: lease.id,
|
||||
priorIdentity: { runnerInstanceId: runner.runnerId, environmentLeaseId: runner.leaseId,
|
||||
runId: priorRunId, normalizedSessionId: runner.sessionId },
|
||||
inspectCheckpoint,
|
||||
})) throw new Error("Plugin Runner resume identity mismatch");
|
||||
// Older providers misclassified suspended checkpoints. A fresh provider
|
||||
// attestation may repair availability, but never supplies stop/owner authority.
|
||||
if (inspectCheckpoint && sources[0]!.metadata?.runnerTaskSessionRetained !== true) {
|
||||
const status = await executeEnvironmentTask(db, workers, { companyId: input.companyId,
|
||||
leaseId: sources[0]!.id, operation: { kind: "status" } });
|
||||
if (status.kind !== "status" || status.executionStopped !== true || status.sessionStateAvailable !== true ||
|
||||
resumeFrom !== await resolvePluginTaskResume(db, { companyId: input.companyId, leaseId: lease.id,
|
||||
priorIdentity: { runnerInstanceId: runner.runnerId, environmentLeaseId: runner.leaseId,
|
||||
runId: priorRunId, normalizedSessionId: runner.sessionId } })) {
|
||||
throw new Error("Plugin Runner retained session is not authorized");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (metadata.driver !== "plugin" || typeof metadata.pluginId !== "string" || typeof metadata.driverKey !== "string") {
|
||||
throw new Error("Plugin-provided Runner execution lease has no pinned plugin driver");
|
||||
@@ -380,6 +394,8 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor
|
||||
export async function resolvePluginTaskResume(db: Db, input: {
|
||||
companyId: string; leaseId: string;
|
||||
priorIdentity: { runnerInstanceId: string; environmentLeaseId: string; runId: string; normalizedSessionId: string };
|
||||
/** Read-only discovery; launching still requires an attested retained checkpoint. */
|
||||
inspectCheckpoint?: boolean;
|
||||
}): Promise<string> {
|
||||
const rows = await db.select({ lease: environmentLeases, run: heartbeatRuns }).from(environmentLeases)
|
||||
.innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId), eq(heartbeatRuns.companyId, input.companyId)))
|
||||
@@ -389,7 +405,8 @@ export async function resolvePluginTaskResume(db: Db, input: {
|
||||
// A retained PRP lease is logical; later provider tasks have different physical lease IDs.
|
||||
const candidates = rows.filter(row => {
|
||||
const identity = row.lease.metadata?.runnerTaskIdentity as Record<string, unknown> | undefined;
|
||||
return row.lease.id !== input.leaseId && row.run.id === input.priorIdentity.runId && row.lease.metadata?.runnerTaskSessionRetained === true &&
|
||||
return row.lease.id !== input.leaseId && row.run.id === input.priorIdentity.runId &&
|
||||
(input.inspectCheckpoint || row.lease.metadata?.runnerTaskSessionRetained === true) &&
|
||||
identity?.runnerId === input.priorIdentity.runnerInstanceId && identity.leaseId === input.priorIdentity.environmentLeaseId &&
|
||||
identity.sessionId === input.priorIdentity.normalizedSessionId && identity.runId === input.priorIdentity.runId &&
|
||||
!rows.some(successor => successor.lease.id !== input.leaseId &&
|
||||
|
||||
@@ -2713,6 +2713,7 @@ export function heartbeatService(
|
||||
mergedConfig,
|
||||
configSnapshot,
|
||||
secretManifest,
|
||||
managedAiCredentialIdentity: managedAiRuntime?.identity,
|
||||
},
|
||||
reuse: {
|
||||
requestedShouldReuseExisting,
|
||||
|
||||
@@ -311,6 +311,26 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () =
|
||||
expect(await coordinators(run.id)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each(["api_key", "subscription"] as const)("keeps managed Claude %s task execution warm", async method => {
|
||||
const { input, run, issue } = await fixture();
|
||||
const adapterConfig = { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6", lifecycleMode: "warm" };
|
||||
input.agent = { ...input.agent, adapterConfig };
|
||||
await db.update(agents).set({ adapterConfig }).where(eq(agents.id, input.agent.id));
|
||||
input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: input.run, enabled: true, runtimeConfig: {}, adapterConfig,
|
||||
agent: input.agent, issue, target: null, workspaceId: null });
|
||||
input.workspace.executionTarget = { ...sandbox, runnerTask: vi.fn(), reusableLeaseConfigured: true,
|
||||
effectiveCapabilities: { reusableLeases: true } as never, runnerLifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 } };
|
||||
const attribution = { connectionId: randomUUID(), grantId: randomUUID(), provider: "anthropic" as const, method,
|
||||
mode: "responsible_user" as const, responsibleUserId: "task-owner" };
|
||||
input.config.managedAiRuntime = { sessionIdentity: "account:epoch-1", identity: "account:generation-1",
|
||||
config: { env: {}, managedAiConnection: { ...attribution, identity: "account:generation-1", sessionIdentity: "account:epoch-1" } },
|
||||
attribution, accountName: "Saved account", accountOwnerUserId: "task-owner", home: instructionRoot, cleanup: async () => {} };
|
||||
const selected = await selectHeartbeatRuntime(db, input);
|
||||
if (!selected.selected || !selected.nativeExecution) throw new Error("Expected native Claude selection");
|
||||
expect(selected.nativeExecution.session.lifecyclePolicy).toEqual({ mode: "warm", idleTimeoutMs: 300_000 });
|
||||
expect((await read(run.id)).runnerProfileJson).toMatchObject({ nativeExecutionInput: { session: { lifecyclePolicy: { mode: "warm" } } } });
|
||||
});
|
||||
|
||||
it("preserves only server-owned admission and dispatch evidence when recording legacy selection", async () => {
|
||||
const { input, run } = await fixture("process");
|
||||
const admission = { version: 1, kind: "fixture" }, dispatch = { entered: true };
|
||||
|
||||
@@ -506,11 +506,10 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
executionTarget.transport === "sandbox"
|
||||
? (executionTarget.runnerLifecyclePolicy ?? null)
|
||||
: null;
|
||||
// Codex refreshes its durable auth home; Copilot owns a fenced home
|
||||
// and an explicitly bound token. Both retire on credential generation
|
||||
// changes. Other managed harnesses require per-turn cleanup.
|
||||
// Task providers own credential lifetime; process-backed managed homes retain their provider-specific cleanup policy.
|
||||
const supportsManagedWarmSession = agent.adapterType === "paperclip_runner" &&
|
||||
supportsManagedNativeWarmSession({ ...nativeRuntimeResolution.profile, acpxAgent: parseObject(agent.adapterConfig).acpxAgent }, managedAiRuntime?.attribution);
|
||||
((executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" && !!executionTarget.runnerTask) ||
|
||||
supportsManagedNativeWarmSession({ ...nativeRuntimeResolution.profile, acpxAgent: parseObject(agent.adapterConfig).acpxAgent }, managedAiRuntime?.attribution));
|
||||
const effectiveLifecyclePolicy = persistedNativeExecutionInput?.session.lifecyclePolicy ??
|
||||
(["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession
|
||||
? { mode: "per_turn" as const, idleTimeoutMs: null }
|
||||
|
||||
@@ -456,10 +456,12 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary",
|
||||
|
||||
it("persists the workspace and issue binding before acquiring its environment", async () => {
|
||||
const { input, source, issue, envOrchestrator } = await fixture();
|
||||
input.config.managedAiCredentialIdentity = "account:generation";
|
||||
const acquire = envOrchestrator.acquireForRun;
|
||||
vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => {
|
||||
expect((await readIssue(issue.id)).executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id);
|
||||
expect((await readRun(input.run.id)).contextSnapshot?.executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id);
|
||||
expect(args.managedAiCredentialIdentity).toBe("account:generation");
|
||||
return acquire(args);
|
||||
});
|
||||
const result = await prepareHeartbeatWorkspace(db, input);
|
||||
|
||||
@@ -129,6 +129,7 @@ export interface HeartbeatWorkspacePreparationInput {
|
||||
mergedConfig: Record<string, unknown>;
|
||||
configSnapshot: WorkspaceMetadataInput["configSnapshot"];
|
||||
secretManifest: WorkspaceMetadataInput["secretManifest"];
|
||||
managedAiCredentialIdentity?: string;
|
||||
};
|
||||
reuse: {
|
||||
requestedShouldReuseExisting: boolean;
|
||||
@@ -1121,6 +1122,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
adapterType: agent.adapterType,
|
||||
adapterConfig: parseObject(agent.adapterConfig),
|
||||
admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode,
|
||||
managedAiCredentialIdentity: input.config.managedAiCredentialIdentity,
|
||||
runnerTaskProjectIds:[...new Set([executionWorkspace.projectId,...resolvedWorkspace.runnerTaskProjectIds ?? [],
|
||||
...(executionWorkspace.additionalWorkspaces ?? []).map(workspace=>workspace.projectId)]
|
||||
.filter((id):id is string=>Boolean(id)))],
|
||||
|
||||
@@ -11128,13 +11128,15 @@ describe("runnerd provider runtime wiring", () => {
|
||||
await rm(isolatedStateDirectory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => {
|
||||
it.each((["codex", "claude"] as const).flatMap(provider => (["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const).filter(scenario => provider === "codex" || scenario.startsWith("plugin-")).map(scenario => [provider, scenario] as const)))("rotates an owned warm %s remote runner before checking changed configuration: %s", async (provider, scenario) => {
|
||||
const plugin = scenario === "plugin-replaced" || scenario === "plugin-managed-github";
|
||||
const reusable = scenario === "unchanged" || scenario === "plugin-managed-github";
|
||||
const first = { ...scopedNativeExecution(`warm-admission-${scenario}`),
|
||||
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
|
||||
const first = { ...scopedNativeExecution(`warm-admission-${provider}-${scenario}`),
|
||||
schema: "paperclip.native-execution-input.v5" as const,
|
||||
provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
|
||||
provider: provider === "claude" ? { kind: "acpx" as const, agent: "claude" as const, model: "claude-sonnet-4-6", permissionMode: "approve-all" as const, profile: claudeProfile } : { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
|
||||
executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() };
|
||||
if (provider === "claude") first.session.driverKind = "acpx_runtime";
|
||||
first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 };
|
||||
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` },
|
||||
session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner", "plugin-replaced", "plugin-managed-github"].includes(scenario) ? 60_000 : 90_000 } } };
|
||||
@@ -11161,7 +11163,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
if(plugin) {
|
||||
await mkdir(home,{recursive:true});await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600});
|
||||
}
|
||||
const pluginOptions=plugin ? {runnerIngressAuthorized:true,managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home,
|
||||
const pluginOptions=plugin ? {runnerIngressAuthorized:true,runnerEnvironment:provider === "claude" ? {ANTHROPIC_API_KEY:"synthetic-fixture"} : {},managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home,
|
||||
managedAiCredentialIdentity:"revision",managedAiAccountIdentity:"account"} : {};
|
||||
const resume=scenario === "plugin-replaced" ? vi.spyOn(environmentTaskRuntime,"resolvePluginTaskResume").mockResolvedValue("task-first") : undefined;
|
||||
const createOwner = pluginTaskRuntime.createPluginRunnerTaskOwner;
|
||||
@@ -11862,6 +11864,103 @@ describe("runnerd provider runtime wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))(
|
||||
"recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => {
|
||||
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
|
||||
const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`),
|
||||
schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, runtimeContext: nativeRuntimeContextFixture(),
|
||||
...(provider === "claude" ? { provider: { kind: "acpx", agent: "claude", model: "claude-sonnet-4-6", permissionMode: "approve-all", profile: claudeProfile } } : { provider: { kind: "codex", model: null, approvalPolicy: "never" } }),
|
||||
session: { ...execution.session, driverKind: provider === "claude" ? "acpx_runtime" : "codex_app_server",
|
||||
normalizedSessionId: `plugin-${provider}-${policy}-${mode}`, lifecyclePolicy: policy === "warm" ? { mode: "warm", idleTimeoutMs: 60_000 } : { mode: "per_turn", idleTimeoutMs: null } },
|
||||
} as NativeExecutionInput;
|
||||
parseNativeExecutionInput(first);
|
||||
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-reopened` } };
|
||||
const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId!,
|
||||
runnerInstanceId: "plugin-runner", environmentLeaseId: "old-lease", turnId: "old-turn", itemId: "old-item" };
|
||||
const scoped = await initializeScopedRunnerRoot(first, identity.runnerInstanceId);
|
||||
await mkdir(join(scoped, "control-plane"), { recursive: true });
|
||||
const journal = JSON.stringify({ ...durableControlPlaneState(identity), commands: [{ status: mode.endsWith("pending-command") ? "pending" : "completed" }],
|
||||
committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }] });
|
||||
await writeFile(join(scoped, "control-plane/control-plane-state.json"), journal);
|
||||
if (mode.startsWith("empty-runner") || mode === "nonempty-runner") {
|
||||
await mkdir(join(scoped, "runner"));
|
||||
if (mode === "nonempty-runner") await writeFile(join(scoped, "runner", "unknown.json"), "{}");
|
||||
}
|
||||
if (mode.startsWith("local-receipt")) await writeFile(join(scoped, "plugin-runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended")));
|
||||
let root = scoped;
|
||||
if (["quarantined", "empty-runner-quarantined", "ambiguous"].includes(mode)) {
|
||||
await mkdir(join(isolatedStateDirectory, "quarantine"), { recursive: true });
|
||||
root = join(isolatedStateDirectory, "quarantine", `${scoped.split("/").at(-1)}.identity_indeterminate.1`);
|
||||
await rename(scoped, root);
|
||||
if (mode === "ambiguous") await cp(root, `${root}-second`, { recursive: true });
|
||||
}
|
||||
if (mode === "contradictory") {
|
||||
await mkdir(join(root, "runner"));
|
||||
await writeFile(join(root, "runner/runner-state.json"), "{");
|
||||
}
|
||||
const db = leaseDb(next);
|
||||
const ordinarySelect = db.select.bind(db);
|
||||
db.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
|
||||
? priorNativeRunDb(mode === "foreign-scope" ? { ...first, binding: { ...first.binding, agentId: "foreign" } } : first,
|
||||
mode === "active" ? "running" : "succeeded").select(fields as never)
|
||||
: ordinarySelect(fields as never)) as Db["select"];
|
||||
const throughExecutor = policy === "warm" && ["canonical", "local-receipt", "pending-outbox"].includes(mode);
|
||||
let inspected = false;
|
||||
const resume = vi.spyOn(environmentTaskRuntime, "resolvePluginTaskResume").mockImplementation(async (_db, request) => {
|
||||
if (request.inspectCheckpoint) inspected = true;
|
||||
else if (throughExecutor && !inspected) throw new Error("stale negative checkpoint availability");
|
||||
if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized");
|
||||
return "old-task";
|
||||
});
|
||||
const task = vi.fn(async (operation: any) => {
|
||||
environmentTaskOperationSchema.parse(operation);
|
||||
if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: {
|
||||
...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"),
|
||||
outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null,
|
||||
...(mode === "wrong-identity" ? { itemId: "foreign" } : {}),
|
||||
} };
|
||||
return { kind: "description", taskId: "replacement-task", launchPreparationVersion: 1, projectAuthCheckVersion: 1, warmTaskVersion: 1,
|
||||
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } };
|
||||
});
|
||||
const owner = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner");
|
||||
state.createBackend.mockClear();
|
||||
const home = join(isolatedStateDirectory, "checkpoint-account");
|
||||
await mkdir(home, { recursive: true });
|
||||
await writeFile(join(home, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "synthetic-fixture" }), { mode: 0o600 });
|
||||
try {
|
||||
const launch = { db, execution: next, runnerInstanceId: "replacement-runner", runnerIngressAuthorized: true,
|
||||
managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account",
|
||||
runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" },
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task,
|
||||
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never };
|
||||
if (throughExecutor) {
|
||||
const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done",
|
||||
completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] },
|
||||
evidence: [], verification: [], artifacts: [], attentionRequests: [] });
|
||||
if (!validated.ok) throw new Error("Invalid completion fixture");
|
||||
state.execute.mockResolvedValueOnce({ result: validated.result, terminal: { runTerminalState: "succeeded" }, turnId: "turn",
|
||||
normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind,
|
||||
driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null });
|
||||
}
|
||||
const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch);
|
||||
if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
|
||||
await expect(backend).resolves.toBeDefined();
|
||||
expect(state.createBackend).toHaveBeenCalledOnce();
|
||||
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity }));
|
||||
expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
|
||||
expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" });
|
||||
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
|
||||
} else {
|
||||
await expect(backend).rejects.toThrow();
|
||||
expect(state.createBackend).not.toHaveBeenCalled();
|
||||
if (["local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) {
|
||||
expect(task).not.toHaveBeenCalled();
|
||||
}
|
||||
if (mode === "ambiguous") expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
|
||||
}
|
||||
} finally { owner.mockRestore(); resume.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["verified", "wrong-state", "no-stop-proof", "disposition-retry", "disposition-wrong-state", "disposition-no-stop-proof"] as const)("admits only an authorized same-run plugin checkpoint: %s", async mode => {
|
||||
const current = scopedNativeExecution(`plugin-crash-${mode}`);
|
||||
const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!,
|
||||
@@ -11969,10 +12068,11 @@ describe("runnerd provider runtime wiring", () => {
|
||||
} finally { owner.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx"] as const)("negotiates task-only warm execution: %s", async mode => {
|
||||
it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx", "installed-codex", "installed-claude"] as const)("negotiates task-only execution: %s", async mode => {
|
||||
const current = scopedNativeExecution(`plugin-warm-${mode}`);
|
||||
current.session.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000};
|
||||
if(mode==="acpx") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
|
||||
const installed = mode.startsWith("installed-");
|
||||
current.session.lifecyclePolicy = installed ? {mode:"per_turn",idleTimeoutMs:null} : {mode:"warm",idleTimeoutMs:300000};
|
||||
if(mode==="acpx" || mode==="installed-claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
|
||||
const home=await mkdtemp(join(tmpdir(),"plugin-warm-account-"));
|
||||
await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600});
|
||||
const owner=vi.spyOn(pluginTaskRuntime,"createPluginRunnerTaskOwner");
|
||||
@@ -11983,20 +12083,22 @@ describe("runnerd provider runtime wiring", () => {
|
||||
state.createBackend.mockClear();state.createTransport.mockClear();
|
||||
try {
|
||||
const backend=createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"runner",runnerIngressAuthorized:true,
|
||||
managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" ? undefined : "revision",managedAiAccountIdentity:"account",
|
||||
runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" || installed ? undefined : "revision",managedAiAccountIdentity:installed ? undefined : "account",
|
||||
runnerExecutionTarget:{kind:"remote",transport:"sandbox",leaseId:"lease",remoteCwd:".",runnerTask:task,
|
||||
...(mode==="fresh" ? {sandboxLeaseAcquisition:{outcome:"resumed",providerLeaseId:"old-task"}} : {})}});
|
||||
if(mode!=="codex" && mode!=="fresh") {await expect(backend).rejects.toThrow(mode==="acpx" ? "Codex workspace" : "capability or account");return;}
|
||||
if(mode==="missing-capability" || mode==="missing-account") {await expect(backend).rejects.toThrow("capability or account");return;}
|
||||
await backend;
|
||||
if(mode==="fresh") expect(task.mock.calls.map(([operation])=>operation.kind)).toEqual(["stop","status","complete","describe"]);
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:{mode:"warm",idleTimeoutMs:300000}});
|
||||
expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,accountIdentity:"account",onCredentials:undefined});
|
||||
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:current.session.lifecyclePolicy});
|
||||
expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,onCredentials:undefined});
|
||||
expect(owner.mock.calls.at(-1)![0].accountIdentity).toBe(installed ? undefined : "account");
|
||||
} finally {owner.mockRestore();await rm(home,{recursive:true,force:true});}
|
||||
});
|
||||
|
||||
it.each([true,false])("reconnects task-only warm recovery under the saved identity before reuse or retirement: %s", async reuseProcess => {
|
||||
const current=scopedNativeExecution(`plugin-restart-${reuseProcess}`);
|
||||
it.each((["codex", "claude"] as const).flatMap(provider => [true, false].map(reuseProcess => [provider, reuseProcess] as const)))("reconnects task-only %s warm recovery under the saved identity before reuse or retirement: %s", async (provider, reuseProcess) => {
|
||||
const current=scopedNativeExecution(`plugin-restart-${provider}-${reuseProcess}`);
|
||||
if(provider === "claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
|
||||
current.session.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000};
|
||||
const identity={runnerInstanceId:"old-runner",environmentLeaseId:"old-lease",runId:"old-run",
|
||||
normalizedSessionId:current.session.normalizedSessionId!,turnId:"old-turn",itemId:"old-item"};
|
||||
@@ -12008,7 +12110,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
state.createBackend.mockClear();state.createTransport.mockClear();
|
||||
try {
|
||||
await createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"new-runner",runnerIngressAuthorized:true,
|
||||
managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account",
|
||||
runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account",
|
||||
managedAiCredentialWriteback:{} as never,
|
||||
remoteWarmRecovery:{identity,reuseProcess,previousExecution:current,checkpoint:{} as never,
|
||||
remote:{providerLeaseId:"saved-task",remoteCwd:"."},revalidate:async()=>{}},
|
||||
|
||||
@@ -2062,29 +2062,34 @@ function savePluginRunnerCheckpoint(root: string, state: Record<string, unknown>
|
||||
renameSync(temporary, path);
|
||||
}
|
||||
|
||||
function pluginRunnerdAuthorityLifecycle(
|
||||
root: string,
|
||||
identity: RunnerdDurableIdentity,
|
||||
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
|
||||
try {
|
||||
const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"),
|
||||
NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8")));
|
||||
if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle))
|
||||
|| !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate";
|
||||
return state.lifecycle === "suspended" ? "suspended" : "not_suspended";
|
||||
} catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; }
|
||||
}
|
||||
|
||||
function runnerdAuthorityLifecycle(
|
||||
root: string,
|
||||
identity: RunnerdDurableIdentity,
|
||||
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
|
||||
const runnerRoot = resolve(root, "runner");
|
||||
if (!existsSync(runnerRoot)) {
|
||||
try {
|
||||
const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"),
|
||||
NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8")));
|
||||
if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle))
|
||||
|| !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate";
|
||||
return state.lifecycle === "suspended" ? "suspended" : "not_suspended";
|
||||
} catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; }
|
||||
}
|
||||
if (!existsSync(runnerRoot)) return pluginRunnerdAuthorityLifecycle(root, identity);
|
||||
if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate";
|
||||
const statePath = resolve(runnerRoot, "runner-state.json");
|
||||
if (!existsSync(statePath)) {
|
||||
try {
|
||||
// Older remote transports created this controller-side placeholder even
|
||||
// though runner state was owned by the sandbox. It carries no authority,
|
||||
// so a verified failover backup may be consulted. Any non-empty direct
|
||||
// so a verified plugin receipt or failover backup may be consulted. Any non-empty direct
|
||||
// directory remains indeterminate and therefore blocks fallback.
|
||||
return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate";
|
||||
return readdirSync(runnerRoot).length === 0 ? pluginRunnerdAuthorityLifecycle(root, identity) : "indeterminate";
|
||||
} catch {
|
||||
return "indeterminate";
|
||||
}
|
||||
@@ -4894,6 +4899,72 @@ async function verifyWarmTransitionRestart(input: {
|
||||
return verified;
|
||||
}
|
||||
|
||||
async function recoverPluginRunnerCheckpoint(input: {
|
||||
db: Db;
|
||||
execution: NativeExecutionInput;
|
||||
runnerExecutionTarget?: AdapterExecutionTarget | null;
|
||||
}, scoped: string): Promise<void> {
|
||||
const target = input.runnerExecutionTarget;
|
||||
if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask || !target.leaseId ||
|
||||
target.sandboxLeaseAcquisition?.outcome === "resumed") return;
|
||||
let root = scoped;
|
||||
const quarantined = !existsSync(scoped);
|
||||
const deny = (): never => { throw new Error("runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven"); };
|
||||
if (quarantined) {
|
||||
const quarantine = resolve(runnerdStateBase(), "quarantine");
|
||||
if (!existsSync(quarantine)) return;
|
||||
if (!isSafeNativeStateDirectory(quarantine)) return deny();
|
||||
const candidates = readdirSync(quarantine).filter(name => name.startsWith(`${basename(scoped)}.identity_indeterminate.`));
|
||||
if (candidates.length === 0) return;
|
||||
if (candidates.length !== 1) return deny();
|
||||
root = resolve(quarantine, candidates[0]!);
|
||||
}
|
||||
if (!existsSync(root)) return;
|
||||
if (!isSafeNativeStateDirectory(root)) return deny();
|
||||
if (hasRetainedWarmTransitionEvidence(root)) {
|
||||
if (quarantined) return deny();
|
||||
return;
|
||||
}
|
||||
const identity = readRunnerdDurableIdentity(root);
|
||||
if (!durableIdentityMatchesSession(identity, input.execution) || identity.runId === input.execution.binding.runId) {
|
||||
if (quarantined) return deny();
|
||||
return;
|
||||
}
|
||||
const direct = runnerdAuthorityLifecycle(root, identity);
|
||||
const verifiedPluginReceipt = direct === "suspended" &&
|
||||
!existsSync(resolve(root, "runner", "runner-state.json")) && pluginRunnerdAuthorityLifecycle(root, identity) === "suspended";
|
||||
// Never hide contradictory direct authority behind a remote checkpoint.
|
||||
if (direct !== "absent" && !(quarantined && direct === "suspended") && !verifiedPluginReceipt) {
|
||||
if (quarantined) return deny();
|
||||
return;
|
||||
}
|
||||
await assertCleanupActivationCommitted(input.db, root, input.execution);
|
||||
const prior = await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true });
|
||||
if (prior !== "terminal_state_indeterminate" && !((quarantined || verifiedPluginReceipt) && prior === "verified")) {
|
||||
if (quarantined) return deny();
|
||||
return;
|
||||
}
|
||||
const control = record(JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"),
|
||||
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8")));
|
||||
const terminal = record(Array.isArray(control.committedEvents) ? control.committedEvents.at(-1) : null);
|
||||
if (!Array.isArray(control.commands) || control.commands.some(command => record(command).status !== "completed") ||
|
||||
terminal.eventType !== "run.terminal" || record(terminal.envelope).runId !== identity.runId ||
|
||||
record(terminal.envelope).normalizedSessionId !== identity.normalizedSessionId) return deny();
|
||||
const resumeFrom = await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId,
|
||||
leaseId: target.leaseId, priorIdentity: identity, inspectCheckpoint: true });
|
||||
const checkpoint = environmentTaskResultSchema.parse(await target.runnerTask({ kind: "read_state", identity, resumeFrom }));
|
||||
if (checkpoint.kind !== "state" || checkpoint.state.lifecycle !== "suspended" ||
|
||||
!Array.isArray(checkpoint.state.outbox) || checkpoint.state.outbox.length !== 0 ||
|
||||
checkpoint.state.pendingTerminalDelivery != null || checkpoint.state.pendingProviderCleanup != null ||
|
||||
checkpoint.state.warmTransition != null) return deny();
|
||||
savePluginRunnerCheckpoint(root, checkpoint.state, identity);
|
||||
if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny();
|
||||
if (quarantined) {
|
||||
if (existsSync(scoped)) return deny();
|
||||
renameSync(root, scoped);
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateRunnerdStateRootForExecution(input: {
|
||||
db: Db;
|
||||
execution: NativeExecutionInput;
|
||||
@@ -4913,6 +4984,9 @@ async function migrateRunnerdStateRootForExecution(input: {
|
||||
issueId: input.execution.binding.issueId,
|
||||
stateKey: basename(scoped),
|
||||
});
|
||||
if (!input.restartRecovery && !input.retainedWarmOwnerRunId) {
|
||||
await recoverPluginRunnerCheckpoint(input, scoped);
|
||||
}
|
||||
if (
|
||||
input.allowLocalRecovery &&
|
||||
!input.retainedWarmOwnerRunId &&
|
||||
@@ -8000,10 +8074,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
// Inspection can replace a stopped retained task before deciding between adoption and checkpoint recovery.
|
||||
await taskTarget.runnerTask({kind:"describe"});
|
||||
if (taskTarget.sandboxLeaseAcquisition?.outcome !== "resumed") {
|
||||
const resumeFrom=await resolvePluginTaskResume(input.db,{companyId:input.execution.binding.companyId,leaseId:taskTarget.leaseId,priorIdentity:identity});
|
||||
const checkpoint=environmentTaskResultSchema.parse(await taskTarget.runnerTask({kind:"read_state",identity,resumeFrom}));
|
||||
if(checkpoint.kind!=="state") throw new Error("Plugin Runner checkpoint unavailable");
|
||||
savePluginRunnerCheckpoint(root,checkpoint.state,identity);
|
||||
await recoverPluginRunnerCheckpoint(input, root);
|
||||
}
|
||||
}
|
||||
if (durableIdentityMatchesSession(identity, input.execution) && identity.runId !== input.execution.binding.runId &&
|
||||
@@ -11802,8 +11873,8 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
const remoteTarget = target.kind === "remote" ? target : null;
|
||||
const pluginTask = remoteTarget?.transport === "sandbox" ? remoteTarget.runnerTask : undefined;
|
||||
if (pluginTask && input.runnerIngressAuthorized !== true) throw new Error("runner_ingress_unavailable");
|
||||
if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8" || (input.execution.session.lifecyclePolicy.mode === "warm" && input.execution.provider.kind !== "codex"))) {
|
||||
throw new Error("Plugin warm Runner tasks require a Codex workspace session");
|
||||
if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8")) {
|
||||
throw new Error("Plugin Runner tasks require a workspace session");
|
||||
}
|
||||
const remoteCommandRunner = remoteTarget
|
||||
? remoteTarget.transport === "ssh"
|
||||
@@ -11992,7 +12063,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
.where(and(eq(environmentLeases.id, remoteTarget.leaseId!), eq(environmentLeases.companyId, input.execution.binding.companyId)));
|
||||
}, onDispatched: input.onTaskDispatched,
|
||||
};
|
||||
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin Codex warm task capability or account unavailable");
|
||||
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin warm task capability or account unavailable");
|
||||
const owner = retainedOwner ?? createPluginRunnerTaskOwner(ownerInput);
|
||||
if (retainedOwner) retainedOwner.rebind(ownerInput);
|
||||
if (description.workspaceFileReadVersion === 1) {
|
||||
|
||||
@@ -23,8 +23,11 @@ const requirements: RunnerProviderLaunchRequirements = {
|
||||
settings: {}, approvalPolicy: "never", readOnly: false, preparedConversation: false,
|
||||
includeCollaborationModeInstructions: false,
|
||||
};
|
||||
async function prepare(owner: ReturnType<typeof createPluginRunnerTaskOwner>) {
|
||||
await owner.driver.prepareLaunch!(requirements);
|
||||
function launchRequirements(harness: string): RunnerProviderLaunchRequirements {
|
||||
return harness === "claude" ? { ...requirements, provider: "acpx", settings: { acpxAgent: "claude" } } : requirements;
|
||||
}
|
||||
async function prepare(owner: ReturnType<typeof createPluginRunnerTaskOwner>, current = requirements) {
|
||||
await owner.driver.prepareLaunch!(current);
|
||||
}
|
||||
function provider() {
|
||||
let stopped = false;
|
||||
@@ -323,14 +326,14 @@ describe("plugin task workspace file transfer", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses a warm Codex process across run authorities without idle status polling", async () => {
|
||||
it.each(["codex", "claude"])("reuses a warm %s process across run authorities without idle status polling", async harness => {
|
||||
const {task} = provider();
|
||||
ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}});
|
||||
const retained = vi.fn(async()=>{});
|
||||
const input = {task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
|
||||
const input = {task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
|
||||
accountIdentity:"account",credentialIdentity:"revision",pollMs:1,onSessionRetained:retained};
|
||||
const owner = createPluginRunnerTaskOwner(input);
|
||||
await prepare(owner);
|
||||
await prepare(owner, launchRequirements(harness));
|
||||
const registration = await owner.registration({} as never,"run");
|
||||
const spec = launchSpec(); spec.nativeRunner!.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000};
|
||||
const handle = owner.launcher(spec); registration.activate(); await registration.ready();
|
||||
@@ -345,7 +348,7 @@ it("reuses a warm Codex process across run authorities without idle status polli
|
||||
owner.rebind({...input,task:nextTask});
|
||||
const nextRegistration=await owner.registration({} as never,"next-run");
|
||||
const nextIdentity={...requirements.identity,runId:"next-run",turnId:"next-turn",itemId:"next-item"};
|
||||
await owner.driver.prepareLaunch!({...requirements,identity:nextIdentity});
|
||||
await owner.driver.prepareLaunch!({...launchRequirements(harness),identity:nextIdentity});
|
||||
expect(nextTask.mock.calls[0][0]).toMatchObject({kind:"prepare",priorIdentity:requirements.identity,accountIdentity:"account"});
|
||||
expect(nextTask.mock.calls.map(([op])=>op.kind)).toEqual(["prepare","check_project_auth"]);
|
||||
await registration.release(); nextRegistration.activate(); await nextRegistration.ready();
|
||||
@@ -355,13 +358,13 @@ it("reuses a warm Codex process across run authorities without idle status polli
|
||||
expect(nextTask.mock.calls.filter(([op])=>op.kind==="stop")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("adopts a verified retained Codex task without submitting a replacement", async () => {
|
||||
it.each(["codex", "claude"])("adopts a verified retained %s task without submitting a replacement", async harness => {
|
||||
const {task}=provider();
|
||||
ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}});
|
||||
const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness:"codex",projectAuthCheckVersion:1,
|
||||
const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness,projectAuthCheckVersion:1,
|
||||
warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity,pollMs:1});
|
||||
const recovered = {configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}};
|
||||
await expect(owner.driver.prepareLaunch!(requirements,recovered)).resolves.toEqual(recovered);
|
||||
await expect(owner.driver.prepareLaunch!(launchRequirements(harness),recovered)).resolves.toEqual(recovered);
|
||||
expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]);
|
||||
expect(task.mock.calls[0][0]).toMatchObject({configurationId:"saved",projectIds:[]});
|
||||
expect(await owner.driver.adoptExistingRunner!.isAlive()).toBe(true);
|
||||
@@ -372,28 +375,30 @@ it("adopts a verified retained Codex task without submitting a replacement", asy
|
||||
expect(task.mock.calls.some(([op])=>op.kind==="complete")).toBe(true);
|
||||
});
|
||||
|
||||
it("denies retained process adoption when current project authentication is revoked",async()=>{
|
||||
it.each(["codex", "claude"])("denies retained %s process adoption when current project authentication is revoked",async harness=>{
|
||||
const {task}=provider();
|
||||
task.mockImplementation(async op=>op.kind === "check_project_auth"
|
||||
? {kind:"project_auth",taskId:"task",configurationId:op.configurationId,provider:op.provider,allowed:false} as never
|
||||
: {kind:"accepted",taskId:"task"} as never);
|
||||
const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness:"codex",projectAuthCheckVersion:1,
|
||||
const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness,projectAuthCheckVersion:1,
|
||||
warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity});
|
||||
await expect(owner.driver.prepareLaunch!(requirements,{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}))
|
||||
await expect(owner.driver.prepareLaunch!(launchRequirements(harness),{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}))
|
||||
.rejects.toThrow("project authentication conflicts");
|
||||
expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]);
|
||||
});
|
||||
|
||||
it("refuses warm account or credential generation changes", async()=>{
|
||||
it.each(["codex", "claude"])("refuses warm %s account, harness, project, or credential generation changes", async harness=>{
|
||||
const {task}=provider();
|
||||
const input={task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
|
||||
const input={task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
|
||||
accountIdentity:"account",credentialIdentity:"revision",pollMs:1};
|
||||
const owner=createPluginRunnerTaskOwner(input);
|
||||
await prepare(owner);
|
||||
await prepare(owner, launchRequirements(harness));
|
||||
const spec=launchSpec();spec.nativeRunner!.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000};
|
||||
const process=owner.launcher(spec);
|
||||
expect(()=>owner.rebind({...input,accountIdentity:"other"})).toThrow("cannot be rebound");
|
||||
expect(()=>owner.rebind({...input,credentialIdentity:"rotated"})).toThrow("cannot be rebound");
|
||||
expect(()=>owner.rebind({...input,harness:harness === "codex" ? "claude" : "codex"})).toThrow("cannot be rebound");
|
||||
expect(()=>owner.rebind({...input,projectIds:["foreign"]})).toThrow("cannot be rebound");
|
||||
const registration=await owner.registration({} as never,"run");
|
||||
await registration.checkpoint("unsettled"); await process.completion;
|
||||
});
|
||||
@@ -128,7 +128,7 @@ export function createPluginRunnerTaskOwner(input: {
|
||||
if (launched || released) throw new Error("Plugin Runner replacement requires a new task lease");
|
||||
const native = spec.nativeRunner;
|
||||
if (!preparedLaunchId) throw new Error("Plugin Runner launch requires preparation");
|
||||
if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.harness === "codex" && input.accountIdentity && input.warmTaskVersion === 1))
|
||||
if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.accountIdentity && input.warmTaskVersion === 1))
|
||||
|| !preparedIdentity || Object.entries(preparedIdentity).some(([key, value]) => native.identity[key as keyof DurableRecoveryIdentity] !== value)) {
|
||||
throw new Error("Plugin Runner requires a structured per-turn native launch");
|
||||
}
|
||||
@@ -295,7 +295,7 @@ export function createPluginRunnerTaskOwner(input: {
|
||||
};
|
||||
return { launcher, registration, driver, readWorkspaceFile,
|
||||
rebind(next: typeof input) {
|
||||
if (!launched || stopRequested || stoppedState || detached || next.harness !== "codex" ||
|
||||
if (!launched || stopRequested || stoppedState || detached || next.harness !== input.harness ||
|
||||
input.accountIdentity !== next.accountIdentity || input.credentialIdentity !== next.credentialIdentity ||
|
||||
JSON.stringify(input.projectIds) !== JSON.stringify(next.projectIds)) throw new Error("Plugin warm task cannot be rebound");
|
||||
input = next;
|
||||
|
||||
@@ -1,8 +1,20 @@
|
||||
import {expect,it} from "vitest";
|
||||
import {resolveRunnerEnvironmentForRun} from "./runner-environment-lifecycle.js";
|
||||
it("opts Codex plugin tasks into warm retention without opting ACPX in",()=>{
|
||||
|
||||
it.each([undefined,"codex","acpx","opencode"])("defaults plugin Runner execution to warm without provider gates: %s",provider=>{
|
||||
const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{}}};
|
||||
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider:"codex",lifecycleMode:"warm",idleTimeoutMs:1234})).toMatchObject({config:{runnerLifecycleMode:"warm",runnerIdleTimeoutMs:1234}});
|
||||
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider:"acpx",lifecycleMode:"warm"})).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
|
||||
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider,idleTimeoutMs:1234},undefined,"selected-account:generation")).toMatchObject({config:{runnerLifecycleMode:"warm",runnerIdleTimeoutMs:1234}});
|
||||
expect(environment.config).not.toHaveProperty("runnerLifecycleMode");
|
||||
});
|
||||
it("keeps plugin-owned credentials per-turn without changing admitted recovery",()=>{
|
||||
const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{},runnerLifecycleMode:"warm"}};
|
||||
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{lifecycleMode:"warm"})).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
|
||||
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{},"warm")).toMatchObject({config:{runnerLifecycleMode:"warm"}});
|
||||
});
|
||||
it("preserves explicit per-turn settings and the admitted recovery policy",()=>{
|
||||
const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{}}};
|
||||
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{lifecycleMode:"per_turn"},undefined,"account:generation")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
|
||||
expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{},"per_turn")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
|
||||
expect(resolveRunnerEnvironmentForRun({...environment,config:{...environment.config,runnerLifecycleMode:"per_turn"}},"paperclip_runner",{lifecycleMode:"warm"},undefined,"account:generation")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}});
|
||||
expect(resolveRunnerEnvironmentForRun(environment,"claude_local",{})).toBe(environment);
|
||||
});
|
||||
@@ -10,14 +10,16 @@ export function resolveRunnerEnvironmentForRun<T extends Pick<Environment, "driv
|
||||
adapterType: string,
|
||||
adapterConfig: Record<string, unknown> = {},
|
||||
admittedLifecycleMode?: "warm" | "per_turn",
|
||||
managedAiCredentialIdentity?: string,
|
||||
): T {
|
||||
if (adapterType !== "paperclip_runner" || !["sandbox", "plugin"].includes(environment.driver)) return environment;
|
||||
const config = environment.config ?? {};
|
||||
const lifecycleMode = admittedLifecycleMode ?? (config.runnerLifecycleMode === "warm" || config.runnerLifecycleMode === "per_turn"
|
||||
? config.runnerLifecycleMode
|
||||
: adapterConfig.lifecycleMode);
|
||||
// Warm tasks need a managed credential identity; recovery keeps its admitted policy.
|
||||
if (environment.driver === "plugin") return { ...environment, config: { ...config,
|
||||
runnerLifecycleMode: lifecycleMode === "warm" && (adapterConfig.provider ?? "codex") === "codex" ? "warm" : "per_turn",
|
||||
runnerLifecycleMode: admittedLifecycleMode ?? (managedAiCredentialIdentity && lifecycleMode !== "per_turn" ? "warm" : "per_turn"),
|
||||
runnerIdleTimeoutMs: resolvePaperclipRunnerIdleTimeoutMs(config.runnerIdleTimeoutMs ?? adapterConfig.idleTimeoutMs),
|
||||
} };
|
||||
if (lifecycleMode !== "warm") return environment;
|
||||
|
||||
Reference in new issue
Block a user