diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index d1e567873c8..ae04f0ab6d2 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -23,7 +23,7 @@ bounded `read_file` operation for attachments and deliverables. Core supports per-turn workspace sessions with Codex, OpenCode, and ACPX. OpenCode and ACPX require the existing qualified provider profiles. -Workspace-free providers are not supported here. Codex warm retention is opt-in through `warmTaskVersion: 1`; ACPX and OpenCode remain per-turn. +Workspace-free providers are not supported here. Drivers opt into warm retention through `warmTaskVersion: 1`. Core prefers warm execution for new task runs on capable drivers, including ACPX; existing provider admission still determines which harnesses the driver supports. The plugin owns project workspace preparation. Core does not prepare or reuse a Core Git worktree for a task-only environment. Selecting a plain task directory does not add redundant default configuration. @@ -164,13 +164,20 @@ The existing agent-independent `inspect` and `remove` operations continue to protect storage maintenance. Providers that do not declare finalization version 1 retain the inspection-based completion path. No stored binding migration is required. -## Codex warm task retention +## Warm task retention A driver declares `warmTaskVersion: 1`, implements `onEnvironmentResumeLease`, and returns `warmTaskVersion: 1` from `describe` only when its execution host -supports authorized run attachment. Select the agent's warm lifecycle or set -`runnerLifecycleMode: "warm"` and `runnerIdleTimeoutMs` on its plugin environment. -Core requires an explicitly selected account. Setup probes remain ephemeral. +supports authorized run attachment. Newly hired agents targeting such a driver +are configured for warm execution. Existing agents keep their explicit policy; +plugin runs without an explicit policy prefer warm execution. There is no +provider-specific warm gate or additional setting. Existing runs retain their +admitted lifecycle policy during restart recovery. The existing +`runnerIdleTimeoutMs` controls bounded retention. Fresh warm admission requires +a Core-managed credential identity before lease acquisition. Plugin-owned installed +accounts continue per-turn until such an identity is available; the saved warm +preference does not bypass account ownership checks. Drivers without warm +capability and setup probes remain ephemeral. Plugin task execution does not create a Core-managed GitHub broker. Warm reuse and restart recovery use the same broker-ownership rule as initial startup. @@ -208,4 +215,5 @@ Changes to account, credential generation, workspace, runtime or process policy retire the old owner before recovery. A new authority epoch archives the stopped checkpoint under its stable archive key and preserves provider session files. Warm status monitoring relies on PRP after startup and does not poll an idle -Launcher. General ACPX warm support is deliberately excluded. +Launcher. ACPX uses this task-provider contract; other execution transports retain +their provider-specific lifecycle policies. diff --git a/doc/testing-native-remote-continuity.md b/doc/testing-native-remote-continuity.md index 434b179e149..8c9577fdfbd 100644 --- a/doc/testing-native-remote-continuity.md +++ b/doc/testing-native-remote-continuity.md @@ -16,6 +16,20 @@ Each recovery case includes another healthy turn after the recovered turn and asserts that it does not start another runner or provider process. It does not qualify a live model or a sandbox provider API. +Per-turn plugin continuation uses a stopped checkpoint on a replacement lease. +The executor tests cover Claude and Codex, including reopening blocked work. +The lease-authority integration test uses PostgreSQL to check that checkpoint +discovery cannot launch work or bypass a termination receipt. A stale negative +availability flag requires a fresh provider status before the checkpoint can +be read for continuation. + +The executor can restore one quarantined journal in the same session scope. +It requires a terminal prior run, completed commands, a terminal event, and an +identity-matched suspended plugin checkpoint with no pending delivery or cleanup. +Multiple candidates, contradictory direct state, and unfinished work still fail +closed. Recovery preserves the journal and the provider's existing state owner. +Do not delete a quarantine or change database ownership to force continuation. + Use a disposable Linux container with Node.js, `sh`, `tar`, `base64`, `ps`, and `setsid`. Build `paperclip-runnerd` and `fake-codex-app-server` from the checkout's `packages/paperclip-runner/runner/Cargo.toml` for that container's architecture. diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index 85591e1070a..4329d374ac5 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -764,6 +764,20 @@ describe("environmentRunOrchestrator — realizeForRun", () => { describe("native runner lifecycle changes before lease acquisition", () => { + it.each([undefined, "account:generation"])("uses a warm plugin lease only with a managed credential identity: %s", async managedAiCredentialIdentity => { + const environment = { ...makeEnvironment("plugin"), config: { pluginKey: "fixture", driverKey: "tasks", driverConfig: {} } }; + mockGetEnvironment.mockResolvedValue(environment); + const acquireRunLease = vi.fn(async input => ({ environment: input.environment, + lease: makeLease({ leasePolicy: input.environment.config.runnerLifecycleMode === "warm" ? "reuse_by_environment" : "ephemeral" }), leaseContext: {} })); + const orchestrator = environmentRunOrchestrator({} as never, { environmentRuntime: makeMockRuntime({ acquireRunLease }) }); + const result = await orchestrator.acquireForRun({ companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local", + adapterType: "paperclip_runner", adapterConfig: { lifecycleMode: "warm" }, managedAiCredentialIdentity, + issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1", persistedExecutionWorkspace: null, executionWorkspaceSettings: null }); + expect(result.environment.config.runnerLifecycleMode).toBe(managedAiCredentialIdentity ? "warm" : "per_turn"); + expect(result.lease.leasePolicy).toBe(managedAiCredentialIdentity ? "reuse_by_environment" : "ephemeral"); + expect(environment.config).not.toHaveProperty("runnerLifecycleMode"); + }); + it("requests a reusable lease when an existing task switches from per-turn to warm", async () => { const environment = { ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } }; mockGetEnvironment.mockResolvedValue(environment); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index b926be7e975..14b968a0ba5 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -824,7 +824,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { const cold = await runtime.acquireRunLease({ ...input, environment, heartbeatRunId: seeded.runId }); expect(cold.lease.leasePolicy).toBe("ephemeral"); await runtime.releaseRunLeases(seeded.runId, "released"); - const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" }); + const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" }, undefined, "selected-account:generation"); const warmRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: warmRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); const warm = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: warmRunId }); diff --git a/server/src/__tests__/environment-task-authority.integration.test.ts b/server/src/__tests__/environment-task-authority.integration.test.ts index 446c590012f..d01611e1972 100644 --- a/server/src/__tests__/environment-task-authority.integration.test.ts +++ b/server/src/__tests__/environment-task-authority.integration.test.ts @@ -59,16 +59,41 @@ databaseSuite("environment task lease authority", () => { await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, oldLeaseId)); await expect(authorize()).rejects.toThrow("not authorized"); await db.update(environmentLeases).set({ cleanupStatus: "success" }).where(eq(environmentLeases.id, oldLeaseId)); + let sessionStateAvailable = false; + const operations: string[] = []; const workers = { getWorker: () => ({ supportedMethods: ["environmentTask"] }), call: async (_id: string, _method: string, params: any) => ["read_state", "archive_state"].includes(params.operation.kind) ? { kind: "state", taskId: params.taskId, state: { schema: "paperclip.runner.durable.state.v1", lifecycle: "suspended", ...(params.operation.identity ?? params.operation.priorIdentity) } } + : params.operation.kind === "status" + ? (operations.push("status"), { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true, sessionStateAvailable }) : { kind: "accepted", taskId: params.taskId }, } as unknown as PluginWorkerManager; const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers }); const runner = { runnerId: "runner", leaseId: oldLeaseId, sessionId: "session", runId: nextRunId, turnId: "turn", itemId: "item", harness: "codex", protocolMin: 1, protocolMax: 3 }; const checkpointIdentity = { ...priorIdentity, turnId: "prior-turn", itemId: "prior-item" }; + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, oldLeaseId)); + await db.update(environmentLeases).set({ metadata: { ...source.metadata, runnerTaskSessionRetained: false } }).where(eq(environmentLeases.id, oldLeaseId)); + await expect(authorize()).rejects.toThrow("not authorized"); + await expect(resolvePluginTaskResume(db, { companyId, leaseId: nextLeaseId, priorIdentity, inspectCheckpoint: true })).resolves.toBe("old-task"); + const inspect = () => runtime.task({ companyId, leaseId: nextLeaseId, operation: { + kind: "read_state", identity: checkpointIdentity, resumeFrom: "old-task", + } }); + await expect(inspect()).rejects.toThrow("not authorized"); + await expect(authorize()).rejects.toThrow("not authorized"); + // Discovery cannot launch, and cannot bypass the original termination receipt. + await expect(runtime.task({ companyId, leaseId: nextLeaseId, operation: { + kind: "submit", preparedLaunchId: "fixture", projectIds: [], bootstrapTicket: "fixture", runner, resumeFrom: "old-task", + } })).rejects.toThrow("resume identity mismatch"); + await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, oldLeaseId)); + await expect(inspect()).rejects.toThrow("not authorized"); + expect(operations).toEqual(["status"]); + await db.update(environmentLeases).set({ cleanupStatus: "success" }).where(eq(environmentLeases.id, oldLeaseId)); + sessionStateAvailable = true; + await expect(inspect()).resolves.toMatchObject({ kind: "state" }); + expect(operations).toEqual(["status", "status"]); + await expect(authorize()).resolves.toBe("old-task"); for (const operation of [ { kind: "read_state" as const, identity: checkpointIdentity, resumeFrom: "old-task" }, { kind: "archive_state" as const, priorIdentity: checkpointIdentity, archiveKey: "a".repeat(24), resumeFrom: "old-task" }, diff --git a/server/src/__tests__/heartbeat-plugin-environment.test.ts b/server/src/__tests__/heartbeat-plugin-environment.test.ts index 99c5abfa58b..f41e8aea8cf 100644 --- a/server/src/__tests__/heartbeat-plugin-environment.test.ts +++ b/server/src/__tests__/heartbeat-plugin-environment.test.ts @@ -93,7 +93,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { await stopDb?.(); }); - it("dispatches a plugin-only project without Core workspace files through the native heartbeat boundary", async () => { + it("dispatches a plugin-owned account per-turn without Core workspace files through the native heartbeat boundary", async () => { const companyId = randomUUID(), projectId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sourceId = randomUUID(); const environmentId = randomUUID(), pluginId = randomUUID(), pluginKey = `test.tasks.${pluginId}`; const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-task-heartbeat-")); tempRoots.push(root); @@ -112,11 +112,11 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { capabilities: ["environment.drivers.register", "workspaces.manage", "events.subscribe"], resourceLifecycle: ["project"], workspaceProvider: { modes: ["shared_workspace", "isolated_workspace", "operator_branch"], strategies: ["project_primary", "git_worktree"] }, entrypoints: { worker: "worker.js" }, - environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" } }] } }); + environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, warmTaskVersion: 1, configSchema: { type: "object" } }] } }); await db.insert(environments).values({ id: environmentId, companyId, name: "Tasks", driver: "plugin", status: "active", config: { pluginKey, driverKey: "tasks", driverConfig: {} } }); await db.insert(agents).values({ id: agentId, companyId, name: "Task agent", role: "engineer", status: "idle", - adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol" }, runtimeConfig: {}, + adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol", lifecycleMode: "warm" }, runtimeConfig: {}, defaultEnvironmentId: environmentId, permissions: {} }); await db.insert(issues).values({ id: issueId, companyId, projectId, projectWorkspaceId: sourceId, title: "Task-only dispatch", status: "in_progress", assigneeAgentId: agentId, issueNumber: 1, identifier: "TASK-1" }); @@ -129,14 +129,15 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { if (method === "environmentReleaseLease" || method === "environmentDestroyLease") return undefined; throw new Error(`Unexpected task provider call: ${method}`); }); - const workerManager = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["workspaceOperation", "environmentTask", "environmentAcquireLease", "environmentReleaseLease", "environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; + const workerManager = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["workspaceOperation", "environmentTask", "environmentAcquireLease", "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; const entered = vi.fn(() => { throw new Error("synthetic_native_boundary_reached"); }); const heartbeat = heartbeatService(db, { pluginWorkerManager: workerManager, nativeSessionBackendFactory: entered }); heartbeats.push(heartbeat); const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", contextSnapshot: { issueId, projectId } }); expect(run).not.toBeNull(); await vi.waitFor(async () => { expect((await heartbeat.getRun(run!.id))?.status).toBe("failed"); }, { timeout: 10_000 }); - expect(entered).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({ runId: run!.id, companyId, issueId }) })); + expect(entered).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({ runId: run!.id, companyId, issueId }), + session: expect.objectContaining({ lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }) })); expect((await heartbeat.getRun(run!.id))?.error).toContain("synthetic_native_boundary_reached"); await heartbeat.drainActiveRunExecutions(); expect(call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: "heartbeat-task-attempt" })); diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index d9b7dcf8e0c..8f494cc6fa7 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -48,6 +48,24 @@ const support = await getEmbeddedPostgresTestSupport(); async function hire(status = "idle") { return createAgentLifecycle(db).requestHire(companyId, { name: "Agent", status, adapterType: "process" }); } async function current(id: string) { return (await db.select().from(agents).where(eq(agents.id, id)))[0]!; } + it.each((["codex", "claude"] as const).flatMap(provider => ["direct", "setup", "approval"].map(path => [provider, path] as const)))("defaults a newly hired %s agent on a warm task environment to warm execution through %s", async (provider, path) => { + const pluginId = randomUUID(), environmentId = randomUUID(); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "warm-hire-fixture", version: "1.0.0", status: "ready", + manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true, warmTaskVersion: 1 }] } as never }); + await db.insert(environments).values({ id: environmentId, name: `Warm computer ${environmentId}`, driver: "plugin", + config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }); + const adapterConfig = provider === "claude" ? { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6", lifecycleMode: "per_turn" } + : { provider: "codex", model: "gpt-5.6-luna", lifecycleMode: "per_turn" }; + const lifecycle = createAgentLifecycle(db); + const input = { name: "Warm hire", adapterType: "paperclip_runner", adapterConfig, defaultEnvironmentId: environmentId }; + const draft = path === "setup" ? await lifecycle.requestHire(companyId, input, { setupOnly: true }) : null; + const hired = await lifecycle.requestHire(companyId, { ...input, ...(path === "approval" ? { status: "pending_approval" } : {}) }, draft ? { setupAgentId: draft.id } : undefined); + if (path === "approval") await lifecycle.approveHire(hired.id, "board"); + expect((await current(hired.id)).adapterConfig).toMatchObject({ ...adapterConfig, lifecycleMode: "warm" }); + await createAgentLifecycle(db).updateConfiguration(hired.id, { adapterConfig: { ...(await current(hired.id)).adapterConfig, lifecycleMode: "per_turn" } }); + expect((await current(hired.id)).adapterConfig.lifecycleMode).toBe("per_turn"); + }); + it("rejects an outdated configuration snapshot under the write lock", async () => { const agent = await hire(); const snapshot = await current(agent.id); diff --git a/server/src/services/agent-hiring.ts b/server/src/services/agent-hiring.ts index 00ed74c6ab3..8a08fa6052e 100644 --- a/server/src/services/agent-hiring.ts +++ b/server/src/services/agent-hiring.ts @@ -12,7 +12,7 @@ import { secretService, assertClaudeOAuthBindingInvariant } from "./secrets.js"; import { agentCredentialService } from "./agent-credentials.js"; import { agentIdentityService } from "./agent-identity.js"; import { initializePrimaryAgent } from "./primary-agent.js"; -import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; +import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise { const { runner, ...data } = input; @@ -44,6 +44,10 @@ export async function prepareAgentHire(db: Db, companyId: string, input: CreateA ? await secretService(db).normalizeAdapterConfigForPersistence(companyId, data.adapterConfig, { adapterType }) : {}; const adapterConfig = normalizePaperclipRunnerAdapterConfig(adapterType, rawAdapterConfig); + if (adapterType === "paperclip_runner" && !["openai_dot", "grok_bot", "muse"].includes(String(adapterConfig.provider)) && + (await resolveAgentRunnerTaskDriverForCompany(db, companyId, data.defaultEnvironmentId))?.warmTaskVersion === 1) { + adapterConfig.lifecycleMode = "warm"; + } await agentCredentialService(db).enforceClaudeOAuthBindingClaim(db, { companyId, consume: true, environmentId: data.defaultEnvironmentId ?? null, childAdapterConfig: adapterConfig, claudeLogin: options?.claudeLogin, diff --git a/server/src/services/agent-runner-selection.test.ts b/server/src/services/agent-runner-selection.test.ts index afc3949acdf..042c6084f45 100644 --- a/server/src/services/agent-runner-selection.test.ts +++ b/server/src/services/agent-runner-selection.test.ts @@ -12,7 +12,7 @@ vi.mock("./environment-config.js", async () => ({ })); vi.mock("./plugin-registry.js", () => ({ pluginRegistryService: () => ({ getByKey: state.pluginByKey }) })); vi.mock("@paperclipai/adapter-utils/ssh", () => ({ runSshCommand: state.ssh })); -import { AGENT_RUNNER_SSH_PLATFORM_COMMAND, agentRunnerAvailability, resolveAgentRunnerTargetForCompany, resolveNewAgentRunner, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; +import { AGENT_RUNNER_SSH_PLATFORM_COMMAND, agentRunnerAvailability, resolveAgentRunnerTaskDriverForCompany, resolveAgentRunnerTargetForCompany, resolveNewAgentRunner, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; import { buildCodexAcpConfig, resolveCodexExecutionEngine } from "@paperclipai/adapter-codex-local/server"; import { secretService } from "./secrets.js"; import * as localPlatform from "../lib/agent-runner-platform.js"; @@ -522,6 +522,24 @@ describe("Runner-only plugin environments", () => { expect(agentRunnerAvailability("gemini_local", target)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" }); expect(state.ssh).not.toHaveBeenCalled(); }); + it("carries warm task capability from the selected driver without provider gates", async () => { + state.pluginByKey.mockResolvedValue(plugin({ supportsTasks: true, warmTaskVersion: 1 })); + expect(await resolveAgentRunnerTargetForCompany({} as never, "company", "computer")) + .toEqual({ driver: "plugin", runnerTasks: true, warmTaskVersion: 1 }); + expect(await resolveAgentRunnerTaskDriverForCompany({} as never, "company", "computer")) + .toMatchObject({ driverKey: "tasks", supportsTasks: true, warmTaskVersion: 1 }); + }); + + it.each(["ssh", "sandbox"])("does not qualify a %s runner while inspecting hire defaults", async driver => { + state.getEnvironment.mockResolvedValue({ ...pluginEnvironment, driver, config: { provider: "fake" } }); + expect(await resolveAgentRunnerTaskDriverForCompany({} as never, "company", "computer")).toBeUndefined(); + expect(state.ssh).not.toHaveBeenCalled(); + expect(state.sandboxQualification).not.toHaveBeenCalled(); + expect(state.pluginByKey).not.toHaveBeenCalled(); + if (driver === "sandbox") await expect(resolveAgentRunnerTargetForCompany({} as never, "company", "computer")) + .rejects.toThrow("The fake sandbox provider cannot execute runs."); + }); + it("saves a Claude agent on the plugin environment as Paperclip Runner", async () => { const saved = await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "claude_local", defaultEnvironmentId: "computer" }); expect(saved).toMatchObject({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "claude" } }); diff --git a/server/src/services/agent-runner-selection.ts b/server/src/services/agent-runner-selection.ts index 904e3c7aae3..77ecf01cfe7 100644 --- a/server/src/services/agent-runner-selection.ts +++ b/server/src/services/agent-runner-selection.ts @@ -15,6 +15,7 @@ export type RunnerTarget = { driver: string; platform?: string; architecture?: string; libc?: "glibc" | "musl" | "unknown"; glibcVersion?: string; /** A plugin environment whose driver declares Runner task execution. It runs agents only through Paperclip Runner. */ runnerTasks?: boolean; + warmTaskVersion?: 1; }; export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi"; export interface AgentRunnerSelection { @@ -293,7 +294,10 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId); let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget() : { driver }; - if (driver === "plugin" && environment && await pluginEnvironmentRunsRunnerTasks(db, environment)) target = { driver, runnerTasks: true }; + if (driver === "plugin" && environment) { + const taskDriver = await pluginEnvironmentRunnerTaskDriver(db, environment); + if (taskDriver) target = { driver, runnerTasks: true, ...(taskDriver.warmTaskVersion === 1 ? { warmTaskVersion: 1 as const } : {}) }; + } if (driver === "sandbox" && environment) { const { readSandboxPlatformQualification } = await import("./environment-probe.js"); const qualified = await readSandboxPlatformQualification(db, environment, companyId); @@ -312,6 +316,18 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri } export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) { + const resolved = await resolveAgentEnvironmentForCompany(db, companyId, defaultEnvironmentId); + if (resolved.environment?.driver === "sandbox" && resolved.environment.config?.provider === "fake") throw unprocessable("The fake sandbox provider cannot execute runs."); + return resolved; +} + +// Hiring defaults inspect capabilities without probing or qualifying an already selected runner. +export async function resolveAgentRunnerTaskDriverForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) { + const { environment, driver } = await resolveAgentEnvironmentForCompany(db, companyId, defaultEnvironmentId); + return driver === "plugin" && environment ? pluginEnvironmentRunnerTaskDriver(db, environment) : undefined; +} + +async function resolveAgentEnvironmentForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) { const { instanceSettingsService } = await import("./instance-settings.js"); const { environmentService } = await import("./environments.js"); const settings = await instanceSettingsService(db).get(); @@ -331,7 +347,6 @@ export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId: const boundCompanyIds = await envs.listBoundCompanyIds(environment.id); if (boundCompanyIds.length > 0 && !boundCompanyIds.includes(companyId)) throw forbidden("The selected environment belongs to another company.", { code: "environment_company_mismatch" }); if (environment.status === "archived") throw unprocessable("Environment is archived."); - if (environment.driver === "sandbox" && environment.config?.provider === "fake") throw unprocessable("The fake sandbox provider cannot execute runs."); } return { environment, driver: environment?.driver ?? "local" }; } @@ -339,17 +354,17 @@ export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId: // The plugin's manifest declaration decides whether its environment runs // Runner tasks. Run admission separately verifies a ready plugin and a live // worker, so a plugin that is down still selects the Runner and fails its run. -async function pluginEnvironmentRunsRunnerTasks(db: Db, environment: Parameters[0]): Promise { +async function pluginEnvironmentRunnerTaskDriver(db: Db, environment: Parameters[0]) { let config: { pluginKey: string; driverKey: string }; try { const parsed = parseEnvironmentDriverConfig(environment); - if (parsed.driver !== "plugin") return false; + if (parsed.driver !== "plugin") return undefined; config = parsed.config; } catch { - return false; + return undefined; } const { pluginRegistryService } = await import("./plugin-registry.js"); const plugin = await pluginRegistryService(db).getByKey(config.pluginKey); - return Boolean(plugin?.manifestJson.capabilities.includes("environment.drivers.register") - && plugin.manifestJson.environmentDrivers?.some(driver => driver.driverKey === config.driverKey && driver.supportsTasks === true)); + return plugin?.manifestJson.capabilities.includes("environment.drivers.register") + ? plugin.manifestJson.environmentDrivers?.find(driver => driver.driverKey === config.driverKey && driver.supportsTasks === true) : undefined; } diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index caa024c79a4..065108ee6b8 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -273,6 +273,7 @@ export function environmentRunOrchestrator( adapterType: string; adapterConfig?: Record; admittedLifecycleMode?: "warm" | "per_turn"; + managedAiCredentialIdentity?: string; issueId: string | null; heartbeatRunId: string; agentId: string; @@ -292,7 +293,7 @@ export function environmentRunOrchestrator( }); const environment = resolveRunnerEnvironmentForRun( - selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, + selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, input.managedAiCredentialIdentity, ); if (input.recoverPluginTask && environment.driver === "plugin" && input.adapterType === "paperclip_runner") { diff --git a/server/src/services/environment-task-runtime.ts b/server/src/services/environment-task-runtime.ts index e58491b01fa..020cbca36d6 100644 --- a/server/src/services/environment-task-runtime.ts +++ b/server/src/services/environment-task-runtime.ts @@ -85,10 +85,11 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage throw new Error("Plugin-provided Runner execution identity mismatch"); } if (runner && resumeFrom) { - const sources = await db.select({ runId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }).from(environmentLeases).where(and( + const inspectCheckpoint = operation.kind === "read_state"; + const sources = await db.select({ id: environmentLeases.id, runId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }).from(environmentLeases).where(and( eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.provider, lease.provider!), eq(environmentLeases.providerLeaseId, resumeFrom), - sql`${environmentLeases.metadata}->>'runnerTaskSessionRetained' = 'true'`, + inspectCheckpoint ? undefined : sql`${environmentLeases.metadata}->>'runnerTaskSessionRetained' = 'true'`, checkpointIdentity ? eq(environmentLeases.heartbeatRunId, checkpointIdentity.runId) : undefined, )).limit(2); const priorRunId = sources.length === 1 ? sources[0]!.runId : null; @@ -98,7 +99,20 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage companyId: input.companyId, leaseId: lease.id, priorIdentity: { runnerInstanceId: runner.runnerId, environmentLeaseId: runner.leaseId, runId: priorRunId, normalizedSessionId: runner.sessionId }, + inspectCheckpoint, })) throw new Error("Plugin Runner resume identity mismatch"); + // Older providers misclassified suspended checkpoints. A fresh provider + // attestation may repair availability, but never supplies stop/owner authority. + if (inspectCheckpoint && sources[0]!.metadata?.runnerTaskSessionRetained !== true) { + const status = await executeEnvironmentTask(db, workers, { companyId: input.companyId, + leaseId: sources[0]!.id, operation: { kind: "status" } }); + if (status.kind !== "status" || status.executionStopped !== true || status.sessionStateAvailable !== true || + resumeFrom !== await resolvePluginTaskResume(db, { companyId: input.companyId, leaseId: lease.id, + priorIdentity: { runnerInstanceId: runner.runnerId, environmentLeaseId: runner.leaseId, + runId: priorRunId, normalizedSessionId: runner.sessionId } })) { + throw new Error("Plugin Runner retained session is not authorized"); + } + } } if (metadata.driver !== "plugin" || typeof metadata.pluginId !== "string" || typeof metadata.driverKey !== "string") { throw new Error("Plugin-provided Runner execution lease has no pinned plugin driver"); @@ -380,6 +394,8 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor export async function resolvePluginTaskResume(db: Db, input: { companyId: string; leaseId: string; priorIdentity: { runnerInstanceId: string; environmentLeaseId: string; runId: string; normalizedSessionId: string }; + /** Read-only discovery; launching still requires an attested retained checkpoint. */ + inspectCheckpoint?: boolean; }): Promise { const rows = await db.select({ lease: environmentLeases, run: heartbeatRuns }).from(environmentLeases) .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId), eq(heartbeatRuns.companyId, input.companyId))) @@ -389,7 +405,8 @@ export async function resolvePluginTaskResume(db: Db, input: { // A retained PRP lease is logical; later provider tasks have different physical lease IDs. const candidates = rows.filter(row => { const identity = row.lease.metadata?.runnerTaskIdentity as Record | undefined; - return row.lease.id !== input.leaseId && row.run.id === input.priorIdentity.runId && row.lease.metadata?.runnerTaskSessionRetained === true && + return row.lease.id !== input.leaseId && row.run.id === input.priorIdentity.runId && + (input.inspectCheckpoint || row.lease.metadata?.runnerTaskSessionRetained === true) && identity?.runnerId === input.priorIdentity.runnerInstanceId && identity.leaseId === input.priorIdentity.environmentLeaseId && identity.sessionId === input.priorIdentity.normalizedSessionId && identity.runId === input.priorIdentity.runId && !rows.some(successor => successor.lease.id !== input.leaseId && diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 72a79b53199..59d2bd04a61 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2713,6 +2713,7 @@ export function heartbeatService( mergedConfig, configSnapshot, secretManifest, + managedAiCredentialIdentity: managedAiRuntime?.identity, }, reuse: { requestedShouldReuseExisting, diff --git a/server/src/services/heartbeat/runtime-selection.test.ts b/server/src/services/heartbeat/runtime-selection.test.ts index f3ce34c7759..737f69beffa 100644 --- a/server/src/services/heartbeat/runtime-selection.test.ts +++ b/server/src/services/heartbeat/runtime-selection.test.ts @@ -311,6 +311,26 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = expect(await coordinators(run.id)).toHaveLength(1); }); + it.each(["api_key", "subscription"] as const)("keeps managed Claude %s task execution warm", async method => { + const { input, run, issue } = await fixture(); + const adapterConfig = { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-4-6", lifecycleMode: "warm" }; + input.agent = { ...input.agent, adapterConfig }; + await db.update(agents).set({ adapterConfig }).where(eq(agents.id, input.agent.id)); + input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: input.run, enabled: true, runtimeConfig: {}, adapterConfig, + agent: input.agent, issue, target: null, workspaceId: null }); + input.workspace.executionTarget = { ...sandbox, runnerTask: vi.fn(), reusableLeaseConfigured: true, + effectiveCapabilities: { reusableLeases: true } as never, runnerLifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 } }; + const attribution = { connectionId: randomUUID(), grantId: randomUUID(), provider: "anthropic" as const, method, + mode: "responsible_user" as const, responsibleUserId: "task-owner" }; + input.config.managedAiRuntime = { sessionIdentity: "account:epoch-1", identity: "account:generation-1", + config: { env: {}, managedAiConnection: { ...attribution, identity: "account:generation-1", sessionIdentity: "account:epoch-1" } }, + attribution, accountName: "Saved account", accountOwnerUserId: "task-owner", home: instructionRoot, cleanup: async () => {} }; + const selected = await selectHeartbeatRuntime(db, input); + if (!selected.selected || !selected.nativeExecution) throw new Error("Expected native Claude selection"); + expect(selected.nativeExecution.session.lifecyclePolicy).toEqual({ mode: "warm", idleTimeoutMs: 300_000 }); + expect((await read(run.id)).runnerProfileJson).toMatchObject({ nativeExecutionInput: { session: { lifecyclePolicy: { mode: "warm" } } } }); + }); + it("preserves only server-owned admission and dispatch evidence when recording legacy selection", async () => { const { input, run } = await fixture("process"); const admission = { version: 1, kind: "fixture" }, dispatch = { entered: true }; diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index 05a7b039a2f..032f3875580 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -506,11 +506,10 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele executionTarget.transport === "sandbox" ? (executionTarget.runnerLifecyclePolicy ?? null) : null; - // Codex refreshes its durable auth home; Copilot owns a fenced home - // and an explicitly bound token. Both retire on credential generation - // changes. Other managed harnesses require per-turn cleanup. + // Task providers own credential lifetime; process-backed managed homes retain their provider-specific cleanup policy. const supportsManagedWarmSession = agent.adapterType === "paperclip_runner" && - supportsManagedNativeWarmSession({ ...nativeRuntimeResolution.profile, acpxAgent: parseObject(agent.adapterConfig).acpxAgent }, managedAiRuntime?.attribution); + ((executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" && !!executionTarget.runnerTask) || + supportsManagedNativeWarmSession({ ...nativeRuntimeResolution.profile, acpxAgent: parseObject(agent.adapterConfig).acpxAgent }, managedAiRuntime?.attribution)); const effectiveLifecyclePolicy = persistedNativeExecutionInput?.session.lifecyclePolicy ?? (["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession ? { mode: "per_turn" as const, idleTimeoutMs: null } diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index eb0f0e2abc3..d3c9944abd5 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -456,10 +456,12 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", it("persists the workspace and issue binding before acquiring its environment", async () => { const { input, source, issue, envOrchestrator } = await fixture(); + input.config.managedAiCredentialIdentity = "account:generation"; const acquire = envOrchestrator.acquireForRun; vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => { expect((await readIssue(issue.id)).executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id); expect((await readRun(input.run.id)).contextSnapshot?.executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id); + expect(args.managedAiCredentialIdentity).toBe("account:generation"); return acquire(args); }); const result = await prepareHeartbeatWorkspace(db, input); diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index f56bde29125..d81b24cd919 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -129,6 +129,7 @@ export interface HeartbeatWorkspacePreparationInput { mergedConfig: Record; configSnapshot: WorkspaceMetadataInput["configSnapshot"]; secretManifest: WorkspaceMetadataInput["secretManifest"]; + managedAiCredentialIdentity?: string; }; reuse: { requestedShouldReuseExisting: boolean; @@ -1121,6 +1122,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace adapterType: agent.adapterType, adapterConfig: parseObject(agent.adapterConfig), admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode, + managedAiCredentialIdentity: input.config.managedAiCredentialIdentity, runnerTaskProjectIds:[...new Set([executionWorkspace.projectId,...resolvedWorkspace.runnerTaskProjectIds ?? [], ...(executionWorkspace.additionalWorkspaces ?? []).map(workspace=>workspace.projectId)] .filter((id):id is string=>Boolean(id)))], diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index de8105859b1..30d8792a64d 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -11128,13 +11128,15 @@ describe("runnerd provider runtime wiring", () => { await rm(isolatedStateDirectory, { recursive: true, force: true }); }); - it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => { + it.each((["codex", "claude"] as const).flatMap(provider => (["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const).filter(scenario => provider === "codex" || scenario.startsWith("plugin-")).map(scenario => [provider, scenario] as const)))("rotates an owned warm %s remote runner before checking changed configuration: %s", async (provider, scenario) => { const plugin = scenario === "plugin-replaced" || scenario === "plugin-managed-github"; const reusable = scenario === "unchanged" || scenario === "plugin-managed-github"; - const first = { ...scopedNativeExecution(`warm-admission-${scenario}`), + const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude; + const first = { ...scopedNativeExecution(`warm-admission-${provider}-${scenario}`), schema: "paperclip.native-execution-input.v5" as const, - provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const }, + provider: provider === "claude" ? { kind: "acpx" as const, agent: "claude" as const, model: "claude-sonnet-4-6", permissionMode: "approve-all" as const, profile: claudeProfile } : { kind: "codex" as const, model: null, approvalPolicy: "never" as const }, executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() }; + if (provider === "claude") first.session.driverKind = "acpx_runtime"; first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 }; const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` }, session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner", "plugin-replaced", "plugin-managed-github"].includes(scenario) ? 60_000 : 90_000 } } }; @@ -11161,7 +11163,7 @@ describe("runnerd provider runtime wiring", () => { if(plugin) { await mkdir(home,{recursive:true});await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600}); } - const pluginOptions=plugin ? {runnerIngressAuthorized:true,managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home, + const pluginOptions=plugin ? {runnerIngressAuthorized:true,runnerEnvironment:provider === "claude" ? {ANTHROPIC_API_KEY:"synthetic-fixture"} : {},managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home, managedAiCredentialIdentity:"revision",managedAiAccountIdentity:"account"} : {}; const resume=scenario === "plugin-replaced" ? vi.spyOn(environmentTaskRuntime,"resolvePluginTaskResume").mockResolvedValue("task-first") : undefined; const createOwner = pluginTaskRuntime.createPluginRunnerTaskOwner; @@ -11862,6 +11864,103 @@ describe("runnerd provider runtime wiring", () => { } }); + it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))( + "recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => { + const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude; + const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`), + schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, runtimeContext: nativeRuntimeContextFixture(), + ...(provider === "claude" ? { provider: { kind: "acpx", agent: "claude", model: "claude-sonnet-4-6", permissionMode: "approve-all", profile: claudeProfile } } : { provider: { kind: "codex", model: null, approvalPolicy: "never" } }), + session: { ...execution.session, driverKind: provider === "claude" ? "acpx_runtime" : "codex_app_server", + normalizedSessionId: `plugin-${provider}-${policy}-${mode}`, lifecyclePolicy: policy === "warm" ? { mode: "warm", idleTimeoutMs: 60_000 } : { mode: "per_turn", idleTimeoutMs: null } }, + } as NativeExecutionInput; + parseNativeExecutionInput(first); + const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-reopened` } }; + const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId!, + runnerInstanceId: "plugin-runner", environmentLeaseId: "old-lease", turnId: "old-turn", itemId: "old-item" }; + const scoped = await initializeScopedRunnerRoot(first, identity.runnerInstanceId); + await mkdir(join(scoped, "control-plane"), { recursive: true }); + const journal = JSON.stringify({ ...durableControlPlaneState(identity), commands: [{ status: mode.endsWith("pending-command") ? "pending" : "completed" }], + committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }] }); + await writeFile(join(scoped, "control-plane/control-plane-state.json"), journal); + if (mode.startsWith("empty-runner") || mode === "nonempty-runner") { + await mkdir(join(scoped, "runner")); + if (mode === "nonempty-runner") await writeFile(join(scoped, "runner", "unknown.json"), "{}"); + } + if (mode.startsWith("local-receipt")) await writeFile(join(scoped, "plugin-runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended"))); + let root = scoped; + if (["quarantined", "empty-runner-quarantined", "ambiguous"].includes(mode)) { + await mkdir(join(isolatedStateDirectory, "quarantine"), { recursive: true }); + root = join(isolatedStateDirectory, "quarantine", `${scoped.split("/").at(-1)}.identity_indeterminate.1`); + await rename(scoped, root); + if (mode === "ambiguous") await cp(root, `${root}-second`, { recursive: true }); + } + if (mode === "contradictory") { + await mkdir(join(root, "runner")); + await writeFile(join(root, "runner/runner-state.json"), "{"); + } + const db = leaseDb(next); + const ordinarySelect = db.select.bind(db); + db.select = ((fields?: Record) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson + ? priorNativeRunDb(mode === "foreign-scope" ? { ...first, binding: { ...first.binding, agentId: "foreign" } } : first, + mode === "active" ? "running" : "succeeded").select(fields as never) + : ordinarySelect(fields as never)) as Db["select"]; + const throughExecutor = policy === "warm" && ["canonical", "local-receipt", "pending-outbox"].includes(mode); + let inspected = false; + const resume = vi.spyOn(environmentTaskRuntime, "resolvePluginTaskResume").mockImplementation(async (_db, request) => { + if (request.inspectCheckpoint) inspected = true; + else if (throughExecutor && !inspected) throw new Error("stale negative checkpoint availability"); + if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized"); + return "old-task"; + }); + const task = vi.fn(async (operation: any) => { + environmentTaskOperationSchema.parse(operation); + if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: { + ...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"), + outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null, + ...(mode === "wrong-identity" ? { itemId: "foreign" } : {}), + } }; + return { kind: "description", taskId: "replacement-task", launchPreparationVersion: 1, projectAuthCheckVersion: 1, warmTaskVersion: 1, + runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; + }); + const owner = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner"); + state.createBackend.mockClear(); + const home = join(isolatedStateDirectory, "checkpoint-account"); + await mkdir(home, { recursive: true }); + await writeFile(join(home, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "synthetic-fixture" }), { mode: 0o600 }); + try { + const launch = { db, execution: next, runnerInstanceId: "replacement-runner", runnerIngressAuthorized: true, + managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account", + runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task, + sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never }; + if (throughExecutor) { + const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done", + completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] }, + evidence: [], verification: [], artifacts: [], attentionRequests: [] }); + if (!validated.ok) throw new Error("Invalid completion fixture"); + state.execute.mockResolvedValueOnce({ result: validated.result, terminal: { runTerminalState: "succeeded" }, turnId: "turn", + normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind, + driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }); + } + const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch); + if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) { + await expect(backend).resolves.toBeDefined(); + expect(state.createBackend).toHaveBeenCalledOnce(); + expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity })); + expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true })); + expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" }); + expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); + } else { + await expect(backend).rejects.toThrow(); + expect(state.createBackend).not.toHaveBeenCalled(); + if (["local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) { + expect(task).not.toHaveBeenCalled(); + } + if (mode === "ambiguous") expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal); + } + } finally { owner.mockRestore(); resume.mockRestore(); } + }); + it.each(["verified", "wrong-state", "no-stop-proof", "disposition-retry", "disposition-wrong-state", "disposition-no-stop-proof"] as const)("admits only an authorized same-run plugin checkpoint: %s", async mode => { const current = scopedNativeExecution(`plugin-crash-${mode}`); const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!, @@ -11969,10 +12068,11 @@ describe("runnerd provider runtime wiring", () => { } finally { owner.mockRestore(); } }); - it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx"] as const)("negotiates task-only warm execution: %s", async mode => { + it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx", "installed-codex", "installed-claude"] as const)("negotiates task-only execution: %s", async mode => { const current = scopedNativeExecution(`plugin-warm-${mode}`); - current.session.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000}; - if(mode==="acpx") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never; + const installed = mode.startsWith("installed-"); + current.session.lifecyclePolicy = installed ? {mode:"per_turn",idleTimeoutMs:null} : {mode:"warm",idleTimeoutMs:300000}; + if(mode==="acpx" || mode==="installed-claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never; const home=await mkdtemp(join(tmpdir(),"plugin-warm-account-")); await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600}); const owner=vi.spyOn(pluginTaskRuntime,"createPluginRunnerTaskOwner"); @@ -11983,20 +12083,22 @@ describe("runnerd provider runtime wiring", () => { state.createBackend.mockClear();state.createTransport.mockClear(); try { const backend=createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"runner",runnerIngressAuthorized:true, - managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" ? undefined : "revision",managedAiAccountIdentity:"account", + runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" || installed ? undefined : "revision",managedAiAccountIdentity:installed ? undefined : "account", runnerExecutionTarget:{kind:"remote",transport:"sandbox",leaseId:"lease",remoteCwd:".",runnerTask:task, ...(mode==="fresh" ? {sandboxLeaseAcquisition:{outcome:"resumed",providerLeaseId:"old-task"}} : {})}}); - if(mode!=="codex" && mode!=="fresh") {await expect(backend).rejects.toThrow(mode==="acpx" ? "Codex workspace" : "capability or account");return;} + if(mode==="missing-capability" || mode==="missing-account") {await expect(backend).rejects.toThrow("capability or account");return;} await backend; if(mode==="fresh") expect(task.mock.calls.map(([operation])=>operation.kind)).toEqual(["stop","status","complete","describe"]); state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); - expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:{mode:"warm",idleTimeoutMs:300000}}); - expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,accountIdentity:"account",onCredentials:undefined}); + expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:current.session.lifecyclePolicy}); + expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,onCredentials:undefined}); + expect(owner.mock.calls.at(-1)![0].accountIdentity).toBe(installed ? undefined : "account"); } finally {owner.mockRestore();await rm(home,{recursive:true,force:true});} }); - it.each([true,false])("reconnects task-only warm recovery under the saved identity before reuse or retirement: %s", async reuseProcess => { - const current=scopedNativeExecution(`plugin-restart-${reuseProcess}`); + it.each((["codex", "claude"] as const).flatMap(provider => [true, false].map(reuseProcess => [provider, reuseProcess] as const)))("reconnects task-only %s warm recovery under the saved identity before reuse or retirement: %s", async (provider, reuseProcess) => { + const current=scopedNativeExecution(`plugin-restart-${provider}-${reuseProcess}`); + if(provider === "claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never; current.session.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000}; const identity={runnerInstanceId:"old-runner",environmentLeaseId:"old-lease",runId:"old-run", normalizedSessionId:current.session.normalizedSessionId!,turnId:"old-turn",itemId:"old-item"}; @@ -12008,7 +12110,7 @@ describe("runnerd provider runtime wiring", () => { state.createBackend.mockClear();state.createTransport.mockClear(); try { await createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"new-runner",runnerIngressAuthorized:true, - managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account", + runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account", managedAiCredentialWriteback:{} as never, remoteWarmRecovery:{identity,reuseProcess,previousExecution:current,checkpoint:{} as never, remote:{providerLeaseId:"saved-task",remoteCwd:"."},revalidate:async()=>{}}, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 4817a631bf9..15f54f4a5bb 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -2062,29 +2062,34 @@ function savePluginRunnerCheckpoint(root: string, state: Record renameSync(temporary, path); } +function pluginRunnerdAuthorityLifecycle( + root: string, + identity: RunnerdDurableIdentity, +): "absent" | "suspended" | "not_suspended" | "indeterminate" { + try { + const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"), + NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8"))); + if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle)) + || !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate"; + return state.lifecycle === "suspended" ? "suspended" : "not_suspended"; + } catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; } +} + function runnerdAuthorityLifecycle( root: string, identity: RunnerdDurableIdentity, ): "absent" | "suspended" | "not_suspended" | "indeterminate" { const runnerRoot = resolve(root, "runner"); - if (!existsSync(runnerRoot)) { - try { - const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"), - NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8"))); - if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle)) - || !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate"; - return state.lifecycle === "suspended" ? "suspended" : "not_suspended"; - } catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; } - } + if (!existsSync(runnerRoot)) return pluginRunnerdAuthorityLifecycle(root, identity); if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate"; const statePath = resolve(runnerRoot, "runner-state.json"); if (!existsSync(statePath)) { try { // Older remote transports created this controller-side placeholder even // though runner state was owned by the sandbox. It carries no authority, - // so a verified failover backup may be consulted. Any non-empty direct + // so a verified plugin receipt or failover backup may be consulted. Any non-empty direct // directory remains indeterminate and therefore blocks fallback. - return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate"; + return readdirSync(runnerRoot).length === 0 ? pluginRunnerdAuthorityLifecycle(root, identity) : "indeterminate"; } catch { return "indeterminate"; } @@ -4894,6 +4899,72 @@ async function verifyWarmTransitionRestart(input: { return verified; } +async function recoverPluginRunnerCheckpoint(input: { + db: Db; + execution: NativeExecutionInput; + runnerExecutionTarget?: AdapterExecutionTarget | null; +}, scoped: string): Promise { + const target = input.runnerExecutionTarget; + if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask || !target.leaseId || + target.sandboxLeaseAcquisition?.outcome === "resumed") return; + let root = scoped; + const quarantined = !existsSync(scoped); + const deny = (): never => { throw new Error("runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven"); }; + if (quarantined) { + const quarantine = resolve(runnerdStateBase(), "quarantine"); + if (!existsSync(quarantine)) return; + if (!isSafeNativeStateDirectory(quarantine)) return deny(); + const candidates = readdirSync(quarantine).filter(name => name.startsWith(`${basename(scoped)}.identity_indeterminate.`)); + if (candidates.length === 0) return; + if (candidates.length !== 1) return deny(); + root = resolve(quarantine, candidates[0]!); + } + if (!existsSync(root)) return; + if (!isSafeNativeStateDirectory(root)) return deny(); + if (hasRetainedWarmTransitionEvidence(root)) { + if (quarantined) return deny(); + return; + } + const identity = readRunnerdDurableIdentity(root); + if (!durableIdentityMatchesSession(identity, input.execution) || identity.runId === input.execution.binding.runId) { + if (quarantined) return deny(); + return; + } + const direct = runnerdAuthorityLifecycle(root, identity); + const verifiedPluginReceipt = direct === "suspended" && + !existsSync(resolve(root, "runner", "runner-state.json")) && pluginRunnerdAuthorityLifecycle(root, identity) === "suspended"; + // Never hide contradictory direct authority behind a remote checkpoint. + if (direct !== "absent" && !(quarantined && direct === "suspended") && !verifiedPluginReceipt) { + if (quarantined) return deny(); + return; + } + await assertCleanupActivationCommitted(input.db, root, input.execution); + const prior = await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }); + if (prior !== "terminal_state_indeterminate" && !((quarantined || verifiedPluginReceipt) && prior === "verified")) { + if (quarantined) return deny(); + return; + } + const control = record(JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"), + NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8"))); + const terminal = record(Array.isArray(control.committedEvents) ? control.committedEvents.at(-1) : null); + if (!Array.isArray(control.commands) || control.commands.some(command => record(command).status !== "completed") || + terminal.eventType !== "run.terminal" || record(terminal.envelope).runId !== identity.runId || + record(terminal.envelope).normalizedSessionId !== identity.normalizedSessionId) return deny(); + const resumeFrom = await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId, + leaseId: target.leaseId, priorIdentity: identity, inspectCheckpoint: true }); + const checkpoint = environmentTaskResultSchema.parse(await target.runnerTask({ kind: "read_state", identity, resumeFrom })); + if (checkpoint.kind !== "state" || checkpoint.state.lifecycle !== "suspended" || + !Array.isArray(checkpoint.state.outbox) || checkpoint.state.outbox.length !== 0 || + checkpoint.state.pendingTerminalDelivery != null || checkpoint.state.pendingProviderCleanup != null || + checkpoint.state.warmTransition != null) return deny(); + savePluginRunnerCheckpoint(root, checkpoint.state, identity); + if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny(); + if (quarantined) { + if (existsSync(scoped)) return deny(); + renameSync(root, scoped); + } +} + async function migrateRunnerdStateRootForExecution(input: { db: Db; execution: NativeExecutionInput; @@ -4913,6 +4984,9 @@ async function migrateRunnerdStateRootForExecution(input: { issueId: input.execution.binding.issueId, stateKey: basename(scoped), }); + if (!input.restartRecovery && !input.retainedWarmOwnerRunId) { + await recoverPluginRunnerCheckpoint(input, scoped); + } if ( input.allowLocalRecovery && !input.retainedWarmOwnerRunId && @@ -8000,10 +8074,7 @@ async function executePaperclipNativeSessionWithinScope( // Inspection can replace a stopped retained task before deciding between adoption and checkpoint recovery. await taskTarget.runnerTask({kind:"describe"}); if (taskTarget.sandboxLeaseAcquisition?.outcome !== "resumed") { - const resumeFrom=await resolvePluginTaskResume(input.db,{companyId:input.execution.binding.companyId,leaseId:taskTarget.leaseId,priorIdentity:identity}); - const checkpoint=environmentTaskResultSchema.parse(await taskTarget.runnerTask({kind:"read_state",identity,resumeFrom})); - if(checkpoint.kind!=="state") throw new Error("Plugin Runner checkpoint unavailable"); - savePluginRunnerCheckpoint(root,checkpoint.state,identity); + await recoverPluginRunnerCheckpoint(input, root); } } if (durableIdentityMatchesSession(identity, input.execution) && identity.runId !== input.execution.binding.runId && @@ -11802,8 +11873,8 @@ async function createRunnerdBackendWithinSessionClaim( const remoteTarget = target.kind === "remote" ? target : null; const pluginTask = remoteTarget?.transport === "sandbox" ? remoteTarget.runnerTask : undefined; if (pluginTask && input.runnerIngressAuthorized !== true) throw new Error("runner_ingress_unavailable"); - if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8" || (input.execution.session.lifecyclePolicy.mode === "warm" && input.execution.provider.kind !== "codex"))) { - throw new Error("Plugin warm Runner tasks require a Codex workspace session"); + if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8")) { + throw new Error("Plugin Runner tasks require a workspace session"); } const remoteCommandRunner = remoteTarget ? remoteTarget.transport === "ssh" @@ -11992,7 +12063,7 @@ async function createRunnerdBackendWithinSessionClaim( .where(and(eq(environmentLeases.id, remoteTarget.leaseId!), eq(environmentLeases.companyId, input.execution.binding.companyId))); }, onDispatched: input.onTaskDispatched, }; - if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin Codex warm task capability or account unavailable"); + if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin warm task capability or account unavailable"); const owner = retainedOwner ?? createPluginRunnerTaskOwner(ownerInput); if (retainedOwner) retainedOwner.rebind(ownerInput); if (description.workspaceFileReadVersion === 1) { diff --git a/server/src/services/native-runtime/plugin-runner-task.test.ts b/server/src/services/native-runtime/plugin-runner-task.test.ts index 27f7f4f5364..a1f4bae2d98 100644 --- a/server/src/services/native-runtime/plugin-runner-task.test.ts +++ b/server/src/services/native-runtime/plugin-runner-task.test.ts @@ -23,8 +23,11 @@ const requirements: RunnerProviderLaunchRequirements = { settings: {}, approvalPolicy: "never", readOnly: false, preparedConversation: false, includeCollaborationModeInstructions: false, }; -async function prepare(owner: ReturnType) { - await owner.driver.prepareLaunch!(requirements); +function launchRequirements(harness: string): RunnerProviderLaunchRequirements { + return harness === "claude" ? { ...requirements, provider: "acpx", settings: { acpxAgent: "claude" } } : requirements; +} +async function prepare(owner: ReturnType, current = requirements) { + await owner.driver.prepareLaunch!(current); } function provider() { let stopped = false; @@ -323,14 +326,14 @@ describe("plugin task workspace file transfer", () => { }); }); -it("reuses a warm Codex process across run authorities without idle status polling", async () => { +it.each(["codex", "claude"])("reuses a warm %s process across run authorities without idle status polling", async harness => { const {task} = provider(); ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}}); const retained = vi.fn(async()=>{}); - const input = {task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const, + const input = {task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const, accountIdentity:"account",credentialIdentity:"revision",pollMs:1,onSessionRetained:retained}; const owner = createPluginRunnerTaskOwner(input); - await prepare(owner); + await prepare(owner, launchRequirements(harness)); const registration = await owner.registration({} as never,"run"); const spec = launchSpec(); spec.nativeRunner!.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000}; const handle = owner.launcher(spec); registration.activate(); await registration.ready(); @@ -345,7 +348,7 @@ it("reuses a warm Codex process across run authorities without idle status polli owner.rebind({...input,task:nextTask}); const nextRegistration=await owner.registration({} as never,"next-run"); const nextIdentity={...requirements.identity,runId:"next-run",turnId:"next-turn",itemId:"next-item"}; - await owner.driver.prepareLaunch!({...requirements,identity:nextIdentity}); + await owner.driver.prepareLaunch!({...launchRequirements(harness),identity:nextIdentity}); expect(nextTask.mock.calls[0][0]).toMatchObject({kind:"prepare",priorIdentity:requirements.identity,accountIdentity:"account"}); expect(nextTask.mock.calls.map(([op])=>op.kind)).toEqual(["prepare","check_project_auth"]); await registration.release(); nextRegistration.activate(); await nextRegistration.ready(); @@ -355,13 +358,13 @@ it("reuses a warm Codex process across run authorities without idle status polli expect(nextTask.mock.calls.filter(([op])=>op.kind==="stop")).toHaveLength(1); }); -it("adopts a verified retained Codex task without submitting a replacement", async () => { +it.each(["codex", "claude"])("adopts a verified retained %s task without submitting a replacement", async harness => { const {task}=provider(); ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}}); - const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness:"codex",projectAuthCheckVersion:1, + const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness,projectAuthCheckVersion:1, warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity,pollMs:1}); const recovered = {configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}; - await expect(owner.driver.prepareLaunch!(requirements,recovered)).resolves.toEqual(recovered); + await expect(owner.driver.prepareLaunch!(launchRequirements(harness),recovered)).resolves.toEqual(recovered); expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]); expect(task.mock.calls[0][0]).toMatchObject({configurationId:"saved",projectIds:[]}); expect(await owner.driver.adoptExistingRunner!.isAlive()).toBe(true); @@ -372,28 +375,30 @@ it("adopts a verified retained Codex task without submitting a replacement", asy expect(task.mock.calls.some(([op])=>op.kind==="complete")).toBe(true); }); -it("denies retained process adoption when current project authentication is revoked",async()=>{ +it.each(["codex", "claude"])("denies retained %s process adoption when current project authentication is revoked",async harness=>{ const {task}=provider(); task.mockImplementation(async op=>op.kind === "check_project_auth" ? {kind:"project_auth",taskId:"task",configurationId:op.configurationId,provider:op.provider,allowed:false} as never : {kind:"accepted",taskId:"task"} as never); - const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness:"codex",projectAuthCheckVersion:1, + const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness,projectAuthCheckVersion:1, warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity}); - await expect(owner.driver.prepareLaunch!(requirements,{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}})) + await expect(owner.driver.prepareLaunch!(launchRequirements(harness),{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}})) .rejects.toThrow("project authentication conflicts"); expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]); }); -it("refuses warm account or credential generation changes", async()=>{ +it.each(["codex", "claude"])("refuses warm %s account, harness, project, or credential generation changes", async harness=>{ const {task}=provider(); - const input={task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const, + const input={task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const, accountIdentity:"account",credentialIdentity:"revision",pollMs:1}; const owner=createPluginRunnerTaskOwner(input); - await prepare(owner); + await prepare(owner, launchRequirements(harness)); const spec=launchSpec();spec.nativeRunner!.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000}; const process=owner.launcher(spec); expect(()=>owner.rebind({...input,accountIdentity:"other"})).toThrow("cannot be rebound"); expect(()=>owner.rebind({...input,credentialIdentity:"rotated"})).toThrow("cannot be rebound"); + expect(()=>owner.rebind({...input,harness:harness === "codex" ? "claude" : "codex"})).toThrow("cannot be rebound"); + expect(()=>owner.rebind({...input,projectIds:["foreign"]})).toThrow("cannot be rebound"); const registration=await owner.registration({} as never,"run"); await registration.checkpoint("unsettled"); await process.completion; }); diff --git a/server/src/services/native-runtime/plugin-runner-task.ts b/server/src/services/native-runtime/plugin-runner-task.ts index 8cbb01a9f4c..ca5e6f87cde 100644 --- a/server/src/services/native-runtime/plugin-runner-task.ts +++ b/server/src/services/native-runtime/plugin-runner-task.ts @@ -128,7 +128,7 @@ export function createPluginRunnerTaskOwner(input: { if (launched || released) throw new Error("Plugin Runner replacement requires a new task lease"); const native = spec.nativeRunner; if (!preparedLaunchId) throw new Error("Plugin Runner launch requires preparation"); - if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.harness === "codex" && input.accountIdentity && input.warmTaskVersion === 1)) + if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.accountIdentity && input.warmTaskVersion === 1)) || !preparedIdentity || Object.entries(preparedIdentity).some(([key, value]) => native.identity[key as keyof DurableRecoveryIdentity] !== value)) { throw new Error("Plugin Runner requires a structured per-turn native launch"); } @@ -295,7 +295,7 @@ export function createPluginRunnerTaskOwner(input: { }; return { launcher, registration, driver, readWorkspaceFile, rebind(next: typeof input) { - if (!launched || stopRequested || stoppedState || detached || next.harness !== "codex" || + if (!launched || stopRequested || stoppedState || detached || next.harness !== input.harness || input.accountIdentity !== next.accountIdentity || input.credentialIdentity !== next.credentialIdentity || JSON.stringify(input.projectIds) !== JSON.stringify(next.projectIds)) throw new Error("Plugin warm task cannot be rebound"); input = next; diff --git a/server/src/services/runner-environment-lifecycle.test.ts b/server/src/services/runner-environment-lifecycle.test.ts index d6aaeb0d9bf..41aaac73124 100644 --- a/server/src/services/runner-environment-lifecycle.test.ts +++ b/server/src/services/runner-environment-lifecycle.test.ts @@ -1,8 +1,20 @@ import {expect,it} from "vitest"; import {resolveRunnerEnvironmentForRun} from "./runner-environment-lifecycle.js"; -it("opts Codex plugin tasks into warm retention without opting ACPX in",()=>{ + +it.each([undefined,"codex","acpx","opencode"])("defaults plugin Runner execution to warm without provider gates: %s",provider=>{ const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{}}}; - expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider:"codex",lifecycleMode:"warm",idleTimeoutMs:1234})).toMatchObject({config:{runnerLifecycleMode:"warm",runnerIdleTimeoutMs:1234}}); - expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider:"acpx",lifecycleMode:"warm"})).toMatchObject({config:{runnerLifecycleMode:"per_turn"}}); + expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{provider,idleTimeoutMs:1234},undefined,"selected-account:generation")).toMatchObject({config:{runnerLifecycleMode:"warm",runnerIdleTimeoutMs:1234}}); expect(environment.config).not.toHaveProperty("runnerLifecycleMode"); }); +it("keeps plugin-owned credentials per-turn without changing admitted recovery",()=>{ + const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{},runnerLifecycleMode:"warm"}}; + expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{lifecycleMode:"warm"})).toMatchObject({config:{runnerLifecycleMode:"per_turn"}}); + expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{},"warm")).toMatchObject({config:{runnerLifecycleMode:"warm"}}); +}); +it("preserves explicit per-turn settings and the admitted recovery policy",()=>{ + const environment={driver:"plugin" as const,config:{pluginKey:"fixture",driverKey:"tasks",driverConfig:{}}}; + expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{lifecycleMode:"per_turn"},undefined,"account:generation")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}}); + expect(resolveRunnerEnvironmentForRun(environment,"paperclip_runner",{},"per_turn")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}}); + expect(resolveRunnerEnvironmentForRun({...environment,config:{...environment.config,runnerLifecycleMode:"per_turn"}},"paperclip_runner",{lifecycleMode:"warm"},undefined,"account:generation")).toMatchObject({config:{runnerLifecycleMode:"per_turn"}}); + expect(resolveRunnerEnvironmentForRun(environment,"claude_local",{})).toBe(environment); +}); diff --git a/server/src/services/runner-environment-lifecycle.ts b/server/src/services/runner-environment-lifecycle.ts index b2908196eb7..52cfb782aa0 100644 --- a/server/src/services/runner-environment-lifecycle.ts +++ b/server/src/services/runner-environment-lifecycle.ts @@ -10,14 +10,16 @@ export function resolveRunnerEnvironmentForRun = {}, admittedLifecycleMode?: "warm" | "per_turn", + managedAiCredentialIdentity?: string, ): T { if (adapterType !== "paperclip_runner" || !["sandbox", "plugin"].includes(environment.driver)) return environment; const config = environment.config ?? {}; const lifecycleMode = admittedLifecycleMode ?? (config.runnerLifecycleMode === "warm" || config.runnerLifecycleMode === "per_turn" ? config.runnerLifecycleMode : adapterConfig.lifecycleMode); + // Warm tasks need a managed credential identity; recovery keeps its admitted policy. if (environment.driver === "plugin") return { ...environment, config: { ...config, - runnerLifecycleMode: lifecycleMode === "warm" && (adapterConfig.provider ?? "codex") === "codex" ? "warm" : "per_turn", + runnerLifecycleMode: admittedLifecycleMode ?? (managedAiCredentialIdentity && lifecycleMode !== "per_turn" ? "warm" : "per_turn"), runnerIdleTimeoutMs: resolvePaperclipRunnerIdleTimeoutMs(config.runnerIdleTimeoutMs ?? adapterConfig.idleTimeoutMs), } }; if (lifecycleMode !== "warm") return environment;