Preserve plugin session continuity through warm execution and checkpoint recovery (#17820)

This commit is contained in:
Nicky Leach authored and GitHub committed 2026-10-26 20:25:10 -07:00
1 parent dcfc2acfa8
commit ea727dc3c2
23 files changed
+433 -82

No files matched your search

@@ -11128,13 +11128,15 @@ describe("runnerd provider runtime wiring", () => {
await rm(isolatedStateDirectory, { recursive: true, force: true });
});
it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => {
it.each((["codex", "claude"] as const).flatMap(provider => (["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const).filter(scenario => provider === "codex" || scenario.startsWith("plugin-")).map(scenario => [provider, scenario] as const)))("rotates an owned warm %s remote runner before checking changed configuration: %s", async (provider, scenario) => {
const plugin = scenario === "plugin-replaced" || scenario === "plugin-managed-github";
const reusable = scenario === "unchanged" || scenario === "plugin-managed-github";
const first = { ...scopedNativeExecution(`warm-admission-${scenario}`),
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
const first = { ...scopedNativeExecution(`warm-admission-${provider}-${scenario}`),
schema: "paperclip.native-execution-input.v5" as const,
provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
provider: provider === "claude" ? { kind: "acpx" as const, agent: "claude" as const, model: "claude-sonnet-4-6", permissionMode: "approve-all" as const, profile: claudeProfile } : { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() };
if (provider === "claude") first.session.driverKind = "acpx_runtime";
first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 };
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` },
session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner", "plugin-replaced", "plugin-managed-github"].includes(scenario) ? 60_000 : 90_000 } } };
@@ -11161,7 +11163,7 @@ describe("runnerd provider runtime wiring", () => {
if(plugin) {
await mkdir(home,{recursive:true});await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600});
}
const pluginOptions=plugin ? {runnerIngressAuthorized:true,managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home,
const pluginOptions=plugin ? {runnerIngressAuthorized:true,runnerEnvironment:provider === "claude" ? {ANTHROPIC_API_KEY:"synthetic-fixture"} : {},managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home,
managedAiCredentialIdentity:"revision",managedAiAccountIdentity:"account"} : {};
const resume=scenario === "plugin-replaced" ? vi.spyOn(environmentTaskRuntime,"resolvePluginTaskResume").mockResolvedValue("task-first") : undefined;
const createOwner = pluginTaskRuntime.createPluginRunnerTaskOwner;
@@ -11862,6 +11864,103 @@ describe("runnerd provider runtime wiring", () => {
}
});
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))(
"recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => {
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`),
schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, runtimeContext: nativeRuntimeContextFixture(),
...(provider === "claude" ? { provider: { kind: "acpx", agent: "claude", model: "claude-sonnet-4-6", permissionMode: "approve-all", profile: claudeProfile } } : { provider: { kind: "codex", model: null, approvalPolicy: "never" } }),
session: { ...execution.session, driverKind: provider === "claude" ? "acpx_runtime" : "codex_app_server",
normalizedSessionId: `plugin-${provider}-${policy}-${mode}`, lifecyclePolicy: policy === "warm" ? { mode: "warm", idleTimeoutMs: 60_000 } : { mode: "per_turn", idleTimeoutMs: null } },
} as NativeExecutionInput;
parseNativeExecutionInput(first);
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-reopened` } };
const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId!,
runnerInstanceId: "plugin-runner", environmentLeaseId: "old-lease", turnId: "old-turn", itemId: "old-item" };
const scoped = await initializeScopedRunnerRoot(first, identity.runnerInstanceId);
await mkdir(join(scoped, "control-plane"), { recursive: true });
const journal = JSON.stringify({ ...durableControlPlaneState(identity), commands: [{ status: mode.endsWith("pending-command") ? "pending" : "completed" }],
committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }] });
await writeFile(join(scoped, "control-plane/control-plane-state.json"), journal);
if (mode.startsWith("empty-runner") || mode === "nonempty-runner") {
await mkdir(join(scoped, "runner"));
if (mode === "nonempty-runner") await writeFile(join(scoped, "runner", "unknown.json"), "{}");
}
if (mode.startsWith("local-receipt")) await writeFile(join(scoped, "plugin-runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended")));
let root = scoped;
if (["quarantined", "empty-runner-quarantined", "ambiguous"].includes(mode)) {
await mkdir(join(isolatedStateDirectory, "quarantine"), { recursive: true });
root = join(isolatedStateDirectory, "quarantine", `${scoped.split("/").at(-1)}.identity_indeterminate.1`);
await rename(scoped, root);
if (mode === "ambiguous") await cp(root, `${root}-second`, { recursive: true });
}
if (mode === "contradictory") {
await mkdir(join(root, "runner"));
await writeFile(join(root, "runner/runner-state.json"), "{");
}
const db = leaseDb(next);
const ordinarySelect = db.select.bind(db);
db.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
? priorNativeRunDb(mode === "foreign-scope" ? { ...first, binding: { ...first.binding, agentId: "foreign" } } : first,
mode === "active" ? "running" : "succeeded").select(fields as never)
: ordinarySelect(fields as never)) as Db["select"];
const throughExecutor = policy === "warm" && ["canonical", "local-receipt", "pending-outbox"].includes(mode);
let inspected = false;
const resume = vi.spyOn(environmentTaskRuntime, "resolvePluginTaskResume").mockImplementation(async (_db, request) => {
if (request.inspectCheckpoint) inspected = true;
else if (throughExecutor && !inspected) throw new Error("stale negative checkpoint availability");
if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized");
return "old-task";
});
const task = vi.fn(async (operation: any) => {
environmentTaskOperationSchema.parse(operation);
if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: {
...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"),
outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null,
...(mode === "wrong-identity" ? { itemId: "foreign" } : {}),
} };
return { kind: "description", taskId: "replacement-task", launchPreparationVersion: 1, projectAuthCheckVersion: 1, warmTaskVersion: 1,
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } };
});
const owner = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner");
state.createBackend.mockClear();
const home = join(isolatedStateDirectory, "checkpoint-account");
await mkdir(home, { recursive: true });
await writeFile(join(home, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "synthetic-fixture" }), { mode: 0o600 });
try {
const launch = { db, execution: next, runnerInstanceId: "replacement-runner", runnerIngressAuthorized: true,
managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account",
runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" },
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task,
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never };
if (throughExecutor) {
const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done",
completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] },
evidence: [], verification: [], artifacts: [], attentionRequests: [] });
if (!validated.ok) throw new Error("Invalid completion fixture");
state.execute.mockResolvedValueOnce({ result: validated.result, terminal: { runTerminalState: "succeeded" }, turnId: "turn",
normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind,
driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null });
}
const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch);
if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
await expect(backend).resolves.toBeDefined();
expect(state.createBackend).toHaveBeenCalledOnce();
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity }));
expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" });
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
} else {
await expect(backend).rejects.toThrow();
expect(state.createBackend).not.toHaveBeenCalled();
if (["local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) {
expect(task).not.toHaveBeenCalled();
}
if (mode === "ambiguous") expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
}
} finally { owner.mockRestore(); resume.mockRestore(); }
});
it.each(["verified", "wrong-state", "no-stop-proof", "disposition-retry", "disposition-wrong-state", "disposition-no-stop-proof"] as const)("admits only an authorized same-run plugin checkpoint: %s", async mode => {
const current = scopedNativeExecution(`plugin-crash-${mode}`);
const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!,
@@ -11969,10 +12068,11 @@ describe("runnerd provider runtime wiring", () => {
} finally { owner.mockRestore(); }
});
it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx"] as const)("negotiates task-only warm execution: %s", async mode => {
it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx", "installed-codex", "installed-claude"] as const)("negotiates task-only execution: %s", async mode => {
const current = scopedNativeExecution(`plugin-warm-${mode}`);
current.session.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000};
if(mode==="acpx") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
const installed = mode.startsWith("installed-");
current.session.lifecyclePolicy = installed ? {mode:"per_turn",idleTimeoutMs:null} : {mode:"warm",idleTimeoutMs:300000};
if(mode==="acpx" || mode==="installed-claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
const home=await mkdtemp(join(tmpdir(),"plugin-warm-account-"));
await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600});
const owner=vi.spyOn(pluginTaskRuntime,"createPluginRunnerTaskOwner");
@@ -11983,20 +12083,22 @@ describe("runnerd provider runtime wiring", () => {
state.createBackend.mockClear();state.createTransport.mockClear();
try {
const backend=createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"runner",runnerIngressAuthorized:true,
managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" ? undefined : "revision",managedAiAccountIdentity:"account",
runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" || installed ? undefined : "revision",managedAiAccountIdentity:installed ? undefined : "account",
runnerExecutionTarget:{kind:"remote",transport:"sandbox",leaseId:"lease",remoteCwd:".",runnerTask:task,
...(mode==="fresh" ? {sandboxLeaseAcquisition:{outcome:"resumed",providerLeaseId:"old-task"}} : {})}});
if(mode!=="codex" && mode!=="fresh") {await expect(backend).rejects.toThrow(mode==="acpx" ? "Codex workspace" : "capability or account");return;}
if(mode==="missing-capability" || mode==="missing-account") {await expect(backend).rejects.toThrow("capability or account");return;}
await backend;
if(mode==="fresh") expect(task.mock.calls.map(([operation])=>operation.kind)).toEqual(["stop","status","complete","describe"]);
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:{mode:"warm",idleTimeoutMs:300000}});
expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,accountIdentity:"account",onCredentials:undefined});
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:current.session.lifecyclePolicy});
expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,onCredentials:undefined});
expect(owner.mock.calls.at(-1)![0].accountIdentity).toBe(installed ? undefined : "account");
} finally {owner.mockRestore();await rm(home,{recursive:true,force:true});}
});
it.each([true,false])("reconnects task-only warm recovery under the saved identity before reuse or retirement: %s", async reuseProcess => {
const current=scopedNativeExecution(`plugin-restart-${reuseProcess}`);
it.each((["codex", "claude"] as const).flatMap(provider => [true, false].map(reuseProcess => [provider, reuseProcess] as const)))("reconnects task-only %s warm recovery under the saved identity before reuse or retirement: %s", async (provider, reuseProcess) => {
const current=scopedNativeExecution(`plugin-restart-${provider}-${reuseProcess}`);
if(provider === "claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
current.session.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000};
const identity={runnerInstanceId:"old-runner",environmentLeaseId:"old-lease",runId:"old-run",
normalizedSessionId:current.session.normalizedSessionId!,turnId:"old-turn",itemId:"old-item"};
@@ -12008,7 +12110,7 @@ describe("runnerd provider runtime wiring", () => {
state.createBackend.mockClear();state.createTransport.mockClear();
try {
await createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"new-runner",runnerIngressAuthorized:true,
managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account",
runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account",
managedAiCredentialWriteback:{} as never,
remoteWarmRecovery:{identity,reuseProcess,previousExecution:current,checkpoint:{} as never,
remote:{providerLeaseId:"saved-task",remoteCwd:"."},revalidate:async()=>{}},
@@ -2062,29 +2062,34 @@ function savePluginRunnerCheckpoint(root: string, state: Record<string, unknown>
renameSync(temporary, path);
}
function pluginRunnerdAuthorityLifecycle(
root: string,
identity: RunnerdDurableIdentity,
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
try {
const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"),
NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8")));
if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle))
|| !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate";
return state.lifecycle === "suspended" ? "suspended" : "not_suspended";
} catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; }
}
function runnerdAuthorityLifecycle(
root: string,
identity: RunnerdDurableIdentity,
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
const runnerRoot = resolve(root, "runner");
if (!existsSync(runnerRoot)) {
try {
const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"),
NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8")));
if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle))
|| !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate";
return state.lifecycle === "suspended" ? "suspended" : "not_suspended";
} catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; }
}
if (!existsSync(runnerRoot)) return pluginRunnerdAuthorityLifecycle(root, identity);
if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate";
const statePath = resolve(runnerRoot, "runner-state.json");
if (!existsSync(statePath)) {
try {
// Older remote transports created this controller-side placeholder even
// though runner state was owned by the sandbox. It carries no authority,
// so a verified failover backup may be consulted. Any non-empty direct
// so a verified plugin receipt or failover backup may be consulted. Any non-empty direct
// directory remains indeterminate and therefore blocks fallback.
return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate";
return readdirSync(runnerRoot).length === 0 ? pluginRunnerdAuthorityLifecycle(root, identity) : "indeterminate";
} catch {
return "indeterminate";
}
@@ -4894,6 +4899,72 @@ async function verifyWarmTransitionRestart(input: {
return verified;
}
async function recoverPluginRunnerCheckpoint(input: {
db: Db;
execution: NativeExecutionInput;
runnerExecutionTarget?: AdapterExecutionTarget | null;
}, scoped: string): Promise<void> {
const target = input.runnerExecutionTarget;
if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask || !target.leaseId ||
target.sandboxLeaseAcquisition?.outcome === "resumed") return;
let root = scoped;
const quarantined = !existsSync(scoped);
const deny = (): never => { throw new Error("runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven"); };
if (quarantined) {
const quarantine = resolve(runnerdStateBase(), "quarantine");
if (!existsSync(quarantine)) return;
if (!isSafeNativeStateDirectory(quarantine)) return deny();
const candidates = readdirSync(quarantine).filter(name => name.startsWith(`${basename(scoped)}.identity_indeterminate.`));
if (candidates.length === 0) return;
if (candidates.length !== 1) return deny();
root = resolve(quarantine, candidates[0]!);
}
if (!existsSync(root)) return;
if (!isSafeNativeStateDirectory(root)) return deny();
if (hasRetainedWarmTransitionEvidence(root)) {
if (quarantined) return deny();
return;
}
const identity = readRunnerdDurableIdentity(root);
if (!durableIdentityMatchesSession(identity, input.execution) || identity.runId === input.execution.binding.runId) {
if (quarantined) return deny();
return;
}
const direct = runnerdAuthorityLifecycle(root, identity);
const verifiedPluginReceipt = direct === "suspended" &&
!existsSync(resolve(root, "runner", "runner-state.json")) && pluginRunnerdAuthorityLifecycle(root, identity) === "suspended";
// Never hide contradictory direct authority behind a remote checkpoint.
if (direct !== "absent" && !(quarantined && direct === "suspended") && !verifiedPluginReceipt) {
if (quarantined) return deny();
return;
}
await assertCleanupActivationCommitted(input.db, root, input.execution);
const prior = await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true });
if (prior !== "terminal_state_indeterminate" && !((quarantined || verifiedPluginReceipt) && prior === "verified")) {
if (quarantined) return deny();
return;
}
const control = record(JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"),
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8")));
const terminal = record(Array.isArray(control.committedEvents) ? control.committedEvents.at(-1) : null);
if (!Array.isArray(control.commands) || control.commands.some(command => record(command).status !== "completed") ||
terminal.eventType !== "run.terminal" || record(terminal.envelope).runId !== identity.runId ||
record(terminal.envelope).normalizedSessionId !== identity.normalizedSessionId) return deny();
const resumeFrom = await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId,
leaseId: target.leaseId, priorIdentity: identity, inspectCheckpoint: true });
const checkpoint = environmentTaskResultSchema.parse(await target.runnerTask({ kind: "read_state", identity, resumeFrom }));
if (checkpoint.kind !== "state" || checkpoint.state.lifecycle !== "suspended" ||
!Array.isArray(checkpoint.state.outbox) || checkpoint.state.outbox.length !== 0 ||
checkpoint.state.pendingTerminalDelivery != null || checkpoint.state.pendingProviderCleanup != null ||
checkpoint.state.warmTransition != null) return deny();
savePluginRunnerCheckpoint(root, checkpoint.state, identity);
if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny();
if (quarantined) {
if (existsSync(scoped)) return deny();
renameSync(root, scoped);
}
}
async function migrateRunnerdStateRootForExecution(input: {
db: Db;
execution: NativeExecutionInput;
@@ -4913,6 +4984,9 @@ async function migrateRunnerdStateRootForExecution(input: {
issueId: input.execution.binding.issueId,
stateKey: basename(scoped),
});
if (!input.restartRecovery && !input.retainedWarmOwnerRunId) {
await recoverPluginRunnerCheckpoint(input, scoped);
}
if (
input.allowLocalRecovery &&
!input.retainedWarmOwnerRunId &&
@@ -8000,10 +8074,7 @@ async function executePaperclipNativeSessionWithinScope(
// Inspection can replace a stopped retained task before deciding between adoption and checkpoint recovery.
await taskTarget.runnerTask({kind:"describe"});
if (taskTarget.sandboxLeaseAcquisition?.outcome !== "resumed") {
const resumeFrom=await resolvePluginTaskResume(input.db,{companyId:input.execution.binding.companyId,leaseId:taskTarget.leaseId,priorIdentity:identity});
const checkpoint=environmentTaskResultSchema.parse(await taskTarget.runnerTask({kind:"read_state",identity,resumeFrom}));
if(checkpoint.kind!=="state") throw new Error("Plugin Runner checkpoint unavailable");
savePluginRunnerCheckpoint(root,checkpoint.state,identity);
await recoverPluginRunnerCheckpoint(input, root);
}
}
if (durableIdentityMatchesSession(identity, input.execution) && identity.runId !== input.execution.binding.runId &&
@@ -11802,8 +11873,8 @@ async function createRunnerdBackendWithinSessionClaim(
const remoteTarget = target.kind === "remote" ? target : null;
const pluginTask = remoteTarget?.transport === "sandbox" ? remoteTarget.runnerTask : undefined;
if (pluginTask && input.runnerIngressAuthorized !== true) throw new Error("runner_ingress_unavailable");
if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8" || (input.execution.session.lifecyclePolicy.mode === "warm" && input.execution.provider.kind !== "codex"))) {
throw new Error("Plugin warm Runner tasks require a Codex workspace session");
if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8")) {
throw new Error("Plugin Runner tasks require a workspace session");
}
const remoteCommandRunner = remoteTarget
? remoteTarget.transport === "ssh"
@@ -11992,7 +12063,7 @@ async function createRunnerdBackendWithinSessionClaim(
.where(and(eq(environmentLeases.id, remoteTarget.leaseId!), eq(environmentLeases.companyId, input.execution.binding.companyId)));
}, onDispatched: input.onTaskDispatched,
};
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin Codex warm task capability or account unavailable");
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin warm task capability or account unavailable");
const owner = retainedOwner ?? createPluginRunnerTaskOwner(ownerInput);
if (retainedOwner) retainedOwner.rebind(ownerInput);
if (description.workspaceFileReadVersion === 1) {
@@ -23,8 +23,11 @@ const requirements: RunnerProviderLaunchRequirements = {
settings: {}, approvalPolicy: "never", readOnly: false, preparedConversation: false,
includeCollaborationModeInstructions: false,
};
async function prepare(owner: ReturnType<typeof createPluginRunnerTaskOwner>) {
await owner.driver.prepareLaunch!(requirements);
function launchRequirements(harness: string): RunnerProviderLaunchRequirements {
return harness === "claude" ? { ...requirements, provider: "acpx", settings: { acpxAgent: "claude" } } : requirements;
}
async function prepare(owner: ReturnType<typeof createPluginRunnerTaskOwner>, current = requirements) {
await owner.driver.prepareLaunch!(current);
}
function provider() {
let stopped = false;
@@ -323,14 +326,14 @@ describe("plugin task workspace file transfer", () => {
});
});
it("reuses a warm Codex process across run authorities without idle status polling", async () => {
it.each(["codex", "claude"])("reuses a warm %s process across run authorities without idle status polling", async harness => {
const {task} = provider();
ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}});
const retained = vi.fn(async()=>{});
const input = {task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
const input = {task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
accountIdentity:"account",credentialIdentity:"revision",pollMs:1,onSessionRetained:retained};
const owner = createPluginRunnerTaskOwner(input);
await prepare(owner);
await prepare(owner, launchRequirements(harness));
const registration = await owner.registration({} as never,"run");
const spec = launchSpec(); spec.nativeRunner!.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000};
const handle = owner.launcher(spec); registration.activate(); await registration.ready();
@@ -345,7 +348,7 @@ it("reuses a warm Codex process across run authorities without idle status polli
owner.rebind({...input,task:nextTask});
const nextRegistration=await owner.registration({} as never,"next-run");
const nextIdentity={...requirements.identity,runId:"next-run",turnId:"next-turn",itemId:"next-item"};
await owner.driver.prepareLaunch!({...requirements,identity:nextIdentity});
await owner.driver.prepareLaunch!({...launchRequirements(harness),identity:nextIdentity});
expect(nextTask.mock.calls[0][0]).toMatchObject({kind:"prepare",priorIdentity:requirements.identity,accountIdentity:"account"});
expect(nextTask.mock.calls.map(([op])=>op.kind)).toEqual(["prepare","check_project_auth"]);
await registration.release(); nextRegistration.activate(); await nextRegistration.ready();
@@ -355,13 +358,13 @@ it("reuses a warm Codex process across run authorities without idle status polli
expect(nextTask.mock.calls.filter(([op])=>op.kind==="stop")).toHaveLength(1);
});
it("adopts a verified retained Codex task without submitting a replacement", async () => {
it.each(["codex", "claude"])("adopts a verified retained %s task without submitting a replacement", async harness => {
const {task}=provider();
ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}});
const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness:"codex",projectAuthCheckVersion:1,
const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness,projectAuthCheckVersion:1,
warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity,pollMs:1});
const recovered = {configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}};
await expect(owner.driver.prepareLaunch!(requirements,recovered)).resolves.toEqual(recovered);
await expect(owner.driver.prepareLaunch!(launchRequirements(harness),recovered)).resolves.toEqual(recovered);
expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]);
expect(task.mock.calls[0][0]).toMatchObject({configurationId:"saved",projectIds:[]});
expect(await owner.driver.adoptExistingRunner!.isAlive()).toBe(true);
@@ -372,28 +375,30 @@ it("adopts a verified retained Codex task without submitting a replacement", asy
expect(task.mock.calls.some(([op])=>op.kind==="complete")).toBe(true);
});
it("denies retained process adoption when current project authentication is revoked",async()=>{
it.each(["codex", "claude"])("denies retained %s process adoption when current project authentication is revoked",async harness=>{
const {task}=provider();
task.mockImplementation(async op=>op.kind === "check_project_auth"
? {kind:"project_auth",taskId:"task",configurationId:op.configurationId,provider:op.provider,allowed:false} as never
: {kind:"accepted",taskId:"task"} as never);
const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness:"codex",projectAuthCheckVersion:1,
const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness,projectAuthCheckVersion:1,
warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity});
await expect(owner.driver.prepareLaunch!(requirements,{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}))
await expect(owner.driver.prepareLaunch!(launchRequirements(harness),{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}))
.rejects.toThrow("project authentication conflicts");
expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]);
});
it("refuses warm account or credential generation changes", async()=>{
it.each(["codex", "claude"])("refuses warm %s account, harness, project, or credential generation changes", async harness=>{
const {task}=provider();
const input={task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
const input={task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
accountIdentity:"account",credentialIdentity:"revision",pollMs:1};
const owner=createPluginRunnerTaskOwner(input);
await prepare(owner);
await prepare(owner, launchRequirements(harness));
const spec=launchSpec();spec.nativeRunner!.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000};
const process=owner.launcher(spec);
expect(()=>owner.rebind({...input,accountIdentity:"other"})).toThrow("cannot be rebound");
expect(()=>owner.rebind({...input,credentialIdentity:"rotated"})).toThrow("cannot be rebound");
expect(()=>owner.rebind({...input,harness:harness === "codex" ? "claude" : "codex"})).toThrow("cannot be rebound");
expect(()=>owner.rebind({...input,projectIds:["foreign"]})).toThrow("cannot be rebound");
const registration=await owner.registration({} as never,"run");
await registration.checkpoint("unsettled"); await process.completion;
});
@@ -128,7 +128,7 @@ export function createPluginRunnerTaskOwner(input: {
if (launched || released) throw new Error("Plugin Runner replacement requires a new task lease");
const native = spec.nativeRunner;
if (!preparedLaunchId) throw new Error("Plugin Runner launch requires preparation");
if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.harness === "codex" && input.accountIdentity && input.warmTaskVersion === 1))
if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.accountIdentity && input.warmTaskVersion === 1))
|| !preparedIdentity || Object.entries(preparedIdentity).some(([key, value]) => native.identity[key as keyof DurableRecoveryIdentity] !== value)) {
throw new Error("Plugin Runner requires a structured per-turn native launch");
}
@@ -295,7 +295,7 @@ export function createPluginRunnerTaskOwner(input: {
};
return { launcher, registration, driver, readWorkspaceFile,
rebind(next: typeof input) {
if (!launched || stopRequested || stoppedState || detached || next.harness !== "codex" ||
if (!launched || stopRequested || stoppedState || detached || next.harness !== input.harness ||
input.accountIdentity !== next.accountIdentity || input.credentialIdentity !== next.credentialIdentity ||
JSON.stringify(input.projectIds) !== JSON.stringify(next.projectIds)) throw new Error("Plugin warm task cannot be rebound");
input = next;