mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-30 13:20:54 +01:00
Preserve plugin session continuity through warm execution and checkpoint recovery (#17820)
This commit is contained in:
1 parent
dcfc2acfa8
commit
ea727dc3c2
23 files changed
+433
-82
No files matched your search
@@ -11128,13 +11128,15 @@ describe("runnerd provider runtime wiring", () => {
|
||||
await rm(isolatedStateDirectory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => {
|
||||
it.each((["codex", "claude"] as const).flatMap(provider => (["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const).filter(scenario => provider === "codex" || scenario.startsWith("plugin-")).map(scenario => [provider, scenario] as const)))("rotates an owned warm %s remote runner before checking changed configuration: %s", async (provider, scenario) => {
|
||||
const plugin = scenario === "plugin-replaced" || scenario === "plugin-managed-github";
|
||||
const reusable = scenario === "unchanged" || scenario === "plugin-managed-github";
|
||||
const first = { ...scopedNativeExecution(`warm-admission-${scenario}`),
|
||||
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
|
||||
const first = { ...scopedNativeExecution(`warm-admission-${provider}-${scenario}`),
|
||||
schema: "paperclip.native-execution-input.v5" as const,
|
||||
provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
|
||||
provider: provider === "claude" ? { kind: "acpx" as const, agent: "claude" as const, model: "claude-sonnet-4-6", permissionMode: "approve-all" as const, profile: claudeProfile } : { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
|
||||
executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() };
|
||||
if (provider === "claude") first.session.driverKind = "acpx_runtime";
|
||||
first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 };
|
||||
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` },
|
||||
session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner", "plugin-replaced", "plugin-managed-github"].includes(scenario) ? 60_000 : 90_000 } } };
|
||||
@@ -11161,7 +11163,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
if(plugin) {
|
||||
await mkdir(home,{recursive:true});await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600});
|
||||
}
|
||||
const pluginOptions=plugin ? {runnerIngressAuthorized:true,managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home,
|
||||
const pluginOptions=plugin ? {runnerIngressAuthorized:true,runnerEnvironment:provider === "claude" ? {ANTHROPIC_API_KEY:"synthetic-fixture"} : {},managedGitHub:scenario === "plugin-managed-github",managedAiCredentialHome:home,
|
||||
managedAiCredentialIdentity:"revision",managedAiAccountIdentity:"account"} : {};
|
||||
const resume=scenario === "plugin-replaced" ? vi.spyOn(environmentTaskRuntime,"resolvePluginTaskResume").mockResolvedValue("task-first") : undefined;
|
||||
const createOwner = pluginTaskRuntime.createPluginRunnerTaskOwner;
|
||||
@@ -11862,6 +11864,103 @@ describe("runnerd provider runtime wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))(
|
||||
"recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => {
|
||||
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
|
||||
const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`),
|
||||
schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, runtimeContext: nativeRuntimeContextFixture(),
|
||||
...(provider === "claude" ? { provider: { kind: "acpx", agent: "claude", model: "claude-sonnet-4-6", permissionMode: "approve-all", profile: claudeProfile } } : { provider: { kind: "codex", model: null, approvalPolicy: "never" } }),
|
||||
session: { ...execution.session, driverKind: provider === "claude" ? "acpx_runtime" : "codex_app_server",
|
||||
normalizedSessionId: `plugin-${provider}-${policy}-${mode}`, lifecyclePolicy: policy === "warm" ? { mode: "warm", idleTimeoutMs: 60_000 } : { mode: "per_turn", idleTimeoutMs: null } },
|
||||
} as NativeExecutionInput;
|
||||
parseNativeExecutionInput(first);
|
||||
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-reopened` } };
|
||||
const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId!,
|
||||
runnerInstanceId: "plugin-runner", environmentLeaseId: "old-lease", turnId: "old-turn", itemId: "old-item" };
|
||||
const scoped = await initializeScopedRunnerRoot(first, identity.runnerInstanceId);
|
||||
await mkdir(join(scoped, "control-plane"), { recursive: true });
|
||||
const journal = JSON.stringify({ ...durableControlPlaneState(identity), commands: [{ status: mode.endsWith("pending-command") ? "pending" : "completed" }],
|
||||
committedEvents: mode.endsWith("missing-terminal") ? [] : [{ eventType: "run.terminal", envelope: { ...identity, payload: { reportedWorkDisposition: "blocked" } } }] });
|
||||
await writeFile(join(scoped, "control-plane/control-plane-state.json"), journal);
|
||||
if (mode.startsWith("empty-runner") || mode === "nonempty-runner") {
|
||||
await mkdir(join(scoped, "runner"));
|
||||
if (mode === "nonempty-runner") await writeFile(join(scoped, "runner", "unknown.json"), "{}");
|
||||
}
|
||||
if (mode.startsWith("local-receipt")) await writeFile(join(scoped, "plugin-runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended")));
|
||||
let root = scoped;
|
||||
if (["quarantined", "empty-runner-quarantined", "ambiguous"].includes(mode)) {
|
||||
await mkdir(join(isolatedStateDirectory, "quarantine"), { recursive: true });
|
||||
root = join(isolatedStateDirectory, "quarantine", `${scoped.split("/").at(-1)}.identity_indeterminate.1`);
|
||||
await rename(scoped, root);
|
||||
if (mode === "ambiguous") await cp(root, `${root}-second`, { recursive: true });
|
||||
}
|
||||
if (mode === "contradictory") {
|
||||
await mkdir(join(root, "runner"));
|
||||
await writeFile(join(root, "runner/runner-state.json"), "{");
|
||||
}
|
||||
const db = leaseDb(next);
|
||||
const ordinarySelect = db.select.bind(db);
|
||||
db.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
|
||||
? priorNativeRunDb(mode === "foreign-scope" ? { ...first, binding: { ...first.binding, agentId: "foreign" } } : first,
|
||||
mode === "active" ? "running" : "succeeded").select(fields as never)
|
||||
: ordinarySelect(fields as never)) as Db["select"];
|
||||
const throughExecutor = policy === "warm" && ["canonical", "local-receipt", "pending-outbox"].includes(mode);
|
||||
let inspected = false;
|
||||
const resume = vi.spyOn(environmentTaskRuntime, "resolvePluginTaskResume").mockImplementation(async (_db, request) => {
|
||||
if (request.inspectCheckpoint) inspected = true;
|
||||
else if (throughExecutor && !inspected) throw new Error("stale negative checkpoint availability");
|
||||
if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized");
|
||||
return "old-task";
|
||||
});
|
||||
const task = vi.fn(async (operation: any) => {
|
||||
environmentTaskOperationSchema.parse(operation);
|
||||
if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: {
|
||||
...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"),
|
||||
outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null,
|
||||
...(mode === "wrong-identity" ? { itemId: "foreign" } : {}),
|
||||
} };
|
||||
return { kind: "description", taskId: "replacement-task", launchPreparationVersion: 1, projectAuthCheckVersion: 1, warmTaskVersion: 1,
|
||||
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } };
|
||||
});
|
||||
const owner = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner");
|
||||
state.createBackend.mockClear();
|
||||
const home = join(isolatedStateDirectory, "checkpoint-account");
|
||||
await mkdir(home, { recursive: true });
|
||||
await writeFile(join(home, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "synthetic-fixture" }), { mode: 0o600 });
|
||||
try {
|
||||
const launch = { db, execution: next, runnerInstanceId: "replacement-runner", runnerIngressAuthorized: true,
|
||||
managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account",
|
||||
runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" },
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task,
|
||||
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never };
|
||||
if (throughExecutor) {
|
||||
const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done",
|
||||
completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] },
|
||||
evidence: [], verification: [], artifacts: [], attentionRequests: [] });
|
||||
if (!validated.ok) throw new Error("Invalid completion fixture");
|
||||
state.execute.mockResolvedValueOnce({ result: validated.result, terminal: { runTerminalState: "succeeded" }, turnId: "turn",
|
||||
normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind,
|
||||
driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null });
|
||||
}
|
||||
const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch);
|
||||
if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
|
||||
await expect(backend).resolves.toBeDefined();
|
||||
expect(state.createBackend).toHaveBeenCalledOnce();
|
||||
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity }));
|
||||
expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
|
||||
expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" });
|
||||
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
|
||||
} else {
|
||||
await expect(backend).rejects.toThrow();
|
||||
expect(state.createBackend).not.toHaveBeenCalled();
|
||||
if (["local-receipt-missing-terminal", "local-receipt-pending-command", "active", "foreign-scope", "pending-command", "missing-terminal", "ambiguous", "contradictory", "nonempty-runner", "no-stop-proof"].includes(mode)) {
|
||||
expect(task).not.toHaveBeenCalled();
|
||||
}
|
||||
if (mode === "ambiguous") expect(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
|
||||
}
|
||||
} finally { owner.mockRestore(); resume.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["verified", "wrong-state", "no-stop-proof", "disposition-retry", "disposition-wrong-state", "disposition-no-stop-proof"] as const)("admits only an authorized same-run plugin checkpoint: %s", async mode => {
|
||||
const current = scopedNativeExecution(`plugin-crash-${mode}`);
|
||||
const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!,
|
||||
@@ -11969,10 +12068,11 @@ describe("runnerd provider runtime wiring", () => {
|
||||
} finally { owner.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx"] as const)("negotiates task-only warm execution: %s", async mode => {
|
||||
it.each(["codex", "fresh", "missing-capability", "missing-account", "acpx", "installed-codex", "installed-claude"] as const)("negotiates task-only execution: %s", async mode => {
|
||||
const current = scopedNativeExecution(`plugin-warm-${mode}`);
|
||||
current.session.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000};
|
||||
if(mode==="acpx") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
|
||||
const installed = mode.startsWith("installed-");
|
||||
current.session.lifecyclePolicy = installed ? {mode:"per_turn",idleTimeoutMs:null} : {mode:"warm",idleTimeoutMs:300000};
|
||||
if(mode==="acpx" || mode==="installed-claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
|
||||
const home=await mkdtemp(join(tmpdir(),"plugin-warm-account-"));
|
||||
await writeFile(join(home,"auth.json"),JSON.stringify({OPENAI_API_KEY:"synthetic-fixture"}),{mode:0o600});
|
||||
const owner=vi.spyOn(pluginTaskRuntime,"createPluginRunnerTaskOwner");
|
||||
@@ -11983,20 +12083,22 @@ describe("runnerd provider runtime wiring", () => {
|
||||
state.createBackend.mockClear();state.createTransport.mockClear();
|
||||
try {
|
||||
const backend=createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"runner",runnerIngressAuthorized:true,
|
||||
managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" ? undefined : "revision",managedAiAccountIdentity:"account",
|
||||
runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:mode==="missing-account" || installed ? undefined : "revision",managedAiAccountIdentity:installed ? undefined : "account",
|
||||
runnerExecutionTarget:{kind:"remote",transport:"sandbox",leaseId:"lease",remoteCwd:".",runnerTask:task,
|
||||
...(mode==="fresh" ? {sandboxLeaseAcquisition:{outcome:"resumed",providerLeaseId:"old-task"}} : {})}});
|
||||
if(mode!=="codex" && mode!=="fresh") {await expect(backend).rejects.toThrow(mode==="acpx" ? "Codex workspace" : "capability or account");return;}
|
||||
if(mode==="missing-capability" || mode==="missing-account") {await expect(backend).rejects.toThrow("capability or account");return;}
|
||||
await backend;
|
||||
if(mode==="fresh") expect(task.mock.calls.map(([operation])=>operation.kind)).toEqual(["stop","status","complete","describe"]);
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:{mode:"warm",idleTimeoutMs:300000}});
|
||||
expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,accountIdentity:"account",onCredentials:undefined});
|
||||
expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({lifecyclePolicy:current.session.lifecyclePolicy});
|
||||
expect(owner.mock.calls.at(-1)![0]).toMatchObject({warmTaskVersion:1,onCredentials:undefined});
|
||||
expect(owner.mock.calls.at(-1)![0].accountIdentity).toBe(installed ? undefined : "account");
|
||||
} finally {owner.mockRestore();await rm(home,{recursive:true,force:true});}
|
||||
});
|
||||
|
||||
it.each([true,false])("reconnects task-only warm recovery under the saved identity before reuse or retirement: %s", async reuseProcess => {
|
||||
const current=scopedNativeExecution(`plugin-restart-${reuseProcess}`);
|
||||
it.each((["codex", "claude"] as const).flatMap(provider => [true, false].map(reuseProcess => [provider, reuseProcess] as const)))("reconnects task-only %s warm recovery under the saved identity before reuse or retirement: %s", async (provider, reuseProcess) => {
|
||||
const current=scopedNativeExecution(`plugin-restart-${provider}-${reuseProcess}`);
|
||||
if(provider === "claude") current.provider={kind:"acpx",agent:"claude",model:"fixture",permissionMode:"deny-all"} as never;
|
||||
current.session.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000};
|
||||
const identity={runnerInstanceId:"old-runner",environmentLeaseId:"old-lease",runId:"old-run",
|
||||
normalizedSessionId:current.session.normalizedSessionId!,turnId:"old-turn",itemId:"old-item"};
|
||||
@@ -12008,7 +12110,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
state.createBackend.mockClear();state.createTransport.mockClear();
|
||||
try {
|
||||
await createRunnerdBackend({db:leaseDb(current),execution:current,runnerInstanceId:"new-runner",runnerIngressAuthorized:true,
|
||||
managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account",
|
||||
runnerEnvironment:{ANTHROPIC_API_KEY:"synthetic-fixture"},managedAiCredentialHome:home,managedAiCredentialIdentity:reuseProcess ? "old-revision" : "new-revision",managedAiAccountIdentity:"account",
|
||||
managedAiCredentialWriteback:{} as never,
|
||||
remoteWarmRecovery:{identity,reuseProcess,previousExecution:current,checkpoint:{} as never,
|
||||
remote:{providerLeaseId:"saved-task",remoteCwd:"."},revalidate:async()=>{}},
|
||||
|
||||
@@ -2062,29 +2062,34 @@ function savePluginRunnerCheckpoint(root: string, state: Record<string, unknown>
|
||||
renameSync(temporary, path);
|
||||
}
|
||||
|
||||
function pluginRunnerdAuthorityLifecycle(
|
||||
root: string,
|
||||
identity: RunnerdDurableIdentity,
|
||||
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
|
||||
try {
|
||||
const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"),
|
||||
NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8")));
|
||||
if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle))
|
||||
|| !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate";
|
||||
return state.lifecycle === "suspended" ? "suspended" : "not_suspended";
|
||||
} catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; }
|
||||
}
|
||||
|
||||
function runnerdAuthorityLifecycle(
|
||||
root: string,
|
||||
identity: RunnerdDurableIdentity,
|
||||
): "absent" | "suspended" | "not_suspended" | "indeterminate" {
|
||||
const runnerRoot = resolve(root, "runner");
|
||||
if (!existsSync(runnerRoot)) {
|
||||
try {
|
||||
const state = record(JSON.parse(readBoundedNativeFile(resolve(root, "plugin-runner-state.json"),
|
||||
NATIVE_RUNNER_STATE_MAX_BYTES, "runner_state_too_large").toString("utf8")));
|
||||
if (state.schema !== RUNNERD_STATE_SCHEMA || !RUNNERD_STATE_LIFECYCLES.has(String(state.lifecycle))
|
||||
|| !Object.entries(identity).every(([key, value]) => state[key] === value)) return "indeterminate";
|
||||
return state.lifecycle === "suspended" ? "suspended" : "not_suspended";
|
||||
} catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "absent" : "indeterminate"; }
|
||||
}
|
||||
if (!existsSync(runnerRoot)) return pluginRunnerdAuthorityLifecycle(root, identity);
|
||||
if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate";
|
||||
const statePath = resolve(runnerRoot, "runner-state.json");
|
||||
if (!existsSync(statePath)) {
|
||||
try {
|
||||
// Older remote transports created this controller-side placeholder even
|
||||
// though runner state was owned by the sandbox. It carries no authority,
|
||||
// so a verified failover backup may be consulted. Any non-empty direct
|
||||
// so a verified plugin receipt or failover backup may be consulted. Any non-empty direct
|
||||
// directory remains indeterminate and therefore blocks fallback.
|
||||
return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate";
|
||||
return readdirSync(runnerRoot).length === 0 ? pluginRunnerdAuthorityLifecycle(root, identity) : "indeterminate";
|
||||
} catch {
|
||||
return "indeterminate";
|
||||
}
|
||||
@@ -4894,6 +4899,72 @@ async function verifyWarmTransitionRestart(input: {
|
||||
return verified;
|
||||
}
|
||||
|
||||
async function recoverPluginRunnerCheckpoint(input: {
|
||||
db: Db;
|
||||
execution: NativeExecutionInput;
|
||||
runnerExecutionTarget?: AdapterExecutionTarget | null;
|
||||
}, scoped: string): Promise<void> {
|
||||
const target = input.runnerExecutionTarget;
|
||||
if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runnerTask || !target.leaseId ||
|
||||
target.sandboxLeaseAcquisition?.outcome === "resumed") return;
|
||||
let root = scoped;
|
||||
const quarantined = !existsSync(scoped);
|
||||
const deny = (): never => { throw new Error("runner_state_identity_mismatch: plugin_checkpoint_recovery_unproven"); };
|
||||
if (quarantined) {
|
||||
const quarantine = resolve(runnerdStateBase(), "quarantine");
|
||||
if (!existsSync(quarantine)) return;
|
||||
if (!isSafeNativeStateDirectory(quarantine)) return deny();
|
||||
const candidates = readdirSync(quarantine).filter(name => name.startsWith(`${basename(scoped)}.identity_indeterminate.`));
|
||||
if (candidates.length === 0) return;
|
||||
if (candidates.length !== 1) return deny();
|
||||
root = resolve(quarantine, candidates[0]!);
|
||||
}
|
||||
if (!existsSync(root)) return;
|
||||
if (!isSafeNativeStateDirectory(root)) return deny();
|
||||
if (hasRetainedWarmTransitionEvidence(root)) {
|
||||
if (quarantined) return deny();
|
||||
return;
|
||||
}
|
||||
const identity = readRunnerdDurableIdentity(root);
|
||||
if (!durableIdentityMatchesSession(identity, input.execution) || identity.runId === input.execution.binding.runId) {
|
||||
if (quarantined) return deny();
|
||||
return;
|
||||
}
|
||||
const direct = runnerdAuthorityLifecycle(root, identity);
|
||||
const verifiedPluginReceipt = direct === "suspended" &&
|
||||
!existsSync(resolve(root, "runner", "runner-state.json")) && pluginRunnerdAuthorityLifecycle(root, identity) === "suspended";
|
||||
// Never hide contradictory direct authority behind a remote checkpoint.
|
||||
if (direct !== "absent" && !(quarantined && direct === "suspended") && !verifiedPluginReceipt) {
|
||||
if (quarantined) return deny();
|
||||
return;
|
||||
}
|
||||
await assertCleanupActivationCommitted(input.db, root, input.execution);
|
||||
const prior = await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true });
|
||||
if (prior !== "terminal_state_indeterminate" && !((quarantined || verifiedPluginReceipt) && prior === "verified")) {
|
||||
if (quarantined) return deny();
|
||||
return;
|
||||
}
|
||||
const control = record(JSON.parse(readBoundedNativeFile(resolve(root, "control-plane", "control-plane-state.json"),
|
||||
NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8")));
|
||||
const terminal = record(Array.isArray(control.committedEvents) ? control.committedEvents.at(-1) : null);
|
||||
if (!Array.isArray(control.commands) || control.commands.some(command => record(command).status !== "completed") ||
|
||||
terminal.eventType !== "run.terminal" || record(terminal.envelope).runId !== identity.runId ||
|
||||
record(terminal.envelope).normalizedSessionId !== identity.normalizedSessionId) return deny();
|
||||
const resumeFrom = await resolvePluginTaskResume(input.db, { companyId: input.execution.binding.companyId,
|
||||
leaseId: target.leaseId, priorIdentity: identity, inspectCheckpoint: true });
|
||||
const checkpoint = environmentTaskResultSchema.parse(await target.runnerTask({ kind: "read_state", identity, resumeFrom }));
|
||||
if (checkpoint.kind !== "state" || checkpoint.state.lifecycle !== "suspended" ||
|
||||
!Array.isArray(checkpoint.state.outbox) || checkpoint.state.outbox.length !== 0 ||
|
||||
checkpoint.state.pendingTerminalDelivery != null || checkpoint.state.pendingProviderCleanup != null ||
|
||||
checkpoint.state.warmTransition != null) return deny();
|
||||
savePluginRunnerCheckpoint(root, checkpoint.state, identity);
|
||||
if (await verifyPriorRunnerdStateForSessionScope({ ...input, root, identity, allowVerifiedBackup: true }) !== "verified") return deny();
|
||||
if (quarantined) {
|
||||
if (existsSync(scoped)) return deny();
|
||||
renameSync(root, scoped);
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateRunnerdStateRootForExecution(input: {
|
||||
db: Db;
|
||||
execution: NativeExecutionInput;
|
||||
@@ -4913,6 +4984,9 @@ async function migrateRunnerdStateRootForExecution(input: {
|
||||
issueId: input.execution.binding.issueId,
|
||||
stateKey: basename(scoped),
|
||||
});
|
||||
if (!input.restartRecovery && !input.retainedWarmOwnerRunId) {
|
||||
await recoverPluginRunnerCheckpoint(input, scoped);
|
||||
}
|
||||
if (
|
||||
input.allowLocalRecovery &&
|
||||
!input.retainedWarmOwnerRunId &&
|
||||
@@ -8000,10 +8074,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
// Inspection can replace a stopped retained task before deciding between adoption and checkpoint recovery.
|
||||
await taskTarget.runnerTask({kind:"describe"});
|
||||
if (taskTarget.sandboxLeaseAcquisition?.outcome !== "resumed") {
|
||||
const resumeFrom=await resolvePluginTaskResume(input.db,{companyId:input.execution.binding.companyId,leaseId:taskTarget.leaseId,priorIdentity:identity});
|
||||
const checkpoint=environmentTaskResultSchema.parse(await taskTarget.runnerTask({kind:"read_state",identity,resumeFrom}));
|
||||
if(checkpoint.kind!=="state") throw new Error("Plugin Runner checkpoint unavailable");
|
||||
savePluginRunnerCheckpoint(root,checkpoint.state,identity);
|
||||
await recoverPluginRunnerCheckpoint(input, root);
|
||||
}
|
||||
}
|
||||
if (durableIdentityMatchesSession(identity, input.execution) && identity.runId !== input.execution.binding.runId &&
|
||||
@@ -11802,8 +11873,8 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
const remoteTarget = target.kind === "remote" ? target : null;
|
||||
const pluginTask = remoteTarget?.transport === "sandbox" ? remoteTarget.runnerTask : undefined;
|
||||
if (pluginTask && input.runnerIngressAuthorized !== true) throw new Error("runner_ingress_unavailable");
|
||||
if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8" || (input.execution.session.lifecyclePolicy.mode === "warm" && input.execution.provider.kind !== "codex"))) {
|
||||
throw new Error("Plugin warm Runner tasks require a Codex workspace session");
|
||||
if (pluginTask && (input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8")) {
|
||||
throw new Error("Plugin Runner tasks require a workspace session");
|
||||
}
|
||||
const remoteCommandRunner = remoteTarget
|
||||
? remoteTarget.transport === "ssh"
|
||||
@@ -11992,7 +12063,7 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
.where(and(eq(environmentLeases.id, remoteTarget.leaseId!), eq(environmentLeases.companyId, input.execution.binding.companyId)));
|
||||
}, onDispatched: input.onTaskDispatched,
|
||||
};
|
||||
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin Codex warm task capability or account unavailable");
|
||||
if (input.execution.session.lifecyclePolicy.mode === "warm" && (!credentials.accountIdentity || description.warmTaskVersion !== 1)) throw new Error("Plugin warm task capability or account unavailable");
|
||||
const owner = retainedOwner ?? createPluginRunnerTaskOwner(ownerInput);
|
||||
if (retainedOwner) retainedOwner.rebind(ownerInput);
|
||||
if (description.workspaceFileReadVersion === 1) {
|
||||
|
||||
@@ -23,8 +23,11 @@ const requirements: RunnerProviderLaunchRequirements = {
|
||||
settings: {}, approvalPolicy: "never", readOnly: false, preparedConversation: false,
|
||||
includeCollaborationModeInstructions: false,
|
||||
};
|
||||
async function prepare(owner: ReturnType<typeof createPluginRunnerTaskOwner>) {
|
||||
await owner.driver.prepareLaunch!(requirements);
|
||||
function launchRequirements(harness: string): RunnerProviderLaunchRequirements {
|
||||
return harness === "claude" ? { ...requirements, provider: "acpx", settings: { acpxAgent: "claude" } } : requirements;
|
||||
}
|
||||
async function prepare(owner: ReturnType<typeof createPluginRunnerTaskOwner>, current = requirements) {
|
||||
await owner.driver.prepareLaunch!(current);
|
||||
}
|
||||
function provider() {
|
||||
let stopped = false;
|
||||
@@ -323,14 +326,14 @@ describe("plugin task workspace file transfer", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses a warm Codex process across run authorities without idle status polling", async () => {
|
||||
it.each(["codex", "claude"])("reuses a warm %s process across run authorities without idle status polling", async harness => {
|
||||
const {task} = provider();
|
||||
ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}});
|
||||
const retained = vi.fn(async()=>{});
|
||||
const input = {task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
|
||||
const input = {task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
|
||||
accountIdentity:"account",credentialIdentity:"revision",pollMs:1,onSessionRetained:retained};
|
||||
const owner = createPluginRunnerTaskOwner(input);
|
||||
await prepare(owner);
|
||||
await prepare(owner, launchRequirements(harness));
|
||||
const registration = await owner.registration({} as never,"run");
|
||||
const spec = launchSpec(); spec.nativeRunner!.lifecyclePolicy = {mode:"warm",idleTimeoutMs:300000};
|
||||
const handle = owner.launcher(spec); registration.activate(); await registration.ready();
|
||||
@@ -345,7 +348,7 @@ it("reuses a warm Codex process across run authorities without idle status polli
|
||||
owner.rebind({...input,task:nextTask});
|
||||
const nextRegistration=await owner.registration({} as never,"next-run");
|
||||
const nextIdentity={...requirements.identity,runId:"next-run",turnId:"next-turn",itemId:"next-item"};
|
||||
await owner.driver.prepareLaunch!({...requirements,identity:nextIdentity});
|
||||
await owner.driver.prepareLaunch!({...launchRequirements(harness),identity:nextIdentity});
|
||||
expect(nextTask.mock.calls[0][0]).toMatchObject({kind:"prepare",priorIdentity:requirements.identity,accountIdentity:"account"});
|
||||
expect(nextTask.mock.calls.map(([op])=>op.kind)).toEqual(["prepare","check_project_auth"]);
|
||||
await registration.release(); nextRegistration.activate(); await nextRegistration.ready();
|
||||
@@ -355,13 +358,13 @@ it("reuses a warm Codex process across run authorities without idle status polli
|
||||
expect(nextTask.mock.calls.filter(([op])=>op.kind==="stop")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("adopts a verified retained Codex task without submitting a replacement", async () => {
|
||||
it.each(["codex", "claude"])("adopts a verified retained %s task without submitting a replacement", async harness => {
|
||||
const {task}=provider();
|
||||
ingress.connect.mockReturnValue({ready:Promise.resolve(),close:async()=>{}});
|
||||
const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness:"codex",projectAuthCheckVersion:1,
|
||||
const owner=createPluginRunnerTaskOwner({task,projectIds:[],harness,projectAuthCheckVersion:1,
|
||||
warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity,pollMs:1});
|
||||
const recovered = {configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}};
|
||||
await expect(owner.driver.prepareLaunch!(requirements,recovered)).resolves.toEqual(recovered);
|
||||
await expect(owner.driver.prepareLaunch!(launchRequirements(harness),recovered)).resolves.toEqual(recovered);
|
||||
expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]);
|
||||
expect(task.mock.calls[0][0]).toMatchObject({configurationId:"saved",projectIds:[]});
|
||||
expect(await owner.driver.adoptExistingRunner!.isAlive()).toBe(true);
|
||||
@@ -372,28 +375,30 @@ it("adopts a verified retained Codex task without submitting a replacement", asy
|
||||
expect(task.mock.calls.some(([op])=>op.kind==="complete")).toBe(true);
|
||||
});
|
||||
|
||||
it("denies retained process adoption when current project authentication is revoked",async()=>{
|
||||
it.each(["codex", "claude"])("denies retained %s process adoption when current project authentication is revoked",async harness=>{
|
||||
const {task}=provider();
|
||||
task.mockImplementation(async op=>op.kind === "check_project_auth"
|
||||
? {kind:"project_auth",taskId:"task",configurationId:op.configurationId,provider:op.provider,allowed:false} as never
|
||||
: {kind:"accepted",taskId:"task"} as never);
|
||||
const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness:"codex",projectAuthCheckVersion:1,
|
||||
const owner=createPluginRunnerTaskOwner({task,projectIds:["project"],harness,projectAuthCheckVersion:1,
|
||||
warmTaskVersion:1,accountIdentity:"account",adoptIdentity:requirements.identity});
|
||||
await expect(owner.driver.prepareLaunch!(requirements,{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}))
|
||||
await expect(owner.driver.prepareLaunch!(launchRequirements(harness),{configurationId:"saved",requirementsDigest:`sha256:${"a".repeat(64)}`,provider:{}}))
|
||||
.rejects.toThrow("project authentication conflicts");
|
||||
expect(task.mock.calls.map(([op])=>op.kind)).toEqual(["check_project_auth"]);
|
||||
});
|
||||
|
||||
it("refuses warm account or credential generation changes", async()=>{
|
||||
it.each(["codex", "claude"])("refuses warm %s account, harness, project, or credential generation changes", async harness=>{
|
||||
const {task}=provider();
|
||||
const input={task,projectIds:[],harness:"codex",projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
|
||||
const input={task,projectIds:[],harness,projectAuthCheckVersion:1 as const,warmTaskVersion:1 as const,
|
||||
accountIdentity:"account",credentialIdentity:"revision",pollMs:1};
|
||||
const owner=createPluginRunnerTaskOwner(input);
|
||||
await prepare(owner);
|
||||
await prepare(owner, launchRequirements(harness));
|
||||
const spec=launchSpec();spec.nativeRunner!.lifecyclePolicy={mode:"warm",idleTimeoutMs:300000};
|
||||
const process=owner.launcher(spec);
|
||||
expect(()=>owner.rebind({...input,accountIdentity:"other"})).toThrow("cannot be rebound");
|
||||
expect(()=>owner.rebind({...input,credentialIdentity:"rotated"})).toThrow("cannot be rebound");
|
||||
expect(()=>owner.rebind({...input,harness:harness === "codex" ? "claude" : "codex"})).toThrow("cannot be rebound");
|
||||
expect(()=>owner.rebind({...input,projectIds:["foreign"]})).toThrow("cannot be rebound");
|
||||
const registration=await owner.registration({} as never,"run");
|
||||
await registration.checkpoint("unsettled"); await process.completion;
|
||||
});
|
||||
@@ -128,7 +128,7 @@ export function createPluginRunnerTaskOwner(input: {
|
||||
if (launched || released) throw new Error("Plugin Runner replacement requires a new task lease");
|
||||
const native = spec.nativeRunner;
|
||||
if (!preparedLaunchId) throw new Error("Plugin Runner launch requires preparation");
|
||||
if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.harness === "codex" && input.accountIdentity && input.warmTaskVersion === 1))
|
||||
if (!native || (native.lifecyclePolicy?.mode !== "per_turn" && !(native.lifecyclePolicy?.mode === "warm" && input.accountIdentity && input.warmTaskVersion === 1))
|
||||
|| !preparedIdentity || Object.entries(preparedIdentity).some(([key, value]) => native.identity[key as keyof DurableRecoveryIdentity] !== value)) {
|
||||
throw new Error("Plugin Runner requires a structured per-turn native launch");
|
||||
}
|
||||
@@ -295,7 +295,7 @@ export function createPluginRunnerTaskOwner(input: {
|
||||
};
|
||||
return { launcher, registration, driver, readWorkspaceFile,
|
||||
rebind(next: typeof input) {
|
||||
if (!launched || stopRequested || stoppedState || detached || next.harness !== "codex" ||
|
||||
if (!launched || stopRequested || stoppedState || detached || next.harness !== input.harness ||
|
||||
input.accountIdentity !== next.accountIdentity || input.credentialIdentity !== next.credentialIdentity ||
|
||||
JSON.stringify(input.projectIds) !== JSON.stringify(next.projectIds)) throw new Error("Plugin warm task cannot be rebound");
|
||||
input = next;
|
||||
|
||||
Reference in new issue
Block a user