fix(native): verify warm ownership before configuration rotation

Track the exact prior warm-owner run ID separately from mutable configuration. Keep configuration retirement, terminal-state checks and failed-close fences. Add current-v5 handoff regressions and document the recovery boundary.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-10-15 10:38:46 -05:00
1 parent 03f22f4cb9
commit afe5b3418d
4 files changed
+169 -28

No files matched your search

@@ -35,6 +35,7 @@ import {
type Db,
} from "@paperclipai/db";
import {
canonicalNativeRuntimeContextDigest,
CURSOR_DISTRIBUTION_PINS,
QUALIFIED_ACPX_PROFILES,
acpxRuntimeSessionDirectoryName,
@@ -5158,7 +5159,7 @@ type LeaseCoordinator = {
};
function leaseDb(
boundExecution: NativeExecutionInputV1 = execution,
boundExecution: NativeExecutionInput = execution,
coordinatorOverrides: Partial<LeaseCoordinator> = {},
runResultJson: Record<string, unknown> = {},
updates: Array<{ table: unknown; values: Record<string, unknown> }> = [],
@@ -9727,7 +9728,7 @@ describe("runnerd provider runtime wiring", () => {
};
}
function priorNativeRunDb(priorExecution: NativeExecutionInputV1, status = "succeeded"): Db {
function priorNativeRunDb(priorExecution: NativeExecutionInput, status = "succeeded"): Db {
return {
select: () => ({ from: () => ({ where: () => ({ limit: async () => [{
status,
@@ -9736,7 +9737,7 @@ describe("runnerd provider runtime wiring", () => {
} as unknown as Db;
}
async function initializeScopedRunnerRoot(current: NativeExecutionInputV1, runnerInstanceId: string) {
async function initializeScopedRunnerRoot(current: NativeExecutionInput, runnerInstanceId: string) {
state.createBackend.mockClear();
state.createTransport.mockClear();
await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId });
@@ -9771,6 +9772,107 @@ describe("runnerd provider runtime wiring", () => {
await rm(isolatedStateDirectory, { recursive: true, force: true });
});
it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => {
const first = { ...scopedNativeExecution(`warm-admission-${scenario}`),
schema: "paperclip.native-execution-input.v5" as const,
provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() };
first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 };
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` },
session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner"].includes(scenario) ? 60_000 : 90_000 } } };
if (scenario === "runtime-context") {
next.runtimeContext = { ...first.runtimeContext,
mcp: { ...first.runtimeContext.mcp, digest: "1".repeat(64) } };
next.runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(next.runtimeContext);
}
const environmentId = `warm-admission-${scenario}`;
const target = { kind: "remote" as const, transport: "sandbox" as const, environmentId,
remoteCwd: "/home/daytona/workspace", runner: { execute: vi.fn() } };
const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId,
runnerInstanceId: `runner-${scenario}`, environmentLeaseId: first.binding.executionWorkspaceId };
const root = await initializeScopedRunnerRoot(first, identity.runnerInstanceId);
await writeRunnerAuthority(root, identity, "ready");
const order: string[] = [];
let failRetirement = scenario === "retirement-failed";
const firstSession = { close: vi.fn(async () => {
order.push("retire");
if (failRetirement) throw new Error("retirement unconfirmed");
await writeRunnerAuthority(root, identity, "suspended");
}) };
const nextSession = { close: vi.fn(async () => undefined) };
const result = { result: { summary: "done" }, terminal: { runTerminalState: "succeeded" },
turnId: "turn", normalizedSessionId: first.session.normalizedSessionId,
providerSessionId: "provider", driverKind: "test", driverVersion: "1",
nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
state.execute.mockReset().mockImplementationOnce(async options => {
await options.onSession?.(firstSession);
return result;
}).mockImplementationOnce(async options => {
order.push("execute-next");
expect(options.existingSession).toBe(scenario === "unchanged" ? firstSession : undefined);
await options.onSession?.(scenario === "unchanged" ? firstSession : nextSession);
return result;
});
const older = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-older` } };
const nextDb = leaseDb(next);
const ordinarySelect = nextDb.select.bind(nextDb);
nextDb.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status
&& fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
? priorNativeRunDb(scenario === "stale-owner" ? older : first, scenario === "active-owner" ? "running" : "succeeded").select(fields as never)
: ordinarySelect(fields as never)) as Db["select"];
try {
await executePaperclipNativeSession({ db: leaseDb(first), execution: first,
runnerInstanceId: identity.runnerInstanceId, useRunnerd: true, runnerExecutionTarget: target as never });
if (scenario === "stale-owner") await writeRunnerAuthority(root, { ...identity, runId: older.binding.runId }, "ready");
if (scenario === "missing-owner") {
await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "fixture owner loss" });
await writeRunnerAuthority(root, identity, "ready");
order.length = 0;
firstSession.close.mockClear();
}
const continuation = executePaperclipNativeSession({ db: nextDb, execution: next,
runnerInstanceId: `runner-next-${scenario}`, useRunnerd: true,
runnerExecutionTarget: (scenario === "foreign-environment" ? { ...target, environmentId: `${environmentId}-other` } : target) as never });
if (["stale-owner", "active-owner", "missing-owner", "foreign-environment"].includes(scenario)) {
await expect(continuation).rejects.toThrow(scenario === "active-owner"
? "prior_owner_active" : "prior_owner_terminal_state_indeterminate");
expect(state.execute).toHaveBeenCalledOnce();
expect(firstSession.close).not.toHaveBeenCalled();
expect(order).toEqual([]);
} else if (scenario === "retirement-failed") {
await expect(continuation).rejects.toThrow("retirement unconfirmed");
expect(state.execute).toHaveBeenCalledOnce();
expect(order).toEqual(["retire"]);
} else {
await expect(continuation).resolves.toMatchObject({ exitCode: 0 });
expect(order).toEqual(scenario === "unchanged" ? ["execute-next"] : ["retire", "execute-next"]);
if (scenario === "unchanged") {
await writeRunnerAuthority(root, { ...identity, runId: next.binding.runId }, "ready");
const third = { ...next, binding: { ...next.binding, runId: `${next.binding.runId}-third` } };
const thirdDb = leaseDb(third);
const thirdSelect = thirdDb.select.bind(thirdDb);
thirdDb.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status
&& fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
? priorNativeRunDb(next).select(fields as never)
: thirdSelect(fields as never)) as Db["select"];
state.execute.mockImplementationOnce(async options => {
expect(options.existingSession).toBe(firstSession);
await options.onSession?.(firstSession);
return result;
});
await expect(executePaperclipNativeSession({ db: thirdDb, execution: third,
runnerInstanceId: `runner-third-${scenario}`, useRunnerd: true,
runnerExecutionTarget: target as never })).resolves.toMatchObject({ exitCode: 0 });
expect(firstSession.close).not.toHaveBeenCalled();
expect(state.execute).toHaveBeenCalledTimes(3);
}
}
} finally {
failRetirement = false;
await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "test cleanup" });
}
});
it.each([
["open", "before-close"],
["open", "during-close"],