mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 08:22:20 +02:00
fix(native): verify warm ownership before configuration rotation
Track the exact prior warm-owner run ID separately from mutable configuration. Keep configuration retirement, terminal-state checks and failed-close fences. Add current-v5 handoff regressions and document the recovery boundary. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
03f22f4cb9
commit
afe5b3418d
4 files changed
+169
-28
No files matched your search
@@ -35,6 +35,7 @@ import {
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
canonicalNativeRuntimeContextDigest,
|
||||
CURSOR_DISTRIBUTION_PINS,
|
||||
QUALIFIED_ACPX_PROFILES,
|
||||
acpxRuntimeSessionDirectoryName,
|
||||
@@ -5158,7 +5159,7 @@ type LeaseCoordinator = {
|
||||
};
|
||||
|
||||
function leaseDb(
|
||||
boundExecution: NativeExecutionInputV1 = execution,
|
||||
boundExecution: NativeExecutionInput = execution,
|
||||
coordinatorOverrides: Partial<LeaseCoordinator> = {},
|
||||
runResultJson: Record<string, unknown> = {},
|
||||
updates: Array<{ table: unknown; values: Record<string, unknown> }> = [],
|
||||
@@ -9727,7 +9728,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
};
|
||||
}
|
||||
|
||||
function priorNativeRunDb(priorExecution: NativeExecutionInputV1, status = "succeeded"): Db {
|
||||
function priorNativeRunDb(priorExecution: NativeExecutionInput, status = "succeeded"): Db {
|
||||
return {
|
||||
select: () => ({ from: () => ({ where: () => ({ limit: async () => [{
|
||||
status,
|
||||
@@ -9736,7 +9737,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
} as unknown as Db;
|
||||
}
|
||||
|
||||
async function initializeScopedRunnerRoot(current: NativeExecutionInputV1, runnerInstanceId: string) {
|
||||
async function initializeScopedRunnerRoot(current: NativeExecutionInput, runnerInstanceId: string) {
|
||||
state.createBackend.mockClear();
|
||||
state.createTransport.mockClear();
|
||||
await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId });
|
||||
@@ -9771,6 +9772,107 @@ describe("runnerd provider runtime wiring", () => {
|
||||
await rm(isolatedStateDirectory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => {
|
||||
const first = { ...scopedNativeExecution(`warm-admission-${scenario}`),
|
||||
schema: "paperclip.native-execution-input.v5" as const,
|
||||
provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const },
|
||||
executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() };
|
||||
first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 };
|
||||
const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` },
|
||||
session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner"].includes(scenario) ? 60_000 : 90_000 } } };
|
||||
if (scenario === "runtime-context") {
|
||||
next.runtimeContext = { ...first.runtimeContext,
|
||||
mcp: { ...first.runtimeContext.mcp, digest: "1".repeat(64) } };
|
||||
next.runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(next.runtimeContext);
|
||||
}
|
||||
const environmentId = `warm-admission-${scenario}`;
|
||||
const target = { kind: "remote" as const, transport: "sandbox" as const, environmentId,
|
||||
remoteCwd: "/home/daytona/workspace", runner: { execute: vi.fn() } };
|
||||
const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId,
|
||||
runnerInstanceId: `runner-${scenario}`, environmentLeaseId: first.binding.executionWorkspaceId };
|
||||
const root = await initializeScopedRunnerRoot(first, identity.runnerInstanceId);
|
||||
await writeRunnerAuthority(root, identity, "ready");
|
||||
const order: string[] = [];
|
||||
let failRetirement = scenario === "retirement-failed";
|
||||
const firstSession = { close: vi.fn(async () => {
|
||||
order.push("retire");
|
||||
if (failRetirement) throw new Error("retirement unconfirmed");
|
||||
await writeRunnerAuthority(root, identity, "suspended");
|
||||
}) };
|
||||
const nextSession = { close: vi.fn(async () => undefined) };
|
||||
const result = { result: { summary: "done" }, terminal: { runTerminalState: "succeeded" },
|
||||
turnId: "turn", normalizedSessionId: first.session.normalizedSessionId,
|
||||
providerSessionId: "provider", driverKind: "test", driverVersion: "1",
|
||||
nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
|
||||
state.execute.mockReset().mockImplementationOnce(async options => {
|
||||
await options.onSession?.(firstSession);
|
||||
return result;
|
||||
}).mockImplementationOnce(async options => {
|
||||
order.push("execute-next");
|
||||
expect(options.existingSession).toBe(scenario === "unchanged" ? firstSession : undefined);
|
||||
await options.onSession?.(scenario === "unchanged" ? firstSession : nextSession);
|
||||
return result;
|
||||
});
|
||||
const older = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-older` } };
|
||||
const nextDb = leaseDb(next);
|
||||
const ordinarySelect = nextDb.select.bind(nextDb);
|
||||
nextDb.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status
|
||||
&& fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
|
||||
? priorNativeRunDb(scenario === "stale-owner" ? older : first, scenario === "active-owner" ? "running" : "succeeded").select(fields as never)
|
||||
: ordinarySelect(fields as never)) as Db["select"];
|
||||
try {
|
||||
await executePaperclipNativeSession({ db: leaseDb(first), execution: first,
|
||||
runnerInstanceId: identity.runnerInstanceId, useRunnerd: true, runnerExecutionTarget: target as never });
|
||||
if (scenario === "stale-owner") await writeRunnerAuthority(root, { ...identity, runId: older.binding.runId }, "ready");
|
||||
if (scenario === "missing-owner") {
|
||||
await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "fixture owner loss" });
|
||||
await writeRunnerAuthority(root, identity, "ready");
|
||||
order.length = 0;
|
||||
firstSession.close.mockClear();
|
||||
}
|
||||
const continuation = executePaperclipNativeSession({ db: nextDb, execution: next,
|
||||
runnerInstanceId: `runner-next-${scenario}`, useRunnerd: true,
|
||||
runnerExecutionTarget: (scenario === "foreign-environment" ? { ...target, environmentId: `${environmentId}-other` } : target) as never });
|
||||
if (["stale-owner", "active-owner", "missing-owner", "foreign-environment"].includes(scenario)) {
|
||||
await expect(continuation).rejects.toThrow(scenario === "active-owner"
|
||||
? "prior_owner_active" : "prior_owner_terminal_state_indeterminate");
|
||||
expect(state.execute).toHaveBeenCalledOnce();
|
||||
expect(firstSession.close).not.toHaveBeenCalled();
|
||||
expect(order).toEqual([]);
|
||||
} else if (scenario === "retirement-failed") {
|
||||
await expect(continuation).rejects.toThrow("retirement unconfirmed");
|
||||
expect(state.execute).toHaveBeenCalledOnce();
|
||||
expect(order).toEqual(["retire"]);
|
||||
} else {
|
||||
await expect(continuation).resolves.toMatchObject({ exitCode: 0 });
|
||||
expect(order).toEqual(scenario === "unchanged" ? ["execute-next"] : ["retire", "execute-next"]);
|
||||
if (scenario === "unchanged") {
|
||||
await writeRunnerAuthority(root, { ...identity, runId: next.binding.runId }, "ready");
|
||||
const third = { ...next, binding: { ...next.binding, runId: `${next.binding.runId}-third` } };
|
||||
const thirdDb = leaseDb(third);
|
||||
const thirdSelect = thirdDb.select.bind(thirdDb);
|
||||
thirdDb.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status
|
||||
&& fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
|
||||
? priorNativeRunDb(next).select(fields as never)
|
||||
: thirdSelect(fields as never)) as Db["select"];
|
||||
state.execute.mockImplementationOnce(async options => {
|
||||
expect(options.existingSession).toBe(firstSession);
|
||||
await options.onSession?.(firstSession);
|
||||
return result;
|
||||
});
|
||||
await expect(executePaperclipNativeSession({ db: thirdDb, execution: third,
|
||||
runnerInstanceId: `runner-third-${scenario}`, useRunnerd: true,
|
||||
runnerExecutionTarget: target as never })).resolves.toMatchObject({ exitCode: 0 });
|
||||
expect(firstSession.close).not.toHaveBeenCalled();
|
||||
expect(state.execute).toHaveBeenCalledTimes(3);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
failRetirement = false;
|
||||
await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "test cleanup" });
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["open", "before-close"],
|
||||
["open", "during-close"],
|
||||
|
||||
Reference in new issue
Block a user