mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 01:50:13 +01:00
Reject failover backups behind the accepted event cursor (#19613)
A verified failover backup can still trail events already accepted by the controller. Compare the backup produced cursor with the registered controller state, or the supported local journal before registration, and reject stale files before remote sync. Keep the permanent replay-conflict classification ahead of generic Runner errors.
Validation: 1,192 native session and execution-driver tests and 10 workflow tests pass locally, together with the maintained low-memory Runner test project typecheck and fresh server declarations. All applicable checks on cd8c9cd07c are green (56 success, three skipped); exact-head Apex review is 5/5 with no unresolved findings. One e2e shard was retried after GitHub reported lost self-hosted runner communication, and the same-head rerun passed. No acknowledgement was lowered, provider replay authorized, or recovery files discarded.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-Authored-By: Paperclip <Paperclip-Paperclip@users.noreply.github.com>
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
3 files changed
+165
-3
No files matched your search
@@ -18216,6 +18216,124 @@ describe("runnerd provider runtime wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ backup: "stale", nextSourceSeq: 120, controlSchema: "paperclip.runner.durable.control-plane-state.v1", registered: false },
|
||||
{ backup: "stale", nextSourceSeq: 120, controlSchema: "paperclip.runner.durable.control-plane-state.v2", registered: false },
|
||||
{ backup: "current", nextSourceSeq: 1518, controlSchema: "paperclip.runner.durable.control-plane-state.v2", registered: false },
|
||||
{ backup: "stale", nextSourceSeq: 120, controlSchema: "paperclip.runner.durable.control-plane-state.v2", registered: true },
|
||||
{ backup: "current", nextSourceSeq: 1518, controlSchema: "paperclip.runner.durable.control-plane-state.v2", registered: true },
|
||||
])("checks the controller cursor before restoring a $backup failover backup ($controlSchema, registered=$registered)", async ({ backup, nextSourceSeq, controlSchema, registered }) => {
|
||||
const remoteCwd = join(isolatedStateDirectory, "remote");
|
||||
await mkdir(join(remoteCwd, ".paperclip-runtime", "paperclip-runner"), { recursive: true });
|
||||
const syncIn = vi.fn(async () => undefined);
|
||||
const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => {
|
||||
if (command.args?.[0] === "--build-metadata") return {
|
||||
exitCode: 0, timedOut: false, stdout: JSON.stringify({
|
||||
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
|
||||
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"],
|
||||
prpTransportModes: ["listen_ws"],
|
||||
}), stderr: "",
|
||||
};
|
||||
if (command.args?.[0] === "--version") return {
|
||||
exitCode: 0, timedOut: false, stdout: "codex-cli 0.160.0", stderr: "",
|
||||
};
|
||||
if (command.args?.[1]?.includes("base64")) return {
|
||||
exitCode: 1, timedOut: false, stdout: "", stderr: "",
|
||||
};
|
||||
return { exitCode: 0, timedOut: false, stdout: "", stderr: "" };
|
||||
});
|
||||
const checkpointSink = vi.fn(async () => undefined);
|
||||
const checkpointRegistration = registered ? vi.spyOn(runnerCheckpoint, "nativeRunnerCheckpointRegistration")
|
||||
.mockReturnValue(async authority => {
|
||||
authority.setCheckpointPersistence(checkpointSink);
|
||||
await authority.flushCheckpoint();
|
||||
}) : undefined;
|
||||
let controller: DurablePrpControlPlane | undefined;
|
||||
let registration: Awaited<ReturnType<NonNullable<import("../../vendor/paperclip-runner/index.js").RunnerExecutionDriver["controlPlaneRegistration"]>>> | undefined;
|
||||
onTestFinished(async () => { await registration?.release(); await controller?.stop(); checkpointRegistration?.mockRestore(); });
|
||||
const controllerArtifact = join(isolatedStateDirectory, "failover-cursor-controller");
|
||||
if (registered) await writeFile(controllerArtifact, "fixture bootstrap artifact");
|
||||
const runnerInstanceId = "runner-failover-cursor";
|
||||
await createRunnerdBackend({
|
||||
db: leaseDb(execution), execution, runnerInstanceId,
|
||||
...(registered ? { controllerPersistence: { bootId: "failover-cursor-boot" },
|
||||
runnerRemoteBinaryPath: controllerArtifact, runnerRemoteCodexPath: controllerArtifact } : {}),
|
||||
runnerIngressAuthorized: true,
|
||||
runnerExecutionTarget: {
|
||||
kind: "remote", transport: "sandbox", remoteCwd, environmentId: "environment",
|
||||
leaseId: "lease-replacement", providerKey: "daytona",
|
||||
effectiveCapabilities: { runnerWebSocketIngress: true },
|
||||
getRunnerIngressEndpoint: async () => ({ kind: "authenticated_websocket", websocketUrl: "wss://sandbox.invalid/runner",
|
||||
secretHeaders: [{ name: "Authorization", value: "fixture-ingress" }], generation: "1", refresh: vi.fn(), close: vi.fn(async () => undefined) }),
|
||||
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "sandbox-replacement", reason: "not_found" },
|
||||
runner: { execute: remoteExecute, syncIn },
|
||||
} as never,
|
||||
});
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & {
|
||||
prepareExternalRunnerState: () => Promise<void>;
|
||||
controlPlaneRegistration: NonNullable<import("../../vendor/paperclip-runner/index.js").RunnerExecutionDriver["controlPlaneRegistration"]>;
|
||||
};
|
||||
const root = options.stateDirectory!;
|
||||
await mkdir(join(root, "control-plane"), { recursive: true });
|
||||
if (!registered) await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify({
|
||||
schema: controlSchema,
|
||||
identity: options.prpIdentity, ackedSourceSeq: 1517, connectionCount: 1, committedEvents: [], commands: [],
|
||||
}));
|
||||
const current = join(root, "failover-backups", "current");
|
||||
await mkdir(join(current, "runner"), { recursive: true });
|
||||
await mkdir(join(current, "codex-home", "sessions"), { recursive: true });
|
||||
await writeFile(join(current, "runner", "runner-state.json"), JSON.stringify({
|
||||
schema: "paperclip.runner.durable.state.v1", ...options.prpIdentity, runnerInstanceId,
|
||||
lifecycle: "suspended", nextSourceSeq, ackedSourceSeq: nextSourceSeq - 1, outbox: [],
|
||||
}));
|
||||
await writeFile(join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state");
|
||||
await writeFile(join(current, "manifest.json"), JSON.stringify(buildNativeHarnessBackupManifest({
|
||||
backupRoot: current, execution, runnerInstanceId,
|
||||
providerSessionIdentity: { providerSessionId: "thread-1", providerBackendSessionId: "session-1", providerSessionIdentity: null },
|
||||
sourceProviderLeaseId: "sandbox-previous",
|
||||
})));
|
||||
|
||||
if (registered) {
|
||||
controller = new DurablePrpControlPlane({ stateDirectory: join(root, "control-plane"),
|
||||
identity: options.prpIdentity! as ConstructorParameters<typeof DurablePrpControlPlane>[0]["identity"],
|
||||
expectedRunnerVersion: "failover-cursor-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
controller.store.state.ackedSourceSeq = 1517;
|
||||
await controller.flushCheckpoint();
|
||||
registration = await options.controlPlaneRegistration(controller);
|
||||
expect(checkpointRegistration).toHaveBeenCalled();
|
||||
expect(checkpointSink).toHaveBeenCalled();
|
||||
const marker = JSON.parse(await readFile(join(root, "control-plane", "control-plane-state.json"), "utf8"));
|
||||
expect(marker.schema).toBe("paperclip.runner.external-control-plane-state.v1");
|
||||
expect(marker.ackedSourceSeq).toBeUndefined();
|
||||
syncIn.mockClear();
|
||||
}
|
||||
const prepared = options.prepareExternalRunnerState();
|
||||
if (backup === "stale") {
|
||||
await expect(prepared).rejects.toThrow(
|
||||
"native_event_replay_conflict: failover backup is behind the controller cursor (backup produced 119, controller acknowledged 1517)",
|
||||
);
|
||||
expect(syncIn).not.toHaveBeenCalled();
|
||||
const error = await prepared.catch((caught: unknown) => caught);
|
||||
expect(nativeSessionFailureSourceCode(new Error(`native_session_recovery_failed: ${String(error)}`)))
|
||||
.toBe("native_event_replay_conflict");
|
||||
expect(nativeSessionFailureDisposition(1, new Date(), "native_event_replay_conflict").phase).toBe("terminal_failure");
|
||||
} else {
|
||||
await expect(prepared).rejects.toThrow("runner_harness_state_mismatch: restored_provider_identity_changed");
|
||||
expect(syncIn).toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps an integrity conflict permanent when a runner exit wraps it", () => {
|
||||
expect(nativeSessionFailureSourceCode(new Error(
|
||||
"native_session_recovery_failed: runnerd exited unexpectedly: native_event_replay_conflict: failover backup is behind the controller cursor",
|
||||
))).toBe("native_event_replay_conflict");
|
||||
expect(nativeSessionFailureSourceCode(new Error(
|
||||
"native_session_recovery_failed: Error: runnerd exited while waiting for runner.drain",
|
||||
))).toBe("native_runner_process_exited");
|
||||
});
|
||||
|
||||
it.each([
|
||||
...["current", "preinstalled-exact", "preinstalled-mismatch", "preinstalled-error", "preinstalled-timeout", "stale", "missing", "retained", "retained-mismatch", "retained-error", "retained-timeout", "retained-explicit"]
|
||||
.map((image) => ({ image, version: "0.160.0" })),
|
||||
|
||||
Reference in new issue
Block a user