mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-12 22:56:58 +01:00
Add explicit Runner session release and resume support (#19683)
This commit is contained in:
1 parent
f09b4704b6
commit
03498f90ee
15 files changed
+1131
-8
No files matched your search
@@ -366,6 +366,67 @@ describe("runner provider-ingress WebSocket wire", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["readiness", "authenticated observer"])("does not start liveness when quiesce wins %s", async mode => {
|
||||
vi.useFakeTimers();
|
||||
const create = sockets.create, socket = new FakeSocket();
|
||||
sockets.create = () => openingSocket(socket);
|
||||
const endpoint = { kind: "authenticated_websocket" as const,
|
||||
websocketUrl: "wss://fixture.example.test/connect", secretHeaders: [], generation: "fixture",
|
||||
refresh: vi.fn(async () => endpoint), close: vi.fn(async () => {}) };
|
||||
const handle = connectRunnerPrpIngress({ authority: {
|
||||
attachWireConnection: () => ({ isAuthenticated: () => {
|
||||
if (mode === "readiness") handle.quiesce();
|
||||
return true;
|
||||
} }),
|
||||
} as unknown as DurablePrpControlPlane, endpoint,
|
||||
onStateChange: state => { if (state === "authenticated" && mode === "authenticated observer") handle.quiesce(); } });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(100_000);
|
||||
expect(socket.ping).not.toHaveBeenCalled();
|
||||
expect(socket.terminate).not.toHaveBeenCalled();
|
||||
expect(socket.close).not.toHaveBeenCalled();
|
||||
expect(endpoint.refresh).not.toHaveBeenCalled();
|
||||
} finally { await handle.close(); sockets.create = create; vi.useRealTimers(); }
|
||||
});
|
||||
|
||||
it("quiets liveness and reconnect while preserving the final receive/ACK path", async () => {
|
||||
vi.useFakeTimers();
|
||||
const create = sockets.create;
|
||||
const socket = new FakeSocket();
|
||||
let opens = 0;
|
||||
sockets.create = () => { opens += 1; return openingSocket(socket); };
|
||||
const endpoint = { kind: "authenticated_websocket" as const,
|
||||
websocketUrl: "wss://fixture.example.test/connect", secretHeaders: [], generation: "fixture",
|
||||
refresh: vi.fn(async () => endpoint), close: vi.fn(async () => {}) };
|
||||
const received = vi.fn();
|
||||
const attach = vi.fn((wire: WsJsonWireConnection) => {
|
||||
wire.onJson(value => { received(value); wire.sendJson({ kind: "ack" }); });
|
||||
return { isAuthenticated: () => true };
|
||||
});
|
||||
const handle = connectRunnerPrpIngress({ authority: { attachWireConnection: attach } as unknown as DurablePrpControlPlane,
|
||||
endpoint, livenessIntervalMs: 10, livenessTimeoutMs: 30 });
|
||||
const failure = vi.fn();
|
||||
void handle.failure.catch(failure);
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
await handle.ready;
|
||||
handle.quiesce();
|
||||
socket.ping.mockClear();
|
||||
socket.emit("message", Buffer.from(JSON.stringify({ kind: "last_event" })), false);
|
||||
expect(received).toHaveBeenCalledWith({ kind: "last_event" });
|
||||
expect(socket.send).toHaveBeenCalledWith(JSON.stringify({ kind: "ack" }));
|
||||
await vi.advanceTimersByTimeAsync(100_000);
|
||||
expect(socket.ping).not.toHaveBeenCalled();
|
||||
expect(socket.terminate).not.toHaveBeenCalled();
|
||||
expect(socket.close).not.toHaveBeenCalled();
|
||||
socket.close();
|
||||
await vi.advanceTimersByTimeAsync(100_000);
|
||||
expect(opens).toBe(1);
|
||||
expect(endpoint.refresh).not.toHaveBeenCalled();
|
||||
expect(failure).not.toHaveBeenCalled();
|
||||
} finally { await handle.close(); sockets.create = create; vi.useRealTimers(); }
|
||||
});
|
||||
|
||||
it("does not report intentional close as an active-run ingress failure", async () => {
|
||||
const endpoint = {
|
||||
kind: "authenticated_websocket" as const,
|
||||
|
||||
@@ -305,6 +305,8 @@ export interface RunnerPrpOutboundHandle {
|
||||
readonly ready: Promise<void>;
|
||||
/** Rejects only when active-run recovery has exhausted its fixed budget. */
|
||||
readonly failure: Promise<never>;
|
||||
/** Stop liveness and reconnect work, preserving an established receive/ACK path. */
|
||||
quiesce(): void;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -337,6 +339,7 @@ export function connectRunnerPrpIngress(input: {
|
||||
let readyResolve!: () => void;
|
||||
let readyReject!: (error: unknown) => void;
|
||||
let readySettled = false;
|
||||
let stopLiveness = () => {};
|
||||
const ready = new Promise<void>((resolve, reject) => {
|
||||
readyResolve = resolve;
|
||||
readyReject = reject;
|
||||
@@ -386,22 +389,26 @@ export function connectRunnerPrpIngress(input: {
|
||||
"Runner ingress did not complete PRP authentication before the deadline.",
|
||||
);
|
||||
}
|
||||
if (abort.signal.aborted) return;
|
||||
if (!readySettled) {
|
||||
readySettled = true;
|
||||
readyResolve();
|
||||
}
|
||||
input.onStateChange?.("authenticated");
|
||||
if (abort.signal.aborted) return;
|
||||
attempt = 0;
|
||||
leaseFailureCode = null;
|
||||
const stopLiveness = opened.wire.monitorLiveness({
|
||||
stopLiveness = opened.wire.monitorLiveness({
|
||||
intervalMs: input.livenessIntervalMs ?? DEFAULT_LIVENESS_INTERVAL_MS,
|
||||
timeoutMs: input.livenessTimeoutMs ?? DEFAULT_LIVENESS_TIMEOUT_MS,
|
||||
});
|
||||
if (abort.signal.aborted) { stopLiveness(); return; }
|
||||
try {
|
||||
await socketClosed(opened.wire);
|
||||
} finally {
|
||||
stopLiveness();
|
||||
}
|
||||
if (abort.signal.aborted) return;
|
||||
recoveryDeadline = Date.now() + recoveryGraceMs;
|
||||
const leaseFailure = input.authority.connectionLeaseFailure ?? null;
|
||||
if (leaseFailure) {
|
||||
@@ -480,6 +487,11 @@ export function connectRunnerPrpIngress(input: {
|
||||
return {
|
||||
ready,
|
||||
failure,
|
||||
quiesce() {
|
||||
stopLiveness();
|
||||
// Established wires removed their handshake abort listener. Keep their ACK path.
|
||||
abort.abort();
|
||||
},
|
||||
async close() {
|
||||
abort.abort();
|
||||
await Promise.all([...ownedWires].map(closeOwnedWire));
|
||||
|
||||
Reference in new issue
Block a user