Add explicit Runner session release and resume support (#19683)

This commit is contained in:
Nicky Leach authored and GitHub committed 2026-11-06 18:31:23 -08:00
1 parent f09b4704b6
commit 03498f90ee
15 files changed
+1131 -8

No files matched your search

@@ -366,6 +366,67 @@ describe("runner provider-ingress WebSocket wire", () => {
}
});
it.each(["readiness", "authenticated observer"])("does not start liveness when quiesce wins %s", async mode => {
vi.useFakeTimers();
const create = sockets.create, socket = new FakeSocket();
sockets.create = () => openingSocket(socket);
const endpoint = { kind: "authenticated_websocket" as const,
websocketUrl: "wss://fixture.example.test/connect", secretHeaders: [], generation: "fixture",
refresh: vi.fn(async () => endpoint), close: vi.fn(async () => {}) };
const handle = connectRunnerPrpIngress({ authority: {
attachWireConnection: () => ({ isAuthenticated: () => {
if (mode === "readiness") handle.quiesce();
return true;
} }),
} as unknown as DurablePrpControlPlane, endpoint,
onStateChange: state => { if (state === "authenticated" && mode === "authenticated observer") handle.quiesce(); } });
try {
await vi.advanceTimersByTimeAsync(100_000);
expect(socket.ping).not.toHaveBeenCalled();
expect(socket.terminate).not.toHaveBeenCalled();
expect(socket.close).not.toHaveBeenCalled();
expect(endpoint.refresh).not.toHaveBeenCalled();
} finally { await handle.close(); sockets.create = create; vi.useRealTimers(); }
});
it("quiets liveness and reconnect while preserving the final receive/ACK path", async () => {
vi.useFakeTimers();
const create = sockets.create;
const socket = new FakeSocket();
let opens = 0;
sockets.create = () => { opens += 1; return openingSocket(socket); };
const endpoint = { kind: "authenticated_websocket" as const,
websocketUrl: "wss://fixture.example.test/connect", secretHeaders: [], generation: "fixture",
refresh: vi.fn(async () => endpoint), close: vi.fn(async () => {}) };
const received = vi.fn();
const attach = vi.fn((wire: WsJsonWireConnection) => {
wire.onJson(value => { received(value); wire.sendJson({ kind: "ack" }); });
return { isAuthenticated: () => true };
});
const handle = connectRunnerPrpIngress({ authority: { attachWireConnection: attach } as unknown as DurablePrpControlPlane,
endpoint, livenessIntervalMs: 10, livenessTimeoutMs: 30 });
const failure = vi.fn();
void handle.failure.catch(failure);
try {
await vi.advanceTimersByTimeAsync(0);
await handle.ready;
handle.quiesce();
socket.ping.mockClear();
socket.emit("message", Buffer.from(JSON.stringify({ kind: "last_event" })), false);
expect(received).toHaveBeenCalledWith({ kind: "last_event" });
expect(socket.send).toHaveBeenCalledWith(JSON.stringify({ kind: "ack" }));
await vi.advanceTimersByTimeAsync(100_000);
expect(socket.ping).not.toHaveBeenCalled();
expect(socket.terminate).not.toHaveBeenCalled();
expect(socket.close).not.toHaveBeenCalled();
socket.close();
await vi.advanceTimersByTimeAsync(100_000);
expect(opens).toBe(1);
expect(endpoint.refresh).not.toHaveBeenCalled();
expect(failure).not.toHaveBeenCalled();
} finally { await handle.close(); sockets.create = create; vi.useRealTimers(); }
});
it("does not report intentional close as an active-run ingress failure", async () => {
const endpoint = {
kind: "authenticated_websocket" as const,
+13 -1
View File
@@ -305,6 +305,8 @@ export interface RunnerPrpOutboundHandle {
readonly ready: Promise<void>;
/** Rejects only when active-run recovery has exhausted its fixed budget. */
readonly failure: Promise<never>;
/** Stop liveness and reconnect work, preserving an established receive/ACK path. */
quiesce(): void;
close(): Promise<void>;
}
@@ -337,6 +339,7 @@ export function connectRunnerPrpIngress(input: {
let readyResolve!: () => void;
let readyReject!: (error: unknown) => void;
let readySettled = false;
let stopLiveness = () => {};
const ready = new Promise<void>((resolve, reject) => {
readyResolve = resolve;
readyReject = reject;
@@ -386,22 +389,26 @@ export function connectRunnerPrpIngress(input: {
"Runner ingress did not complete PRP authentication before the deadline.",
);
}
if (abort.signal.aborted) return;
if (!readySettled) {
readySettled = true;
readyResolve();
}
input.onStateChange?.("authenticated");
if (abort.signal.aborted) return;
attempt = 0;
leaseFailureCode = null;
const stopLiveness = opened.wire.monitorLiveness({
stopLiveness = opened.wire.monitorLiveness({
intervalMs: input.livenessIntervalMs ?? DEFAULT_LIVENESS_INTERVAL_MS,
timeoutMs: input.livenessTimeoutMs ?? DEFAULT_LIVENESS_TIMEOUT_MS,
});
if (abort.signal.aborted) { stopLiveness(); return; }
try {
await socketClosed(opened.wire);
} finally {
stopLiveness();
}
if (abort.signal.aborted) return;
recoveryDeadline = Date.now() + recoveryGraceMs;
const leaseFailure = input.authority.connectionLeaseFailure ?? null;
if (leaseFailure) {
@@ -480,6 +487,11 @@ export function connectRunnerPrpIngress(input: {
return {
ready,
failure,
quiesce() {
stopLiveness();
// Established wires removed their handshake abort listener. Keep their ACK path.
abort.abort();
},
async close() {
abort.abort();
await Promise.all([...ownedWires].map(closeOwnedWire));