diff --git a/doc/live-transcript-delivery.md b/doc/live-transcript-delivery.md index d17e4f45da5..98362e8d139 100644 --- a/doc/live-transcript-delivery.md +++ b/doc/live-transcript-delivery.md @@ -21,11 +21,13 @@ setting and the run's registered secret values. An event never uses a read that began before the event was queued. An event that arrives while a read is in flight starts a new set. A set is also replaced after -one second, so slow per-event work cannot extend it. A failed read is not shared; +250 ms or 100 events, so a large queue or slow per-event work cannot extend it. A failed read is not shared; the next event reads again. The set is discarded when the queue empties, so decrypted secret values are not retained by an idle connection. An isolated event therefore costs the same reads as before, and a revocation applies to the first -event published after it. +event published after it. Events that were already queued when access was revoked +can still be delivered until their set is replaced: at most 100 events or 250 ms +of output produced before the revocation. ## Additive negotiation diff --git a/server/src/realtime/live-event-delivery.test.ts b/server/src/realtime/live-event-delivery.test.ts index d46255525d8..7d3dd8c40b3 100644 --- a/server/src/realtime/live-event-delivery.test.ts +++ b/server/src/realtime/live-event-delivery.test.ts @@ -235,14 +235,14 @@ it("shares one permission read among events queued behind it, never with an even expect(read).toHaveBeenCalledTimes(4); f.delivery.dispose(); }); -it("stops sharing a read after one second and never reuses a failed read", async () => { +it("stops sharing a read after 250 ms and never reuses a failed read", async () => { let now = 1_000_000; const clock = vi.spyOn(Date, "now").mockImplementation(() => now); try { const read = vi.fn(async () => true); const f = fixture(vi.fn(async (value: LiveEvent, reads?: LiveDeliveryReads) => { const allowed = await sharedDeliveryRead(reads, "permission", read); - now += 400; + now += 100; return allowed ? value : null; })); f.subscribe(["run-1"]); @@ -261,3 +261,45 @@ it("stops sharing a read after one second and never reuses a failed read", async g.delivery.dispose(); } finally { clock.mockRestore(); } }); +it("serves exactly 100 events from one read set, so the 101st sees a revocation", async () => { + let allowed = true; + const read = vi.fn(async () => allowed); + const sets = new Map(); + const f = fixture(vi.fn(async (value: LiveEvent, reads?: LiveDeliveryReads) => { + sets.set(value.payload.seq as number, reads); + const visible = await sharedDeliveryRead(reads, "permission", read); + if (value.payload.seq === 2) allowed = false; + return visible ? value : null; + })); + f.subscribe(["run-1"]); + for (let seq = 1; seq <= 102; seq++) f.delivery.enqueue(event(seq)); + await flush(); + expect(read).toHaveBeenCalledTimes(3); + expect(sets.get(2)).not.toBe(sets.get(1)); + for (let seq = 3; seq <= 101; seq++) expect(sets.get(seq)).toBe(sets.get(2)); + expect(sets.get(102)).not.toBe(sets.get(101)); + expect(f.frames.filter(frame => frame.type === "heartbeat.run.log").map(frame => frame.payload.seq)) + .toEqual(Array.from({ length: 101 }, (_, index) => index + 1)); + f.delivery.dispose(); +}); +it("keeps a read set at 249 ms and replaces it at 250 ms", async () => { + let now = 1_000_000; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + try { + const steps = new Map([[2, 249], [3, 1]]); + const sets = new Map(); + const f = fixture(vi.fn(async (value: LiveEvent, reads?: LiveDeliveryReads) => { + const seq = value.payload.seq as number; + sets.set(seq, reads); + now += steps.get(seq) ?? 0; + return value; + })); + f.subscribe(["run-1"]); + for (let seq = 1; seq <= 4; seq++) f.delivery.enqueue(event(seq)); + await flush(); + expect(sets.get(3)).toBe(sets.get(2)); + expect(sets.get(4)).not.toBe(sets.get(3)); + expect(f.frames.filter(frame => frame.type === "heartbeat.run.log")).toHaveLength(4); + f.delivery.dispose(); + } finally { clock.mockRestore(); } +}); diff --git a/server/src/realtime/live-event-delivery.ts b/server/src/realtime/live-event-delivery.ts index f4aad8b7938..3c49e992bfb 100644 --- a/server/src/realtime/live-event-delivery.ts +++ b/server/src/realtime/live-event-delivery.ts @@ -4,7 +4,8 @@ import { beginIdleTrackedWork } from "../services/task-admission.js"; export const LIVE_DELIVERY_MAX_ITEMS = 1_000; export const LIVE_DELIVERY_MAX_BYTES = 1024 * 1024; const CONGESTION_DEADLINE_MS = 5_000; -const SHARED_READS_MAX_AGE_MS = 1_000; +const SHARED_READS_MAX_AGE_MS = 250; +const SHARED_READS_MAX_EVENTS = 100; export type LiveDeliveryReads = Map>; export function sharedDeliveryRead(reads: LiveDeliveryReads | undefined, key: string, read: () => Promise): Promise { if (!reads) return read(); @@ -39,11 +40,13 @@ export function createLiveEventDelivery(input: { let reads: LiveDeliveryReads = new Map(); let readsCoverSeq = 0; let readsOpenedAt = 0; + let readsServed = 0; function readsFor(seq: number) { const now = Date.now(); - if (seq > readsCoverSeq || now - readsOpenedAt >= SHARED_READS_MAX_AGE_MS) { - reads = new Map(); readsCoverSeq = enqueuedSeq; readsOpenedAt = now; + if (seq > readsCoverSeq || readsServed >= SHARED_READS_MAX_EVENTS || now - readsOpenedAt >= SHARED_READS_MAX_AGE_MS) { + reads = new Map(); readsCoverSeq = enqueuedSeq; readsOpenedAt = now; readsServed = 0; } + readsServed += 1; return reads; } function closeReads() { reads = new Map(); readsCoverSeq = 0; }