From f9447d6b1ac95af9e629e5333565e5cd729f2059 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 24 Sep 2026 15:53:10 -0700 Subject: [PATCH] fix(runner): admit bounded event-rich continuation state Read control-plane identity with the transport's 64 MiB bound. Preserve full scope and lifecycle checks, and cover verbose valid histories plus oversized valid JSON rejection. Co-Authored-By: Paperclip --- .../native-session-executor.test.ts | 79 ++++++++++++++++--- .../native-runtime/native-session-executor.ts | 13 +-- 2 files changed, 77 insertions(+), 15 deletions(-) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c94aabfd83b..a9d2876f8ab 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -8538,7 +8538,7 @@ describe("runnerd provider runtime wiring", () => { } }); - it("migrates a suspended prior-run authority only when its persisted execution has the same full session scope", async () => { + it.each([0, 2048])("migrates a suspended prior-run authority in the same full session scope with %i retained events", async (eventCount) => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-prior-run-session-state-"), ); @@ -8596,16 +8596,45 @@ describe("runnerd provider runtime wiring", () => { try { await mkdir(join(legacyRoot, "control-plane"), { recursive: true }); await mkdir(join(legacyRoot, "runner"), { recursive: true }); + const identity = { + runId: priorExecution.binding.runId, + normalizedSessionId: priorExecution.session.normalizedSessionId, + runnerInstanceId: "runner-prior-run-scope", + environmentLeaseId: "lease-prior-run-scope", + }; + const controlPlaneBytes = JSON.stringify({ + ...durableControlPlaneState(identity), + // The identity shares a file with the retained PRP event window. A + // verbose valid turn can exceed the old 2 MiB identity-read limit. + committedEvents: Array.from({ length: eventCount }, (_, index) => ({ + sourceSeq: index + 1, + sourceEventId: `event-${index + 1}`, + eventType: "item.delta", + priority: 1, + envelope: { + schema: "paperclip.prp.event.v1", + schemaVersion: 1, + sourceKind: "runner", + sourceInstanceId: identity.runnerInstanceId, + sourceEventId: `event-${index + 1}`, + sourceSeq: index + 1, + normalizedSessionId: identity.normalizedSessionId, + runId: identity.runId, + turnId: "turn-prior-run-scope", + itemId: "item-prior-run-scope", + eventType: "item.delta", + priority: 1, + emittedAt: "2026-09-24T00:00:00.000Z", + payload: { delta: "x".repeat(1024) }, + }, + deliveryCount: 1, + logicalEffectCount: 1, + })), + }); + if (eventCount > 0) expect(Buffer.byteLength(controlPlaneBytes)).toBeGreaterThan(2 * 1024 * 1024); await writeFile( join(legacyRoot, "control-plane", "control-plane-state.json"), - JSON.stringify( - durableControlPlaneState({ - runId: priorExecution.binding.runId, - normalizedSessionId: priorExecution.session.normalizedSessionId, - runnerInstanceId: "runner-prior-run-scope", - environmentLeaseId: "lease-prior-run-scope", - }), - ), + controlPlaneBytes, ); await writeFile( join(legacyRoot, "runner", "runner-state.json"), @@ -8634,6 +8663,17 @@ describe("runnerd provider runtime wiring", () => { state.createTransport.mock.calls[0]![0].stateDirectory!; expect(migratedRoot).not.toBe(legacyRoot); await expect(access(legacyRoot)).rejects.toThrow(); + // Re-enter through the canonical scoped root as ordinary continuation + // does, not only through the legacy migration path above. + state.createBackend.mockClear(); + state.createTransport.mockClear(); + await createRunnerdBackend({ + db: priorRunDb, + execution: currentExecution, + runnerInstanceId: "runner-current-run-scope", + }); + state.createBackend.mock.calls[0]![1].codexTransportFactory!(); + expect(state.createTransport.mock.calls[0]![0].stateDirectory).toBe(migratedRoot); expect(state.createTransport.mock.calls[0]![0].prpIdentity).toEqual( expect.objectContaining({ runId: currentExecution.binding.runId, @@ -9804,7 +9844,7 @@ describe("runnerd provider runtime wiring", () => { }, ); - it.each(["missing", "malformed", "unknown_schema", "mismatched"] as const)( + it.each(["missing", "malformed", "unknown_schema", "mismatched", "oversized"] as const)( "fails closed on %s durable identity in an existing scoped root", async (caseName) => { const stateBase = await mkdtemp( @@ -9865,6 +9905,25 @@ describe("runnerd provider runtime wiring", () => { ), ); } + if (caseName === "oversized") { + const identity = { + runId: scopedExecution.binding.runId, + normalizedSessionId: scopedExecution.session.normalizedSessionId, + runnerInstanceId: `runner-${caseName}-scoped-state`, + environmentLeaseId: scopedExecution.binding.executionWorkspaceId, + }; + // Valid JSON and valid ready authority: only the byte bound rejects + // this file. Malformed sparse padding would not test that boundary. + await writeFile( + join(scopedRoot, "control-plane", "control-plane-state.json"), + JSON.stringify(durableControlPlaneState(identity)).padEnd(64 * 1024 * 1024 + 1, " "), + ); + await mkdir(join(scopedRoot, "runner"), { recursive: true }); + await writeFile( + join(scopedRoot, "runner", "runner-state.json"), + JSON.stringify(durableRunnerState(identity, "ready")), + ); + } state.createBackend.mockClear(); state.createTransport.mockClear(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index d2706c5a112..06287417d74 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -280,7 +280,10 @@ export async function detachNativeSessionsForRestart( const MAX_REMOTE_CHECKPOINT_ARCHIVE_BYTES = 64 * 1024 * 1024; const MAX_REMOTE_CHECKPOINT_EXPANDED_BYTES = 64 * 1024 * 1024; const MAX_REMOTE_CHECKPOINT_ENTRIES = 20_000; -const NATIVE_DURABLE_IDENTITY_MAX_BYTES = 2 * 1024 * 1024; +// Identity is read from the complete control-plane state, including its PRP +// event window. Match the transport's bounded reader so verbose valid turns do +// not lose continuation authority merely because their history exceeds 2 MiB. +const NATIVE_CONTROL_PLANE_STATE_MAX_BYTES = 64 * 1024 * 1024; const NATIVE_RUNNER_STATE_MAX_BYTES = 16 * 1024 * 1024; const NATIVE_WARM_CHECKPOINT_MAX_BYTES = 8 * 1024 * 1024; const CODEX_HOME_NON_PERSISTENT_ENTRIES = [ @@ -4105,7 +4108,7 @@ function hasRetainedWarmTransitionEvidence(root: string): boolean { [ "control-plane", "control-plane-state.json", - NATIVE_DURABLE_IDENTITY_MAX_BYTES, + NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, ], ["runner", "runner-state.json", NATIVE_RUNNER_STATE_MAX_BYTES], ] as const) { @@ -4162,7 +4165,7 @@ function readWarmTransitionSnapshot(root: string) { } const core = readBoundedNativeFile( resolve(root, "control-plane", "control-plane-state.json"), - NATIVE_DURABLE_IDENTITY_MAX_BYTES, + NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "native_runner_warm_transition_recovery_unproven", ); const runner = readBoundedNativeFile( @@ -5003,7 +5006,7 @@ export function runnerdStateProvesIncompleteBootstrap(root: string): boolean { JSON.parse( readBoundedNativeFile( statePath, - NATIVE_DURABLE_IDENTITY_MAX_BYTES, + NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large", ).toString("utf8"), ), @@ -5056,7 +5059,7 @@ function readRunnerdDurableIdentity( JSON.parse( readBoundedNativeFile( statePath, - NATIVE_DURABLE_IDENTITY_MAX_BYTES, + NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large", ).toString("utf8"), ),