diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index e54d096968f..351ea9b579b 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1775,6 +1775,18 @@ starts by itself once the gate clears. A Retry accepted during the pre-deploy drain stays saved and is dispatched by the deferred-receipt worker after the restart. +After the provider accepts a turn, the transport waits for that turn's +`turn/started` before it reports the turn identity. Codex starts its MCP +servers before it starts the turn, so a healthy cold start in a loaded sandbox +can take most of a minute: on staging, submit to `turn/started` measured p50 +25 s, p90 50 s and max 76 s across 68 successful starts, while a fixed 30 s wait +hit 52 of 198 recent runs with `runnerd did not report the provider turn identity`. +The default wait is 90 s. Each provider notice, or runner diagnostic naming the +accepted turn, restarts that window, up to 180 s in all. A provider relaunch +does not count as progress. A silent start with no proof that the accepted turn +is still live fails when its window ends, and a Runner exit or a runner +failure of the pending turn still fails at once. + When a remote native run fails and its close cannot prove that the Runner suspended (for example, the Runner is unreachable after `provider_transport_failed` or a `turn.start` timeout), the controller no longer force-stops that Runner. It diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index f3bd0906d73..0ff1bc7b086 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -1989,7 +1989,24 @@ fn run() -> Result<(), Box> { } // A loaded host can accept a turn long before it starts it. if let Some(delay) = argument(&args, "--turn-started-delay-ms") { - thread::sleep(Duration::from_millis(delay.parse()?)); + let delay = Duration::from_millis(delay.parse()?); + let started = Instant::now(); + if let Some(interval) = argument(&args, "--turn-started-progress-ms") { + let interval = Duration::from_millis(interval.parse()?); + let mut server = 0; + while started.elapsed() + interval < delay { + thread::sleep(interval); + server += 1; + send( + json!({"method": "mcpServer/startupStatus/updated", "params": { + "name": format!("startup-{server}"), + "status": "failed", + "error": "fixture server unavailable" + }}), + )?; + } + } + thread::sleep(delay.saturating_sub(started.elapsed())); } // A test can hold the start until it has detached the controller. if let Some(gate) = argument(&args, "--turn-started-gate").map(PathBuf::from) { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts index b03eee9506e..d8fe12f51be 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts @@ -106,6 +106,96 @@ it("fails a pending turn start at once with the runner's own terminal failure", } }, 60_000); +async function withUnprovableLiveness(run: (unprove: () => void) => Promise): Promise { + let unproven = false; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function(options: ConstructorParameters[0]) { + const core = new OriginalCore(options); + const queueCommand = core.queueCommand.bind(core); + core.queueCommand = ((type: string, ...rest: unknown[]) => { + if (unproven && type === "session.snapshot") throw new Error("runner snapshot unavailable"); + return (queueCommand as (...args: unknown[]) => unknown)(type, ...rest); + }) as typeof core.queueCommand; + return core; + } as unknown as typeof OriginalCore); + try { + return await run(() => { unproven = true; }); + } finally { + coreSpy.mockRestore(); + } +} + +async function startSlowCodexTurn(input: { + label: string; + delayMs: number; + progressMs?: number; + turnStartTimeoutMs?: number; + turnStartProgressCeilingMs?: number; +}) { + const stateDirectory = await mkdtemp(join(tmpdir(), `runnerd-${input.label}-`)); + const callsPath = join(stateDirectory, "calls.log"); + return withUnprovableLiveness(async (unprove) => { + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--turn-started-delay-ms", String(input.delayMs), + ...(input.progressMs === undefined ? [] : ["--turn-started-progress-ms", String(input.progressMs)]), + "--call-log", callsPath), + stateDirectory, + ...(input.turnStartTimeoutMs === undefined ? {} : { turnStartTimeoutMs: input.turnStartTimeoutMs }), + ...(input.turnStartProgressCeilingMs === undefined ? {} : { turnStartProgressCeilingMs: input.turnStartProgressCeilingMs }), + }); + try { + await bundle.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [] }); + unprove(); + const started = Date.now(); + const outcome = await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Start slowly." }] }) + .then(result => ({ result, error: null }), (error: unknown) => ({ result: null, error })); + const elapsedMs = Date.now() - started; + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + return { ...outcome, elapsedMs, turnStarts: calls.filter(call => call === "turn/start").length, diagnostics: bundle.evidence().diagnostics }; + } finally { + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } + }); +} + +it("keeps waiting for an accepted Codex turn while its startup keeps reporting progress", async () => { + const outcome = await startSlowCodexTurn({ label: "progressing-turn-start", delayMs: 12_000, progressMs: 1_500, turnStartTimeoutMs: 4_000 }); + expect(outcome.error).toBeNull(); + expect(outcome.result).toMatchObject({ turn: { id: "provider-turn-1" } }); + expect(outcome.elapsedMs).toBeGreaterThan(10_000); + expect(outcome.turnStarts).toBe(1); + expect(outcome.diagnostics.join("\n")).not.toContain("adopting the durably accepted"); +}, 60_000); + +it("fails an accepted Codex start at the total ceiling while progress never reaches turn/started", async () => { + const outcome = await startSlowCodexTurn({ + label: "endless-progress-turn-start", delayMs: 30_000, progressMs: 1_000, + turnStartTimeoutMs: 3_000, turnStartProgressCeilingMs: 8_000, + }); + expect(String(outcome.error)).toContain("runnerd did not report the provider turn identity"); + expect(outcome.elapsedMs).toBeGreaterThan(7_000); + expect(outcome.elapsedMs).toBeLessThan(8_000 + 2_000); + expect(outcome.turnStarts).toBe(1); +}, 60_000); + +it("still fails a silent accepted Codex start at its bound when liveness cannot be proven", async () => { + const outcome = await startSlowCodexTurn({ label: "silent-turn-start", delayMs: 20_000, turnStartTimeoutMs: 3_000 }); + expect(String(outcome.error)).toContain("runnerd did not report the provider turn identity"); + expect(outcome.elapsedMs).toBeLessThan(3_000 + 1_500); + expect(outcome.turnStarts).toBe(1); +}, 60_000); + +it("gives a silent Codex cold start the measured default start window", async () => { + const outcome = await startSlowCodexTurn({ label: "cold-turn-start", delayMs: 40_000 }); + expect(outcome.error).toBeNull(); + expect(outcome.result).toMatchObject({ turn: { id: "provider-turn-1" } }); + expect(outcome.elapsedMs).toBeGreaterThan(38_000); + expect(outcome.turnStarts).toBe(1); +}, 90_000); + it("keeps a quiet active Codex turn in the same process across connection lease expiry", async () => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-renew-active-")); const callsPath = join(stateDirectory, "calls.log"); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 548666ec590..d24560d5cbe 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -962,6 +962,8 @@ function terminalMayBelongToPendingTurnStart(input: { /** Part of the turn-start bound kept for the accepted-turn adoption check. */ const TURN_START_ADOPTION_CHECK_RESERVE_MS = 5_000; +const TURN_START_EVENT_DEFAULT_TIMEOUT_MS = 90_000; +const TURN_START_PROGRESS_CEILING_MS = 180_000; /** * The runner's own final failure of a turn whose start is still pending: a @@ -1669,6 +1671,7 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD closeGraceMs?: number; /** Bounded provider turn-admission wait; defaults to 30 seconds. */ turnStartTimeoutMs?: number; + turnStartProgressCeilingMs?: number; /** Controller-owned opt-in operation timing. No exporter lives in Runner. */ measureOperation?(phase: string, work: () => Promise, attributes?: Record): Promise; recordTimingReceipt?(receipt: unknown): void; @@ -7618,13 +7621,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }): boolean { if (input.snapshot?.activeProviderTurnId !== input.expectedProviderTurnId) return false; if (!Number.isSafeInteger(input.acceptingCommandSeq)) return false; - const relaunched = (this.#core?.store.state.committedEvents ?? []).some((event) => { - if (event.eventType !== "harness.diagnostic") return false; - const payload = record(record(event.envelope.payload).payload); - return payload.code === "provider_startup_ownership" && - Number(record(record(payload.startup).command).controllerSeq) > input.acceptingCommandSeq; - }); - if (relaunched) return false; + if (this.#providerRelaunchedAfter(input.acceptingCommandSeq)) return false; if (input.acceptingProviderPid === null || this.#evidence.providerPid !== input.acceptingProviderPid) return false; // A remote provider's pid names no local process; the runner's own // launch reports above are its replacement evidence. @@ -7632,6 +7629,37 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return readLocalProcessStartedAt(input.acceptingProviderPid) === input.acceptingProviderStartedAt; } + #providerRelaunchedAfter(acceptingCommandSeq: number): boolean { + return (this.#core?.store.state.committedEvents ?? []).some((event) => { + if (event.eventType !== "harness.diagnostic") return false; + const payload = record(record(event.envelope.payload).payload); + return payload.code === "provider_startup_ownership" && + Number(record(record(payload.startup).command).controllerSeq) > acceptingCommandSeq; + }); + } + + #latestCommittedSourceSeq(): number { + return this.#core?.store.state.committedEvents.at(-1)?.sourceSeq ?? 0; + } + + #pendingTurnStartProgressSince(sourceSeq: number, expectedProviderTurnId: string): { progressed: boolean; latestSeq: number } { + const events = this.#core?.store.state.committedEvents ?? []; + let latestSeq = sourceSeq; + let progressed = false; + for (let index = events.length - 1; index >= 0; index -= 1) { + const event = events[index]!; + if (event.sourceSeq <= sourceSeq) break; + if (event.sourceSeq > latestSeq) latestSeq = event.sourceSeq; + if (progressed) continue; + if (event.eventType === "provider.notice.recorded") progressed = true; + else if (event.eventType === "harness.diagnostic") { + const payload = record(record(event.envelope.payload).payload); + progressed = payload.expectedTurnId === expectedProviderTurnId; + } + } + return { progressed, latestSeq }; + } + async #startTurnWork( params: Record, ): Promise> { @@ -7757,10 +7785,19 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // command's durable result so a delayed prior-turn event cannot satisfy // the new response fence. ACPX echoes the requested identity, while // Codex and OpenCode return their provider-assigned identity. - const deadline = + const quietWindowMs = this.options.turnStartTimeoutMs === undefined - ? this.#waitNow() + 30_000 - : commandDeadline; + ? TURN_START_EVENT_DEFAULT_TIMEOUT_MS + : Math.max(0, commandDeadline - this.#waitNow()); + let deadline = this.#waitNow() + quietWindowMs; + const progressCeiling = this.#waitNow() + Math.max(quietWindowMs, this.options.turnStartProgressCeilingMs ?? TURN_START_PROGRESS_CEILING_MS); + let progressSeq = this.#latestCommittedSourceSeq(); + const extendOnProgress = () => { + const progress = this.#pendingTurnStartProgressSince(progressSeq, expectedProviderTurnId); + progressSeq = progress.latestSeq; + if (progress.progressed && !this.#providerRelaunchedAfter(acceptingCommandSeq)) + deadline = Math.min(progressCeiling, Math.max(deadline, this.#waitNow() + quietWindowMs)); + }; const providerTurnStarted = () => turnStartResponseReady({ responseEpoch, @@ -7768,14 +7805,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { expectedProviderTurnId, boundTurnId: this.#turnId, }); - // Keep the adoption check inside the configured bound: stop waiting for - // the start event early enough to read the runner's snapshot before it. - const adoptionCheckAt = deadline - Math.min( + const adoptionReserveMs = Math.min( TURN_START_ADOPTION_CHECK_RESERVE_MS, - Math.floor(Math.max(0, deadline - this.#waitNow()) / 4), + Math.floor(quietWindowMs / 4), ); - // The runner can end the accepted turn before it starts. Its terminal is - // held for the pending start, so report that real cause at once. const throwIfRunnerFailedStart = () => { const failure = runnerdPendingTurnStartFailure([ ...this.#deferredTurnStartEvents, @@ -7787,59 +7820,41 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { throw new Error(`runnerd failed the provider turn before it started: ${failure}`); }; const waitForStarted = async () => { - while (!providerTurnStarted() && this.#waitNow() < adoptionCheckAt) { - this.#throwIfFailed(); - this.#pumpEvents(); - if (providerTurnStarted()) break; - throwIfRunnerFailedStart(); - if (await this.#runnerHasExited()) - throw new Error("runnerd exited before provider turn startup"); - await new Promise((resolveWait) => setTimeout(resolveWait, 10)); - } - if (!providerTurnStarted()) { - // The durable command result already carries the provider's own - // acceptance and turn id. A loaded provider can start that turn after - // this bound, and it still runs it. Failing would abandon a live turn - // that no controller owns, so its tools and close could never settle. - // Bind the exact accepted identity when the runner proves it is live; - // its turn/started is then accepted in order, and the turn's own - // terminal, timeout and Stop paths own it from here. - this.#throwIfFailed(); - if (await this.#runnerHasExited()) - throw new Error("runnerd exited before provider turn startup"); - const snapshot = await this.#commandResult( - "session.snapshot", {}, deadline, - ).catch(() => null); - // Ingest any runner-reported provider launch committed before it. A - // pump that rejected a stale start has failed the transport; never - // report a successful start on it. - this.#pumpEventsSafely(); - this.#throwIfFailed(); - if (!providerTurnStarted()) throwIfRunnerFailedStart(); - if (!providerTurnStarted() && this.#acceptedTurnProvablyLive({ - snapshot, expectedProviderTurnId, acceptingProviderPid, - acceptingProviderStartedAt, acceptingCommandSeq, - })) { - this.#turnId = expectedProviderTurnId; - this.#observedTurnStartEpoch = responseEpoch; - this.#diagnostic( - "provider turn start outlived its bound; adopting the durably accepted active provider turn", - ); - } - // Not provably live: keep the original contract and wait for the - // start event until the configured bound, then fail as before. - while (!providerTurnStarted() && this.#waitNow() < deadline) { + let adoptionCheckedFor: number | null = null; + while (!providerTurnStarted()) { this.#throwIfFailed(); this.#pumpEvents(); if (providerTurnStarted()) break; throwIfRunnerFailedStart(); + extendOnProgress(); if (await this.#runnerHasExited()) throw new Error("runnerd exited before provider turn startup"); + if (this.#waitNow() >= deadline - adoptionReserveMs && adoptionCheckedFor !== deadline) { + adoptionCheckedFor = deadline; + const snapshot = await this.#commandResult( + "session.snapshot", {}, deadline, + ).catch(() => null); + this.#pumpEventsSafely(); + this.#throwIfFailed(); + if (providerTurnStarted()) break; + throwIfRunnerFailedStart(); + if (this.#acceptedTurnProvablyLive({ + snapshot, expectedProviderTurnId, acceptingProviderPid, + acceptingProviderStartedAt, acceptingCommandSeq, + })) { + this.#turnId = expectedProviderTurnId; + this.#observedTurnStartEpoch = responseEpoch; + this.#diagnostic( + "provider turn start outlived its bound; adopting the durably accepted active provider turn", + ); + break; + } + continue; + } + if (this.#waitNow() >= deadline) + throw new Error("runnerd did not report the provider turn identity"); await new Promise((resolveWait) => setTimeout(resolveWait, 10)); } - if (!providerTurnStarted()) - throw new Error("runnerd did not report the provider turn identity"); - } }; if (this.options.measureOperation) await this.options.measureOperation("runner.transport.started_event", waitForStarted); else await waitForStarted();