diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 53d63488edf..925936fb915 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -199,8 +199,14 @@ also grades "no leaked sandboxes / clean warm shutdown" (`warm-shutdown.ts`). It the wave when the environment delete stayed blocked, when a wave sandbox is still running after cleanup, or when the server log shows a warm session that did not close on environment delete or controller shutdown. Idle closes that needed a forced Runner -stop, or whose sandbox was already destroyed, are counted but do not fail. The result -is in `/warm-shutdown.json` and in the wave's `SUMMARY.md`. +stop, or whose sandbox was already destroyed, are counted but do not fail. Both counts +are distinct (run, reason) failures, not log lines: the harness's environment-delete +retries re-log the same failed close every few seconds (one session logged 27 times in +w-20261116T001158-s7106). Each failure keeps its line count. On a Daytona warm wave, the +first idle-close failure in the server output triggers one sandbox capture +(`daytona-runner/idle-close-failure-/`: processes, Runner state and logs), listed +under `sandboxCaptures`. The result is in `/warm-shutdown.json` and in the wave's +`SUMMARY.md`. ## Hosted mode (`--hosted-base-url`, `--hosted-allow`) @@ -411,7 +417,7 @@ part of U3 is checked only in waves run with this harness. temporary instance is deleted. - Faults are serialized; each one hits every task running at that moment. - A `todo` or `in_progress` task with no live run and no named wait is not a rest. The observer keeps watching it until it resumes or passes the R5 stall limit. -- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. A quiet run is still live while a tool it started has not ended (for example a 4-minute `bench.sh`) or while its recorded local runner process still runs with the same PID and start time. While a provider turn is active and no tool runs, one model request is outstanding: R5 allows up to 10 minutes of quiet before it counts a dead run. A slow OpenRouter request took 303.6 s in w-20261115T044649-s7106/t028, and common provider clients time out a request at 10 minutes. +- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. A quiet run is still live while a tool it started has not ended (for example a 4-minute `bench.sh`) or while its recorded local runner process still runs with the same PID and start time. While a provider turn is active and no tool runs, one model request is outstanding: R5 allows up to 10 minutes of quiet before it counts a dead run. A slow OpenRouter request took 303.6 s in w-20261115T044649-s7106/t028, and common provider clients time out a request at 10 minutes. A run with no run event yet (`preparing`: its events read back empty) is still acquiring or preparing its environment, because "run started" is written only after that. R5 allows it 10 minutes too: in w-20261116T001158-s7106/t010 a Daytona restore of an archived sandbox took 251 s, including one 90 s plugin call timeout, and the Daytona provider bounds each start or restore at 5 minutes by default, so 10 minutes covers one timed-out restore and its retry. U1 uses the same bound (`PREPARATION_LIMIT_MS`) before it stops counting a preparing run as the task's owner. - The wave fingerprints the checkout and `packages/paperclip-runner/dist` at start and after every fault. A build, install or branch switch during a wave would make later restarts run different code, so it fails the wave as a harness error (`worktree_changed_mid_wave`). The baseline is taken after the harness's own setup (preflight rebuilds provider entrypoints) and covers the tracked diff, untracked file contents, the runner `dist` and the runnerd binary. Build before the wave, never during it. - `faults.json` and `task-faults.json` (each task's launch time and the faults that crossed it) are written after every fault, so `--replay` of a task in an interrupted or wedged wave, or a storm task, re-fires the faults that crossed it at the same offsets. - A fault that fails to complete fails the wave as a harness error (exit 2), even when recovery brings the server back. diff --git a/tests/restart-chaos/daytona-cleanup.ts b/tests/restart-chaos/daytona-cleanup.ts index 8dfb0ee2b20..d4d4c328bd0 100644 --- a/tests/restart-chaos/daytona-cleanup.ts +++ b/tests/restart-chaos/daytona-cleanup.ts @@ -21,9 +21,9 @@ export function daytonaCleanupLine(sweep: DaytonaSweepResult, cleanup: WaveClean } export async function finishDaytonaCleanup(input: { sweeper: Pick; sweep: DaytonaSweepResult | null; waveDir: string; - timeline: { add(kind: string, data: Record): void }; log?: (line: string) => void }): Promise { + timeline: { add(kind: string, data: Record): void }; log?: (line: string) => void; beforeSandboxDelete?: () => Promise }): Promise { const sweep = input.sweep ?? await input.sweeper.sweep(0); - const cleanup = await input.sweeper.cleanUp(); + const cleanup = await input.sweeper.cleanUp(undefined, undefined, input.beforeSandboxDelete); input.timeline.add("daytona_cleanup", cleanup); (input.log ?? console.info)(daytonaCleanupLine(sweep, cleanup)); if (cleanup.environmentDeleteBlocked) input.timeline.add("daytona_environment_delete_blocked", { detail: cleanup.environmentDeleteBlocked }); diff --git a/tests/restart-chaos/daytona-sweep.test.ts b/tests/restart-chaos/daytona-sweep.test.ts index e28282f7103..cc05176e8f1 100644 --- a/tests/restart-chaos/daytona-sweep.test.ts +++ b/tests/restart-chaos/daytona-sweep.test.ts @@ -61,6 +61,22 @@ describe("cleanup through Paperclip", () => { expect(rows.find(row => row.id === "other-env")!.state).toBe("archived"); }); + it("waits for a capture the environment delete triggered before it deletes any wave sandbox", async () => { + const rows = waveRows(["started"]); + const order: string[] = []; + const { client, removeSandbox } = deletingDaytona(rows); + const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, + api: { delete: async () => { order.push("environment delete"); } } as never, + removeSandbox: async id => { order.push(`delete ${id}`); await removeSandbox(id); } }); + (sweeper as unknown as { client: unknown }).client = client; + const result = await sweeper.cleanUp(0, 1_000, async () => { await new Promise(resolve => setTimeout(resolve, 20)); order.push("capture"); }); + expect(order).toEqual(["environment delete", "capture", "delete started"]); + expect(result.errors).toEqual([]); + const failing = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: { delete: async () => undefined } as never }); + (failing as unknown as { client: unknown }).client = readOnlyDaytona(listing); + expect((await failing.cleanUp(0, 1_000, async () => { throw new Error("probe failed"); })).errors).toEqual(["before sandbox delete: probe failed"]); + }); + it("fails cleanup when a wave sandbox survives deletion, and never deletes a fleet snapshot even with the wave's labels", async () => { const rows = [...waveRows(["archived"]), ...waveRows(["fleet"], { snapshot: "fleet-sbx-9", state: "archived" })]; const { client, removeSandbox, removed } = deletingDaytona(rows); diff --git a/tests/restart-chaos/daytona-sweep.ts b/tests/restart-chaos/daytona-sweep.ts index 2e31241faee..243b5455ac7 100644 --- a/tests/restart-chaos/daytona-sweep.ts +++ b/tests/restart-chaos/daytona-sweep.ts @@ -215,7 +215,7 @@ export class DaytonaWaveSweeper { } } - async cleanUp(settleMs = 90_000, deleteWindowMs = R6_SETTLE_MS): Promise { + async cleanUp(settleMs = 90_000, deleteWindowMs = R6_SETTLE_MS, beforeSandboxDelete?: () => Promise): Promise { const errors: string[] = []; let environmentDeleted = false; let environmentDeleteBlocked: string | null = null; @@ -229,6 +229,7 @@ export class DaytonaWaveSweeper { if (/returned 409\b/.test(message)) environmentDeleteBlocked = message; else errors.push(`environment delete: ${message}`); } + await beforeSandboxDelete?.().catch(error => { errors.push(`before sandbox delete: ${error instanceof Error ? error.message : String(error)}`); }); let remainingSandboxes: string[] = []; let archivedSandboxes: string[] = []; // Provider teardown is asynchronous; give it a bounded settle before reporting. diff --git a/tests/restart-chaos/fixtures/a1-warm-close-failures.log b/tests/restart-chaos/fixtures/a1-warm-close-failures.log new file mode 100644 index 00000000000..68a9f0d01e3 --- /dev/null +++ b/tests/restart-chaos/fixtures/a1-warm-close-failures.log @@ -0,0 +1,34 @@ +[18:30:55] WARN: idle warm native session close failed {"runId":"300f0d61-548b-4630-8f6a-f79292ed9d24","issueId":"6577ff15-9c90-4b0c-812c-ee1f6e2226b0","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"287cbe35-0b93-46f4-89f4-5a0430592870","closeFailure":"runner_remote_process_identity_unavailable: runner_remote_process_identity_unavailable [process_verification/process_identity_changed]","closeAttempt":1} +[18:31:00] WARN: idle warm native session close failed {"runId":"300f0d61-548b-4630-8f6a-f79292ed9d24","issueId":"6577ff15-9c90-4b0c-812c-ee1f6e2226b0","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"287cbe35-0b93-46f4-89f4-5a0430592870","closeFailure":"runner_remote_process_identity_unavailable: runner_remote_process_identity_unavailable [process_verification/process_identity_changed]","closeAttempt":2} +[18:31:05] WARN: idle warm native session close failed {"runId":"300f0d61-548b-4630-8f6a-f79292ed9d24","issueId":"6577ff15-9c90-4b0c-812c-ee1f6e2226b0","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"287cbe35-0b93-46f4-89f4-5a0430592870","closeFailure":"runner_remote_process_identity_unavailable: runner_remote_process_identity_unavailable [process_verification/process_identity_changed]","closeAttempt":3} +[18:31:10] WARN: idle warm native session close failed {"runId":"300f0d61-548b-4630-8f6a-f79292ed9d24","issueId":"6577ff15-9c90-4b0c-812c-ee1f6e2226b0","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"287cbe35-0b93-46f4-89f4-5a0430592870","closeFailure":"runner_remote_process_identity_unavailable: runner_remote_process_identity_unavailable [process_verification/process_identity_changed]","closeAttempt":4} +[18:32:50] WARN: idle warm native sessions did not close {"reason":"controller restart","failures":[{"runId":"300f0d61-548b-4630-8f6a-f79292ed9d24","issueId":"6577ff15-9c90-4b0c-812c-ee1f6e2226b0","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"287cbe35-0b93-46f4-89f4-5a0430592870","reason":"close failed: runner_remote_process_identity_unavailable: runner_remote_process_identity_unavailable [process_verification/process_identity_changed]"}]} +[18:32:50] WARN: idle native session cleanup failed before controller shutdown {"idleSessions":{"closed":0,"busy":0,"failed":1,"failures":[{"runId":"300f0d61-548b-4630-8f6a-f79292ed9d24","issueId":"6577ff15-9c90-4b0c-812c-ee1f6e2226b0","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"287cbe35-0b93-46f4-89f4-5a0430592870","reason":"close failed: runner_remote_process_identity_unavailable: runner_remote_process_identity_unavailable [process_verification/process_identity_changed]"}]}} +[19:11:36] WARN: idle warm native session close failed {"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","closeFailure":"native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}","closeAttempt":1} +[19:12:13] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:14] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:16] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:19] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:22] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:25] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:28] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:31] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:34] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:40] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:43] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:46] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:49] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:52] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:55] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:12:58] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:01] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:04] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:12] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:15] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:18] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:21] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:24] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:27] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:30] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:33] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} +[19:13:36] WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[{"runId":"70d909cc-8b04-40ae-a0bf-0846df140cf0","issueId":"0605dbde-cd34-4728-bc68-3c1fe9ca4c06","environmentId":"d729f12b-4d52-4e10-8cae-e03bdf73b2a3","providerLeaseId":"2f139bcc-8503-4722-8943-0defefeecbab","reason":"close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint {\"runnerSuspended\":false,\"runnerRetained\":true,\"providerDrained\":true,\"suspensionState\":{\"commandStatus\":\"failed\",\"runnerLifecycle\":\"ready\",\"runnerIdentityMatches\":true},\"semanticTools\":{\"persistenceFailed\":false,\"failures\":[],\"pending\":[],\"deliveries\":[]},\"finalProviderState\":null,\"timings\":{\"exitProbeMs\":0,\"stopMs\":0,\"drainMs\":225,\"suspendQueuedAfterMs\":225,\"proofReadMs\":4290,\"barrierExitProbeMs\":0,\"suspendMs\":5239,\"retainProbeMs\":0,\"releaseMs\":0,\"retireMs\":0,\"checkpointMs\":591,\"totalMs\":6055},\"proofWindowEnd\":\"suspend_not_completed\"}"}]} diff --git a/tests/restart-chaos/fixtures/t010-slow-environment-acquire.json b/tests/restart-chaos/fixtures/t010-slow-environment-acquire.json new file mode 100644 index 00000000000..797727db15c --- /dev/null +++ b/tests/restart-chaos/fixtures/t010-slow-environment-acquire.json @@ -0,0 +1 @@ +{"source":"w-20261116T001158-s7106/t010 (CHA-14): follow-up run aedc1f6e spent 251 s in environment.acquire (Daytona restore of an archived sandbox, one 90 s plugin RPC timeout); its first run event (\"run started\") was written after acquire; the run then succeeded","runId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","firstRunEventAt":"2026-11-16T01:08:19.648Z","samples":[{"at":1794791045930,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791049786,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791053308,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791056820,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791060250,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791063734,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791067437,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791071043,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791074554,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791078071,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791081605,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791085129,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791088725,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791092293,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791096048,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791099622,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791103213,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791106728,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791110419,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791113916,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791117514,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791121018,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791124503,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791128091,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791131569,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791135106,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791138664,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791142108,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791145618,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791149091,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791152654,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791156173,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791159798,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791163380,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791166998,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791170482,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791173996,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791177644,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791181447,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791185024,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791188501,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791191879,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791195507,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791198998,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791202459,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791206044,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791209546,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791213060,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791216838,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791220341,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791223885,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791227341,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791230710,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791234016,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791237276,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791240549,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791243826,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791247101,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791250552,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791253975,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791257391,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791260728,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791264071,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791267534,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791270731,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791273943,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791277190,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791280387,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791283571,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791286739,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791289940,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791293122,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791296289,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791299422,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:04:05.882Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791302519,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:21.990Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791305659,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:24.885Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791308760,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:27.515Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791311883,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:31.824Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791314996,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:32.415Z","providerTurnActive":true},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791318122,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:37.668Z","providerTurnActive":true},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791321226,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:41.181Z","toolInFlight":true,"providerTurnActive":true},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791324310,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:44.329Z","providerTurnActive":true},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791327385,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:47.430Z","providerTurnActive":true},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791330464,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:49.194Z","providerTurnActive":true},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791333536,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:53.543Z","toolInFlight":true,"providerTurnActive":true},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791336656,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"running","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-16T01:08:56.730Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]},{"at":1794791339766,"reachable":true,"serverReady":true,"issueStatus":"done","interruption":{"state":"resolved","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"aedc1f6e-3a5e-42d2-97ab-295f19e70bb8","status":"succeeded","createdAt":"2026-11-16T00:59:12.296Z","startedAt":"2026-11-16T01:04:05.882Z","finishedAt":"2026-11-16T01:08:58.812Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"completed","lastEvidenceAt":"2026-11-16T01:08:58.892Z"},{"id":"91d61efb-4a8b-4ed2-87ba-f640d6065cbf","status":"succeeded","createdAt":"2026-11-16T00:26:58.075Z","startedAt":"2026-11-16T00:26:58.346Z","finishedAt":"2026-11-16T00:38:17.936Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-16T00:38:17.875Z"}]}]} \ No newline at end of file diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 5c2c3ff6ea4..1c7ad539cb7 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -8,6 +8,7 @@ import { enforcedUniversalFailures, gradeUniversal, type SlotFree, type Universa export const ACTIVE_RUN_STATUSES = new Set(["queued", "running", "scheduled_retry"]); export const STALL_LIMIT_MS = 3 * 60_000; export const PROVIDER_TURN_LIMIT_MS = 10 * 60_000; +export const PREPARATION_LIMIT_MS = 10 * 60_000; const REST_STATUSES_WITH_WORK = new Set(["todo", "in_progress"]); const ms = (iso: string | null | undefined) => (iso ? Date.parse(iso) : NaN); @@ -211,7 +212,7 @@ export function deadRunningRun(sample: TaskSample, readySince: number): { id: st if (run.status !== "running") continue; // Quiet is not dead while a tool is still in flight or the run's process is verifiably alive. if (run.toolInFlight || run.processAlive) continue; - const limit = run.providerTurnActive ? PROVIDER_TURN_LIMIT_MS : STALL_LIMIT_MS; + const limit = run.preparing ? PREPARATION_LIMIT_MS : run.providerTurnActive ? PROVIDER_TURN_LIMIT_MS : STALL_LIMIT_MS; const evidence = Math.max(ms(run.lastEvidenceAt) || 0, ms(run.startedAt) || 0, readySince); if (sample.at - evidence > limit) return { id: run.id, idleMs: sample.at - Math.max(ms(run.lastEvidenceAt) || 0, ms(run.startedAt) || 0) }; } diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 67b9c8e0325..f55f3311dc6 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -44,7 +44,7 @@ import { matrixMayLaunch, planMatrixWave } from "./journey-matrix.js"; import { redeployWave } from "./redeploy.js"; import { withT1CrossingCheck } from "./workloads.js"; import { parseRecoveryStepReceipts, withRecoveryStepCheck } from "./recovery-step-faults.js"; -import { captureWaveRunnerLogs, executeInWaveSandboxes } from "./daytona-runner-logs.js"; +import { captureWaveRunnerLogs, executeInWaveSandboxes, RUNNER_LOG_CAPTURE_DEADLINE_MS } from "./daytona-runner-logs.js"; import { isClassRFault, killedPidsFromOutput, providerKillCommand, watchLabels, withClassRCheck, type ProviderKillRecord } from "./class-r.js"; import { TASK_FAULT_LOG, observedFaultSchedule, planWave, replayOnCurrentBuild, replayRecordFor, replayWave, type TaskFaultLog } from "./scheduler.js"; import { chaosLifecycleOverrides, seedChaosLab } from "./seed.js"; @@ -59,7 +59,7 @@ import { guardedRunnerOutputs, worktreeFingerprint } from "./wave-guard.js"; import { checkCrossServer, readCrossServerEvidence } from "./multi-server.js"; import { ReleaseSwapWave, stormRunFor } from "./release-swap-setup.js"; import { runnerBindFaultServerEnvironment, sessionFaultServerEnvironment } from "./session-faults.js"; -import { controllerLogNames, gradeWarmShutdown, isWarmWave, warmShutdownMarkdown, type WarmShutdownCleanup } from "./warm-shutdown.js"; +import { controllerLogNames, gradeWarmShutdown, isWarmWave, waitBounded, warmShutdownMarkdown, watchFirstIdleCloseFailure, type WarmShutdownCleanup } from "./warm-shutdown.js"; import type { FaultKind, FaultReceipt, OracleArtifacts, TriggerId, WavePlan } from "./types.js"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); @@ -318,6 +318,12 @@ async function main(): Promise { let storageSection: string | null = null; let warmShutdownSection: string | null = null; let daytonaCleanup: WarmShutdownCleanup | null = null; + let idleCloseCapture: Promise | null = null; + const idleCloseCaptures: string[] = []; + const waitForIdleCloseCapture = async (phase: string) => { + const outcome = await waitBounded(() => idleCloseCapture, RUNNER_LOG_CAPTURE_DEADLINE_MS + 5_000); + if (outcome === "timed_out") timeline.add("idle_close_failure_capture_wait_timed_out", { phase }); + }; let harnessError: unknown; process.on("uncaughtException", error => { if (isSocketTypeOfServiceRace(error)) { timeline.add("harness_socket_race_ignored", { error: String(error) }); return; } @@ -461,6 +467,12 @@ async function main(): Promise { if (sandboxEvidence) { runnerLogTarget = { companyId: owned.companyId, environmentId: owned.environmentId }; if (redeployWave(plan)) supervisor.redeployEvidence = (faultId, phase) => captureRunnerLogs(`${faultId}-${phase}`); + if (isWarmWave(plan.options)) supervisor.onServerOutput = watchFirstIdleCloseFailure(failure => { + const label = `idle-close-failure-${(failure.runId ?? "unknown").slice(0, 8)}`; + idleCloseCaptures.push(label); + timeline.add("idle_close_failure_seen", { runId: failure.runId, label, line: failure.line.slice(0, 2_000) }); + idleCloseCapture = captureRunnerLogs(label).catch(error => { timeline.add("idle_close_failure_capture_failed", { label, error: String(error) }); }); + }); // `--sandbox-watch-seconds`: capture on a clock too, for Runner exits that no fault announces (W2 re-wake). if (plan.options.sandboxWatchMs) faultWork.push(watchSandboxes(plan.options.sandboxWatchMs)); } @@ -919,8 +931,10 @@ async function main(): Promise { if (supervisor.serverReady) await recordWaveSpend().catch(() => undefined); await board?.close().catch(() => undefined); await closeJourneyApps(); + await waitForIdleCloseCapture("before_cleanup"); try { if (sweeper) try { - const outcome = await finishDaytonaCleanup({ sweeper: sweeper as DaytonaWaveSweeper, sweep: sandboxSweep, waveDir, timeline }); + const outcome = await finishDaytonaCleanup({ sweeper: sweeper as DaytonaWaveSweeper, sweep: sandboxSweep, waveDir, timeline, + beforeSandboxDelete: () => waitForIdleCloseCapture("before_sandbox_delete") }); sandboxSweep = outcome.sweep; daytonaCleanup = outcome.cleanup; if (outcome.failed) anyFailure = true; @@ -942,7 +956,8 @@ async function main(): Promise { const logDir = path.dirname(supervisor.paths.serverLogPath); const controllerLogs = controllerLogNames(await readdir(logDir).catch(() => [] as string[])); const serverLog = (await Promise.all(controllerLogs.map(name => readFile(path.join(logDir, name), "utf8").catch(() => "")))).join("\n"); - const warmShutdown = gradeWarmShutdown({ cleanup: daytonaCleanup, serverLog }); + await waitForIdleCloseCapture("before_grade"); + const warmShutdown = gradeWarmShutdown({ cleanup: daytonaCleanup, serverLog, sandboxCaptures: idleCloseCaptures }); timeline.add("warm_shutdown", { ...warmShutdown }); if (warmShutdown.verdict === "fail") anyFailure = true; warmShutdownSection = warmShutdownMarkdown(warmShutdown); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 0d10c55eb35..948df3a9ccc 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -555,8 +555,10 @@ export class TaskObserver { private async runEvidence(run: RunSample): Promise { if (run.status !== "running") return run; const cursor = this.runEventCursors.get(run.id) ?? { seq: 0, lastAt: null, openTools: 0, process: null }; + let eventsRead = false; for (let page = 0; page < 3; page += 1) { const events = await this.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${cursor.seq}&limit=${RUN_EVENT_PAGE}`, 5_000).catch(() => null); + if (Array.isArray(events)) eventsRead = true; if (!events?.length) break; for (const event of events) { if (typeof event.seq === "number" && event.seq > cursor.seq) cursor.seq = event.seq; @@ -571,6 +573,7 @@ export class TaskObserver { } this.runEventCursors.set(run.id, cursor); return { ...run, lastEvidenceAt: latestIso([run.lastEvidenceAt, cursor.lastAt]), + ...(eventsRead && cursor.seq === 0 ? { preparing: true } : {}), ...(cursor.openTools > 0 ? { toolInFlight: true } : {}), ...(cursor.turnActive ? { providerTurnActive: true } : {}), ...(cursor.process && this.processAlive(cursor.process.pid, cursor.process.startedAt) ? { processAlive: true } : {}) }; diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts index 61062e49628..cff69c77097 100644 --- a/tests/restart-chaos/supervisor.ts +++ b/tests/restart-chaos/supervisor.ts @@ -183,6 +183,7 @@ export class ChaosSupervisor { deployShape: { overlap: boolean; freshHome: boolean } = { overlap: false, freshHome: false }; ephemeralServer = false; swapHook: ((info: SwapPhaseInfo) => Promise) | null = null; + onServerOutput: ((chunk: Buffer | string, source: string) => void) | null = null; private rollingRedeploys = 0; /** `--k15-fresh-home`: the instance home and config of the serving generation (generation 0 uses `paths`). */ private serverHome: { home: string; configPath: string }; @@ -232,6 +233,10 @@ export class ChaosSupervisor { /** K14 waves: whether the older build or this checkout is serving. */ get serverBuild(): "upgrade_from" | "current" { return this.serverRoot === this.repositoryRoot ? "current" : "upgrade_from"; } + private noteServerOutput(chunk: Buffer | string, source: string): void { + try { this.onServerOutput?.(chunk, source); } catch (error) { this.timeline.add("server_output_hook_failed", { error: String(error) }); } + } + serverLogOffset(): number { try { return statSync(this.paths.serverLogPath).size + this.log.writableLength; } catch { return 0; } } @@ -321,8 +326,8 @@ export class ChaosSupervisor { this.child = child; const window = this.trackWindow(child, "primary"); - child.stdout?.on("data", chunk => this.log.write(chunk)); - child.stderr?.on("data", chunk => this.log.write(chunk)); + child.stdout?.on("data", chunk => { this.log.write(chunk); this.noteServerOutput(chunk, `${child.pid}:stdout`); }); + child.stderr?.on("data", chunk => { this.log.write(chunk); this.noteServerOutput(chunk, `${child.pid}:stderr`); }); child.once("error", error => { this.spawnErrors.set(child, error); }); child.once("exit", (code, signal) => { if (window) window.exitedAt = Date.now(); @@ -741,8 +746,8 @@ export class ChaosSupervisor { record.secondary = { pid: child.pid ?? null, port, spawnedAt: Date.now(), readyAt: null, stoppedAt: null }; const secondary = { child, port, log, exit: null as string | null }; this.secondary = secondary; - child.stdout?.on("data", chunk => log.write(chunk)); - child.stderr?.on("data", chunk => log.write(chunk)); + child.stdout?.on("data", chunk => { log.write(chunk); this.noteServerOutput(chunk, `${child.pid}:stdout`); }); + child.stderr?.on("data", chunk => { log.write(chunk); this.noteServerOutput(chunk, `${child.pid}:stderr`); }); child.once("error", error => { this.spawnErrors.set(child, error); }); child.once("exit", (code, signal) => { if (window) window.exitedAt = Date.now(); diff --git a/tests/restart-chaos/triggers.test.ts b/tests/restart-chaos/triggers.test.ts index 2aa1415f659..79c90bfa9ab 100644 --- a/tests/restart-chaos/triggers.test.ts +++ b/tests/restart-chaos/triggers.test.ts @@ -6,7 +6,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { isSocketTypeOfServiceRace, type ChaosApi } from "./api.js"; import { parseCommand } from "./config.js"; -import { gradeR5 } from "./grader.js"; +import { gradeR5, PREPARATION_LIMIT_MS } from "./grader.js"; +import { DEFAULT_UNIVERSAL_BOUNDS, gradeU1 } from "./universal.js"; import { latestIso, localProcessAlive, noteLivenessEvent, QUEUED_TRIGGER_MAX_EMPTY_SAMPLES, TaskObserver, TRIGGER_LATE_MS } from "./observer.js"; import { observedFaultSchedule, planTasks, taskFaultSchedule, TRIGGER_DELAY_RANGE_MS } from "./scheduler.js"; import type { ChaosLab } from "./seed.js"; @@ -424,7 +425,7 @@ describe("R5 live evidence", () => { expect(gradeR5(record(slow.samples)).verdict).toBe("pass"); }); - const quietTurn = (live: { providerTurnActive?: boolean }, quietMs: number) => { + const quietTurn = (live: { providerTurnActive?: boolean; preparing?: boolean }, quietMs: number) => { const start = Date.parse("2026-11-15T05:00:00Z"); const run = { id: "r1", status: "running", createdAt: null, startedAt: new Date(start - 600_000).toISOString(), finishedAt: null, errorCode: null, retryOfRunId: null, wakeCommentIds: [], livenessState: null, lastEvidenceAt: new Date(start).toISOString(), ...live }; @@ -440,6 +441,62 @@ describe("R5 live evidence", () => { detail: expect.stringMatching(/^silent_dead_run r1 .* for 630s/) }); }); + it("does not flag a run that is still acquiring its environment before its first run event (w-20261116T001158-s7106/t010)", () => { + const slow = JSON.parse(readFileSync(path.join(import.meta.dirname, "fixtures/t010-slow-environment-acquire.json"), "utf8")) as + { runId: string; firstRunEventAt: string; samples: TaskSample[] }; + const firstEventAt = Date.parse(slow.firstRunEventAt); + const quiet = slow.samples.filter(sample => sample.at < firstEventAt); + expect(quiet.at(-1)!.at - quiet[0]!.at).toBeGreaterThan(240_000); + expect(quiet.every(sample => sample.runs!.find(run => run.id === slow.runId)!.lastEvidenceAt + === sample.runs!.find(run => run.id === slow.runId)!.startedAt)).toBe(true); + expect(gradeR5(record(slow.samples))).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^silent_dead_run aedc1f6e/) }); + const observed = slow.samples.map(sample => ({ ...sample, runs: sample.runs?.map(run => + run.id === slow.runId && run.status === "running" && sample.at < firstEventAt ? { ...run, preparing: true } : run) })); + expect(gradeR5(record(observed)).verdict).toBe("pass"); + expect(gradeU1(record(observed), DEFAULT_UNIVERSAL_BOUNDS).verdict).toBe("pass"); + }); + + it("R5 and U1 agree on a preparing run: a 6 and 7 minute restore passes both, silence past the preparation bound fails both", () => { + for (const quietMs of [360_000, 420_000]) { + expect(gradeR5(quietTurn({ preparing: true }, quietMs)).verdict).toBe("pass"); + expect(gradeU1(quietTurn({ preparing: true }, quietMs), DEFAULT_UNIVERSAL_BOUNDS).verdict).toBe("pass"); + } + const pastBound = PREPARATION_LIMIT_MS + DEFAULT_UNIVERSAL_BOUNDS.orphanMs + 60_000; + expect(gradeR5(quietTurn({ preparing: true }, pastBound))).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^silent_dead_run r1/) }); + expect(gradeU1(quietTurn({ preparing: true }, pastBound), DEFAULT_UNIVERSAL_BOUNDS)).toMatchObject({ verdict: "fail", + detail: expect.stringMatching(/^no_owner_no_action/) }); + expect(gradeU1(quietTurn({}, 420_000), DEFAULT_UNIVERSAL_BOUNDS).verdict).toBe("fail"); + }); + + it("still fails a run silent past the preparation bound, and a started run silent past the stall bound", () => { + expect(gradeR5(quietTurn({ preparing: true }, 570_000)).verdict).toBe("pass"); + expect(gradeR5(quietTurn({ preparing: true }, 630_000))).toMatchObject({ verdict: "fail", + detail: expect.stringMatching(/^silent_dead_run r1 .* for 630s/) }); + expect(gradeR5(quietTurn({ preparing: false }, 240_000))).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^silent_dead_run r1/) }); + }); + + it("marks a running run as preparing only while its events read back empty", async () => { + const startedAt = new Date().toISOString(); + const state = { runs: [{ runId: "r1", status: "running", startedAt }] as Json[], issue: { id: "i1", status: "in_progress" }, events: [] as Json[] }; + const observer = new TaskObserver({ ...base, workload: "B5" }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + await observer.poll(Date.now()); + expect(observer.record.samples.at(-1)!.runs![0]).toMatchObject({ preparing: true }); + state.events.push({ seq: 1, eventType: "lifecycle", message: "run started", payload: {}, createdAt: startedAt }); + await observer.poll(Date.now() + 3_000); + expect(observer.record.samples.at(-1)!.runs![0]!.preparing).toBeUndefined(); + + const unread = scriptedApi({ ...state, events: [] }); + const failing = { ...unread, get: async (route: string) => { + if (route.includes("/events")) throw new Error("events unavailable"); + return (unread as unknown as { get(route: string): Promise }).get(route); + } } as unknown as ChaosApi; + const blind = new TaskObserver({ ...base, workload: "B5" }, failing, lab, new Timeline(null), ready); + await blind.launch(); + await blind.poll(Date.now()); + expect(blind.record.samples.at(-1)!.runs![0]!.preparing).toBeUndefined(); + }); + it("derives tool-in-flight and process liveness from run events", async () => { const startedAt = new Date().toISOString(); const identity = { processPid: 4242, processGroupId: 4242, processStartedAt: "2026-11-06T10:00:00.000Z", targetKind: "local" }; diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index a9cb8c61f2a..1a90635ab39 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -271,6 +271,7 @@ export interface RunSample { processAlive?: boolean; /** A provider turn started and has no turn terminal yet. */ providerTurnActive?: boolean; + preparing?: boolean; } export interface InterruptionSample { diff --git a/tests/restart-chaos/universal.ts b/tests/restart-chaos/universal.ts index 0fdf88ac07f..99ff4947e11 100644 --- a/tests/restart-chaos/universal.ts +++ b/tests/restart-chaos/universal.ts @@ -12,6 +12,7 @@ * (reported in the grade and ledger, never fails the task). The split comes * from the false-positive audit over saved waves; see README "Universal checks". */ +import { PREPARATION_LIMIT_MS } from "./grader.js"; import type { RunSample, TaskRecord, TaskSample } from "./types.js"; export type UniversalId = "U1" | "U2" | "U3" | "U4" | "U5"; @@ -137,7 +138,8 @@ export function liveOwner(sample: TaskSample, queuedFor: ReadonlyMap { expect(check.verdict).toBe("fail"); expect(check.environmentDeleteBlocked).toBe(blocked); expect(check.leakedSandboxes).toEqual(["sbx-1:started", "sbx-2:started"]); - expect(check.shutdownCloseFailures).toEqual(['[14:20:17] WARN: idle native session cleanup failed before controller shutdown {"idleSessions":{"failed":2}}']); + expect(check.shutdownCloseFailures).toEqual([{ runId: null, lines: 1, + reason: 'WARN: idle native session cleanup failed before controller shutdown {"idleSessions":{"failed":2}}', + firstLine: '[14:20:17] WARN: idle native session cleanup failed before controller shutdown {"idleSessions":{"failed":2}}' }]); expect(check.problems).toHaveLength(3); expect(warmShutdownMarkdown(check)).toContain("Verdict: FAIL"); }); @@ -39,7 +43,7 @@ describe("clean warm shutdown check", () => { it("reports a forced stop without failing when cleanup still finished", () => { const check = gradeWarmShutdown({ cleanup: clean, serverLog: "WARN: warm native session close did not settle; its Runner was stopped and the session discarded {}\n" }); - expect(check).toMatchObject({ verdict: "pass", idleCloseFailures: [expect.stringContaining("did not settle")] }); + expect(check).toMatchObject({ verdict: "pass", idleCloseFailures: [{ firstLine: expect.stringContaining("did not settle") }] }); expect(gradeWarmShutdown({ cleanup: clean, serverLog: "WARN: warm native session's sandbox was already destroyed; released its owner {}\n" }).idleCloseFailures).toHaveLength(1); }); @@ -51,4 +55,66 @@ describe("clean warm shutdown check", () => { expect(gradeWarmShutdown({ cleanup: null, serverLog: 'WARN: idle warm native sessions did not close {"reason":"terminal heartbeat run destroyed its environment lease"}' }).verdict).toBe("pass"); }); + + const a1Log = readFileSync(path.join(import.meta.dirname, "fixtures/a1-warm-close-failures.log"), "utf8"); + const run300 = "300f0d61-548b-4630-8f6a-f79292ed9d24"; + const run70d = "70d909cc-8b04-40ae-a0bf-0846df140cf0"; + + it("reports each distinct (run, reason) once, not each log line (w-20261116T001158-s7106 A1: 2 sessions, 29 lines)", () => { + expect(a1Log.split("\n").filter(line => /did not close|cleanup failed before controller shutdown/.test(line))).toHaveLength(29); + const check = gradeWarmShutdown({ cleanup: clean, serverLog: a1Log }); + expect(check.verdict).toBe("fail"); + expect(check.shutdownCloseFailures.map(({ runId, lines }) => ({ runId, lines }))).toEqual([ + { runId: run300, lines: 2 }, { runId: run70d, lines: 27 }]); + expect(check.shutdownCloseFailures[0]!.reason).toMatch(/process_identity_changed\]$/); + expect(check.shutdownCloseFailures[1]!.reason).toBe( + "close failed: native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint"); + expect(check.idleCloseFailures.map(({ runId, lines }) => ({ runId, lines }))).toEqual([ + { runId: run300, lines: 4 }, { runId: run70d, lines: 1 }]); + expect(check.problems).toHaveLength(2); + expect(warmShutdownMarkdown(check)).toContain("warm sessions that did not close: 2; idle closes that needed a forced stop or failed: 2."); + }); + + it("still fails on one distinct failure, and keeps different reasons for the same run apart", () => { + const one = a1Log.split("\n").filter(line => line.includes(run70d) && line.includes("did not close")); + expect(gradeWarmShutdown({ cleanup: clean, serverLog: one.join("\n") })).toMatchObject({ verdict: "fail", + shutdownCloseFailures: [{ runId: run70d, lines: 27 }] }); + const twoReasons = [ + 'WARN: idle warm native sessions did not close {"reason":"controller restart","failures":[{"runId":"r1","reason":"close failed: a"}]}', + 'WARN: idle warm native sessions did not close {"reason":"controller restart","failures":[{"runId":"r1","reason":"close failed: b"}]}', + ].join("\n"); + expect(gradeWarmShutdown({ cleanup: clean, serverLog: twoReasons }).problems).toHaveLength(2); + const unparsed = gradeWarmShutdown({ cleanup: clean, serverLog: 'WARN: idle warm native sessions did not close {"reason":"environment deleted","failures":[' }); + expect(unparsed).toMatchObject({ verdict: "fail", shutdownCloseFailures: [{ runId: null, lines: 1 }] }); + }); + + it("names the first idle-close failure once, across chunk boundaries and interleaved streams", () => { + const seen: Array<{ runId: string | null }> = []; + const watch = watchFirstIdleCloseFailure(failure => { seen.push(failure); }); + const failed = a1Log.split("\n").find(line => line.includes("idle warm native session close failed") && line.includes(run70d))!; + watch("INFO: server ready\n", "1:stdout"); + watch(failed.slice(0, 40), "1:stdout"); + watch("WARN: unrelated stderr line\n", "1:stderr"); + expect(seen).toEqual([]); + watch(`${failed.slice(40)}\n`, "1:stdout"); + watch(`${failed}\n`, "1:stdout"); + expect(seen).toEqual([expect.objectContaining({ runId: run70d })]); + const quiet: unknown[] = []; + const none = watchFirstIdleCloseFailure(failure => { quiet.push(failure); }); + none("WARN: warm native session close did not settle; its Runner was stopped and the session discarded {}\n"); + expect(quiet).toEqual([]); + }); + + it("lists the sandbox captures taken at the first idle-close failure", () => { + const check = gradeWarmShutdown({ cleanup: clean, serverLog: a1Log, sandboxCaptures: ["idle-close-failure-300f0d61"] }); + expect(check.sandboxCaptures).toEqual(["idle-close-failure-300f0d61"]); + expect(warmShutdownMarkdown(check)).toContain("idle-close-failure-300f0d61"); + }); + + it("waits for a pending capture only up to its bound", async () => { + expect(await waitBounded(() => null, 10)).toBe("none"); + expect(await waitBounded(() => Promise.resolve(), 1_000)).toBe("settled"); + expect(await waitBounded(() => Promise.reject(new Error("probe failed")), 1_000)).toBe("settled"); + expect(await waitBounded(() => new Promise(() => undefined), 20)).toBe("timed_out"); + }); }); diff --git a/tests/restart-chaos/warm-shutdown.ts b/tests/restart-chaos/warm-shutdown.ts index cc8bc6e087d..d87211e8ddf 100644 --- a/tests/restart-chaos/warm-shutdown.ts +++ b/tests/restart-chaos/warm-shutdown.ts @@ -5,19 +5,29 @@ export interface WarmShutdownCleanup { undeletedSandboxes?: string[]; } +export interface WarmCloseFailure { + runId: string | null; + reason: string; + lines: number; + firstLine: string; +} + export interface WarmShutdownCheck { verdict: "pass" | "fail"; environmentDeleted: boolean | null; environmentDeleteBlocked: string | null; leakedSandboxes: string[]; - shutdownCloseFailures: string[]; - idleCloseFailures: string[]; + shutdownCloseFailures: WarmCloseFailure[]; + idleCloseFailures: WarmCloseFailure[]; + sandboxCaptures: string[]; problems: string[]; } const SHUTDOWN_CLOSE_FAILURE = /idle native session cleanup failed before controller shutdown|idle warm native sessions did not close.*"reason":"(environment deleted|controller restart)"/; const IDLE_CLOSE_FAILURE = /idle warm native session (close|detach) failed|warm native session close did not settle|warm native session's sandbox was already destroyed/; +const FIRST_IDLE_CLOSE_FAILURE = /idle warm native session (close|detach) failed|idle warm native sessions did not close|idle native session cleanup failed before controller shutdown/; const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g"); +const MAX_PARTIAL_LINE = 256 * 1024; export function controllerLogNames(names: readonly string[]): string[] { return names.filter(name => /^server(-[a-z0-9]+)?\.log$/.test(name)).sort(); @@ -27,16 +37,75 @@ export function isWarmWave(options: { lifecycle?: string; daytonaWarm?: boolean return options.lifecycle === "warm" || options.daytonaWarm === true; } -export function gradeWarmShutdown(input: { cleanup: WarmShutdownCleanup | null; serverLog: string }): WarmShutdownCheck { - const lines = input.serverLog.split("\n").map(line => line.replace(ANSI, "").trim()).filter(Boolean); - const shutdownCloseFailures = lines.filter(line => SHUTDOWN_CLOSE_FAILURE.test(line)); - const idleCloseFailures = lines.filter(line => IDLE_CLOSE_FAILURE.test(line)); +function cleanLine(line: string): string { + return line.replace(ANSI, "").trim(); +} + +function lineDetail(line: string): Record | null { + const start = line.indexOf("{"); + if (start < 0) return null; + try { + const parsed = JSON.parse(line.slice(start)) as unknown; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : null; + } catch { + return null; + } +} + +function lineMessage(line: string): string { + const brace = line.indexOf("{"); + return (brace < 0 ? line : line.slice(0, brace)).replace(/^\[[^\]]*\]\s*/, "").trim(); +} + +function failureReason(value: unknown): string | null { + if (typeof value !== "string" || !value.trim()) return null; + return value.split(" {")[0]!.trim(); +} + +function lineFailures(line: string): Array<{ runId: string | null; reason: string }> { + const detail = lineDetail(line); + const listed = Array.isArray(detail?.failures) ? detail.failures + : Array.isArray((detail?.idleSessions as Record | undefined)?.failures) + ? (detail!.idleSessions as Record).failures as unknown[] : null; + const fromList = (listed ?? []).flatMap(entry => { + if (!entry || typeof entry !== "object") return []; + const item = entry as Record; + const reason = failureReason(item.reason) ?? failureReason(item.closeFailure); + return [{ runId: typeof item.runId === "string" ? item.runId : null, reason: reason ?? lineMessage(line) }]; + }); + if (fromList.length) return fromList; + if (detail && typeof detail.runId === "string") { + return [{ runId: detail.runId, reason: failureReason(detail.closeFailure) ?? lineMessage(line) }]; + } + const body = line.replace(/^\[[^\]]*\]\s*/, ""); + return [{ runId: null, reason: body }]; +} + +export function distinctCloseFailures(lines: readonly string[]): WarmCloseFailure[] { + const byKey = new Map(); + for (const line of lines) { + for (const failure of lineFailures(line)) { + const key = `${failure.runId ?? ""}\u0000${failure.reason}`; + const existing = byKey.get(key); + if (existing) existing.lines += 1; + else byKey.set(key, { runId: failure.runId, reason: failure.reason, lines: 1, firstLine: line }); + } + } + return [...byKey.values()]; +} + +export function gradeWarmShutdown(input: { cleanup: WarmShutdownCleanup | null; serverLog: string; sandboxCaptures?: readonly string[] }): WarmShutdownCheck { + const lines = input.serverLog.split("\n").map(cleanLine).filter(Boolean); + const shutdownCloseFailures = distinctCloseFailures(lines.filter(line => SHUTDOWN_CLOSE_FAILURE.test(line))); + const idleCloseFailures = distinctCloseFailures(lines.filter(line => IDLE_CLOSE_FAILURE.test(line))); const problems: string[] = []; if (input.cleanup?.environmentDeleteBlocked) problems.push(`environment delete blocked: ${input.cleanup.environmentDeleteBlocked}`); else if (input.cleanup && !input.cleanup.environmentDeleted) problems.push("environment was not deleted"); if (input.cleanup?.remainingSandboxes.length) problems.push(`sandboxes still running after cleanup: ${input.cleanup.remainingSandboxes.join(", ")}`); if (input.cleanup?.undeletedSandboxes?.length) problems.push(`wave sandboxes left after the harness deleted them: ${input.cleanup.undeletedSandboxes.join(", ")}`); - for (const line of shutdownCloseFailures) problems.push(`warm session did not close: ${line}`); + for (const failure of shutdownCloseFailures) { + problems.push(`warm session did not close: run ${failure.runId ?? "unknown"} (${failure.lines} log line${failure.lines === 1 ? "" : "s"}): ${failure.reason}`); + } return { verdict: problems.length ? "fail" : "pass", environmentDeleted: input.cleanup?.environmentDeleted ?? null, @@ -44,15 +113,45 @@ export function gradeWarmShutdown(input: { cleanup: WarmShutdownCleanup | null; leakedSandboxes: [...new Set([...input.cleanup?.remainingSandboxes ?? [], ...input.cleanup?.undeletedSandboxes ?? []])], shutdownCloseFailures, idleCloseFailures, + sandboxCaptures: [...input.sandboxCaptures ?? []], problems, }; } +export function watchFirstIdleCloseFailure(onFailure: (failure: { runId: string | null; line: string }) => void): (chunk: Buffer | string, source?: string) => void { + const partials = new Map(); + let fired = false; + return (chunk, source = "") => { + if (fired) return; + const lines = `${partials.get(source) ?? ""}${typeof chunk === "string" ? chunk : chunk.toString("utf8")}`.split("\n"); + const rest = lines.pop() ?? ""; + partials.set(source, rest.length > MAX_PARTIAL_LINE ? "" : rest); + for (const raw of lines) { + const line = cleanLine(raw); + if (!FIRST_IDLE_CLOSE_FAILURE.test(line)) continue; + fired = true; + onFailure({ runId: lineFailures(line)[0]?.runId ?? null, line }); + return; + } + }; +} + +export async function waitBounded(pending: () => Promise | null, ms: number): Promise<"settled" | "timed_out" | "none"> { + const work = pending(); + if (!work) return "none"; + let timer: NodeJS.Timeout | undefined; + const outcome = await Promise.race([work.then(() => "settled" as const, () => "settled" as const), + new Promise<"timed_out">(resolve => { timer = setTimeout(() => resolve("timed_out"), ms); })]); + clearTimeout(timer); + return outcome; +} + export function warmShutdownMarkdown(check: WarmShutdownCheck): string { const out = ["## Clean warm shutdown (no leaked sandboxes)", "", `Verdict: ${check.verdict.toUpperCase()}. Environment deleted: ${check.environmentDeleted ?? "n/a"}; sandboxes left running: ${check.leakedSandboxes.length}; ` + `warm sessions that did not close: ${check.shutdownCloseFailures.length}; idle closes that needed a forced stop or failed: ${check.idleCloseFailures.length}.`, ""]; for (const problem of check.problems) out.push(`- ${problem.slice(0, 2_000)}`); if (check.problems.length) out.push(""); + if (check.sandboxCaptures.length) out.push(`Sandbox captures at the first idle-close failure: ${check.sandboxCaptures.join(", ")} (daytona-runner/).`, ""); return out.join("\n"); }