diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 99faafee805..aba9ef36c23 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -661,7 +661,7 @@ cap. The quiet period lets a continuation the product wakes after a run ends (fo subtasks finish) start before the read, so the task is graded by that continuation, not as `nothing pending`. A task still running at the cap is not counted in the number; the headline reports how many with the age of their last activity, and the wave fails. The grade uses the same 5-minute threshold: after the drain, a run with no event for 5 minutes, a queued wake, a scheduled retry or a working notice with no -progressing run behind it is not pending work, so the task is `nothing pending`. The single read then covers every task of the wave's company that had a run (probes and agent-created subtasks included), the +progressing run behind it is not pending work, so the task is `nothing pending`. Queued wakes behind a queued or running run wait for that run by design, so the detail does not call them never dispatched. The single read then covers every task of the wave's company that had a run (probes and agent-created subtasks included), the way the board shows it: | Verdict | When | @@ -2155,7 +2155,7 @@ re-arms the task's own triggered faults instead of using their offsets. temporary instance is deleted. - Faults are serialized; each one hits every task running at that moment. - A `todo` or `in_progress` task with no live run and no named wait is not a rest. The observer keeps watching it until it resumes or passes the R5 stall limit. -- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. A quiet run is still live while a tool it started has not ended (for example a 4-minute `bench.sh`) or while its recorded local runner process still runs with the same PID and start time. +- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. A quiet run is still live while a tool it started has not ended (for example a 4-minute `bench.sh`) or while its recorded local runner process still runs with the same PID and start time. While a provider turn is active and no tool runs, one model request is outstanding: R5 allows up to 10 minutes of quiet before it counts a dead run. A slow OpenRouter request took 303.6 s in w-20261115T044649-s7106/t028, and common provider clients time out a request at 10 minutes. - The wave fingerprints the checkout and `packages/paperclip-runner/dist` at start and after every fault. A build, install or branch switch during a wave would make later restarts run different code, so it fails the wave as a harness error (`worktree_changed_mid_wave`). The baseline is taken after the harness's own setup (preflight rebuilds provider entrypoints) and covers the tracked diff, untracked file contents, the runner `dist` and the runnerd binary. Build before the wave, never during it. - `faults.json` and `task-faults.json` (each task's launch time and the faults that crossed it) are written after every fault, so `--replay` of a task in an interrupted or wedged wave, or a storm task, re-fires the faults that crossed it at the same offsets. - A fault that fails to complete fails the wave as a harness error (exit 2), even when recovery brings the server back. diff --git a/tests/restart-chaos/fixtures/t028-slow-provider-turn.json b/tests/restart-chaos/fixtures/t028-slow-provider-turn.json new file mode 100644 index 00000000000..b6e7cd5058d --- /dev/null +++ b/tests/restart-chaos/fixtures/t028-slow-provider-turn.json @@ -0,0 +1 @@ +{"source":"w-20261115T044649-s7106/t028: one model request outstanding for 303.6 s with no tool running; the run then finished normally","samples":[{"at":1794720090451,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720093681,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720096928,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720100201,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720103431,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720106662,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720109887,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720113084,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720116361,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720119564,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720122789,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720125995,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720129213,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720132430,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:20:42.672Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720135668,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720138872,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720142119,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720145347,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720148636,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720151833,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720155027,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720158267,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720161483,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720164772,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720167969,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720171180,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720174443,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720177664,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720180908,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720184170,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720187379,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720190604,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720193821,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720197053,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720200288,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720205769,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720208978,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720212186,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720215410,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720218570,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720221760,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720224934,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720228136,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720231363,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720234546,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720237738,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720240946,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720244141,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720247428,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720250593,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720253784,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720256987,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720260222,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720263428,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720266633,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720269802,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720273030,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720276200,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720279390,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720282558,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720285719,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720288923,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720292143,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720295391,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720298579,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720301778,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720304977,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720308194,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720311477,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720314705,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720317910,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720321160,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720324337,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720327502,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720330656,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720333800,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720336984,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720340189,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720343345,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720346542,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720349711,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720352877,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720356025,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720359227,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720362379,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720365552,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720368707,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720371862,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720375029,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720378183,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720381326,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720384610,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720387771,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720390941,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720394694,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720397848,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720401031,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720404250,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720407415,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720410584,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720413761,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720416917,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720420078,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720423237,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720426381,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720429508,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720432667,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720435805,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:22:13.005Z","providerTurnActive":true}]},{"at":1794720439018,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:27:17.498Z","providerTurnActive":true}]},{"at":1794720442150,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:27:20.825Z","providerTurnActive":true}]},{"at":1794720445276,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:27:25.321Z","providerTurnActive":true}]},{"at":1794720448443,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:27:26.394Z","providerTurnActive":true}]},{"at":1794720451590,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:27:26.394Z","providerTurnActive":true}]},{"at":1794720454752,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:27:33.880Z","toolInFlight":true,"providerTurnActive":true}]},{"at":1794720457882,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"running","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-15T05:27:37.123Z","providerTurnActive":true}]},{"at":1794720461035,"reachable":true,"serverReady":true,"issueStatus":"blocked","interruption":{"state":"resolved","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"7b51b352-85f1-4b6c-9224-c6170b8d4231","userAction":null},"activeRecoveryAction":false,"recoveryOwner":null,"executionBlocker":false,"scheduledRetry":false,"runs":[{"id":"7b51b352-85f1-4b6c-9224-c6170b8d4231","status":"succeeded","createdAt":"2026-11-15T04:50:01.510Z","startedAt":"2026-11-15T05:09:16.731Z","finishedAt":"2026-11-15T05:27:40.304Z","errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":"blocked","lastEvidenceAt":"2026-11-15T05:27:40.246Z"}],"waitingOnChildren":true}]} diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 8c494c7070c..5c2c3ff6ea4 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -7,6 +7,7 @@ import { enforcedUniversalFailures, gradeUniversal, type SlotFree, type Universa export const ACTIVE_RUN_STATUSES = new Set(["queued", "running", "scheduled_retry"]); export const STALL_LIMIT_MS = 3 * 60_000; +export const PROVIDER_TURN_LIMIT_MS = 10 * 60_000; const REST_STATUSES_WITH_WORK = new Set(["todo", "in_progress"]); const ms = (iso: string | null | undefined) => (iso ? Date.parse(iso) : NaN); @@ -210,8 +211,9 @@ export function deadRunningRun(sample: TaskSample, readySince: number): { id: st if (run.status !== "running") continue; // Quiet is not dead while a tool is still in flight or the run's process is verifiably alive. if (run.toolInFlight || run.processAlive) continue; + const limit = run.providerTurnActive ? PROVIDER_TURN_LIMIT_MS : STALL_LIMIT_MS; const evidence = Math.max(ms(run.lastEvidenceAt) || 0, ms(run.startedAt) || 0, readySince); - if (sample.at - evidence > STALL_LIMIT_MS) return { id: run.id, idleMs: sample.at - Math.max(ms(run.lastEvidenceAt) || 0, ms(run.startedAt) || 0) }; + if (sample.at - evidence > limit) return { id: run.id, idleMs: sample.at - Math.max(ms(run.lastEvidenceAt) || 0, ms(run.startedAt) || 0) }; } return null; } diff --git a/tests/restart-chaos/release-swap-grade.test.ts b/tests/restart-chaos/release-swap-grade.test.ts index b1313682a9f..c29ed103a06 100644 --- a/tests/restart-chaos/release-swap-grade.test.ts +++ b/tests/restart-chaos/release-swap-grade.test.ts @@ -128,6 +128,20 @@ describe("release swap task grade", () => { expect(gradeReleaseSwapTask(task({ status: "todo", runs: [failed], queuedWakes: 1 })).verdict).toBe("still_working"); }); + it("does not report queued wakes as never dispatched while a run holds them (w-20261115T044649-s7106/t019)", () => { + const now = 10_000_000; + const held = gradeReleaseSwapTask(task({ status: "in_progress", now, staleAfterMs: 60_000, queuedWakes: 2, + interruption: interruption({ state: "working" }), + runs: [{ runId: "9df2283c", status: "running", errorCode: null, createdAt: 0, startedAt: 1, finishedAt: null, lastActivityAt: now - 70_000 }] })); + expect(held.detail).toContain("run 9df2283c running with no activity for 1 min"); + expect(held.detail).not.toContain("never dispatched"); + const queued = gradeReleaseSwapTask(task({ status: "todo", now, staleAfterMs: 60_000, queuedWakes: 1, + runs: [{ runId: "q1", status: "queued", errorCode: null, createdAt: now - 120_000, startedAt: null, finishedAt: null, lastActivityAt: null }] })); + expect(queued.detail).not.toContain("never dispatched"); + const orphan = gradeReleaseSwapTask(task({ status: "todo", now, staleAfterMs: 60_000, runs: [failed], queuedWakes: 2 })); + expect(orphan.detail).toContain("2 queued wake(s) never dispatched"); + }); + it("keeps a task still running past the drain cap out of the number and reports it with its last activity", () => { const running = gradeReleaseSwapTask(task({ status: "in_progress", now: 1_000_000, staleAfterMs: 900_000, runs: [{ runId: "r5", status: "running", errorCode: null, createdAt: 0, startedAt: 1, finishedAt: null, lastActivityAt: 940_000 }] })); diff --git a/tests/restart-chaos/release-swap-grade.ts b/tests/restart-chaos/release-swap-grade.ts index 983f7433913..b8c0568ab1c 100644 --- a/tests/restart-chaos/release-swap-grade.ts +++ b/tests/restart-chaos/release-swap-grade.ts @@ -143,9 +143,10 @@ export function gradeReleaseSwapTask(task: GradeTaskInput): TaskGrade { const lastActivityAgeMs = task.now !== undefined && liveActivity.length ? task.now - Math.max(...liveActivity) : null; const verdict = (value: TaskVerdict, detail: string): TaskGrade => ({ ...base, verdict: value, detail, label: label(value), lastActivityAgeMs }); const stale = staleRuns(task); + const heldByRun = task.runs.some(run => LIVE_RUN.has(run.status)); const idle = task.now === undefined ? [] : [ ...stale.map(run => `run ${run.runId} ${run.status} with no activity for ${Math.round((task.now! - (run.lastActivityAt ?? run.createdAt ?? 0)) / 60_000)} min`), - ...(task.queuedWakes ? [`${task.queuedWakes} queued wake(s) never dispatched`] : []), ...(task.scheduledRetry ? ["a scheduled retry that never ran"] : []), + ...(task.queuedWakes && !heldByRun ? [`${task.queuedWakes} queued wake(s) never dispatched`] : []), ...(task.scheduledRetry ? ["a scheduled retry that never ran"] : []), ...(task.interruption && (task.interruption.state === "working" || task.interruption.state === "retry_scheduled" || task.interruption.retryWait) ? [`the board shows ${task.interruption.retryWait ? "a Retry waiting on the system" : task.interruption.state}`] : []), ...(task.activeRecoveryOwner && task.activeRecoveryOwner !== "board" ? [`a ${task.activeRecoveryOwner} recovery action that made no progress`] : [])]; diff --git a/tests/restart-chaos/triggers.test.ts b/tests/restart-chaos/triggers.test.ts index 25908cf61eb..2aa1415f659 100644 --- a/tests/restart-chaos/triggers.test.ts +++ b/tests/restart-chaos/triggers.test.ts @@ -416,6 +416,30 @@ describe("R5 live evidence", () => { expect(gradeR5(quietB5({}))).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^silent_dead_run r1/) }); }); + it("does not flag a slow model request while the provider turn is active with no tool running (w-20261115T044649-s7106/t028)", () => { + const slow = JSON.parse(readFileSync(path.join(import.meta.dirname, "fixtures/t028-slow-provider-turn.json"), "utf8")) as { samples: TaskSample[] }; + const gap = slow.samples.filter(sample => sample.runs?.[0]?.lastEvidenceAt === "2026-11-15T05:22:13.005Z"); + expect(gap.at(-1)!.at - gap[0]!.at).toBeGreaterThan(300_000); + expect(gap.every(sample => sample.runs![0]!.providerTurnActive && !sample.runs![0]!.toolInFlight)).toBe(true); + expect(gradeR5(record(slow.samples)).verdict).toBe("pass"); + }); + + const quietTurn = (live: { providerTurnActive?: boolean }, quietMs: number) => { + const start = Date.parse("2026-11-15T05:00:00Z"); + const run = { id: "r1", status: "running", createdAt: null, startedAt: new Date(start - 600_000).toISOString(), finishedAt: null, + errorCode: null, retryOfRunId: null, wakeCommentIds: [], livenessState: null, lastEvidenceAt: new Date(start).toISOString(), ...live }; + const offsets = Array.from({ length: Math.floor(quietMs / 30_000) + 1 }, (_, index) => index * 30_000); + return record(offsets.map(offset => ({ at: start + offset, reachable: true, serverReady: true, issueStatus: "in_progress", runs: [run] }))); + }; + + it("still fails a silent run with no active provider turn, and a provider turn silent past the provider-latency bound", () => { + expect(gradeR5(quietTurn({}, 240_000))).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^silent_dead_run r1/) }); + expect(gradeR5(quietTurn({ providerTurnActive: false }, 240_000))).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^silent_dead_run r1/) }); + expect(gradeR5(quietTurn({ providerTurnActive: true }, 570_000)).verdict).toBe("pass"); + expect(gradeR5(quietTurn({ providerTurnActive: true }, 630_000))).toMatchObject({ verdict: "fail", + detail: expect.stringMatching(/^silent_dead_run r1 .* for 630s/) }); + }); + it("derives tool-in-flight and process liveness from run events", async () => { const startedAt = new Date().toISOString(); const identity = { processPid: 4242, processGroupId: 4242, processStartedAt: "2026-11-06T10:00:00.000Z", targetKind: "local" };