diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md
index b457d35b03f..f0e1731365d 100644
--- a/tests/restart-chaos/README.md
+++ b/tests/restart-chaos/README.md
@@ -511,7 +511,10 @@ windows are written to `storage-faults.json` and removed from the speed grade li
Every Retry the grader needs is clicked in the real board UI: the harness opens the issue in a headless
Chromium, waits for the visible Retry button and clicks it. A task whose Retry button is missing fails
R3 as `ui_retry_button_missing (user saw: …)`; a click that sends no retry request fails as
-`ui_retry_no_request`. Crash-point waves serve no UI and click Retry through the API.
+`ui_retry_no_request`. Before it looks for Retry, the harness closes any open modal that covers the page
+("Not now", else Escape) and records it in the attempt's `dismissedDialogs`. A visible Retry that stays
+unreachable behind a modal that will not close fails as `ui_retry_blocked_by_modal`. Crash-point waves
+serve no UI and click Retry through the API.
## Daytona (`--env daytona`)
diff --git a/tests/restart-chaos/board-ui.test.ts b/tests/restart-chaos/board-ui.test.ts
index 2a5ba83b88a..b8897d8f42f 100644
--- a/tests/restart-chaos/board-ui.test.ts
+++ b/tests/restart-chaos/board-ui.test.ts
@@ -4,10 +4,24 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { once } from "node:events";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { BoardUi, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, isRetryRequestFor } from "./board-ui.js";
+import { BoardUi, UI_RETRY_BLOCKED_BY_MODAL, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, isRetryRequestFor } from "./board-ui.js";
const retry = (issue: string) => `fetch('/api/issues/${issue}/retry',{method:'POST'})`;
+const modal = (buttons: string) => `
+
You completed 5 tasks! ${buttons}`;
+const coveredRetry = (issue: string) => `
+ Couldn’t finish this attempt. Retry
`;
+const closeModal = "document.querySelector('[role=dialog]').remove();document.querySelector('main').removeAttribute('aria-hidden')";
+const lateModal = `
+
You completed 5 tasks! Not now `;
+const openLateModal = `document.querySelector('main').setAttribute('aria-hidden','true');document.body.insertAdjacentHTML('beforeend', ${JSON.stringify(lateModal).replace(/"/g, """)})`;
const pages: Record = {
+ "/issues/CHA-8": coveredRetry("i8") + modal(`Not now `),
+ "/issues/CHA-9": coveredRetry("i9") + modal(`No way out `),
+ "/issues/CHA-11": `Couldn’t finish this attempt.
+ Retry
`,
+ "/issues/CHA-10": coveredRetry("i10") + modal(`Press Escape `)
+ + ``,
"/issues/CHA-1": `Couldn’t finish this attempt. Retry
`,
"/issues/CHA-2": `Saving files from the previous attempt
`,
"/issues/CHA-7": `Couldn’t finish this attempt. Retry
`,
@@ -89,6 +103,38 @@ describe("BoardUi.clickRetry", () => {
}
}, 60_000);
+ it("dismisses a prompt that covers the Retry with Not now, then clicks Retry", async () => {
+ await withBoard(async board => {
+ expect(await board.clickRetry("CHA-8", "i8", 5_000, 1_000)).toMatchObject({ clicked: true, httpStatus: 202, accepted: true, error: null,
+ dismissedDialogs: [expect.stringContaining("You completed 5 tasks!")] });
+ });
+ expect(posts).toContain("/api/issues/i8/retry");
+ }, 60_000);
+
+ it("closes a prompt without a Not now button with Escape", async () => {
+ await withBoard(async board => {
+ expect(await board.clickRetry("CHA-10", "i10", 5_000, 1_000)).toMatchObject({ clicked: true, accepted: true,
+ dismissedDialogs: [expect.stringContaining("You completed 5 tasks!")] });
+ });
+ expect(posts).toContain("/api/issues/i10/retry");
+ }, 60_000);
+
+ it("closes a prompt that opens after Retry appeared, then clicks Retry", async () => {
+ await withBoard(async board => {
+ expect(await board.clickRetry("CHA-11", "i11", 5_000, 1_500)).toMatchObject({ clicked: true, httpStatus: 202, accepted: true, error: null,
+ dismissedDialogs: [expect.stringContaining("You completed 5 tasks!")] });
+ });
+ expect(posts).toContain("/api/issues/i11/retry");
+ }, 60_000);
+
+ it("names a modal that keeps a visible Retry unreachable", async () => {
+ await withBoard(async board => {
+ expect(await board.clickRetry("CHA-9", "i9", 2_000, 500)).toMatchObject({ clicked: false, error: UI_RETRY_BLOCKED_BY_MODAL,
+ bannerText: expect.stringContaining("Couldn’t finish this attempt") });
+ });
+ expect(posts).not.toContain("/api/issues/i9/retry");
+ }, 60_000);
+
it("reports a Retry button whose click sends no retry request", async () => {
await withBoard(async board => {
expect(await board.clickRetry("CHA-3", "i3", 1_500, 500)).toMatchObject({ clicked: true, httpStatus: null, error: UI_RETRY_NO_REQUEST });
diff --git a/tests/restart-chaos/board-ui.ts b/tests/restart-chaos/board-ui.ts
index 90390b45288..3677360e8a6 100644
--- a/tests/restart-chaos/board-ui.ts
+++ b/tests/restart-chaos/board-ui.ts
@@ -1,9 +1,9 @@
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { chromium, type Browser, type ElementHandle, type Page } from "@playwright/test";
-import { UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST } from "./types.js";
+import { UI_RETRY_BLOCKED_BY_MODAL, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST } from "./types.js";
-export { UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST };
+export { UI_RETRY_BLOCKED_BY_MODAL, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST };
export interface UiRetryClick {
clicked: boolean;
@@ -12,6 +12,7 @@ export interface UiRetryClick {
error: string | null;
bannerText: string | null;
diagnostics?: { screenshot: string | null; dom: string | null } | null;
+ dismissedDialogs?: string[];
}
export async function saveRetryMissingDiagnostics(page: Page, directory: string, label: string) {
@@ -37,6 +38,28 @@ export async function saveRetryMissingDiagnostics(page: Page, directory: string,
}
const PER_BUTTON_REQUEST_MS = 5_000;
+const DIALOG_SCAN_MS = 1_000;
+const MAX_DISMISSED_DIALOGS = 3;
+const BLOCKING_DIALOG = '[role="dialog"][data-state="open"], [role="alertdialog"][data-state="open"], '
+ + '[aria-modal="true"]:not([data-state="closed"]), dialog[open]';
+const DISMISS_BUTTON = /^(not now|no thanks|maybe later|dismiss)$/i;
+
+const blockingDialog = (page: Page) => page.locator(BLOCKING_DIALOG).filter({ visible: true }).first();
+
+async function dismissBlockingDialog(page: Page): Promise {
+ const dialog = blockingDialog(page);
+ if (!await dialog.isVisible().catch(() => false)) return null;
+ if (await dialog.getByRole("button", { name: "Retry", exact: true }).count().catch(() => 0)) return null;
+ const label = (await dialog.innerText().catch(() => "")).trim().split(/\r?\n/)[0]?.slice(0, 120) || "dialog";
+ const handle = await dialog.elementHandle().catch(() => null);
+ const notNow = dialog.getByRole("button", { name: DISMISS_BUTTON }).filter({ visible: true }).first();
+ const clicked = await notNow.count().catch(() => 0) > 0 && await notNow.click({ timeout: 2_000 }).then(() => true, () => false);
+ if (!clicked) await page.keyboard.press("Escape").catch(() => undefined);
+ const closed = handle
+ ? await handle.waitForElementState("hidden", { timeout: 2_000 }).then(() => true, () => false)
+ : !await blockingDialog(page).isVisible().catch(() => false);
+ return closed ? label : null;
+}
export function isRetryRequestFor(issueRefs: readonly string[], method: string, url: string): boolean {
if (method !== "POST") return false;
@@ -61,28 +84,55 @@ export async function clickRetryOnPage(page: Page, url: string, issueRefs: reado
const texts = await page.locator("[role=status], [role=alert]").allInnerTexts().catch(() => []);
return texts.map(text => text.trim()).filter(Boolean).join(" | ").slice(0, 500) || null;
};
- try {
- await page.getByRole("button", { name: "Retry", exact: true }).filter({ visible: true }).first()
- .waitFor({ state: "visible", timeout: timeoutMs });
- } catch {
+ const reachable = page.getByRole("button", { name: "Retry", exact: true }).filter({ visible: true }).first();
+ const dismissedDialogs: string[] = [];
+ const dismissed = () => (dismissedDialogs.length ? { dismissedDialogs } : {});
+ const dismissDialogs = async () => {
+ const before = dismissedDialogs.length;
+ while (dismissedDialogs.length < MAX_DISMISSED_DIALOGS) {
+ const closed = await dismissBlockingDialog(page);
+ if (closed === null) break;
+ dismissedDialogs.push(closed);
+ }
+ return dismissedDialogs.length > before;
+ };
+ const blockedByModal = async () => await blockingDialog(page).isVisible().catch(() => false)
+ && await page.getByRole("button", { name: "Retry", exact: true, includeHidden: true }).filter({ visible: true }).count().catch(() => 0) > 0;
+ const deadline = Date.now() + timeoutMs;
+ for (;;) {
+ await dismissDialogs();
+ const remaining = deadline - Date.now();
+ if (remaining <= 0) break;
+ if (await reachable.waitFor({ state: "visible", timeout: Math.min(remaining, DIALOG_SCAN_MS) }).then(() => true, () => false)) break;
+ }
+ if (!await reachable.isVisible().catch(() => false)) {
const bannerText = await banner();
const diagnostics = diagnosticsDirectory
? await saveRetryMissingDiagnostics(page, diagnosticsDirectory, issueRefs[0] ?? "issue").catch(() => null) : undefined;
- return { clicked: false, httpStatus: null, accepted: null, error: UI_RETRY_BUTTON_MISSING, bannerText,
- ...(diagnostics !== undefined ? { diagnostics } : {}) };
+ const blocked = await blockedByModal();
+ return { clicked: false, httpStatus: null, accepted: null, error: blocked ? UI_RETRY_BLOCKED_BY_MODAL : UI_RETRY_BUTTON_MISSING, bannerText,
+ ...(diagnostics !== undefined ? { diagnostics } : {}), ...dismissed() };
}
const bannerText = await banner();
for (const button of await retryButtonsBannerFirst(page)) {
- const response = page.waitForResponse(candidate => isRetryRequestFor(issueRefs, candidate.request().method(), candidate.url()),
- { timeout: perButtonMs }).catch(() => null);
- await button.click().catch(() => undefined);
- const answered = await response;
- if (!answered) continue;
- const body = await answered.json().catch(() => null) as { accepted?: unknown } | null;
- return { clicked: true, httpStatus: answered.status(), accepted: answered.ok() ? body?.accepted === true : false,
- error: answered.ok() ? null : `retry request failed with ${answered.status()}`, bannerText };
+ for (let attempt = 0; attempt < 2; attempt++) {
+ await dismissDialogs();
+ const response = page.waitForResponse(candidate => isRetryRequestFor(issueRefs, candidate.request().method(), candidate.url()),
+ { timeout: perButtonMs }).catch(() => null);
+ await button.click({ timeout: perButtonMs }).catch(() => undefined);
+ const answered = await response;
+ if (answered) {
+ const body = await answered.json().catch(() => null) as { accepted?: unknown } | null;
+ return { clicked: true, httpStatus: answered.status(), accepted: answered.ok() ? body?.accepted === true : false,
+ error: answered.ok() ? null : `retry request failed with ${answered.status()}`, bannerText, ...dismissed() };
+ }
+ if (!await dismissDialogs()) break;
+ }
}
- return { clicked: true, httpStatus: null, accepted: null, error: UI_RETRY_NO_REQUEST, bannerText };
+ if (await blockedByModal()) {
+ return { clicked: false, httpStatus: null, accepted: null, error: UI_RETRY_BLOCKED_BY_MODAL, bannerText, ...dismissed() };
+ }
+ return { clicked: true, httpStatus: null, accepted: null, error: UI_RETRY_NO_REQUEST, bannerText, ...dismissed() };
}
export class BoardUi {
diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts
index cf6aed66e04..f6495d49e68 100644
--- a/tests/restart-chaos/grader.test.ts
+++ b/tests/restart-chaos/grader.test.ts
@@ -244,6 +244,8 @@ describe("T-oracles", () => {
retry: { at: 2, observedRunId: "r1", httpStatus: null, accepted: null, error: "ui_retry_button_missing", via: "ui", bannerText: "Saving files" } } as unknown as TaskRecord;
expect(gradeR3(record)).toEqual({ verdict: "fail", detail: "ui_retry_button_missing (user saw: Saving files)" });
expect(gradeR3({ ...record, retry: { ...record.retry!, clicked: true, error: "ui_retry_no_request" } } as TaskRecord).detail).toMatch(/^ui_retry_no_request/);
+ expect(gradeR3({ ...record, retry: { ...record.retry!, error: "ui_retry_blocked_by_modal" } } as TaskRecord).detail)
+ .toBe("ui_retry_blocked_by_modal (user saw: Saving files)");
});
it("reports R2 as not checked for a sandbox-only workspace instead of passing it", () => {
diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts
index 14815988041..1a3730f456e 100644
--- a/tests/restart-chaos/grader.ts
+++ b/tests/restart-chaos/grader.ts
@@ -1,4 +1,4 @@
-import { UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, type FaultReceipt, type OracleResult, type RuleId, type RuleResult, type TaskGrade, type TaskRecord, type TaskSample } from "./types.js";
+import { UI_RETRY_BLOCKED_BY_MODAL, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, type FaultReceipt, type OracleResult, type RuleId, type RuleResult, type TaskGrade, type TaskRecord, type TaskSample } from "./types.js";
import { g1Decision } from "./workloads.js";
import { enforcedUniversalFailures, gradeUniversal, type UniversalDbEvidence } from "./universal.js";
@@ -112,7 +112,7 @@ function gradeOwnRetry(record: TaskRecord): RuleResult {
}
return { verdict: "n/a", detail: "no Retry needed" };
}
- if (record.retry.via === "ui" && (record.retry.error === UI_RETRY_BUTTON_MISSING || record.retry.error === UI_RETRY_NO_REQUEST)) {
+ if (record.retry.via === "ui" && [UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, UI_RETRY_BLOCKED_BY_MODAL].includes(record.retry.error ?? "")) {
return { verdict: "fail", detail: `${record.retry.error}${record.retry.bannerText ? ` (user saw: ${record.retry.bannerText})` : ""}` };
}
if (record.retry.error || record.retry.accepted !== true) {
diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts
index 7ba82e01fda..8618de508c1 100644
--- a/tests/restart-chaos/observer.ts
+++ b/tests/restart-chaos/observer.ts
@@ -718,7 +718,8 @@ export class TaskObserver {
if (this.board) {
const click = await this.board.clickRetry(issueRef, issueId);
Object.assign(attempt, { via: "ui", httpStatus: click.httpStatus, accepted: click.accepted, error: click.error, bannerText: click.bannerText,
- ...(click.diagnostics !== undefined ? { diagnostics: click.diagnostics } : {}) });
+ ...(click.diagnostics !== undefined ? { diagnostics: click.diagnostics } : {}),
+ ...(click.dismissedDialogs?.length ? { dismissedDialogs: click.dismissedDialogs } : {}) });
if (click.accepted) attempt.acceptedAt = Math.max(now, Date.now());
return attempt;
}
diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts
index 0dee982800e..451dc26240e 100644
--- a/tests/restart-chaos/types.ts
+++ b/tests/restart-chaos/types.ts
@@ -256,10 +256,12 @@ export interface RetryAttempt {
via?: "ui" | "api";
bannerText?: string | null;
diagnostics?: { screenshot: string | null; dom: string | null } | null;
+ dismissedDialogs?: string[];
}
export const UI_RETRY_BUTTON_MISSING = "ui_retry_button_missing";
export const UI_RETRY_NO_REQUEST = "ui_retry_no_request";
+export const UI_RETRY_BLOCKED_BY_MODAL = "ui_retry_blocked_by_modal";
export interface FaultReceipt {
faultId: string;