test(evals): match the dedicated breadcrumb task identifier

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-07 00:08:08 -05:00
1 parent cbf1c4fba8
commit 162cc90fda
3 files changed
+15 -5

No files matched your search

@@ -34,6 +34,12 @@ The historical Everyday fixture and prompts remain unchanged. The new fixture an
## Validation and remaining gate
Credential-free checks pass: eval typecheck; 1,801 support tests plus 128 Node tests (one intentional support skip); 51 connection/schema tests; one real-server fixture test; one focused Rust feedback test; and local browser wrong-task/title calibration. Repository typecheck and build pass. The full repository test run is still under inspection after a Telegram integration test failed outside the changed paths.
Credential-free checks pass: eval typecheck; 1,801 support tests plus 128 Node tests (one intentional support skip); 51 connection/schema tests; one real-server fixture test; one focused Rust feedback test; and local browser wrong-task/title calibration. Repository typecheck and build pass. The full repository test run is still under inspection after Telegram and workspace-runtime integration tests failed outside the changed paths. No full local pass is claimed.
A corrected paid baseline is pending. Run an explicit provider-choice canary first, then the other distinct cells only after that evidence is usable. Preserve every attempt, decision, actual run, cost gap and failure. No instruction reduction or general integration quality claim is qualified yet.
## Initial corrected campaign stopped before provider execution
[Campaign 37574251834](https://github.com/paperclipai/paperclip/actions/runs/37574251834) resolved source `cbf1c4fba8eabbf779435f550c455f6a66c5aae5` on trusted master workflow `99a9de9940bf5974352d9dbfbb2f21e62e89689f`. It was cancelled during the shared build after review identified another browser precondition defect. The matrix job has no steps and no runner execution; no provider cell or behavioral grade occurred. Preserve this cancelled campaign separately.
The initial stable-ID check assumed whitespace around the identifier. The real breadcrumb uses adjacent title/identifier spans. A regression reproduces the failure with `Renamed taskRUN-1`; the corrected check targets the visible identifier element under the current breadcrumb and matches its text exactly. All five browser checks now pass, including wrong route, wrong/partial identifier and an identifier present only in the mutable title or elsewhere on the page. Seven focused suite checks and eval typecheck pass. This review correction changes only browser support and this report; production logic is unchanged.
+4 -3
View File
@@ -16,9 +16,10 @@ export async function captureLoadedContinuation(
/** A task title is mutable while an agent runs. Match the route and visible stable identifier. */
export async function waitForTaskIdentity(page: Page, taskPath: string, identifier: string, timeout = 30_000) {
await expect(page).toHaveURL(url => url.pathname === taskPath, { timeout });
const escaped = identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
await expect(page.getByRole("navigation", { name: "breadcrumb" }))
.toContainText(new RegExp(`(?:^|\\s)${escaped}(?:\\s|$)`), { timeout });
const currentIdentifier = page.getByRole("navigation", { name: "breadcrumb" })
.locator('[aria-current="page"] [data-slot="task-title-identifier"]');
await expect(currentIdentifier).toBeVisible({ timeout });
await expect(currentIdentifier).toHaveText(identifier, { timeout });
}
/** Wait for the persisted task projection before taking a browser screenshot. */
@@ -35,7 +35,7 @@ for (const state of ["app-loader", "history-loader", "wrong-task"] as const) {
test("mutable titles do not change task identity, but wrong routes and identifiers fail", async ({ page }) => {
await page.route("https://fixture.invalid/**", route => route.fulfill({ contentType: "text/html",
body: `<nav aria-label="breadcrumb">Renamed task RUN-1</nav>${shell(false, "Renamed task")}` }));
body: `<nav aria-label="breadcrumb"><span aria-current="page"><span>Renamed task</span><span data-slot="task-title-identifier">RUN-1</span></span></nav>${shell(false, "Renamed task")}` }));
await page.goto("https://fixture.invalid/RUN/issues/RUN-1");
await waitForTaskIdentity(page, "/RUN/issues/RUN-1", "RUN-1", 500);
await waitForTaskChatRendered(page, undefined, 500);
@@ -43,4 +43,7 @@ test("mutable titles do not change task identity, but wrong routes and identifie
await expect(waitForTaskIdentity(page, route, identifier, 150)).rejects.toThrow();
}
await expect(waitForTaskChatRendered(page, "Original task title", 150)).rejects.toThrow();
// An identifier in the mutable title or outside the breadcrumb is not identity.
await page.setContent(`<nav aria-label="breadcrumb"><span aria-current="page"><span>RUN-1 renamed task</span><span data-slot="task-title-identifier">RUN-2</span></span></nav><span data-slot="task-title-identifier">RUN-1</span>${shell(false)}`);
await expect(waitForTaskIdentity(page, "/RUN/issues/RUN-1", "RUN-1", 150)).rejects.toThrow();
});