From 96603470733044cbdbd32af132d8c13d64a8263f Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 08:30:57 -0500 Subject: [PATCH] fix(evals): follow scoped runtime session and retry controls --- .../runtime-failed-runs.test.ts | 11 +++++++- .../runtime-failed-runs.ts | 25 ++++++++++++++++++- .../chat-catalog-domains/runtime-session.ts | 4 ++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts index c7ec762ab95..f83449f3f94 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.test.ts @@ -1,5 +1,5 @@ import { it, expect } from "vitest"; -import { gradeFailedAttempt } from "./runtime-failed-runs.js"; +import { exactFailedRunLink, gradeFailedAttempt } from "./runtime-failed-runs.js"; function valid() { const failedRun = { id: "failed-model", status: "failed", errorCode: "native_provider_model_rejected", error: "Selected model unavailable", costUsd: null as number | null, logRef: "actual-log" as string | null }; return { scenarioId: "runs-adapters.inspect-failed-run", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { failedRun, retainedFailure: { ...failedRun }, successfulRun: { id: "actual-success", status: "succeeded" }, successArtifactVerified: true, task: { id: "website" }, failureLogAvailable: true }, @@ -17,3 +17,12 @@ it("requires a missing output disclosure when logs are unavailable", () => { con expect(gradeFailedAttempt(o).passed).toBe(false); o.reply += " Retained log output is unavailable."; expect(gradeFailedAttempt(o).passed).toBe(true); }); it("accepts an actually recorded zero cost without calling it unavailable", () => { const o = valid(); o.state.failedRun.costUsd = 0; o.reply = "The failed-model run failed with a provider model error. The recorded cost was $0.00."; expect(gradeFailedAttempt(o).passed).toBe(true); }); it("rejects invented successful output without verified saved bytes", () => { const o = valid(); o.state.successArtifactVerified = false; expect(gradeFailedAttempt(o).passed).toBe(false); }); + +it("binds the visible retry journey to the exact original actor, company and failed attempt", () => { + const worker = { id: "worker-id", urlKey: "morgan" }; + expect(exactFailedRunLink("/RUN/agents/morgan/runs/failed", "http://owned.invalid", "RUN", worker, "failed")).toBe(true); + expect(exactFailedRunLink("/RUN/agents/worker-id/runs/failed", "http://owned.invalid", "RUN", worker, "failed")).toBe(true); + for (const href of [null, "https://foreign.invalid/RUN/agents/morgan/runs/failed", "/OTHER/agents/morgan/runs/failed", + "/RUN/agents/other/runs/failed", "/RUN/agents/morgan/runs/success", "/RUN/agents/morgan/runs/failed?other=1"]) + expect(exactFailedRunLink(href, "http://owned.invalid", "RUN", worker, "failed")).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts index 9de3b159191..273bb93e499 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-failed-runs.ts @@ -5,6 +5,15 @@ import { checks, type Entity } from "./internal-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js"; import { unavailableNativeModelFailure } from "./runtime-native-failure.js"; import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; +export function exactFailedRunLink(href: string | null, origin: string, companyPrefix: string, worker: { id: string; urlKey?: string }, runId: string) { + if (!href) return false; + try { + const url = new URL(href, origin), parts = url.pathname.split("/").map(decodeURIComponent); + return url.origin === new URL(origin).origin && !url.search && !url.hash && parts.length === 6 + && parts[1] === companyPrefix && parts[2] === "agents" && [worker.id, worker.urlKey].includes(parts[3]) + && parts[4] === "runs" && parts[5] === runId; + } catch { return false; } +} export function gradeFailedAttempt(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("actual-failed-and-successful-native-history", !!o, "An actually admitted native failure and a later real successful execution must remain separately readable."); if (!o) return c.result(); @@ -40,7 +49,21 @@ export const runtimeFailedRunCases = [runtimeDefinition("runs-adapters.inspect-f if (!task) throw new Error("Actual failed native task was not created"); const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); - const retry = ctx.input.page.getByTestId("task-chat-run-failed-try-again").last(); await expect(retry).toBeVisible(); + const marker = ctx.input.page.locator("#main-content").getByTestId("task-chat-collapsible-marker") + .filter({ has: ctx.input.page.getByRole("button", { name: /^Model unavailable(?: ·|$)/ }) }); + await expect(marker).toHaveCount(1); + await marker.getByRole("button", { name: /^Model unavailable(?: ·|$)/ }).click(); + const runLink = marker.getByRole("link", { name: "View run", exact: true }); + const href = await runLink.getAttribute("href"); + if (!exactFailedRunLink(href, ctx.input.api.baseURL, ctx.input.fixtures.company.issuePrefix!, worker, failure.failedRun.id)) + throw new Error("Visible failed-run detail link does not identify the original company, worker and attempt"); + await runLink.click(); + await ctx.input.page.waitForURL(url => url.pathname === new URL(href!, ctx.input.api.baseURL).pathname); + const original = await ctx.input.api.get(`/api/heartbeat-runs/${failure.failedRun.id}`); + if (original.id !== failure.failedRun.id || original.status !== "failed" || original.companyId !== worker.companyId || original.agentId !== worker.id) + throw new Error("The visible retry target does not retain the genuine original failure"); + const retry = ctx.input.page.locator("#main-content").getByRole("button", { name: "Retry", exact: true }); + await expect(retry).toBeVisible(); await retry.click(); const done = await pollUntil({ label: "Real successful task retry after restoring the admitted model", deadlineAt: Date.now() + 240_000, load: async () => ({ task: await ctx.input.api.get(`/api/issues/${task!.id}`), runs: await ctx.allRuns() }), diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-session.ts b/tests/runner-e2e/chat-catalog-domains/runtime-session.ts index 0af01aa9b86..c5f6b319db5 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-session.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-session.ts @@ -52,7 +52,9 @@ export const runtimeSessionCases = [runtimeDefinition("runs-adapters.reset-sessi state.agentResetStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}/runtime-state/reset-session`, "POST", {}); state.sessionPresentBeforeHuman = (await ctx.input.api.get(`/api/agents/${worker.id}/task-sessions`)).length > 0; const source = ctx.input.page.url(); await openRuntimeAgentProfile(ctx, worker, query); - await ctx.input.page.getByRole("button", { name: "Open actions for Morgan", exact: true }).click(); + const profile = ctx.input.page.locator("#main-content"); + await expect(profile.getByRole("heading", { name: worker.name!, exact: true })).toBeVisible(); + await profile.getByRole("button", { name: `Open actions for ${worker.name}`, exact: true }).click(); await ctx.input.page.getByRole("button", { name: "Reset Sessions", exact: true }).click(); await pollUntil({ label: "Human reset clears the real task session", deadlineAt: Date.now() + 30_000, load: () => ctx.input.api.get(`/api/agents/${worker.id}/task-sessions`), accept: rows => rows.length === 0 });