From 8565913082ceb70fc48552b5e74a52af9971d625 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 07:41:42 -0500 Subject: [PATCH] fix(evals): supply the selected historical developer lab link --- .../privileged-companions.test.ts | 9 +++- .../privileged-gated-lab.ts | 45 ++++++++++++++++--- .../runner-e2e/chat-query-attachments.test.ts | 1 + tests/runner-e2e/chat-query-attachments.ts | 1 + 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts index b2bd69b72ce..d5da7efe5e8 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { gradeInstanceSettings } from "./privileged-configuration.js"; import { documentExportCommand, gradeExportDocumentCli } from "./privileged-document-cli.js"; import { inspectRunCommands, gradeInspectRunCli } from "./privileged-run-cli.js"; -import { gradeGatedLab } from "./privileged-gated-lab.js"; +import { historicalLabLink, gradeGatedLab } from "./privileged-gated-lab.js"; const base = { scenarioId: "owned", origin: "http://127.0.0.1:3100", companyId: "company", leadId: "lead", state: {}, tasks: [], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "run", agentId: "lead", status: "succeeded" }], reply: "" }; const block = (body: string) => `\`\`\`bash\n${body}\n\`\`\``; @@ -58,3 +58,10 @@ describe("operator companions use actual saved authority and output", () => { expect(gradeGatedLab({ ...o, reply: "The lab is available in production; it is not a development-only task-chat-lab." }).passed).toBe(false); }); }); + +it("binds factual historical lab links only to the owned production listener and selected company", () => { + expect(historicalLabLink("http://127.0.0.1:43210", "RUN")).toBe("http://127.0.0.1:43210/RUN/dev/task-chat-lab"); + for (const origin of ["https://unowned.invalid", "http://127.0.0.1", "http://127.0.0.1:43210/other", "http://127.0.0.1:43210/?redirect=elsewhere"]) + expect(() => historicalLabLink(origin, "RUN")).toThrow(); + expect(() => historicalLabLink("http://127.0.0.1:43210", "../foreign")).toThrow(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-gated-lab.ts b/tests/runner-e2e/chat-catalog-domains/privileged-gated-lab.ts index a14995301b5..27ccc876e8e 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-gated-lab.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-gated-lab.ts @@ -1,8 +1,18 @@ import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { factualContextQueryAttachment, uploadFactualQueryAttachment } from "../chat-query-attachments.js"; import { checks } from "./internal-support.js"; import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; import { startOwnedProductionUi } from "./privileged-production-ui.js"; +const productionTargets = new WeakMap>>(); +export function historicalLabLink(origin: string, companyPrefix: string | null | undefined) { + const url = new URL(origin); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || !url.port || url.pathname !== "/" + || url.search || url.hash || url.username || url.password || typeof companyPrefix !== "string" || !/^[A-Z][A-Z0-9]*$/.test(companyPrefix)) + throw new Error("Historical lab link must bind the owned production listener and exact company prefix"); + return new URL(`/${companyPrefix}/dev/task-chat-lab`, url).href; +} export function gradeGatedLab(value: unknown) { const c = checks(), o = privilegedObservation(value); c.add("production-route-observation", !!o, "A real production build must deny the historical developer route."); if (!o) return c.result(); @@ -19,13 +29,31 @@ export function gradeGatedLab(value: unknown) { return c.result(); } const gated = privilegedDefinition("developer-labs.gated-lab", "privileged-gated-lab", { - async setup() { return {}; }, + async setup(ctx) { + const target = await startOwnedProductionUi(ctx); + productionTargets.set(ctx, target); + try { + const historicalUrl = historicalLabLink(target.origin, ctx.input.fixtures.company.issuePrefix); + const factualContextAttachment = await uploadFactualQueryAttachment(ctx, { + filename: "historical-lab-link.md", body: `${historicalUrl}\n`, + }); + return { historicalUrl, factualContextAttachment }; + } catch (error) { + productionTargets.delete(ctx); + await target.stop(); + throw error; + } + }, async act(ctx, state) { await settlePrivilegedChat(ctx); - const target = await startOwnedProductionUi(ctx), page = ctx.input.page, source = page.url(); + const target = productionTargets.get(ctx); + if (!target) throw new Error("Missing exact pre-query production target"); + const page = ctx.input.page, source = page.url(); try { - const historicalPath = `/${ctx.input.fixtures.company.issuePrefix}/dev/task-chat-lab`; - const response = await page.goto(target.origin + historicalPath, { waitUntil: "domcontentloaded" }); + const historicalUrl = historicalLabLink(target.origin, ctx.input.fixtures.company.issuePrefix); + if (state.historicalUrl !== historicalUrl) throw new Error("Selected historical link changed after the human query"); + const historicalPath = new URL(historicalUrl).pathname; + const response = await page.goto(historicalUrl, { waitUntil: "domcontentloaded" }); state.productionHeaderMatched = response?.headers()["x-eval-production-ui-build"] === target.buildSha256; await expect(page.getByRole("heading", { name: /not found|page.*exist|couldn.*find/i }).first()).toBeVisible(); await expect(page.getByRole("heading", { name: "Task Chat Lab", exact: true })).toHaveCount(0); @@ -35,10 +63,15 @@ const gated = privilegedDefinition("developer-labs.gated-lab", "privileged-gated state.unrelatedTaskAbsent = await page.getByTestId("task-chat-composer-input").count() === 0; await ctx.input.evidence("chat-catalog-production-lab-gate.json", { ...state, files: target.files, proof: "production build; no second database or provider" }); await ctx.input.capture("production-lab-gate", "Real production build excludes developer lab", "production-lab-gate.png"); - } finally { try { await page.goto(source, { waitUntil: "domcontentloaded" }); } finally { await target.stop(); } } + } finally { try { await page.goto(source, { waitUntil: "domcontentloaded" }); } finally { productionTargets.delete(ctx); await target.stop(); } } }, grade: gradeGatedLab, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, }); gated.attemptTimeoutMs = 600_000; -gated.sourceFiles = [...gated.sourceFiles, "chat-catalog-domains/privileged-production-ui.ts", "../../ui/vite.config.ts", "../../ui/src/App.tsx", "../../ui/src/pages/NotFound.tsx"]; +gated.queryAttachments = factualContextQueryAttachment("historical-lab-link.md"); +gated.cleanup = async ctx => { + const target = productionTargets.get(ctx); + if (target) { productionTargets.delete(ctx); await target.stop(); } +}; +gated.sourceFiles = [...gated.sourceFiles, "chat-catalog-domains/privileged-production-ui.ts", "../../ui/vite.config.ts", "../../ui/src/App.tsx", "../../ui/src/pages/NotFound.tsx", "chat-query-attachments.ts"]; export const privilegedGatedLabCases = [gated]; diff --git a/tests/runner-e2e/chat-query-attachments.test.ts b/tests/runner-e2e/chat-query-attachments.test.ts index dbfef4485ff..799acbc3fa0 100644 --- a/tests/runner-e2e/chat-query-attachments.test.ts +++ b/tests/runner-e2e/chat-query-attachments.test.ts @@ -44,6 +44,7 @@ it("independently resolves declared receipt IDs and rejects undeclared, duplicat }); it.each([ + ["developer-labs.gated-lab", "historical-lab-link.md"], ["documents-artifacts.resolve-feedback", "selected-source.md"], ["documents-artifacts.restore-document", "selected-source.md"], ["documents-artifacts.produce-download", "selected-source.md"], ["documents-artifacts.revise-delivery", "selected-source.md"], ["skills.author-skill", "skill-context.md"], ["skills.studio-test", "skill-context.md"], ["skills.review-update", "skill-context.md"], diff --git a/tests/runner-e2e/chat-query-attachments.ts b/tests/runner-e2e/chat-query-attachments.ts index c9a79908caf..87d8f7c6b8d 100644 --- a/tests/runner-e2e/chat-query-attachments.ts +++ b/tests/runner-e2e/chat-query-attachments.ts @@ -13,6 +13,7 @@ export interface ChatQueryHumanComment { issueId?: string; companyId?: string; createdByRunId?: string | null; } const factualFiles: Readonly> = { + "developer-labs.gated-lab": "historical-lab-link.md", "documents-artifacts.resolve-feedback": "selected-source.md", "documents-artifacts.restore-document": "selected-source.md", "documents-artifacts.produce-download": "selected-source.md",