From 5b54009a03f4593e2f70f669e5da0d7729a2e76d Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 13:11:41 -0500 Subject: [PATCH] test(evals): retain frozen chat scope across merged capability maps Co-Authored-By: Paperclip --- tests/runner-e2e/CHAT-FEATURES.md | 8 ++++++++ tests/runner-e2e/catalog.test.ts | 19 +++++++++++++++---- .../runner-e2e/chat-scenario-catalog.test.ts | 9 +++++++-- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 77c64f02b4b..10babdba4bc 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -229,3 +229,11 @@ The supervisor binds the actual CLI PID, version and CLI/server bytes before set and after an allowed update restart. The operator receipt stays private0600; fingerprints bind public artifact hashes and the full owned dependency tree. Source-local refusal remains the default for every other ordinary chat case. + +## Frozen capability coverage + +The campaign retains its original 244 scenarios and 191 declared capabilities. +Merged production now also maps `runs-adapters/model-accounts`; that capability +is outside this frozen catalog and remains uncovered and unqualified by it. +The setup-health cell is diagnostic only and is excluded from those 244 scenarios +and from `--all`, as is the explicit-only chat catalog. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 09c95dd259a..bf16b459e2e 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; -import { readFileSync } from "node:fs"; +import { readFileSync, readdirSync } from "node:fs"; +import { parseDocument } from "yaml"; import { contextIntegrityTasks } from "./context-integrity-cases.js"; import { normalizePrpResultSignals } from "../../packages/paperclip-runner/src/protocol/result-normalization.js"; import { @@ -219,10 +220,20 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 2, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2, + 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 1, 244, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(769); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(769); + expect(validateRunnerCatalog()).toHaveLength(1012); + const frozenDirectory = new URL("./chat-scenarios/", import.meta.url); + const frozenIds = readdirSync(frozenDirectory).filter(name => name.endsWith(".yaml") && name !== "defaults.yaml") + .flatMap(name => parseDocument(readFileSync(new URL(name, frozenDirectory), "utf8")).toJS().scenarios.map((scenario: { id: string }) => scenario.id)); + expect(frozenIds).toHaveLength(244); + const chat = selectRunnerExecutions(parseRunnerSelectors(["--suite", "chat-features"])); + // These two historical execution aliases retain their exact frozen YAML identity. + const legacyScenarioIds: Record = { "create-backlog": "tasks.create-backlog", "delegate-announcement": "tasks.delegate-announcement" }; + expect(chat.map(cell => legacyScenarioIds[cell.task.id] ?? cell.task.id).sort()).toEqual(frozenIds.sort()); + for (const cell of chat) expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id]))).toEqual([cell]); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => ["chat-features", "chat-setup-health"].includes(cell.suite.id))).toBe(false); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(1012); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/chat-scenario-catalog.test.ts b/tests/runner-e2e/chat-scenario-catalog.test.ts index eef37691e65..ae8c17a2309 100644 --- a/tests/runner-e2e/chat-scenario-catalog.test.ts +++ b/tests/runner-e2e/chat-scenario-catalog.test.ts @@ -36,8 +36,13 @@ it("validates the complete planned chat catalog against its schema and feature m capabilities.forEach(id => mapped.add(id)); } expect(ids.size).toBe(244); - expect([...covered].sort()).toEqual([...mapped].sort()); - expect(mapped.size).toBe(191); + // Added by merged production after this 244-scenario catalog was frozen. + // This capability is explicitly uncovered, not implicitly qualified by this suite. + const outsideFrozenScope = ["runs-adapters/model-accounts"]; + expect([...covered].filter(id => !mapped.has(id))).toEqual([]); + expect([...mapped].filter(id => !covered.has(id)).sort()).toEqual(outsideFrozenScope); + expect(covered.size).toBe(191); + expect(mapped.size).toBe(192); const defaults = parseDocument(readFileSync(new URL("defaults.yaml", catalog), "utf8"), { uniqueKeys: true }); expect(defaults.errors).toEqual([]); expect(defaults.toJS()).toMatchObject({ schema_version: 1, status: "planned" });