test(evals): retain frozen chat scope across merged capability maps

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/CHAT-FEATURES.md

@@ -229,3 +229,11 @@ The supervisor binds the actual CLI PID, version and CLI/server bytes before set
 and after an allowed update restart. The operator receipt stays private0600;
 fingerprints bind public artifact hashes and the full owned dependency tree.
 Source-local refusal remains the default for every other ordinary chat case.
+
+## Frozen capability coverage
+
+The campaign retains its original 244 scenarios and 191 declared capabilities.
+Merged production now also maps `runs-adapters/model-accounts`; that capability
+is outside this frozen catalog and remains uncovered and unqualified by it.
+The setup-health cell is diagnostic only and is excluded from those 244 scenarios
+and from `--all`, as is the explicit-only chat catalog.

tests/runner-e2e/catalog.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "vitest";
-import { readFileSync } from "node:fs";
+import { readFileSync, readdirSync } from "node:fs";
+import { parseDocument } from "yaml";
 import { contextIntegrityTasks } from "./context-integrity-cases.js";
 import { normalizePrpResultSignals } from "../../packages/paperclip-runner/src/protocol/result-normalization.js";
 import {
@@ -219,10 +220,20 @@ describe("runner E2E catalog", () => {
     expect(localIntegrityTasks).toHaveLength(2);
     expect(openRouterBreadthTasks).toHaveLength(3);
     expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
-      2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 2, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2,
+      2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 1, 244, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2,
     ]);
-    expect(validateRunnerCatalog()).toHaveLength(769);
-    expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(769);
+    expect(validateRunnerCatalog()).toHaveLength(1012);
+    const frozenDirectory = new URL("./chat-scenarios/", import.meta.url);
+    const frozenIds = readdirSync(frozenDirectory).filter(name => name.endsWith(".yaml") && name !== "defaults.yaml")
+      .flatMap(name => parseDocument(readFileSync(new URL(name, frozenDirectory), "utf8")).toJS().scenarios.map((scenario: { id: string }) => scenario.id));
+    expect(frozenIds).toHaveLength(244);
+    const chat = selectRunnerExecutions(parseRunnerSelectors(["--suite", "chat-features"]));
+    // These two historical execution aliases retain their exact frozen YAML identity.
+    const legacyScenarioIds: Record<string, string> = { "create-backlog": "tasks.create-backlog", "delegate-announcement": "tasks.delegate-announcement" };
+    expect(chat.map(cell => legacyScenarioIds[cell.task.id] ?? cell.task.id).sort()).toEqual(frozenIds.sort());
+    for (const cell of chat) expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id]))).toEqual([cell]);
+    expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => ["chat-features", "chat-setup-health"].includes(cell.suite.id))).toBe(false);
+    expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(1012);
     expect(
       runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
     ).toHaveLength(48);

tests/runner-e2e/chat-scenario-catalog.test.ts

@@ -36,8 +36,13 @@ it("validates the complete planned chat catalog against its schema and feature m
     capabilities.forEach(id => mapped.add(id));
   }
   expect(ids.size).toBe(244);
-  expect([...covered].sort()).toEqual([...mapped].sort());
-  expect(mapped.size).toBe(191);
+  // Added by merged production after this 244-scenario catalog was frozen.
+  // This capability is explicitly uncovered, not implicitly qualified by this suite.
+  const outsideFrozenScope = ["runs-adapters/model-accounts"];
+  expect([...covered].filter(id => !mapped.has(id))).toEqual([]);
+  expect([...mapped].filter(id => !covered.has(id)).sort()).toEqual(outsideFrozenScope);
+  expect(covered.size).toBe(191);
+  expect(mapped.size).toBe(192);
   const defaults = parseDocument(readFileSync(new URL("defaults.yaml", catalog), "utf8"), { uniqueKeys: true });
   expect(defaults.errors).toEqual([]);
   expect(defaults.toJS()).toMatchObject({ schema_version: 1, status: "planned" });