test: establish frozen chat catalog campaign and executable case modules

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/CHAT-FEATURES.md

@@ -1,9 +1,26 @@
 # Chat feature evals
 
-`chat-features` promotes two entries from the [scenario catalog](chat-scenarios/README.md)
-into real Product E2E executions. The scope is deliberately small: prove the
-fixture, browser interaction, independent grading and report loop before adding
-more features. Catalog coverage is not live acceptance.
+`chat-features` promotes executable entries from the [scenario catalog](chat-scenarios/README.md)
+into real Product E2E executions. The two original task journeys remain intact.
+Additional domain cases use `chat-catalog-contract.ts` and the explicit registry;
+listing a cell or calibrating its oracle is not live acceptance. The complete
+244-scenario denominator and unchanged requests are frozen in
+`chat-campaign-manifest.json`. Its raw-file/defaults inventory hash is
+`9dc9cf6a5426cf1b56bc30a3571851f233f9dd5e93a1292eaee414d33b52df77`.
+
+Domain cases own production API setup, explicit user followups, observations,
+case-specific grading and optional cleanup. The shared driver submits the exact
+catalog query, saves prerequisite state before action, records real API comments,
+run attribution and rendered links, and rechecks after browser reload. Every case
+declares run/time limits and an oracle version. Missing checks and contradictory
+grades fail. Known `coverageGaps` make a diagnostic case fail acceptance until its
+conditions are implemented; they are not blocked cases reported as passing.
+
+The registry contains only executable setups/actions/oracles. Scenarios absent
+from it remain in the denominator as unimplemented; optional external services
+must be real or explicitly labeled simulations. The frozen campaign ledger owns
+implementation readiness and all attempts separately from catalog discovery.
+No model judge supplies a generic pass.
 
 | Scenario | Actual user request | Independent outcome |
 | --- | --- | --- |
@@ -35,9 +52,7 @@ pnpm test:e2e:runner -- --id chat-features.runner-codex-subscription.local.creat
 pnpm test:e2e:runner -- --id chat-features.runner-codex-subscription.local.delegate-announcement --max-automatic-retries 0
 ```
 
-Run creation first and inspect its packaged report before delegation. For initial
-acceptance, run each cell three times against fresh fixtures on the same definition
-revision. Keep every attempt, including failures; a retry does not erase one. A
+Run creation first and inspect its packaged report before delegation. For the historical first-slice repeatability probe, each cell used three fresh fixtures on the same definition revision. The full-catalog campaign instead allows one initial attempt and one documented fix-and-rerun per failing scenario, with automatic retries disabled. Keep every attempt, including failures; a retry does not erase one. A
 fixture or product correction starts a new three-sample measurement. Both cells
 are explicit-only and excluded from `--all`; this does not implement or qualify
 the rest of the YAML inventory.
@@ -89,7 +104,7 @@ accepting the first samples; the existing completion semantic-review rubric stil
 applies. Oracle calibration tests exercise both valid evidence and plausible wrong
 outcomes without calling a model.
 
-Only after this loop is stable should another scenario be promoted. Keep each
+Preserve the historical measurements below when promoting another scenario. Keep each
 starting condition explicit: connected versus missing connection, allowed versus
 approval-required, and so on. Reuse the fixture/report infrastructure and add a
 case-specific outcome oracle; do not make all YAML paragraphs executable through

tests/runner-e2e/catalog.ts

@@ -25,7 +25,8 @@ import { copilotProviderDeathTasks } from "./copilot-active-stop-tasks.js";
 import { instructionPersistenceTask } from "./instruction-persistence.js";
 import { apiResponseReadingTask } from "./api-response-reading.js";
 import { taskTitleTasks, taskTitleDefinitionDigest, TASK_TITLE_BUDGET_CENTS } from "./task-titles.js";
-import { CHAT_FEATURE_SUITE, CHAT_FEATURE_BUDGET_CENTS, chatFeatureTasks, chatFeatureProfile, chatFeatureScenarios, chatFeatureDefinitionDigest } from "./chat-feature-cases.js";
+import { chatCatalogInventoryDigest, chatCatalogScenarios } from "./chat-catalog-inventory.js";
+import { CHAT_FEATURE_SUITE, CHAT_FEATURE_BUDGET_CENTS, chatFeatureTasks, chatCatalogTasks, chatFeatureProfile, chatFeatureScenarios, chatFeatureDefinitionDigest } from "./chat-feature-cases.js";
 import { blockerTasks, blockerProfile } from "./blocker-cases.js";
 import { accountingTasks } from "./accounting-cases.js";
 import { continuationTasks, questionResumeTask } from "./continuation-cases.js";
@@ -1605,16 +1606,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
   },
   {
     id: CHAT_FEATURE_SUITE, label: "Chat feature journeys", manualOnly: true,
-    description: "Natural catalog requests create backlog work or delegate an announcement and return its saved result.",
+    description: "Executable catalog requests through Agent Chat, with independently graded persisted state.",
     groups: ["chat", "native"],
     profiles: [chatFeatureProfile(nativeProfile({ id: "runner-codex-subscription", label: "Native Codex subscription",
       provider: "codex", model: DEFAULT_CODEX_LOCAL_MODEL, credential: "CODEX_AUTH_JSON", supportedEnvironments: ["local"] }))],
-    environments: [localEnvironment], tasks: chatFeatureTasks, expectedMatrixSize: 2,
-    definitionMetadata: { version: 1, definitionDigest: chatFeatureDefinitionDigest, scheduling: "explicit-only",
+    environments: [localEnvironment], tasks: [...chatFeatureTasks, ...chatCatalogTasks], expectedMatrixSize: 2 + chatCatalogTasks.length,
+    definitionMetadata: { version: 2, definitionDigest: chatFeatureDefinitionDigest, inventoryDigest: chatCatalogInventoryDigest, frozenScenarioCount: 244, scheduling: "explicit-only",
       instructions: "production-role-defaults", companyAndAgentBudgetCents: CHAT_FEATURE_BUDGET_CENTS,
-      unpricedUsagePolicy: "allow-through-public-budget-policy", taskRunLimits: { ceo: 3, worker: 1 },
+      unpricedUsagePolicy: "allow-through-public-budget-policy", taskRunLimits: "per-case-declared; agent daily admission ceiling 3",
       maximumAttemptsPerCell: 1, setupProbeUsage: "unavailable-outside-heartbeat-accounting",
-      scenarios: chatFeatureScenarios.map(({ id, capabilities }) => ({ id, capabilities })) },
+      scenarios: [...chatFeatureScenarios, ...chatCatalogScenarios.filter(row => chatCatalogTasks.some(task => task.id === row.id))].map(({ id, capabilities }) => ({ id, capabilities })) },
   },
   {
     id: "agent-chat", label: "Persistent Agent Chat",

tests/runner-e2e/chat-campaign-preflight.py

@@ -0,0 +1,60 @@
+"""Read-only campaign readiness. No case selection, database launch or IPC cleanup."""
+import argparse
+import datetime
+import json
+import os
+import pathlib
+import platform
+import shutil
+import subprocess
+
+
+def command(arguments):
+    result = subprocess.run(arguments, capture_output=True, text=True, timeout=10)
+    return result.returncode, result.stdout
+
+
+def inspect(root):
+    root = pathlib.Path(root).resolve()
+    ipc = {"available": False, "semaphore_headroom": None, "shared_memory_headroom": None}
+    if platform.system() == "Darwin":
+        status, output = command(["ipcs", "-s", "-a"])
+        status_m, output_m = command(["ipcs", "-m", "-a"])
+        status_limits, output_limits = command(["sysctl", "-n", "kern.sysv.semmns", "kern.sysv.shmmni"])
+        if status == status_m == status_limits == 0:
+            semaphores = [line.split() for line in output.splitlines() if line.startswith("s ")]
+            segments = [line for line in output_m.splitlines() if line.startswith("m ")]
+            limits = [int(value) for value in output_limits.split()]
+            consumed = sum(int(row[8]) for row in semaphores)
+            ipc = {"available": True, "semaphore_sets": len(semaphores), "semaphores_consumed": consumed,
+                   "semaphore_limit": limits[0], "semaphore_headroom": limits[0] - consumed,
+                   "shared_memory_segments": len(segments), "shared_memory_limit": limits[1],
+                   "shared_memory_headroom": limits[1] - len(segments)}
+    runner = root / "packages/paperclip-runner/runner/target/debug/paperclip-runnerd"
+    tools = {name: bool(shutil.which(name)) for name in ["node", "pnpm", "python3", "codex"]}
+    credentials = {name: bool(os.getenv(name)) for name in ["CODEX_AUTH_JSON", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "DAYTONA_API_KEY"]}
+    status, revision = command(["git", "-C", str(root), "rev-parse", "HEAD"])
+    status_dirty, dirty = command(["git", "-C", str(root), "status", "--porcelain", "--untracked-files=no"])
+    # A guard against known exhaustion, not proof that a database will start.
+    ipc_guard = ipc["available"] and ipc["semaphore_headroom"] >= 34 and ipc["shared_memory_headroom"] >= 2
+    return {"version": 1, "measuredAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            "scope": "read-only host preflight; no selected-case attempt", "platform": platform.system(),
+            "sourceRevision": revision.strip() if status == 0 else None,
+            "trackedSourceClean": not dirty.strip() if status_dirty == 0 else None,
+            "tools": tools, "runnerExecutable": os.access(runner, os.X_OK),
+            "credentialsPresent": credentials,
+            "privateCodexAuthFilePresent": (pathlib.Path.home() / ".codex/auth.json").is_file(),
+            "localEnvironmentFilePresent": (root / ".env.runner-e2e.local").is_file(),
+            "immutableDaytonaImageConfigured": "@sha256:" in os.getenv("PAPERCLIP_E2E_DAYTONA_IMAGE", ""),
+            "ipc": ipc, "singleInstanceIpcGuardPassed": ipc_guard,
+            "ipcGuardLimits": {"minimumFreeSemaphores": 34, "minimumFreeSharedMemorySlots": 2},
+            "limitations": ["Credential existence does not establish account validity.",
+                            "IPC headroom does not establish database or browser health.",
+                            "No external connected integration is qualified by this inspection."]}
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--checkout", default=str(pathlib.Path(__file__).resolve().parents[2]))
+    arguments = parser.parse_args()
+    print(json.dumps(inspect(arguments.checkout), indent=2))

tests/runner-e2e/chat-campaign.test.ts

@@ -0,0 +1,85 @@
+import { expect, it } from "vitest";
+import { campaignSummary, completeCampaignAttempt, freezeChatCampaign, reserveCampaignAttempt, validateFrozenCampaign, type CampaignAttempt } from "./chat-campaign.js";
+const root = new URL("../../", import.meta.url).pathname;
+const base = "8373d7867963674ff960d2345d0d28086f799bc7";
+function attempt(ordinal: 1 | 2, executionId: string): CampaignAttempt {
+  return { ordinal, outcome: "running", executionId, startedAt: new Date().toISOString(), finishedAt: null,
+    sourceRevision: base, workingTreeDigest: "fixture", definitionDigest: "fixture", suiteDefinitionDigest: "fixture", graderDigest: "fixture",
+    model: "fixture", profile: "fixture", environment: "local", automaticRetries: 0,
+    correction: ordinal === 2 ? "Demonstrated fixture defect corrected" : null, classification: null, observedFailure: null,
+    evidencePaths: [], usage: { costUsd: null, coverage: "unknown", tokens: null }, cleanup: "pending", corePrDependencies: [] };
+}
+it("freezes all intended requests and keeps unrun scenarios in the denominator", () => {
+  const campaign = freezeChatCampaign(root, base);
+  expect(campaign.scenarios).toHaveLength(244);
+  expect(campaign.areas).toBe(35);
+  expect(campaign.capabilities).toBe(191);
+  expect(campaign.scenarios.every(scenario => scenario.query && scenario.context && scenario.expected_outcome)).toBe(true);
+  expect(campaignSummary(campaign)).toMatchObject({ denominator: 244, unrun: 244, finalPassed: 0, requiredPasses: 196, mergeEligible: false });
+});
+it("preserves failures and enforces one correction cycle with no retries after passing", () => {
+  const scenario = freezeChatCampaign(root, base).scenarios.find(scenario => scenario.id === "tasks.create-backlog")!;
+  const first = attempt(1, scenario.executionId!);
+  reserveCampaignAttempt(scenario, first);
+  expect(() => reserveCampaignAttempt(scenario, attempt(2, scenario.executionId!))).toThrow("still running");
+  first.outcome = "failed";
+  expect(() => reserveCampaignAttempt(scenario, { ...attempt(2, scenario.executionId!), correction: null })).toThrow("recorded correction");
+  const second = attempt(2, scenario.executionId!);
+  reserveCampaignAttempt(scenario, second);
+  second.outcome = "passed";
+  expect(scenario.attempts[0]!.outcome).toBe("failed");
+  expect(() => reserveCampaignAttempt(scenario, attempt(2, scenario.executionId!))).toThrow("budget exhausted");
+  const fresh = freezeChatCampaign(root, base).scenarios.find(scenario => scenario.id === "tasks.create-backlog")!;
+  const pass = attempt(1, fresh.executionId!);
+  reserveCampaignAttempt(fresh, pass); pass.outcome = "passed";
+  expect(() => reserveCampaignAttempt(fresh, attempt(2, fresh.executionId!))).toThrow("cannot be rerun");
+});
+it("excludes unmerged core-dependent and unclean results from the merge numerator", () => {
+  const campaign = freezeChatCampaign(root, base);
+  const scenario = campaign.scenarios.find(scenario => scenario.id === "tasks.create-backlog")!;
+  scenario.qualifyingDefinitionDigest = "fixture"; scenario.qualifyingGraderDigest = "fixture";
+  const result = attempt(1, scenario.executionId!); reserveCampaignAttempt(scenario, result);
+  result.outcome = "passed";
+  expect(campaignSummary(campaign).finalPassed).toBe(0);
+  result.cleanup = "passed"; result.corePrDependencies = ["https://github.com/example/core/pull/1"];
+  expect(campaignSummary(campaign).finalPassed).toBe(0);
+  result.corePrDependencies = [];
+  expect(campaignSummary(campaign).finalPassed).toBe(1);
+  scenario.qualifyingDefinitionDigest = "new revision";
+  expect(campaignSummary(campaign).finalPassed).toBe(0);
+});
+
+it("rejects missing evidence and definition drift before accepting a retained outcome", () => {
+  const campaign = freezeChatCampaign(root, base);
+  const scenario = campaign.scenarios.find(scenario => scenario.id === "tasks.create-backlog")!;
+  scenario.qualifyingDefinitionDigest = "fixture"; scenario.qualifyingGraderDigest = "fixture";
+  const reserved = attempt(1, scenario.executionId!); reserveCampaignAttempt(scenario, reserved);
+  const completion = { outcome: "passed" as const, finishedAt: new Date().toISOString(), definitionDigest: "fixture", graderDigest: "fixture",
+    evidencePaths: [] as string[], cleanup: "passed" as const, usage: reserved.usage, classification: null, observedFailure: null, corePrDependencies: [] };
+  expect(() => completeCampaignAttempt(scenario, completion)).toThrow("requires evidence");
+  expect(() => completeCampaignAttempt(scenario, { ...completion, definitionDigest: "changed" })).toThrow("changed during attempt");
+  expect(() => completeCampaignAttempt(scenario, { ...completion, evidencePaths: ["tests/runner-e2e/results/../private.json"] })).toThrow("sanitized result paths");
+  completeCampaignAttempt(scenario, { ...completion, evidencePaths: ["tests/runner-e2e/results/fixture/result.json"] });
+  expect(campaignSummary(campaign).finalPassed).toBe(1);
+  scenario.qualifyingDefinitionDigest = "new revision";
+  expect(campaignSummary(campaign).finalPassed).toBe(0);
+});
+
+it("detects catalog tampering while permitting honest readiness and ownership updates", () => {
+  const campaign = freezeChatCampaign(root, base);
+  campaign.scenarios[0]!.owner = "another-worker";
+  campaign.scenarios[0]!.blocker = "external prerequisite missing";
+  expect(() => validateFrozenCampaign(campaign, root)).not.toThrow();
+  campaign.scenarios[0]!.expected_outcome = "A weaker outcome";
+  expect(() => validateFrozenCampaign(campaign, root)).toThrow("Frozen scenario content changed");
+});
+
+it("does not turn partial implementations or automatic retries into valid campaign attempts", () => {
+  const scenario = freezeChatCampaign(root, base).scenarios.find(scenario => scenario.id === "tasks.create-backlog")!;
+  scenario.readiness = "partial";
+  expect(() => reserveCampaignAttempt(scenario, attempt(1, scenario.executionId!))).toThrow("no executable implementation");
+  scenario.readiness = "implemented";
+  const invalid = { ...attempt(1, scenario.executionId!), automaticRetries: 1 } as unknown as CampaignAttempt;
+  expect(() => reserveCampaignAttempt(scenario, invalid)).toThrow("Invalid attempt reservation");
+  expect(scenario.attempts).toHaveLength(0);
+});

tests/runner-e2e/chat-campaign.ts

@@ -0,0 +1,136 @@
+import { createHash } from "node:crypto";
+import { readFileSync, readdirSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { parseDocument } from "yaml";
+
+export const CHAT_CAMPAIGN_MANIFEST = "tests/runner-e2e/chat-campaign-manifest.json";
+export type CampaignOutcome = "passed" | "failed" | "blocked";
+export interface CampaignAttempt {
+  ordinal: 1 | 2;
+  outcome: CampaignOutcome | "running";
+  executionId: string;
+  startedAt: string;
+  finishedAt: string | null;
+  sourceRevision: string;
+  workingTreeDigest: string;
+  definitionDigest: string;
+  suiteDefinitionDigest: string;
+  graderDigest: string;
+  model: string;
+  profile: string;
+  environment: string;
+  automaticRetries: 0;
+  correction: string | null;
+  classification: string | null;
+  observedFailure: string | null;
+  evidencePaths: string[];
+  usage: { costUsd: number | null; coverage: string; tokens: number | null };
+  cleanup: "pending" | "passed" | "failed" | "not_started";
+  corePrDependencies: string[];
+}
+export interface CampaignScenario {
+  id: string; feature: string; sourceFile: string; sourceSha256: string;
+  capabilities: string[]; context: string; query: string; expected_outcome: string;
+  scenarioSha256: string;
+  qualifyingDefinitionDigest: string | null;
+  qualifyingGraderDigest: string | null;
+  owner: string;
+  readiness: "unimplemented" | "partial" | "implemented" | "blocked";
+  executionId: string | null;
+  blocker: string | null;
+  attempts: CampaignAttempt[];
+}
+export interface ChatCampaign {
+  version: 1; campaignId: string; frozenAt: string; baselineRevision: string;
+  inventoryDigest: string; defaultsSha256: string;
+  denominator: number; areas: number; capabilities: number;
+  requiredPasses: number;
+  corePrs: string[]; evalPr: string | null;
+  scenarios: CampaignScenario[];
+}
+const digest = (value: string) => createHash("sha256").update(value).digest("hex");
+export function freezeChatCampaign(root: string, baselineRevision: string): ChatCampaign {
+  const directory = path.join(root, "tests/runner-e2e/chat-scenarios");
+  const scenarios: CampaignScenario[] = [];
+  const files = readdirSync(directory).filter(name => name.endsWith(".yaml") && name !== "defaults.yaml").sort();
+  for (const file of files) {
+    const text = readFileSync(path.join(directory, file), "utf8");
+    const document = parseDocument(text, { uniqueKeys: true });
+    if (document.errors.length) throw new Error(`Invalid catalog ${file}`);
+    const value = document.toJS();
+    for (const scenario of value.scenarios) {
+      const canonical = { id: scenario.id, capabilities: scenario.capabilities, context: scenario.context, query: scenario.query, expected_outcome: scenario.expected_outcome };
+      if (scenarios.some(entry => entry.id === scenario.id)) throw new Error(`Duplicate scenario ${scenario.id}`);
+      const legacy = ["tasks.create-backlog", "tasks.delegate-announcement"].includes(scenario.id);
+      scenarios.push({ ...canonical, feature: value.feature, sourceFile: `tests/runner-e2e/chat-scenarios/${file}`,
+        sourceSha256: digest(text), scenarioSha256: digest(JSON.stringify(canonical)),
+        qualifyingDefinitionDigest: null, qualifyingGraderDigest: null,
+        owner: legacy ? "eval_integrator" : "unassigned", readiness: legacy ? "implemented" : "unimplemented",
+        executionId: legacy ? `chat-features.runner-codex-subscription.local.${scenario.id.slice(6)}` : null,
+        blocker: null, attempts: [] });
+    }
+  }
+  scenarios.sort((a, b) => a.id.localeCompare(b.id));
+  const capabilities = new Set(scenarios.flatMap(scenario => scenario.capabilities)).size;
+  if (scenarios.length !== 244 || files.length !== 35 || capabilities !== 191) throw new Error("Frozen denominator changed; reconcile explicitly");
+  const defaultsSha256 = digest(readFileSync(path.join(directory, "defaults.yaml"), "utf8"));
+  return { version: 1, campaignId: "chat-catalog-2026-10-26", frozenAt: new Date().toISOString(), baselineRevision,
+    inventoryDigest: digest(JSON.stringify({ defaultsSha256, scenarios: scenarios.map(({ id, scenarioSha256, sourceSha256 }) => ({ id, scenarioSha256, sourceSha256 })) })),
+    defaultsSha256, denominator: scenarios.length, areas: files.length, capabilities, requiredPasses: Math.ceil(scenarios.length * 0.8), corePrs: [], evalPr: null, scenarios };
+}
+export function validateFrozenCampaign(campaign: ChatCampaign, root: string): void {
+  const fresh = freezeChatCampaign(root, campaign.baselineRevision);
+  if (campaign.version !== 1 || campaign.denominator !== 244 || campaign.areas !== 35 || campaign.capabilities !== 191
+    || campaign.requiredPasses !== 196 || campaign.scenarios.length !== 244 || campaign.defaultsSha256 !== fresh.defaultsSha256
+    || campaign.inventoryDigest !== fresh.inventoryDigest) throw new Error("Frozen campaign inventory metadata changed");
+  for (let index = 0; index < campaign.scenarios.length; index++) {
+    const actual = campaign.scenarios[index]!; const expected = fresh.scenarios[index]!;
+    for (const field of ["id", "feature", "sourceFile", "sourceSha256", "scenarioSha256", "capabilities", "context", "query", "expected_outcome"] as const) {
+      if (JSON.stringify(actual[field]) !== JSON.stringify(expected[field])) throw new Error(`Frozen scenario content changed: ${expected.id}`);
+    }
+    if (actual.attempts.length > 2 || actual.attempts.some((attempt, attemptIndex) => attempt.ordinal !== attemptIndex + 1 || attempt.automaticRetries !== 0)) throw new Error(`Invalid attempt history: ${actual.id}`);
+  }
+}
+export function reserveCampaignAttempt(scenario: CampaignScenario, attempt: CampaignAttempt): void {
+  if (scenario.readiness !== "implemented" || !scenario.executionId) throw new Error("Scenario has no executable implementation");
+  if (scenario.attempts.some(entry => entry.outcome === "running")) throw new Error("Previous attempt still running");
+  if (scenario.attempts.length >= 2 || attempt.ordinal !== scenario.attempts.length + 1) throw new Error("Attempt budget exhausted or incorrect ordinal");
+  if (scenario.attempts[0]?.outcome === "passed") throw new Error("Passing scenario cannot be rerun");
+  if (attempt.ordinal === 2 && !attempt.correction?.trim()) throw new Error("Fix cycle requires a recorded correction");
+  if (attempt.automaticRetries !== 0 || attempt.executionId !== scenario.executionId || attempt.outcome !== "running") throw new Error("Invalid attempt reservation");
+  if ([attempt.sourceRevision, attempt.workingTreeDigest, attempt.definitionDigest, attempt.suiteDefinitionDigest, attempt.graderDigest, attempt.model, attempt.profile, attempt.environment].some(value => !value.trim())) throw new Error("Missing attempt provenance");
+  scenario.attempts.push(attempt);
+}
+export function completeCampaignAttempt(scenario: CampaignScenario, completion: {
+  outcome: CampaignOutcome; finishedAt: string; definitionDigest: string; graderDigest: string;
+  evidencePaths: string[]; cleanup: CampaignAttempt["cleanup"]; usage: CampaignAttempt["usage"];
+  classification: string | null; observedFailure: string | null; corePrDependencies: string[];
+}): void {
+  const attempt = scenario.attempts.at(-1);
+  if (!attempt || attempt.outcome !== "running") throw new Error("No running reserved attempt");
+  if (completion.definitionDigest !== attempt.definitionDigest || completion.graderDigest !== attempt.graderDigest) throw new Error("Definition or grader changed during attempt");
+  if (completion.evidencePaths.some(value => !value.startsWith("tests/runner-e2e/results/") || value.split("/").includes(".."))) throw new Error("Evidence must use retained sanitized result paths");
+  if (completion.outcome === "passed" && (!completion.evidencePaths.length || completion.cleanup !== "passed")) throw new Error("Pass requires evidence and successful cleanup");
+  if (Date.parse(completion.finishedAt) < Date.parse(attempt.startedAt) || !Number.isFinite(Date.parse(completion.finishedAt))) throw new Error("Invalid completion timing");
+  Object.assign(attempt, completion);
+}
+export function campaignSummary(campaign: ChatCampaign) {
+  const initialPassed = campaign.scenarios.filter(scenario => { const first = scenario.attempts[0]; return first?.outcome === "passed" && first.cleanup === "passed" && first.corePrDependencies.length === 0; }).length;
+  const passed = campaign.scenarios.filter(scenario => { const final = scenario.attempts.at(-1); return final?.outcome === "passed" && final.corePrDependencies.length === 0 && final.cleanup === "passed" && final.definitionDigest === scenario.qualifyingDefinitionDigest && final.graderDigest === scenario.qualifyingGraderDigest; }).length;
+  const attempted = campaign.scenarios.filter(scenario => scenario.attempts.length).length;
+  const implemented = campaign.scenarios.filter(scenario => scenario.readiness === "implemented").length;
+  return { denominator: campaign.denominator, implemented, attempted, unrun: campaign.denominator - attempted,
+    initialPassed, finalPassed: passed, firstAttemptRate: initialPassed / campaign.denominator,
+    finalRate: passed / campaign.denominator, requiredPasses: campaign.requiredPasses,
+    mergeEligible: implemented === campaign.denominator && attempted === campaign.denominator && passed >= campaign.requiredPasses && !campaign.scenarios.some(scenario => scenario.attempts.some(attempt => attempt.outcome === "running")) };
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+  const [command, baseline] = process.argv.slice(2);
+  if (command === "freeze" && baseline) {
+    writeFileSync(CHAT_CAMPAIGN_MANIFEST, `${JSON.stringify(freezeChatCampaign(process.cwd(), baseline), null, 2)}\n`, { flag: "wx" });
+  } else if (command !== "summary") throw new Error("Usage: chat-campaign.ts freeze <baseline> | summary");
+  const campaign = JSON.parse(readFileSync(CHAT_CAMPAIGN_MANIFEST, "utf8")) as ChatCampaign;
+  validateFrozenCampaign(campaign, process.cwd());
+  process.stdout.write(`${JSON.stringify({ inventoryDigest: campaign.inventoryDigest, ...campaignSummary(campaign) }, null, 2)}\n`);
+}

tests/runner-e2e/chat-catalog-contract.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import { validateChatCatalogGrade } from "./chat-catalog-contract.js";
+import { chatCatalogScenarios, chatCatalogInventoryDigest } from "./chat-catalog-inventory.js";
+import { runnerMatrix } from "./catalog.js";
+import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
+import { chatCatalogCases } from "./chat-catalog-registry.js";
+
+describe("catalog oracle contract", () => {
+  it("accepts an independently checked result and rejects missing or contradictory evidence", () => {
+    const checks = [{ id: "saved-owner", passed: true, detail: "Persisted assignee matches the requested agent" }];
+    expect(validateChatCatalogGrade({ passed: true, checks }).passed).toBe(true);
+    expect(validateChatCatalogGrade({ passed: false, checks: [{ ...checks[0]!, passed: false }] }).passed).toBe(false);
+    for (const value of [{ passed: true, checks: [] }, { passed: true, checks: [{ ...checks[0]!, passed: false }] },
+      { passed: true, checks: [...checks, ...checks] }]) expect(() => validateChatCatalogGrade(value)).toThrow();
+  });
+  it("discovers each concrete implementation explicitly with its exact query and bounds", () => {
+    for (const implementation of chatCatalogCases) {
+      const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && cell.task.id === implementation.scenarioId);
+      expect(cells).toHaveLength(1);
+      const cell = cells[0]!;
+      expect(cell.task.buildPrompt("nonce")).toBe(chatCatalogScenarios.find(row => row.id === implementation.scenarioId)!.query);
+      expect(cell.task.automaticRetryPolicy).toBe("single_attempt");
+      expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id])).map(row => row.id)).toEqual([cell.id]);
+      expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.id === cell.id)).toBe(false);
+    }
+  });
+  it("keeps every frozen scenario in the denominator independently of executable readiness", () => {
+    expect(chatCatalogScenarios).toHaveLength(244);
+    expect(new Set(chatCatalogScenarios.map(row => row.feature)).size).toBe(35);
+    expect(new Set(chatCatalogScenarios.flatMap(row => row.capabilities)).size).toBe(191);
+    expect(chatCatalogInventoryDigest).toBe("9dc9cf6a5426cf1b56bc30a3571851f233f9dd5e93a1292eaee414d33b52df77");
+    for (const implementation of chatCatalogCases) expect(chatCatalogScenarios.some(row => row.id === implementation.scenarioId)).toBe(true);
+  });
+});

tests/runner-e2e/chat-catalog-contract.ts

@@ -0,0 +1,45 @@
+import type { ChatFlowInput, ChatIssue, ChatRun } from "./chat-flow.js";
+import type { ChatScenario } from "./chat-feature-cases.js";
+
+export interface ChatCatalogCheck { id: string; passed: boolean; detail: string }
+export interface ChatCatalogGrade { passed: boolean; checks: ChatCatalogCheck[] }
+export interface ChatCatalogContext {
+  input: ChatFlowInput;
+  scenario: ChatScenario;
+  issue(): ChatIssue | undefined;
+  allRuns(): Promise<ChatRun[]>;
+  refreshIssue(): Promise<void>;
+}
+/** One concrete scenario, with production API setup and a case-specific independent oracle. */
+export interface ChatCatalogCase {
+  scenarioId: string;
+  /** All case/grader helpers must be listed; the suite also fingerprints its complete source tree. */
+  sourceFiles: readonly string[];
+  oracleVersion: string;
+  /** Known missing catalog conditions keep this executable diagnostic case from acceptance. */
+  coverageGaps?: readonly string[];
+  expectedRunCount: number;
+  /** Persisted human comments; explicit followups must be declared before execution. */
+  expectedHumanMessageCount?: number;
+  minimumRunCount?: number;
+  turnTimeoutMs: number;
+  attemptTimeoutMs: number;
+  setup(context: ChatCatalogContext): Promise<Record<string, unknown>>;
+  /** Only explicit scenario user followups/approvals through visible production UI. */
+  act?(context: ChatCatalogContext, state: Record<string, unknown>): Promise<void>;
+  observe(context: ChatCatalogContext, state: Record<string, unknown>): Promise<unknown>;
+  grade(observation: unknown): ChatCatalogGrade;
+  /** Open the actual linked result through the browser and verify its identity. */
+  verifyResult(context: ChatCatalogContext, state: Record<string, unknown>, observation: unknown): Promise<void>;
+  cleanup?(context: ChatCatalogContext, state: Record<string, unknown>): Promise<void>;
+}
+
+export function validateChatCatalogGrade(grade: ChatCatalogGrade): ChatCatalogGrade {
+  if (!grade || !Array.isArray(grade.checks) || grade.checks.length === 0
+    || grade.checks.some(check => !check.id || typeof check.passed !== "boolean" || !check.detail)
+    || new Set(grade.checks.map(check => check.id)).size !== grade.checks.length
+    || grade.passed !== grade.checks.every(check => check.passed)) {
+    throw new Error("Invalid chat catalog oracle result: checks must be nonempty, unique and agree with the verdict");
+  }
+  return grade;
+}

tests/runner-e2e/chat-catalog-domains/internal-support.ts

@@ -0,0 +1,126 @@
+import { expect } from "@playwright/test";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { ChatRun } from "../chat-flow.js";
+
+export interface Entity {
+  id: string; companyId?: string; name?: string; title?: string; description?: string | null;
+  identifier?: string; status?: string; priority?: string; assigneeAgentId?: string | null;
+  projectId?: string | null; parentId?: string | null; goalId?: string | null; goalIds?: string[];
+  level?: string; ownerAgentId?: string | null; archivedAt?: string | null;
+  key?: string; body?: string; latestRevisionId?: string; latestRevisionNumber?: number;
+  createdByAgentId?: string | null; updatedByAgentId?: string | null; createdByRunId?: string | null;
+  createdAt?: string; updatedAt?: string; authorAgentId?: string | null; authorUserId?: string | null;
+  issueId?: string; action?: string; agentId?: string | null; actorId?: string; actorType?: string;
+  entityId?: string; details?: Record<string, unknown>;
+}
+export interface InternalObservation {
+  scenarioId: string; companyId: string; leadId: string; state: Record<string, unknown>;
+  tasks: Entity[]; projects: Entity[]; goals: Entity[]; comments: Entity[];
+  activity: Entity[]; documents: Entity[]; revisions: Entity[]; reply: string; links: string[]; humanQueries: string[]; runs: ChatRun[];
+}
+export function entity(state: Record<string, unknown>, key: string): Entity {
+  const value = state[key];
+  if (!value || typeof value !== "object" || typeof (value as Entity).id !== "string") throw new Error(`Missing fixture ${key}`);
+  return value as Entity;
+}
+export const companyPath = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`;
+export async function project(ctx: ChatCatalogContext, name = "Launch", extra: Record<string, unknown> = {}) {
+  return ctx.input.api.post<Entity>(`${companyPath(ctx)}/projects`, { name, ...extra });
+}
+export async function task(ctx: ChatCatalogContext, title: string, projectId: string, extra: Record<string, unknown> = {}) {
+  // Backlog fixtures never start provider work. These are prerequisite records, not requested outcomes.
+  return ctx.input.api.post<Entity>(`${companyPath(ctx)}/issues`, { title, projectId, status: "backlog", priority: "medium", ...extra });
+}
+export async function goal(ctx: ChatCatalogContext, title: string, extra: Record<string, unknown> = {}) {
+  return ctx.input.api.post<Entity>(`${companyPath(ctx)}/goals`, { title, level: "company", ownerAgentId: ctx.input.fixtures.agent.id, ...extra });
+}
+export async function putDocument(ctx: ChatCatalogContext, issueId: string, body: string, baseRevisionId: string | null = null) {
+  return ctx.input.api.putSensitive<Entity>(`/api/issues/${issueId}/documents/plan`, {
+    title: "Website implementation plan", format: "markdown", body, baseRevisionId,
+  });
+}
+export async function observeInternal(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<InternalObservation> {
+  await ctx.refreshIssue();
+  const { api, fixtures } = ctx.input;
+  const [tasks, projects, goals, runs] = await Promise.all([
+    api.get<Entity[]>(`${companyPath(ctx)}/issues?includeAgentCreatedTasks=true&limit=1000`),
+    api.get<Entity[]>(`${companyPath(ctx)}/projects?includeArchived=true`),
+    api.get<Entity[]>(`${companyPath(ctx)}/goals`), ctx.allRuns(),
+  ]);
+  const source = ctx.issue();
+  const chat = source ? await api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`) : [];
+  const replies = chat.filter(row => row.authorAgentId === fixtures.agent.id && runs.some(run => run.id === row.createdByRunId));
+  const links: string[] = [];
+  for (const reply of replies) {
+    links.push(...await ctx.input.page.locator(`[id=${JSON.stringify(`comment-${reply.id}`)}] a[href]`).evaluateAll(elements => elements.map(element => element.getAttribute("href")!)));
+  }
+  const target = state.task ? entity(state, "task") : undefined;
+  const comments = target ? await api.get<Entity[]>(`/api/issues/${target.id}/comments?order=asc`) : [];
+  const companyActivity = await api.get<Entity[]>(`${companyPath(ctx)}/activity?limit=1000`);
+  const taskActivity = target ? await api.get<Entity[]>(`/api/issues/${target.id}/activity`) : [];
+  const activity = [...new Map([...companyActivity, ...taskActivity].map(row => [row.id, row])).values()];
+  const documents: Entity[] = []; const revisions: Entity[] = [];
+  if (target) {
+    for (const summary of await api.get<Entity[]>(`/api/issues/${target.id}/documents`)) {
+      const endpoint = `/api/issues/${target.id}/documents/${encodeURIComponent(summary.key!)}`;
+      documents.push(await api.get<Entity>(endpoint));
+      revisions.push(...await api.get<Entity[]>(`${endpoint}/revisions`));
+    }
+  }
+  return { scenarioId: ctx.scenario.id, companyId: fixtures.company.id, leadId: fixtures.agent.id,
+    state, tasks: tasks.filter(row => row.id !== source?.id), projects, goals, runs, comments, activity, documents, revisions,
+    humanQueries: chat.filter(row => !row.authorAgentId).map(row => row.body ?? ""), reply: replies.map(row => row.body ?? "").join("\n"), links };
+}
+export function hasLink(o: InternalObservation, kind: "issues" | "projects" | "goals", row: Entity): boolean {
+  return o.links.some(href => {
+    try {
+      const url = new URL(href, "http://fixture.invalid");
+      if (url.origin !== "http://fixture.invalid") return false;
+      const parts = url.pathname.split("/").map(decodeURIComponent); const index = parts.indexOf(kind);
+      return index >= 0 && [row.id, row.identifier].includes(parts[index + 1]);
+    } catch { return false; }
+  });
+}
+export function observation(value: unknown): InternalObservation | undefined {
+  if (!value || typeof value !== "object") return;
+  const o = value as InternalObservation;
+  if (!o.state || ![o.tasks, o.projects, o.goals, o.comments, o.activity, o.documents, o.revisions, o.links, o.humanQueries, o.runs].every(Array.isArray)
+    || typeof o.reply !== "string" || typeof o.companyId !== "string" || typeof o.leadId !== "string") return;
+  return o;
+}
+export function checks() {
+  const rows: Array<{ id: string; passed: boolean; detail: string }> = [];
+  return { add(id: string, passed: boolean, detail: string) { rows.push({ id, passed, detail }); },
+    result() { return { passed: rows.length > 0 && rows.every(row => row.passed), checks: rows }; } };
+}
+export function sameTask(before: Entity, after: Entity | undefined, except: readonly string[] = []) {
+  return !!after && ["id", "companyId", "title", "description", "projectId", "status", "priority", "assigneeAgentId", "goalId"]
+    .filter(key => !except.includes(key)).every(key => before[key as keyof Entity] === after[key as keyof Entity]);
+}
+
+/** Follow the exact rendered chat links, check the real result view, and return to the source. */
+export async function verifyInternalResult(ctx: ChatCatalogContext, _state: Record<string, unknown>, value: unknown) {
+  const o = observation(value);
+  if (!o) throw new Error("Result navigation requires a complete observation");
+  const sourceUrl = ctx.input.page.url();
+  const candidates: Array<{ kind: "issues" | "projects" | "goals"; row: Entity }> = [
+    ...o.tasks.map(row => ({ kind: "issues" as const, row })),
+    ...o.projects.map(row => ({ kind: "projects" as const, row })),
+    ...o.goals.map(row => ({ kind: "goals" as const, row })),
+  ].filter(target => hasLink(o, target.kind, target.row));
+  if (!candidates.length) throw new Error("No independently matched result link was rendered");
+  for (const { kind, row } of candidates) {
+    const href = o.links.find(link => hasLink({ ...o, links: [link] }, kind, row))!;
+    const link = ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first();
+    await expect(link).toBeVisible();
+    await link.click();
+    await ctx.input.page.waitForURL(url => {
+      const parts = url.pathname.split("/").map(decodeURIComponent); const at = parts.indexOf(kind);
+      return at >= 0 && [row.id, row.identifier].includes(parts[at + 1]);
+    });
+    await expect(ctx.input.page.getByRole("heading", { name: row.title ?? row.name!, exact: true }).first()).toBeVisible();
+    await ctx.input.evidence(`chat-catalog-opened-${kind}-${row.id}.json`, { id: row.id, kind, url: ctx.input.page.url(), title: row.title ?? row.name });
+    await ctx.input.page.goto(sourceUrl, { waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
+  }
+}

tests/runner-e2e/chat-catalog-domains/internal.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, it } from "vitest";
+import { gradeInternal, internalChatCatalogCases } from "./internal.js";
+import type { Entity, InternalObservation } from "./internal-support.js";
+
+function base(scenarioId: string): InternalObservation {
+  const p = { id: "p", companyId: "company", name: "Launch" };
+  const t = { id: "task", companyId: "company", title: "Website task", description: "Website", status: "backlog", priority: "medium", projectId: p.id, assigneeAgentId: null };
+  return { scenarioId, companyId: "company", leadId: "ceo", state: { project: p, task: t }, tasks: [t], projects: [p], goals: [], comments: [], activity: [], documents: [], revisions: [], reply: "", humanQueries: [], links: ["/ACORN/issues/task", "/ACORN/projects/p"],
+    runs: [{ id: "run", companyId: "company", agentId: "ceo", status: "succeeded" }] };
+}
+function event(action: string, entityId: string, details?: Record<string, unknown>): Entity {
+  return { id: `${action}-${entityId}`, action, entityId, agentId: "ceo", details };
+}
+function valid(scenario: string): InternalObservation {
+  const o = base(scenario);
+  switch (scenario) {
+    case "projects.create-project": o.state = { firstIds: ["p"], retryQuery: "query" }; o.humanQueries = ["query", "query"]; o.tasks = []; o.activity = [event("project.created", "p")]; break;
+    case "projects.archive-project": {
+      o.projects[0] = { ...o.projects[0]!, archivedAt: "2026-10-26T12:00:00Z" }; o.state.originalBody = "Delivered"; o.state.originalActivityId = "original";
+      o.documents = [{ id: "doc", body: "Delivered" }]; o.activity = [{ id: "original", action: "issue.created", entityId: "task" }, event("project.updated", "p")]; break;
+    }
+    case "tasks.edit-properties": o.state.worker = { id: "morgan" }; o.tasks = [{ ...o.tasks[0]!, priority: "high", assigneeAgentId: "morgan" }]; o.activity = [event("issue.updated", "task")]; break;
+    case "tasks.comment-with-reference": {
+      const design = { id: "design", title: "Approved design", status: "done" }; o.state.design = design; o.tasks.push(design);
+      o.comments = [{ id: "comment", authorAgentId: "ceo", createdByRunId: "run", body: "Follow the [approved design](/ACORN/issues/design)." }]; break;
+    }
+    case "goals.create-goal-tree": {
+      o.state = { firstIds: ["growth", "team"], retryQuery: "query" }; o.humanQueries = ["query", "query"]; o.tasks = []; o.projects = [];
+      o.goals = [{ id: "growth", companyId: "company", title: "Reach a thousand signups", level: "company", parentId: null, ownerAgentId: "ceo" },
+        { id: "team", companyId: "company", title: "Launch the website", level: "team", parentId: "growth", ownerAgentId: "ceo" }];
+      o.activity = o.goals.map(row => event("goal.created", row.id)); o.links = ["/ACORN/goals/growth", "/ACORN/goals/team"]; break;
+    }
+    case "goals.align-existing-work": {
+      const growth = { id: "growth" }; o.state.growth = growth; o.goals = [growth]; o.projects[0] = { ...o.projects[0]!, goalIds: ["growth"] }; o.tasks[0] = { ...o.tasks[0]!, goalId: "growth" };
+      o.activity = [event("issue.updated", "task"), event("project.updated", "p")]; o.links.push("/ACORN/goals/growth"); break;
+    }
+    case "goals.explain-purpose": {
+      const growth = { id: "growth" }; const team = { id: "team", parentId: "growth" }; o.state.growth = growth; o.state.team = team; o.state.rationale = "quiet research teams";
+      o.goals = [growth, team]; o.tasks[0] = { ...o.tasks[0]!, goalId: "team" }; o.state.task = { ...o.tasks[0] }; o.projects[0] = { ...o.projects[0]!, goalIds: ["team"] };
+      o.reply = "The website supports quiet research teams."; o.links.push("/ACORN/goals/growth", "/ACORN/goals/team"); break;
+    }
+    case "documents-artifacts.write-plan": {
+      o.state.requirement = "Orchid 42";
+      const body = "Implement the Orchid 42 website, then verify accessible signup.";
+      o.documents = [{ id: "doc", key: "plan", body, latestRevisionId: "revision" }];
+      o.revisions = [{ id: "revision", body, createdByAgentId: "ceo", createdByRunId: "run" }]; o.activity = [event("issue.document_created", "task")]; break;
+    }
+    case "documents-artifacts.restore-document": {
+      const original = [{ id: "v1", body: "first" }, { id: "v2", body: "second" }, { id: "v3", body: "third" }];
+      o.state.originalRevisions = original; o.revisions = [...original, { id: "v4", body: "second" }]; o.documents = [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v4" }];
+      o.activity = [event("issue.document_restored", "task", { restoredFromRevisionId: "v2" })]; o.reply = "Restored version 2 as revision 4."; break;
+    }
+    case "activity.audit-change": {
+      const events = [{ id: "change1", action: "issue.updated", entityId: "task", actorId: "local-board", actorType: "user", details: { priority: "high" } },
+        { id: "change2", action: "issue.updated", entityId: "task", actorId: "local-board", actorType: "user", details: { priority: "low" } }];
+      o.state.priorityEvents = events; o.activity = events; o.reply = "Board changed the priority to high, then low today."; break;
+    }
+    default: throw new Error(`Missing valid calibration ${scenario}`);
+  }
+  return o;
+}
+const corrupt: Record<string, (o: InternalObservation) => void> = {
+  "projects.create-project": o => o.projects.push({ ...o.projects[0]!, id: "duplicate" }),
+  "projects.archive-project": o => { o.documents = []; },
+  "tasks.edit-properties": o => { o.tasks[0]!.assigneeAgentId = "someone-else"; },
+  "tasks.comment-with-reference": o => { o.comments[0]!.authorAgentId = null; o.comments[0]!.authorUserId = "human"; },
+  "goals.create-goal-tree": o => { o.goals[1]!.parentId = "other-company-goal"; },
+  "goals.align-existing-work": o => { o.tasks[0]!.goalId = null; },
+  "goals.explain-purpose": o => { o.reply = "The website exists to sell advertising, obviously."; },
+  "documents-artifacts.write-plan": o => { o.revisions[0]!.createdByRunId = "unrelated-run"; },
+  "documents-artifacts.restore-document": o => { o.documents[0]!.body = "third"; },
+  "activity.audit-change": o => { o.reply = "Morgan, the current assignee, probably changed it today."; },
+};
+describe("internal chat scenario independent oracles", () => {
+  for (const definition of internalChatCatalogCases) {
+    it(`${definition.scenarioId}: valid result and plausible wrong outcome`, () => {
+      const good = valid(definition.scenarioId);
+      expect(gradeInternal(good), JSON.stringify(gradeInternal(good).checks)).toMatchObject({ passed: true });
+      const wrong = structuredClone(good); corrupt[definition.scenarioId]!(wrong);
+      expect(gradeInternal(wrong).passed).toBe(false);
+      expect(gradeInternal(undefined).passed).toBe(false);
+      const noLinks = structuredClone(good); noLinks.links = [];
+      expect(gradeInternal(noLinks).passed).toBe(false);
+    });
+  }
+  it("never treats an unimplemented scenario as passing", () => expect(gradeInternal(base("inbox-search.archive-done")).passed).toBe(false));
+});

tests/runner-e2e/chat-catalog-domains/internal.ts

@@ -0,0 +1,227 @@
+import type { ChatCatalogCase } from "../chat-catalog-contract.js";
+import { observeInternal, project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, verifyInternalResult, type InternalObservation } from "./internal-support.js";
+
+export function gradeInternal(value: unknown) {
+  const c = checks(); const o = observation(value);
+  c.add("complete-independent-observation", !!o, "Missing API, document, audit or rendered-link evidence cannot pass.");
+  if (!o) return c.result();
+  const before = o.state.task ? entity(o.state, "task") : undefined;
+  const current = before && o.tasks.find(row => row.id === before.id);
+  const savedProject = o.state.project ? entity(o.state, "project") : undefined;
+  const projectNow = savedProject && o.projects.find(row => row.id === savedProject.id);
+  const authored = (action: string, id: string) => o.activity.some(row => row.action === action && row.entityId === id && row.agentId === o.leadId);
+  const linkedTask = () => !!current && hasLink(o, "issues", current);
+  switch (o.scenarioId) {
+    case "projects.create-project": {
+      const created = o.projects.filter(row => row.name === "Launch");
+      c.add("one-company-project", created.length === 1 && created[0]?.companyId === o.companyId, "Exactly one Launch is saved in the active company.");
+      c.add("agent-created-project", created.length === 1 && authored("project.created", created[0]!.id), "Creation is attributed to the acting CEO rather than fixture setup.");
+      c.add("usable-project-link", created.length === 1 && hasLink(o, "projects", created[0]!), "Chat renders the exact saved project link.");
+      c.add("retry-reuses-first-project", Array.isArray(o.state.firstIds) && o.state.firstIds.length === 1 && created[0]?.id === o.state.firstIds[0]
+        && o.humanQueries.filter(query => query === o.state.retryQuery).length === 2, "The exact user retry preserves the first saved project rather than silently creating and replacing it.");
+      c.add("no-invented-ready-work", o.tasks.length === 0 && !/\b(?:deployed|website is ready|repository is ready)\b/i.test(o.reply), "Creation does not invent execution, repository readiness or a deployment.");
+      break;
+    }
+    case "projects.archive-project": {
+      c.add("archived-not-deleted", !!projectNow && Number.isFinite(Date.parse(projectNow.archivedAt ?? "")), "Launch remains retrievable in the supported archived state.");
+      c.add("history-retained", !!before && sameTask(before, current) && o.documents.some(row => row.body === o.state.originalBody)
+        && o.activity.some(row => row.id === o.state.originalActivityId), "The original task, document bytes and creation audit survive archiving.");
+      c.add("attributed-project-update", !!projectNow && authored("project.updated", projectNow.id), "The saved lifecycle change belongs to the acting agent.");
+      c.add("historical-link", !!projectNow && hasLink(o, "projects", projectNow), "Chat supplies a reachable historical project.");
+      break;
+    }
+    case "tasks.edit-properties": {
+      const worker = entity(o.state, "worker");
+      c.add("exact-priority-assignee", !!current && current.priority === "high" && current.assigneeAgentId === worker.id, "The intended landing-page task has one eligible owner and High priority.");
+      c.add("no-unrequested-changes", !!before && sameTask(before, current, ["priority", "assigneeAgentId"]) && o.tasks.length === 1, "Task context and execution state remain intact; no workaround duplicate exists.");
+      c.add("saved-attribution", !!current && authored("issue.updated", current.id), "A real CEO-attributed mutation changed the requested task.");
+      c.add("task-link", linkedTask(), "The reply links to the edited task.");
+      break;
+    }
+    case "tasks.comment-with-reference": {
+      const design = entity(o.state, "design");
+      const added = o.comments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId && o.runs.some(run => run.id === row.createdByRunId));
+      c.add("one-attributed-note", added.length === 1 && /\bfollow\b/i.test(added[0]?.body ?? "")
+        && [design.id, design.identifier].filter(Boolean).some(id => (added[0]?.body ?? "").includes(id!))
+        && /\[[^\]]+\]\([^)]*\/issues\//.test(added[0]?.body ?? ""), "One CEO-authored note retains a usable reference to the actual approved design task.");
+      c.add("records-preserved", !!before && sameTask(before, current) && sameTask(design, o.tasks.find(row => row.id === design.id)) && o.tasks.length === 2,
+        "The note neither changes either task nor creates another task.");
+      c.add("task-link", linkedTask(), "The original conversation links back to the task containing the note.");
+      break;
+    }
+    case "goals.create-goal-tree": {
+      const companyGoals = o.goals.filter(row => row.level === "company" && /(?:1[,.]?000|thousand)\s+signups?/i.test(`${row.title} ${row.description ?? ""}`));
+      const teamGoals = o.goals.filter(row => row.level === "team" && /\b(?:launch|launching)\b.*\bwebsite\b|\bwebsite\b.*\blaunch\b/i.test(`${row.title} ${row.description ?? ""}`));
+      c.add("one-requested-tree", o.goals.length === 2 && companyGoals.length === 1 && teamGoals.length === 1
+        && teamGoals[0]?.parentId === companyGoals[0]?.id && companyGoals[0]?.parentId == null
+        && o.goals.every(row => row.companyId === o.companyId), "The two saved goals have exact levels, a valid parent and company scope.");
+      c.add("attributed-goal-creation", o.goals.length === 2 && o.goals.every(row => authored("goal.created", row.id)), "Both requested goals were created by the CEO through the production path.");
+      c.add("ownership-or-question", o.goals.every(row => row.ownerAgentId === o.leadId) || /\b(?:who|which)\b[^\n]*\b(?:own|owner|responsib)/i.test(o.reply), "Goal ownership is valid or its consequential missing input is explicit.");
+      c.add("hierarchy-link", o.goals.length === 2 && o.goals.every(row => hasLink(o, "goals", row)), "The reply links both saved levels for inspection.");
+      c.add("retry-reuses-first-tree", Array.isArray(o.state.firstIds) && o.state.firstIds.length === 2
+        && o.goals.every(row => (o.state.firstIds as string[]).includes(row.id))
+        && o.humanQueries.filter(query => query === o.state.retryQuery).length === 2, "An explicit unchanged request reuses the initial goal tree.");
+      c.add("no-invented-progress", !/\b\d+\s*%\s*(?:complete|done|progress)|\balready\s+\d+\s+signups?/i.test(o.reply), "New goals do not claim fabricated progress metrics.");
+      break;
+    }
+    case "goals.align-existing-work": {
+      const growth = entity(o.state, "growth");
+      c.add("saved-associations", !!current && current.goalId === growth.id && !!projectNow
+        && (projectNow.goalId === growth.id || projectNow.goalIds?.includes(growth.id) === true), "The intended Launch and website task both reference the saved growth goal.");
+      c.add("context-only", !!before && sameTask(before, current, ["goalId"]) && o.tasks.length === 1 && o.goals.length === 1,
+        "Alignment does not alter priority, ownership, execution state or duplicate records.");
+      c.add("attributed-alignment", !!current && authored("issue.updated", current.id) && !!projectNow && authored("project.updated", projectNow.id), "Both association writes are independently attributed.");
+      c.add("linked-work-and-goal", linkedTask() && !!projectNow && hasLink(o, "projects", projectNow) && hasLink(o, "goals", growth), "Chat gives inspectable task, project and strategic goal references.");
+      break;
+    }
+    case "goals.explain-purpose": {
+      const growth = entity(o.state, "growth"); const team = entity(o.state, "team");
+      c.add("actual-chain", current?.goalId === team.id && team.parentId === growth.id && !!projectNow
+        && (projectNow.goalId === team.id || projectNow.goalIds?.includes(team.id) === true), "The observed task/project/team/company references form the fixture's real purpose chain.");
+      c.add("grounded-rationale", o.reply.includes(String(o.state.rationale)) && /\bwebsite\b/i.test(o.reply)
+        && [growth, team].every(row => hasLink(o, "goals", row)) && linkedTask(), "The explanation uses the unusual saved business rationale and links the actual hierarchy.");
+      c.add("read-only-explanation", !!before && sameTask(before, current) && !o.activity.some(row => row.entityId !== o.state.sourceId && [before.id, growth.id, team.id, savedProject?.id].includes(row.entityId) && row.agentId === o.leadId && /(?:updated|created|deleted)$/.test(row.action ?? "")),
+        "An explanation does not rewrite the work's strategic context.");
+      break;
+    }
+    case "documents-artifacts.write-plan": {
+      const plans = o.documents.filter(row => row.key === "plan"); const plan = plans[0];
+      c.add("saved-implementation-plan", plans.length === 1 && !!plan && !!plan.latestRevisionId && /\b(?:implement|build|develop)\b/i.test(plan.body ?? "")
+        && /\b(?:test|verify|validation)\b/i.test(plan.body ?? "") && (plan.body ?? "").includes(String(o.state.requirement)), "A durable plan reflects the website's actual prerequisite brief and verification, rather than merely chat prose.");
+      c.add("versioned-author", !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.body === plan.body && row.createdByAgentId === o.leadId
+        && o.runs.some(run => run.id === row.createdByRunId)), "The current document revision is attributed to this real agent run.");
+      c.add("plan-audit", !!before && authored("issue.document_created", before.id), "The API audit records the CEO creating the plan after the request.");
+      c.add("task-context-and-link", !!before && sameTask(before, current) && linkedTask(), "Saving a plan preserves the task and makes the saved plan reachable through its task.");
+      break;
+    }
+    case "documents-artifacts.restore-document": {
+      const plan = o.documents.find(row => row.key === "plan"); const revisions = o.state.originalRevisions as Entity[];
+      const previous = revisions?.[1];
+      c.add("restored-exact-previous-bytes", !!previous && !!plan && plan.body === previous.body && plan.latestRevisionId !== previous.id
+        && plan.latestRevisionId !== revisions[2]?.id, "Restoration creates a new current revision with the exact second revision's contents.");
+      c.add("retained-history", Array.isArray(revisions) && revisions.length === 3 && o.revisions.length === 4
+        && revisions.every(original => o.revisions.some(row => row.id === original.id && row.body === original.body)), "All three historical revisions remain unmodified alongside the restoration.");
+      c.add("restore-path-attribution", !!before && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === before.id
+        && row.agentId === o.leadId && row.details?.restoredFromRevisionId === previous?.id), "An independently observed restore audit identifies the exact source revision.");
+      c.add("no-execution-rewind", !!before && sameTask(before, current) && linkedTask() && /\b(?:revision|version)\b/i.test(o.reply), "Chat identifies a document revision and leaves task execution unchanged.");
+      break;
+    }
+    case "activity.audit-change": {
+      const priorityEvents = (o.state.priorityEvents as Entity[] | undefined) ?? [];
+      c.add("real-two-change-history", priorityEvents.length === 2 && priorityEvents.every(event => o.activity.some(row => row.id === event.id)), "Both fixture priority changes remain visible on the exact task.");
+      c.add("attribution-and-values", priorityEvents.length === 2 && priorityEvents.every(event => !!event.actorId && (o.reply.includes(event.actorId) || event.actorType === "user" && /\b(?:board|human|operator)\b/i.test(o.reply))
+        && o.reply.toLowerCase().includes(String(event.details?.priority).toLowerCase())) && /\b(?:today|yesterday|\d{4}-\d{2}-\d{2}|\d{1,2}:\d{2})\b/.test(o.reply),
+        "The answer identifies the recorded editors, priority changes and event times rather than guessing the assignee.");
+      c.add("read-only-history", !!before && sameTask(before, current) && linkedTask() && !o.activity.some(row => row.entityId === before.id && row.agentId === o.leadId), "Reading task audit does not mutate the task or manufacture audit records.");
+      break;
+    }
+    default: c.add("implemented-scenario", false, "No independent oracle is implemented for this scenario.");
+  }
+  return c.result();
+}
+
+const ownedSources = ["chat-catalog-domains/internal.ts", "chat-catalog-domains/internal-support.ts"] as const;
+function define(scenarioId: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase {
+  const coverageGaps = scenarioId === "goals.align-existing-work" ? ["cross-company-goal-rejection"]
+    : scenarioId === "goals.explain-purpose" ? ["missing-association", "inaccessible-goal-nondisclosure"]
+    : scenarioId === "documents-artifacts.restore-document" ? ["concurrent-edit-conflict"]
+    : scenarioId === "tasks.edit-properties" ? ["execution-ownership-denied"] : [];
+  return { scenarioId, setup, coverageGaps, sourceFiles: [...ownedSources, `chat-scenarios/${scenarioId.split(".")[0]}.yaml`],
+    oracleVersion: "paperclip.chat-catalog.internal.v1", expectedRunCount: 1, verifyResult: verifyInternalResult, turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000, observe: observeInternal, grade: gradeInternal };
+}
+
+export const internalChatCatalogCases: readonly ChatCatalogCase[] = [
+  retryDefinition("projects.create-project"),
+  define("projects.archive-project", async ctx => {
+    const p = await project(ctx, "Launch", { status: "completed" });
+    const t = await task(ctx, "Website task", p.id, { status: "done" });
+    const body = `# Delivered website\nHistorical receipt ${ctx.input.nonce}.`;
+    await putDocument(ctx, t.id, body);
+    const activity = await ctx.input.api.get<Entity[]>(`/api/issues/${t.id}/activity`);
+    return { project: p, task: t, originalBody: body, originalActivityId: activity.find(row => row.action === "issue.created")?.id };
+  }),
+  define("tasks.edit-properties", async ctx => {
+    const p = await project(ctx); const worker = await fixtureWorker(ctx);
+    const t = await task(ctx, "Landing page", p.id, { description: "Build the Launch landing page; this work is saved for later." });
+    return { project: p, worker, task: t };
+  }),
+  define("tasks.comment-with-reference", async ctx => {
+    const p = await project(ctx); const design = await task(ctx, "Approved design", p.id, { status: "done", description: "Approved design: semantic HTML, clear contrast, one primary signup action." });
+    const t = await task(ctx, "Landing page", p.id, { description: "Build the launch website." });
+    return { project: p, design, task: t };
+  }),
+  retryDefinition("goals.create-goal-tree"),
+  define("goals.align-existing-work", async ctx => {
+    const growth = await goal(ctx, "Growth: reach 1,000 signups");
+    const p = await project(ctx); const t = await task(ctx, "Website task", p.id);
+    return { growth, project: p, task: t };
+  }),
+  define("goals.explain-purpose", async ctx => {
+    const rationale = `quiet research teams ${ctx.input.nonce}`;
+    const growth = await goal(ctx, "Growth: reach 1,000 signups", { description: `We need a thousand signups to support ${rationale}.` });
+    const team = await goal(ctx, "Launch the website", { level: "team", parentId: growth.id });
+    const p = await project(ctx, "Launch", { goalIds: [team.id] });
+    const t = await task(ctx, "Website task", p.id, { goalId: team.id });
+    return { growth, team, project: p, task: t, rationale };
+  }),
+  define("documents-artifacts.write-plan", async ctx => {
+    const p = await project(ctx); const requirement = `Orchid ${ctx.input.nonce}`;
+    const t = await task(ctx, "Website task", p.id, { assigneeAgentId: ctx.input.fixtures.agent.id,
+      description: `Build a single-page website for ${requirement}, with accessible signup and responsive layout. Verify keyboard navigation and mobile presentation.` });
+    return { project: p, task: t, requirement };
+  }),
+  define("documents-artifacts.restore-document", async ctx => {
+    const p = await project(ctx); const t = await task(ctx, "Website task", p.id, { description: "The website task's plan has three saved revisions. The previous version before the last edit is the desired content." });
+    const revisions: Entity[] = [];
+    for (const body of [`# Plan\nBuild a responsive website for ${ctx.input.nonce}.`, `# Plan\nBuild accessible signup for ${ctx.input.nonce}.\nVerify keyboard operation.`, `# Plan\nReplace signup with an unrelated pricing experiment for ${ctx.input.nonce}.`]) {
+      const saved = await putDocument(ctx, t.id, body, revisions.at(-1)?.id ?? null);
+      revisions.push({ id: saved.latestRevisionId!, body });
+    }
+    return { project: p, task: t, originalRevisions: revisions };
+  }),
+  define("activity.audit-change", async ctx => {
+    const p = await project(ctx); let t = await task(ctx, "Website task", p.id);
+    await ctx.input.api.patch(`/api/issues/${t.id}`, { priority: "high" });
+    t = await ctx.input.api.patch(`/api/issues/${t.id}`, { priority: "low" });
+    const activity = await ctx.input.api.get<Entity[]>(`/api/issues/${t.id}/activity`);
+    return { project: p, task: t, priorityEvents: activity.filter(row => row.action === "issue.updated" && row.details?.priority) };
+  }),
+];
+
+import type { Entity } from "./internal-support.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js";
+import { pollUntil } from "../api.js";
+async function fixtureWorker(ctx: ChatCatalogContext) {
+  const { execution, fixtures, workspacePath, nonce, api } = ctx.input;
+  const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id,
+    workspacePath, secretRefs: fixtures.secretRefs, executionId: nonce });
+  const payload = { ...base, name: "Morgan", role: "engineer", title: "Frontend engineer", reportsTo: fixtures.agent.id,
+    capabilities: "Builds accessible websites.", runtimeConfig: { ...(base.runtimeConfig as Record<string, unknown>), heartbeat: { enabled: false, maxConcurrentRuns: 1 } } };
+  const worker = await api.post<Entity>(`/api/companies/${fixtures.company.id}/agents`, execution.profile.credential === "CODEX_AUTH_JSON"
+    ? codexSubscriptionAgentConfig(payload, process.env, fixtures.company.id) : payload);
+  await pollUntil({ label: "Morgan runtime readiness", deadlineAt: Date.now() + 45_000,
+    load: () => api.get<{ lifecycleState: string; lifecycleError?: string }>(`/api/agents/${worker.id}`),
+    accept: row => row.lifecycleState === "ready", reject: row => row.lifecycleError ? "Morgan runtime setup failed" : undefined });
+  return worker;
+}
+
+function retryDefinition(scenarioId: string): ChatCatalogCase {
+  const definition = define(scenarioId, async ctx => ({ retryQuery: ctx.scenario.query }));
+  return { ...definition, expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000,
+    async act(ctx, state) {
+      const first = await pollUntil({ label: "Initial creation before explicit user retry", deadlineAt: Date.now() + 150_000,
+        load: () => observeInternal(ctx, state),
+        accept: o => {
+          const rows = scenarioId === "projects.create-project" ? o.projects.filter(row => row.name === "Launch") : o.goals;
+          return rows.length === (scenarioId === "projects.create-project" ? 1 : 2) && ctx.issue()?.conversationState === "waiting"
+            && o.runs.length === 1 && o.runs.every(row => row.status === "succeeded");
+        },
+      });
+      state.firstIds = (scenarioId === "projects.create-project" ? first.projects.filter(row => row.name === "Launch") : first.goals).map(row => row.id);
+      await ctx.input.evidence("chat-catalog-before-explicit-retry.json", { firstIds: state.firstIds, observation: first });
+      const { sendChatMessage } = await import("../chat-flow.js");
+      await sendChatMessage(ctx.input.page, ctx.scenario.query);
+    },
+  };
+}

tests/runner-e2e/chat-catalog-flow.ts

@@ -0,0 +1,129 @@
+import { expect } from "@playwright/test";
+import { pollUntil } from "./api.js";
+import { collectChatRunEvidence, sendChatMessage } from "./chat-flow.js";
+import { validateChatCatalogGrade, type ChatCatalogContext, type ChatCatalogGrade } from "./chat-catalog-contract.js";
+import { findChatCatalogCase } from "./chat-catalog-registry.js";
+import { chatCatalogInventoryDigest } from "./chat-catalog-inventory.js";
+
+/** The case owns product semantics; this flow owns the actual chat entry point and evidence lifecycle. */
+export async function runChatCatalogFlow(context: ChatCatalogContext) {
+  const { input, scenario } = context;
+  const implementation = findChatCatalogCase(scenario.id);
+  if (!implementation) throw new Error(`No executable chat scenario: ${scenario.id}`);
+  const deadlineAt = Date.now() + implementation.attemptTimeoutMs - 30_000;
+  let state: Record<string, unknown> | undefined;
+  let observation: unknown;
+  let grade: ChatCatalogGrade | undefined;
+  let failed = false;
+  async function conversation() {
+    await context.refreshIssue();
+    const issue = context.issue();
+    const runs = await context.allRuns();
+    const comments = issue ? await input.api.get<Array<{ id: string; body: string; authorAgentId?: string; createdByRunId?: string }>>(
+      `/api/issues/${issue.id}/comments?order=asc`,
+    ) : [];
+    const renderedLinks = [];
+    for (const comment of comments) {
+      const reply = input.page.locator(`[id=${JSON.stringify(`comment-${comment.id}`)}]`);
+      for (const href of await reply.locator("a[href]").evaluateAll(elements => elements.map(element => element.getAttribute("href"))))
+        renderedLinks.push({ commentId: comment.id, href });
+    }
+    const evidence = { issue, runs, comments, renderedLinks };
+    await input.evidence("chat-catalog-conversation.json", evidence);
+    return evidence;
+  }
+  async function refresh() {
+    const evidence = await conversation();
+    observation = await implementation!.observe(context, state!);
+    grade = validateChatCatalogGrade(implementation!.grade(observation));
+    const human = evidence.comments.filter(comment => !comment.authorAgentId);
+    const checks = [...grade.checks,
+      { id: "catalog-complete-conditions", passed: !implementation!.coverageGaps?.length,
+        detail: implementation!.coverageGaps?.length ? `Missing conditions: ${implementation!.coverageGaps.join(", ")}` : "No known catalog condition is omitted." },
+      { id: "catalog-original-human-query", passed: human[0]?.body.trim() === scenario.query.trim()
+        && human.length === (implementation!.expectedHumanMessageCount ?? 1),
+        detail: "The first persisted human message is the unchanged catalog request and all human turns were declared." },
+      { id: "catalog-bounded-native-runs", passed: evidence.runs.length >= (implementation!.minimumRunCount ?? implementation!.expectedRunCount)
+        && evidence.runs.length <= implementation!.expectedRunCount && evidence.runs.every(run => run.companyId === input.fixtures.company.id
+          && run.runtimeMode === "native" && run.status === "succeeded"),
+        detail: "All observed company provider runs satisfy the declared count, identity and successful native execution bounds." },
+    ];
+    grade = { passed: checks.every(check => check.passed), checks };
+    await input.evidence("chat-catalog-observation.json", { scenarioId: scenario.id, inventoryDigest: chatCatalogInventoryDigest,
+      oracleVersion: implementation!.oracleVersion, observation, grade });
+    return { grade, runs: evidence.runs, issue: evidence.issue };
+  }
+  try {
+    state = await implementation.setup(context);
+    const company = `/api/companies/${input.fixtures.company.id}`;
+    const agents = await input.api.get<Array<{ id: string }>>(`${company}/agents`);
+    for (const agent of agents) await pollUntil({ label: `catalog agent readiness ${agent.id}`,
+      deadlineAt: Math.min(deadlineAt, Date.now() + 45_000), intervalMs: 500,
+      load: () => input.api.get<{ lifecycleState: string; status: string; lifecycleError?: string }>(`/api/agents/${agent.id}`),
+      accept: value => value.lifecycleState === "ready" && ["idle", "active"].includes(value.status),
+      reject: value => value.lifecycleError ? "Agent setup failed; diagnostic details withheld" : undefined,
+    });
+    const scopes = [{ scopeType: "company", scopeId: input.fixtures.company.id },
+      ...agents.map(agent => ({ scopeType: "agent", scopeId: agent.id }))];
+    for (const scope of scopes) await input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500,
+      hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
+    const funding = await input.api.get<{ policies: Array<{ scopeType: string; scopeId: string; amount: number; hardStopEnabled: boolean; isActive: boolean; unpricedUsagePolicy: string }> }>(`${company}/budgets/overview`);
+    expect(scopes.every(scope => funding.policies.some(policy => policy.scopeType === scope.scopeType && policy.scopeId === scope.scopeId
+      && policy.amount === 500 && policy.hardStopEnabled && policy.isActive && policy.unpricedUsagePolicy === "allow")), "Public funding policy persisted").toBe(true);
+    await input.evidence("chat-catalog-funding.json", { scopes, amountCents: 500, hardStopEnabled: true,
+      unpricedUsagePolicy: "allow", setupProbeUsage: "unknown; production hello probes are outside heartbeat accounting" });
+    await input.evidence("chat-catalog-setup.json", { scenario, state, inventoryDigest: chatCatalogInventoryDigest,
+      oracleVersion: implementation.oracleVersion, expectedRunCount: implementation.expectedRunCount,
+      minimumRunCount: implementation.minimumRunCount, turnTimeoutMs: implementation.turnTimeoutMs,
+      attemptTimeoutMs: implementation.attemptTimeoutMs });
+    await sendChatMessage(input.page, scenario.query);
+    if (implementation.act) await implementation.act(context, state);
+    let settledAt: number | undefined;
+    let passedAt: number | undefined;
+    await pollUntil({ label: scenario.id, deadlineAt, intervalMs: 1_000, load: refresh,
+      accept: result => {
+        if (!result.grade.passed) { passedAt = undefined; return false; }
+        passedAt ??= Date.now();
+        return Date.now() - passedAt >= 5_000;
+      },
+      reject: result => {
+        if (result.runs.length > implementation.expectedRunCount) return "Scenario exceeded its declared run bound";
+        if (result.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status))) return "Observed provider run failed";
+        if (result.runs.length && result.runs.every(run => run.status === "succeeded") && result.issue?.conversationState === "waiting") settledAt ??= Date.now();
+        else settledAt = undefined;
+        return settledAt && Date.now() - settledAt >= 15_000
+          ? `Settled scenario failed: ${result.grade.checks.filter(check => !check.passed).map(check => check.id).join(", ")}` : undefined;
+      }, timeoutDetail: result => result?.grade.checks.filter(check => !check.passed).map(check => check.id).join(", "),
+    });
+    const originalIssueId = context.issue()?.id;
+    await input.page.reload({ waitUntil: "domcontentloaded" });
+    await expect(input.page.getByTestId("task-chat-composer-input")).toBeVisible();
+    const persisted = await refresh();
+    expect(context.issue()?.id, "Source chat survives browser reload").toBe(originalIssueId);
+    expect(persisted.grade.passed, "Independent oracle survives browser reload").toBe(true);
+    await implementation.verifyResult(context, state, observation);
+    for (const check of persisted.grade.checks) input.check?.(check.id, check.passed, check.detail);
+    await input.capture("chat-catalog-result", scenario.id, "chat-catalog-result.png");
+  } catch (error) {
+    failed = true;
+    throw error;
+  } finally {
+    try {
+      await conversation();
+      if (state) await refresh();
+      const runs = await context.allRuns();
+      await input.evidence("chat-catalog-runs.json", await Promise.all(runs.map(run => collectChatRunEvidence(input.api, run))));
+    } catch {
+      await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id });
+      if (!failed) throw new Error("Scenario evidence collection failed");
+    } finally {
+      try {
+        if (state) await implementation.cleanup?.(context, state);
+        await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "complete", instanceCleanup: "owned by existing launcher" });
+      } catch (error) {
+        await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "failed" });
+        if (!failed) throw error;
+      }
+    }
+  }
+}

tests/runner-e2e/chat-catalog-inventory.ts

@@ -0,0 +1,38 @@
+import { createHash } from "node:crypto";
+import { readFileSync, readdirSync } from "node:fs";
+import { parseDocument } from "yaml";
+import type { ChatScenario } from "./chat-feature-cases.js";
+
+export interface CatalogChatScenario extends ChatScenario { feature: string; sourceFile: string }
+const directory = new URL("./chat-scenarios/", import.meta.url);
+export const chatCatalogScenarios: readonly CatalogChatScenario[] = readdirSync(directory)
+  .filter(name => name.endsWith(".yaml") && name !== "defaults.yaml").sort().flatMap(name => {
+    const document = parseDocument(readFileSync(new URL(name, directory), "utf8"), { uniqueKeys: true });
+    if (document.errors.length) throw new Error(`Invalid chat catalog file ${name}`);
+    const catalog = document.toJS();
+    const feature = name.slice(0, -5);
+    if (catalog.schema_version !== 1 || catalog.feature !== feature || catalog.status !== "planned" || !Array.isArray(catalog.scenarios))
+      throw new Error(`Unsupported chat catalog file ${name}`);
+    return catalog.scenarios.map((scenario: ChatScenario) => {
+      if (!scenario.id.startsWith(`${feature}.`) || [scenario.context, scenario.query, scenario.expected_outcome].some(value => typeof value !== "string" || !value.trim())
+        || !Array.isArray(scenario.capabilities) || scenario.capabilities.length === 0)
+        throw new Error(`Invalid scenario in ${name}`);
+      return { ...scenario, feature, sourceFile: `tests/runner-e2e/chat-scenarios/${name}` };
+    });
+  }).sort((a, b) => a.id.localeCompare(b.id));
+if (chatCatalogScenarios.length !== 244 || new Set(chatCatalogScenarios.map(row => row.id)).size !== 244)
+  throw new Error("Frozen chat catalog must contain exactly 244 unique scenarios");
+/** Canonical semantic digest; whitespace changes alone do not alter scenario meaning. */
+export const chatCatalogSemanticDigest = createHash("sha256").update(JSON.stringify(chatCatalogScenarios.map(
+  ({ id, capabilities, context, query, expected_outcome }) => ({ id, capabilities, context, query, expected_outcome }),
+))).digest("hex");
+
+/** Campaign freeze also binds original file bytes and shared context defaults. */
+export const chatCatalogInventoryDigest = createHash("sha256").update(JSON.stringify({
+  defaultsSha256: createHash("sha256").update(readFileSync(new URL("defaults.yaml", directory))).digest("hex"),
+  scenarios: chatCatalogScenarios.map(({ id, capabilities, context, query, expected_outcome, sourceFile }) => ({
+    id,
+    scenarioSha256: createHash("sha256").update(JSON.stringify({ id, capabilities, context, query, expected_outcome })).digest("hex"),
+    sourceSha256: createHash("sha256").update(readFileSync(new URL(`../../${sourceFile}`, import.meta.url))).digest("hex"),
+  })),
+})).digest("hex");

tests/runner-e2e/chat-catalog-registry.ts

@@ -0,0 +1,18 @@
+import { internalChatCatalogCases } from "./chat-catalog-domains/internal.js";
+import type { ChatCatalogCase } from "./chat-catalog-contract.js";
+import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
+
+// Import domain arrays here only after they supply executable setup/action/oracle.
+export const chatCatalogCases: readonly ChatCatalogCase[] = [...internalChatCatalogCases];
+const known = new Set(chatCatalogScenarios.map(row => row.id));
+const registered = new Map<string, ChatCatalogCase>();
+for (const implementation of chatCatalogCases) {
+  if (!known.has(implementation.scenarioId) || registered.has(implementation.scenarioId))
+    throw new Error(`Unknown or duplicate chat implementation: ${implementation.scenarioId}`);
+  if (typeof implementation.verifyResult !== "function" || !implementation.oracleVersion || !implementation.sourceFiles.length
+    || implementation.expectedRunCount < 1 || implementation.turnTimeoutMs <= 0 || implementation.attemptTimeoutMs <= 0
+    || (implementation.minimumRunCount ?? implementation.expectedRunCount) > implementation.expectedRunCount)
+    throw new Error(`Incomplete bounds/provenance for ${implementation.scenarioId}`);
+  registered.set(implementation.scenarioId, implementation);
+}
+export function findChatCatalogCase(scenarioId: string): ChatCatalogCase | undefined { return registered.get(scenarioId); }

tests/runner-e2e/chat-feature-cases.ts

@@ -1,6 +1,8 @@
 import { createHash } from "node:crypto";
-import { readFileSync } from "node:fs";
+import { readFileSync, readdirSync } from "node:fs";
 import { parseDocument } from "yaml";
+import { chatCatalogCases } from "./chat-catalog-registry.js";
+import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
 import type { RunnerProfileFixture, RunnerTaskFixture } from "./types.js";
 
 export const CHAT_FEATURE_SUITE = "chat-features";
@@ -34,11 +36,22 @@ export function readChatFeatureScenarios(text = readFileSync(new URL("./chat-sce
   });
 }
 export const chatFeatureScenarios = readChatFeatureScenarios();
+function definitionSources(directory = new URL("./", import.meta.url)): string[] {
+  return readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
+    if (entry.isDirectory()) return ["node_modules", "results", "test-results", "playwright-report", ".results"].includes(entry.name)
+      ? [] : definitionSources(new URL(`${entry.name}/`, directory));
+    return /\.(ts|mjs|yaml|json)$/.test(entry.name) && !entry.name.endsWith(".test.ts") && !entry.name.startsWith("chat-campaign")
+      ? [new URL(entry.name, directory).href] : [];
+  });
+}
+// Includes every domain, helper, matcher and fixture source in this Product E2E family.
 export const chatFeatureDefinitionDigest = createHash("sha256").update([
-  "chat-feature-cases.ts", "chat-feature-flow.ts", "chat-feature-scoring.ts", "chat-flow.ts",
-  "completion-updates.ts", "redaction.ts", "launch.ts", "chat-scenarios/tasks.yaml", "chat-scenarios/defaults.yaml",
-  "../../server/src/services/agent-conversations.ts",
-].map(file => readFileSync(new URL(file, import.meta.url), "utf8")).join("\n")).digest("hex");
+  ...definitionSources(), new URL("../../server/src/services/agent-conversations.ts", import.meta.url).href,
+].map(file => {
+  const relative = new URL(file).pathname.replace(new URL("../../", import.meta.url).pathname, "");
+  const body = readFileSync(new URL(file), "utf8");
+  return JSON.stringify([relative, body]);
+}).join("\n")).digest("hex");
 
 export function chatFeatureProfile(profile: RunnerProfileFixture): RunnerProfileFixture {
   return { ...profile, buildAgent(input) {
@@ -66,3 +79,20 @@ export const chatFeatureTasks: readonly RunnerTaskFixture[] = CHAT_FEATURE_CASES
   buildVisibleMarker: nonce => `ACORN${nonce.replace(/[^a-zA-Z0-9]/g, "")}`,
   buildMatchers: () => [{ kind: "issue_status", expected: "in_review" }],
 }));
+
+
+/** Only executable domain cases become selectable cells. Missing cases remain in the frozen denominator. */
+export const chatCatalogTasks: readonly RunnerTaskFixture[] = chatCatalogCases.map(implementation => {
+  const scenario = chatCatalogScenarios.find(row => row.id === implementation.scenarioId)!;
+  return {
+    id: scenario.id, label: scenario.id, groups: ["chat"], flow: "agent_chat", workMode: "standard",
+    expectedRunCount: implementation.expectedRunCount, minimumExpectedRunCount: implementation.minimumRunCount,
+    automaticRetryPolicy: "single_attempt", turnTimeoutMs: implementation.turnTimeoutMs,
+    attemptTimeoutMs: { local: implementation.attemptTimeoutMs, daytona: implementation.attemptTimeoutMs },
+    expectedTerminalState: { issue: "in_review", run: "succeeded" },
+    buildTitle: nonce => `Chat catalog ${scenario.id} ${nonce}`,
+    buildPrompt: () => scenario.query,
+    buildVisibleMarker: nonce => `CATALOG${nonce.replace(/[^a-zA-Z0-9]/g, "")}`,
+    buildMatchers: () => [{ kind: "issue_status", expected: "in_review" }],
+  };
+});

tests/runner-e2e/chat-feature-flow.ts

@@ -1,3 +1,5 @@
+import { runChatCatalogFlow } from "./chat-catalog-flow.js";
+import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
 import { expect } from "@playwright/test";
 import { pollUntil } from "./api.js";
 import { collectChatRunEvidence, sendChatMessage, type ChatFlowInput, type ChatIssue, type ChatRun } from "./chat-flow.js";
@@ -11,6 +13,11 @@ export async function runChatFeatureFlow(context: {
 }) {
   const { input } = context;
   const { api, page, fixtures: f, execution, nonce } = input;
+  if (!["create-backlog", "delegate-announcement"].includes(execution.task.id)) {
+    const scenario = chatCatalogScenarios.find(row => row.id === execution.task.id);
+    if (!scenario) throw new Error(`Unknown catalog scenario: ${execution.task.id}`);
+    return runChatCatalogFlow({ ...context, scenario });
+  }
   const caseId = execution.task.id as ChatFeatureCase;
   const scenario = chatFeatureScenarios.find(row => row.id === `tasks.${caseId}`);
   if (!scenario) throw new Error(`Unregistered chat feature scenario: ${caseId}`);

tests/runner-e2e/chat-feature-scoring.test.ts

@@ -76,8 +76,8 @@ describe("chat feature saved-state oracle", () => {
 });
 
 describe("chat feature selection", () => {
-  it("registers only the two authored queries as explicit local subscription cells", () => {
-    const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features");
+  it("preserves the two original queries as explicit local subscription cells", () => {
+    const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && ["create-backlog", "delegate-announcement"].includes(cell.task.id));
     expect(cells).toHaveLength(2);
     for (const cell of cells) {
       expect(cell.environment.id).toBe("local");