fix(evals): name runtime-built evidence refusals by their fixed template

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/private-refusal-code.test.ts

@@ -11,7 +11,7 @@ it("names only fixed literal refusals from the trusted evidence modules", () =>
 
 it("never returns the text of a dynamic or system message", () => {
   const leaked = "/home/runner/_work/private/secret-token-abc";
-  expect(privateRefusalCode(new Error(`Secret leak in ${leaked}`))).toBe("dynamic");
+  expect(privateRefusalCode(new Error(`Unexpected state at ${leaked}`))).toBe("dynamic");
   expect(privateRefusalCode(Object.assign(new Error(`ENOENT: no such file, open '${leaked}'`), { code: "ENOENT" }))).toBe("fs-ENOENT");
   expect(privateRefusalCode(new TypeError(`bad ${leaked}`))).toBe("error-type-TypeError");
   expect(privateRefusalCode(leaked)).toBe("non-error-thrown");
@@ -39,3 +39,20 @@ it("never throws while naming a refusal", () => {
   const broken = new Proxy(new Set<string>(), { get() { throw new Error("read failed /private/path"); } });
   expect(privateRefusalCode(new Error("x"), broken)).toBe("unnamed");
 });
+
+it("names runtime-built refusal templates without their variable parts", () => {
+  const secret = "/home/runner/private/token-xyz";
+  expect(privateRefusalCode(new Error(`Secret leak in ${secret}: exact secret value`))).toBe("template-secret-leak");
+  expect(privateRefusalCode(new Error("Private cell archive exceeds the publication boundary (300 files, 70000000 bytes)"))).toBe("template-publication-boundary-exceeded");
+  expect(privateRefusalCode(new Error(`${secret} contains invalid or duplicate file metadata`))).toBe("template-invalid-file-metadata");
+  expect(privateRefusalCode(new Error(`Secret leak in ${secret}: x`))).not.toContain("token");
+});
+it("has a named template for every runtime-built refusal in the trusted modules", () => {
+  for (const file of PRIVATE_REFUSAL_MODULES) {
+    const source = readFileSync(new URL(`./${file}`, import.meta.url), "utf8");
+    for (const match of source.matchAll(/new Error\(`([^`]+)`\)/g)) {
+      const sample = match[1]!.replace(/\$\{[^}]+\}/g, "X").replace(/\(X files, X bytes\)/, "(1 files, 2 bytes)");
+      expect(privateRefusalCode(new Error(sample)), `${file}: ${match[1]}`).not.toBe("dynamic");
+    }
+  }
+});

tests/runner-e2e/private-refusal-code.ts

@@ -7,6 +7,14 @@ export const PRIVATE_REFUSAL_MODULES = [
   "manual-dispatch-authority.ts", "testops-chat-admission.ts",
 ] as const;
 
+/** Fixed templates of runtime-built refusals in the trusted modules (variable parts are not reported). */
+const TEMPLATE_REFUSALS: ReadonlyArray<readonly [RegExp, string]> = [
+  [/^Secret leak in /, "template-secret-leak"],
+  [/ exceeds the publication boundary \(\d+ files, \d+ bytes\)$/, "template-publication-boundary-exceeded"],
+  [/ contains invalid or duplicate file metadata$/, "template-invalid-file-metadata"],
+  [/^Bounded history cannot prove expiration with /, "template-history-expiration-unprovable"],
+];
+
 let known: ReadonlySet<string> | undefined;
 /** Literal `new Error("...")` messages in the trusted modules; dynamic messages are never included. */
 export function fixedRefusalMessages(): ReadonlySet<string> {
@@ -29,6 +37,8 @@ export function privateRefusalCode(error: unknown, fixed?: ReadonlySet<string>):
 function refusalCode(error: unknown, fixed: ReadonlySet<string>): string {
   if (!(error instanceof Error)) return "non-error-thrown";
   if (fixed.has(error.message)) return error.message.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 96);
+  // Runtime-built messages from the trusted modules: name the fixed template, never its variable parts.
+  for (const [pattern, name] of TEMPLATE_REFUSALS) if (pattern.test(error.message)) return name;
   const code = (error as NodeJS.ErrnoException).code;
   if (typeof code === "string" && /^(?:E[A-Z]{2,15}|ERR_[A-Z0-9_]{2,48})$/.test(code)) return `fs-${code}`;
   if (error.name !== "Error" && /^[A-Z][A-Za-z]{2,40}$/.test(error.name)) return `error-type-${error.name}`;