test(evals): name refusals from sealing's dependencies and filesystem errors

The seal still reported 'unclassified': its refusal came from a module it calls
or from a filesystem error. Add closed codes for framework-report, redaction and
retention messages, and report other filesystem errors by their error code.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-07 13:05:19 -06:00
1 parent df57973e38
commit 4751eaf4a8
2 files changed
+22 -1

No files matched your search

@@ -123,7 +123,7 @@ it("retains each remaining fixed scanner reason without dynamic text", () => {
import { readFileSync } from "node:fs";
it("gives every fixed packaging, sealing and report-asset refusal a closed finalization code", () => {
const messages = ["manual-private-handoff.ts", "evidence.ts", "manual-private-report-assets.ts"].flatMap(file =>
const messages = ["manual-private-handoff.ts", "evidence.ts", "manual-private-report-assets.ts", "private-framework-report.ts", "redaction.ts", "manual-private-retention.ts"].flatMap(file =>
[...readFileSync(new URL(`./${file}`, import.meta.url), "utf8").matchAll(/(?:new Error|budgetFailure)\("([^"$`]+)"/g)].map(match => match[1]!));
expect(messages.length).toBeGreaterThan(20);
for (const message of messages) {
@@ -131,3 +131,8 @@ it("gives every fixed packaging, sealing and report-asset refusal a closed final
expect(diagnostic.finalization, message).not.toBe("unclassified");
}
});
it("names other filesystem refusals by their closed error code", () => {
const error = Object.assign(new Error("EEXIST: file already exists, open '/private/path'"), { code: "EEXIST" });
expect(privateAttemptDiagnostic({ stage: "seal-original", error }).finalization).toBe("fs-EEXIST");
expect(JSON.stringify(privateAttemptDiagnostic({ stage: "seal-original", error }))).not.toContain("/private/path");
});
@@ -50,6 +50,21 @@ function carrierReasons(input: unknown, source: "reporter" | "packaging") {
}));
}
const failures = new Map([
["Invalid private framework evidence", "invalid-private-framework-evidence"],
["Docker config custody requires the isolated home scan", "docker-config-custody-requires-the-isolated"],
["Private Codex identity policy requires the isolated home root", "private-codex-identity-policy-requires-the"],
["Private Codex identity policy requires an owner-only temporary root", "private-codex-identity-policy-requires-an"],
["Codex rollout record exceeds private scan bound", "codex-rollout-record-exceeds-private-scan"],
["Invalid exact manual evidence selection", "invalid-exact-manual-evidence-selection"],
["Private manual evidence exceeds the per-run cell limit", "private-manual-evidence-exceeds-the-perrun"],
["Invalid manual private evidence identity", "invalid-manual-private-evidence-identity"],
["Unsupported private manual evidence selection", "unsupported-private-manual-evidence-selection"],
["Invalid protected manual evidence approval", "invalid-protected-manual-evidence-approval"],
["Exact protected manual evidence approval missing or ambiguous", "exact-protected-manual-evidence-approval-missing"],
["Invalid manual evidence execution identity", "invalid-manual-evidence-execution-identity"],
["Invalid manual private storage identity", "invalid-manual-private-storage-identity"],
["Private archive sessions require one exact cell or the report writer", "private-archive-sessions-require-one-exact"],
["Private archive session policy exceeds the STS limit", "private-archive-session-policy-exceeds-the"],
["Invalid private evidence preflight custody", "invalid-private-evidence-preflight-custody"],
["Private evidence preflight does not match this exact attempt", "private-evidence-preflight-does-not-match"],
["Invalid private evidence staging ownership", "invalid-private-evidence-staging-ownership"],
@@ -155,6 +170,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) {
status: member(result.status, ["passed", "failed", "skipped"]),
failureClass: member(result.failureClass, ["candidate_failure", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]),
finalization: failures.get(message) ?? (record(input.error).code === "ENOENT" ? "missing-file"
: typeof record(input.error).code === "string" && /^E[A-Z]{2,15}$/.test(record(input.error).code) ? `fs-${record(input.error).code}`
: /^Secret leak in [\s\S]*: (?:exact secret value|secret-shaped value)$/.test(message) ? "text-secret-finding" : "unclassified"),
phase: phases.find(([label]) => [...errors, message].some(error => error.includes(label!)))?.[1] ?? "unclassified",
privacy: { persistedState: errors.some(error => error.startsWith("Secret leak in persisted ")),