mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
test(evals): name refusals from sealing's dependencies and filesystem errors
The seal still reported 'unclassified': its refusal came from a module it calls or from a filesystem error. Add closed codes for framework-report, redaction and retention messages, and report other filesystem errors by their error code. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df57973e38
commit
4751eaf4a8
2 files changed
+22
-1
No files matched your search
@@ -123,7 +123,7 @@ it("retains each remaining fixed scanner reason without dynamic text", () => {
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
it("gives every fixed packaging, sealing and report-asset refusal a closed finalization code", () => {
|
||||
const messages = ["manual-private-handoff.ts", "evidence.ts", "manual-private-report-assets.ts"].flatMap(file =>
|
||||
const messages = ["manual-private-handoff.ts", "evidence.ts", "manual-private-report-assets.ts", "private-framework-report.ts", "redaction.ts", "manual-private-retention.ts"].flatMap(file =>
|
||||
[...readFileSync(new URL(`./${file}`, import.meta.url), "utf8").matchAll(/(?:new Error|budgetFailure)\("([^"$`]+)"/g)].map(match => match[1]!));
|
||||
expect(messages.length).toBeGreaterThan(20);
|
||||
for (const message of messages) {
|
||||
@@ -131,3 +131,8 @@ it("gives every fixed packaging, sealing and report-asset refusal a closed final
|
||||
expect(diagnostic.finalization, message).not.toBe("unclassified");
|
||||
}
|
||||
});
|
||||
it("names other filesystem refusals by their closed error code", () => {
|
||||
const error = Object.assign(new Error("EEXIST: file already exists, open '/private/path'"), { code: "EEXIST" });
|
||||
expect(privateAttemptDiagnostic({ stage: "seal-original", error }).finalization).toBe("fs-EEXIST");
|
||||
expect(JSON.stringify(privateAttemptDiagnostic({ stage: "seal-original", error }))).not.toContain("/private/path");
|
||||
});
|
||||
@@ -50,6 +50,21 @@ function carrierReasons(input: unknown, source: "reporter" | "packaging") {
|
||||
}));
|
||||
}
|
||||
const failures = new Map([
|
||||
["Invalid private framework evidence", "invalid-private-framework-evidence"],
|
||||
["Docker config custody requires the isolated home scan", "docker-config-custody-requires-the-isolated"],
|
||||
["Private Codex identity policy requires the isolated home root", "private-codex-identity-policy-requires-the"],
|
||||
["Private Codex identity policy requires an owner-only temporary root", "private-codex-identity-policy-requires-an"],
|
||||
["Codex rollout record exceeds private scan bound", "codex-rollout-record-exceeds-private-scan"],
|
||||
["Invalid exact manual evidence selection", "invalid-exact-manual-evidence-selection"],
|
||||
["Private manual evidence exceeds the per-run cell limit", "private-manual-evidence-exceeds-the-perrun"],
|
||||
["Invalid manual private evidence identity", "invalid-manual-private-evidence-identity"],
|
||||
["Unsupported private manual evidence selection", "unsupported-private-manual-evidence-selection"],
|
||||
["Invalid protected manual evidence approval", "invalid-protected-manual-evidence-approval"],
|
||||
["Exact protected manual evidence approval missing or ambiguous", "exact-protected-manual-evidence-approval-missing"],
|
||||
["Invalid manual evidence execution identity", "invalid-manual-evidence-execution-identity"],
|
||||
["Invalid manual private storage identity", "invalid-manual-private-storage-identity"],
|
||||
["Private archive sessions require one exact cell or the report writer", "private-archive-sessions-require-one-exact"],
|
||||
["Private archive session policy exceeds the STS limit", "private-archive-session-policy-exceeds-the"],
|
||||
["Invalid private evidence preflight custody", "invalid-private-evidence-preflight-custody"],
|
||||
["Private evidence preflight does not match this exact attempt", "private-evidence-preflight-does-not-match"],
|
||||
["Invalid private evidence staging ownership", "invalid-private-evidence-staging-ownership"],
|
||||
@@ -155,6 +170,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) {
|
||||
status: member(result.status, ["passed", "failed", "skipped"]),
|
||||
failureClass: member(result.failureClass, ["candidate_failure", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]),
|
||||
finalization: failures.get(message) ?? (record(input.error).code === "ENOENT" ? "missing-file"
|
||||
: typeof record(input.error).code === "string" && /^E[A-Z]{2,15}$/.test(record(input.error).code) ? `fs-${record(input.error).code}`
|
||||
: /^Secret leak in [\s\S]*: (?:exact secret value|secret-shaped value)$/.test(message) ? "text-secret-finding" : "unclassified"),
|
||||
phase: phases.find(([label]) => [...errors, message].some(error => error.includes(label!)))?.[1] ?? "unclassified",
|
||||
privacy: { persistedState: errors.some(error => error.startsWith("Secret leak in persisted ")),
|
||||
|
||||
Reference in new issue
Block a user