mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): name runtime-built evidence refusals by their fixed template (#19920)
## Thinking Path - #19908 made a refused private evidence transfer print a closed refusal code instead of one generic line. - Its first real use printed `refusal: dynamic`: the trusted modules build some refusals at runtime (secret-leak findings, publication-boundary limits), and those were deliberately not named because their text can carry paths or counts. - The fixed part of each template is safe and is exactly what an operator needs to know which check refused. ## Linked Issues or Issue Description **What happened** After #19908, a refused `runs-adapters.inspect-failed-run` attempt (workflow run 42422571919) printed `Restricted manual evidence operation failed (refusal: dynamic)`, so the reason was still unknown. **Expected behavior** A refused transfer still keeps message text private, but names which check refused it with a closed code, so the attempt can be diagnosed. **Steps to reproduce** 1. Dispatch `runner-full-stack-e2e.yml` manually for a chat cell whose private evidence fails transfer validation. 2. Open the "Retain complete approved manual originals" step log: it shows the single fixed failure line with no reason. ## What Changed - `private-refusal-code.ts`: runtime-built refusals from the trusted modules map to fixed template codes (`template-secret-leak`, `template-publication-boundary-exceeded`, `template-invalid-file-metadata`, `template-history-expiration-unprovable`). Variable parts are never printed. - Tests: each template is named without its variable text, and a source scan asserts every runtime-built `new Error` in the trusted modules maps to a named code (not `dynamic`). ## Verification - `pnpm exec vitest run --config tests/runner-e2e/vitest.config.ts tests/runner-e2e/private-refusal-code.test.ts tests/runner-e2e/manual-private-transfer.test.ts`: 20 passed. - `npx tsc --noEmit -p tests/runner-e2e`: clean. ## Risks - Template codes reveal which kind of check refused (for example that a secret-shaped value was found), not where or what. That is the intended operator signal; the finding itself stays private. ## Model Used Anthropic Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code, with tool use (shell, file editing). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2fb4837cca
commit
1de794cb80
2 files changed
+28
-1
No files matched your search
@@ -11,7 +11,7 @@ it("names only fixed literal refusals from the trusted evidence modules", () =>
|
||||
|
||||
it("never returns the text of a dynamic or system message", () => {
|
||||
const leaked = "/home/runner/_work/private/secret-token-abc";
|
||||
expect(privateRefusalCode(new Error(`Secret leak in ${leaked}`))).toBe("dynamic");
|
||||
expect(privateRefusalCode(new Error(`Unexpected state at ${leaked}`))).toBe("dynamic");
|
||||
expect(privateRefusalCode(Object.assign(new Error(`ENOENT: no such file, open '${leaked}'`), { code: "ENOENT" }))).toBe("fs-ENOENT");
|
||||
expect(privateRefusalCode(new TypeError(`bad ${leaked}`))).toBe("error-type-TypeError");
|
||||
expect(privateRefusalCode(leaked)).toBe("non-error-thrown");
|
||||
@@ -39,3 +39,20 @@ it("never throws while naming a refusal", () => {
|
||||
const broken = new Proxy(new Set<string>(), { get() { throw new Error("read failed /private/path"); } });
|
||||
expect(privateRefusalCode(new Error("x"), broken)).toBe("unnamed");
|
||||
});
|
||||
|
||||
it("names runtime-built refusal templates without their variable parts", () => {
|
||||
const secret = "/home/runner/private/token-xyz";
|
||||
expect(privateRefusalCode(new Error(`Secret leak in ${secret}: exact secret value`))).toBe("template-secret-leak");
|
||||
expect(privateRefusalCode(new Error("Private cell archive exceeds the publication boundary (300 files, 70000000 bytes)"))).toBe("template-publication-boundary-exceeded");
|
||||
expect(privateRefusalCode(new Error(`${secret} contains invalid or duplicate file metadata`))).toBe("template-invalid-file-metadata");
|
||||
expect(privateRefusalCode(new Error(`Secret leak in ${secret}: x`))).not.toContain("token");
|
||||
});
|
||||
it("has a named template for every runtime-built refusal in the trusted modules", () => {
|
||||
for (const file of PRIVATE_REFUSAL_MODULES) {
|
||||
const source = readFileSync(new URL(`./${file}`, import.meta.url), "utf8");
|
||||
for (const match of source.matchAll(/new Error\(`([^`]+)`\)/g)) {
|
||||
const sample = match[1]!.replace(/\$\{[^}]+\}/g, "X").replace(/\(X files, X bytes\)/, "(1 files, 2 bytes)");
|
||||
expect(privateRefusalCode(new Error(sample)), `${file}: ${match[1]}`).not.toBe("dynamic");
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -7,6 +7,14 @@ export const PRIVATE_REFUSAL_MODULES = [
|
||||
"manual-dispatch-authority.ts", "testops-chat-admission.ts",
|
||||
] as const;
|
||||
|
||||
/** Fixed templates of runtime-built refusals in the trusted modules (variable parts are not reported). */
|
||||
const TEMPLATE_REFUSALS: ReadonlyArray<readonly [RegExp, string]> = [
|
||||
[/^Secret leak in /, "template-secret-leak"],
|
||||
[/ exceeds the publication boundary \(\d+ files, \d+ bytes\)$/, "template-publication-boundary-exceeded"],
|
||||
[/ contains invalid or duplicate file metadata$/, "template-invalid-file-metadata"],
|
||||
[/^Bounded history cannot prove expiration with /, "template-history-expiration-unprovable"],
|
||||
];
|
||||
|
||||
let known: ReadonlySet<string> | undefined;
|
||||
/** Literal `new Error("...")` messages in the trusted modules; dynamic messages are never included. */
|
||||
export function fixedRefusalMessages(): ReadonlySet<string> {
|
||||
@@ -29,6 +37,8 @@ export function privateRefusalCode(error: unknown, fixed?: ReadonlySet<string>):
|
||||
function refusalCode(error: unknown, fixed: ReadonlySet<string>): string {
|
||||
if (!(error instanceof Error)) return "non-error-thrown";
|
||||
if (fixed.has(error.message)) return error.message.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 96);
|
||||
// Runtime-built messages from the trusted modules: name the fixed template, never its variable parts.
|
||||
for (const [pattern, name] of TEMPLATE_REFUSALS) if (pattern.test(error.message)) return name;
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
if (typeof code === "string" && /^(?:E[A-Z]{2,15}|ERR_[A-Z0-9_]{2,48})$/.test(code)) return `fs-${code}`;
|
||||
if (error.name !== "Error" && /^[A-Z][A-Za-z]{2,40}$/.test(error.name)) return `error-type-${error.name}`;
|
||||
|
||||
Reference in new issue
Block a user