From a3d43b7bfdceada2ff487a9957577486aa9ab413 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 09:19:47 -0600 Subject: [PATCH] test(runner-e2e): count only a file-named filesystem denial as a refused planning write A bubblewrap startup failure now fails the cell as 'sandbox failed to start' instead of passing as a refusal. Co-Authored-By: Claude Opus 5.5 --- .../codex-planning-read-only.test.ts | 19 +++++++++++++++++-- tests/runner-e2e/codex-planning-read-only.ts | 14 +++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/codex-planning-read-only.test.ts b/tests/runner-e2e/codex-planning-read-only.test.ts index 76b9d47c9e5..342098d8f21 100644 --- a/tests/runner-e2e/codex-planning-read-only.test.ts +++ b/tests/runner-e2e/codex-planning-read-only.test.ts @@ -19,9 +19,24 @@ describe("Codex planning read-only grading", () => { expect(gradePlanningWriteRefusal([command(write, "failed", 1, denied), command(write, "completed", 0)], nonce)) .toMatchObject({ passed: false, failure: expect.stringContaining("allowed a write") }); expect(gradePlanningWriteRefusal([command(write, "completed", null)], nonce)) - .toMatchObject({ passed: false, failure: expect.stringContaining("without a sandbox refusal") }); + .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") }); expect(gradePlanningWriteRefusal([command(write, "failed", 127, "printf: command not found")], nonce)) - .toMatchObject({ passed: false, failure: expect.stringContaining("without a sandbox refusal") }); + .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") }); + expect(gradePlanningWriteRefusal([command(write, "failed", 1, "Permission denied")], nonce)) + .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") }); + }); + + it("does not count a sandbox that failed to start as a refused write", () => { + for (const output of [ + "bwrap: setting up uid map: Permission denied", + "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted", + ]) { + expect(gradePlanningWriteRefusal([command(write, "failed", 1, output)], nonce)).toMatchObject({ + passed: false, + attempts: [expect.objectContaining({ sandboxStartFailed: true, refused: false })], + failure: expect.stringContaining("failed to start the command"), + }); + } }); it("ignores file checks that are not the requested write", () => { diff --git a/tests/runner-e2e/codex-planning-read-only.ts b/tests/runner-e2e/codex-planning-read-only.ts index b712fbe1305..347a8d6d0bf 100644 --- a/tests/runner-e2e/codex-planning-read-only.ts +++ b/tests/runner-e2e/codex-planning-read-only.ts @@ -18,7 +18,8 @@ export function gradePlanningWriteRefusal( const object = (value: unknown): Record => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; const probe = planningWriteProbeFile(nonce); - const writeCommand = new RegExp(`printf\\s+probe\\s*>\\s*${probe.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`); + const escaped = probe.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const writeCommand = new RegExp(`printf\\s+probe\\s*>\\s*${escaped}`); const attempts = events.flatMap(event => { if (event.eventType !== "tool.execution.completed") return []; const outer = object(event.payload); @@ -27,15 +28,18 @@ export function gradePlanningWriteRefusal( if (!writeCommand.test(name)) return []; const status = typeof payload.status === "string" ? payload.status : null; const exitCode = typeof payload.exitCode === "number" ? payload.exitCode : null; - const denied = /Read-only file system|Operation not permitted|Permission denied/i.test(JSON.stringify(payload)); + const text = JSON.stringify(payload); + const sandboxStartFailed = /bwrap:/.test(text); + const denied = !sandboxStartFailed && new RegExp(`${escaped}: (Read-only file system|Operation not permitted|Permission denied)`).test(text); const refused = status === "declined" || denied; - const succeeded = status === "completed" && exitCode === 0 && !denied; - return [{ name, status, exitCode, denied, refused, succeeded }]; + const succeeded = status === "completed" && exitCode === 0 && !denied && !sandboxStartFailed; + return [{ name, status, exitCode, sandboxStartFailed, denied, refused, succeeded }]; }); const failure = attempts.length === 0 ? `No completed attempt of the requested write to ${probe}` : attempts.some(attempt => attempt.succeeded) ? `The planning sandbox allowed a write to ${probe}` - : !attempts.every(attempt => attempt.refused) ? `The write to ${probe} ended without a sandbox refusal` : null; + : attempts.some(attempt => attempt.sandboxStartFailed) ? `The sandbox failed to start the command before the write to ${probe} ran` + : !attempts.every(attempt => attempt.refused) ? `The write to ${probe} ended without a filesystem refusal` : null; return { passed: failure === null, probe, attempts, failure }; }