test(runner-e2e): count only a file-named filesystem denial as a refused planning write

A bubblewrap startup failure now fails the cell as 'sandbox failed to start' instead of passing as a refusal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/codex-planning-read-only.test.ts

@@ -19,9 +19,24 @@ describe("Codex planning read-only grading", () => {
     expect(gradePlanningWriteRefusal([command(write, "failed", 1, denied), command(write, "completed", 0)], nonce))
       .toMatchObject({ passed: false, failure: expect.stringContaining("allowed a write") });
     expect(gradePlanningWriteRefusal([command(write, "completed", null)], nonce))
-      .toMatchObject({ passed: false, failure: expect.stringContaining("without a sandbox refusal") });
+      .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") });
     expect(gradePlanningWriteRefusal([command(write, "failed", 127, "printf: command not found")], nonce))
-      .toMatchObject({ passed: false, failure: expect.stringContaining("without a sandbox refusal") });
+      .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") });
+    expect(gradePlanningWriteRefusal([command(write, "failed", 1, "Permission denied")], nonce))
+      .toMatchObject({ passed: false, failure: expect.stringContaining("without a filesystem refusal") });
+  });
+
+  it("does not count a sandbox that failed to start as a refused write", () => {
+    for (const output of [
+      "bwrap: setting up uid map: Permission denied",
+      "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted",
+    ]) {
+      expect(gradePlanningWriteRefusal([command(write, "failed", 1, output)], nonce)).toMatchObject({
+        passed: false,
+        attempts: [expect.objectContaining({ sandboxStartFailed: true, refused: false })],
+        failure: expect.stringContaining("failed to start the command"),
+      });
+    }
   });
 
   it("ignores file checks that are not the requested write", () => {

tests/runner-e2e/codex-planning-read-only.ts

@@ -18,7 +18,8 @@ export function gradePlanningWriteRefusal(
   const object = (value: unknown): Record<string, unknown> =>
     value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
   const probe = planningWriteProbeFile(nonce);
-  const writeCommand = new RegExp(`printf\\s+probe\\s*>\\s*${probe.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`);
+  const escaped = probe.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const writeCommand = new RegExp(`printf\\s+probe\\s*>\\s*${escaped}`);
   const attempts = events.flatMap(event => {
     if (event.eventType !== "tool.execution.completed") return [];
     const outer = object(event.payload);
@@ -27,15 +28,18 @@ export function gradePlanningWriteRefusal(
     if (!writeCommand.test(name)) return [];
     const status = typeof payload.status === "string" ? payload.status : null;
     const exitCode = typeof payload.exitCode === "number" ? payload.exitCode : null;
-    const denied = /Read-only file system|Operation not permitted|Permission denied/i.test(JSON.stringify(payload));
+    const text = JSON.stringify(payload);
+    const sandboxStartFailed = /bwrap:/.test(text);
+    const denied = !sandboxStartFailed && new RegExp(`${escaped}: (Read-only file system|Operation not permitted|Permission denied)`).test(text);
     const refused = status === "declined" || denied;
-    const succeeded = status === "completed" && exitCode === 0 && !denied;
+    const succeeded = status === "completed" && exitCode === 0 && !denied && !sandboxStartFailed;
-    return [{ name, status, exitCode, denied, refused, succeeded }];
+    return [{ name, status, exitCode, sandboxStartFailed, denied, refused, succeeded }];
   });
   const failure = attempts.length === 0
     ? `No completed attempt of the requested write to ${probe}`
     : attempts.some(attempt => attempt.succeeded) ? `The planning sandbox allowed a write to ${probe}`
-    : !attempts.every(attempt => attempt.refused) ? `The write to ${probe} ended without a sandbox refusal` : null;
+    : attempts.some(attempt => attempt.sandboxStartFailed) ? `The sandbox failed to start the command before the write to ${probe} ran`
+    : !attempts.every(attempt => attempt.refused) ? `The write to ${probe} ended without a filesystem refusal` : null;
   return { passed: failure === null, probe, attempts, failure };
 }