test(runner-e2e): hide the attempt's private temp dirs and Playwright's compiled graders

Playwright cached the compiled harness and graders, with source maps, in
/tmp/playwright-transform-cache-<uid>, and the attempt root under /tmp held
artifacts-private, control and selected-private-inputs. The sandbox skipped
everything under /tmp, because hiding a child there would stop agents
creating files in /tmp.

A sandboxed attempt now creates its root in /var/tmp. The launcher points
PWTEST_CACHE_DIR at a hidden child of that root and removes the default
cache. The hidden list covers the four private children, including ones
created after runnerd starts, and the setup probe also checks that the
default Playwright cache is not listable.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/SECURITY.md

@@ -332,8 +332,18 @@ its own read and grep tools, but not arbitrary shell commands.
 
 `agent-filesystem-sandbox.ts` closes that gap for local native cells on Linux.
 The launcher selects the hidden paths: `tests/`, the trusted
-`.manual-private-trusted/` copy, the private evidence directory and
-`RUNNER_TEMP`. The E2E server entry starts each `paperclip-runnerd` through
+`.manual-private-trusted/` copy, the private evidence directory, `RUNNER_TEMP`,
+and the attempt root's `artifacts-private/`, `control/`,
+`selected-private-inputs/` and `playwright-transform-cache/` children.
+Playwright compiles the harness and graders (with source maps) into
+`PWTEST_CACHE_DIR`, which the launcher points at that hidden child. The
+launcher also removes Playwright's default cache in the shared temporary
+directory.
+
+Landlock hides a path by leaving its ancestors ungranted, and an ungranted
+directory accepts no new entries. Agents must keep creating files in `/tmp`, so
+a sandboxed attempt creates its root in `/var/tmp` instead of `/tmp`. The
+agent's workspace, home and Paperclip home stay readable inside that root. The E2E server entry starts each `paperclip-runnerd` through
 `agent-filesystem-sandbox.py`, which applies a Landlock domain and then execs
 runnerd with its original PID, argv, cwd and environment. Runnerd, the provider,
 its tools and every shell command inherit the domain. The domain also sets
@@ -341,7 +351,7 @@ its tools and every shell command inherit the domain. The domain also sets
 unrestricted because it loads its E2E entry from the harness.
 
 Before the product starts an agent, a probe runs a sandboxed shell. The probe
-must be denied the harness and every hidden path, and it must read the product
+must be denied the harness, every hidden path and Playwright's default cache, and it must read the product
 package and write a temporary file. The server log records the probe result and
 each wrapped runnerd launch. In CI a missing or ineffective sandbox stops the
 attempt. Landlock does not hide that a path exists (`stat` still works); it
@@ -351,7 +361,8 @@ Limits: Codex cells keep Codex's own bubblewrap sandbox, which cannot mount
 inside Landlock. Legacy adapters, installed-release launches and macOS run
 without this sandbox. A process that can reach the Docker daemon can still
 mount the checkout into a container, and `ps` shows the launcher's command
-line. Daytona cells run the agent in a remote sandbox that holds only the
+line. Case fixtures that create their own private directories directly in the
+shared temporary directory stay readable. Daytona cells run the agent in a remote sandbox that holds only the
 staged runnerd and provider pack, not the checkout.
 
 Fixture names that an agent can read (company, agent, environment, secret,

tests/runner-e2e/agent-filesystem-sandbox.py

@@ -188,11 +188,12 @@ def main():
     required, hidden_inputs, unset, assignments, command = parse(sys.argv[1:])
     hidden = []
     for path in hidden_inputs:
-        if os.path.lexists(path):
-            real = os.path.realpath(path)
-            if real == "/":
-                fail("refusing to hide the filesystem root", 2)
-            hidden.append(real)
+        # Keep paths that do not exist yet: their ancestors stay ungranted, so
+        # a later mkdir there is unreachable too.
+        real = os.path.realpath(path)
+        if real == "/":
+            fail("refusing to hide the filesystem root", 2)
+        hidden.append(real)
     for name in unset:
         os.environ.pop(name, None)
     for name, value in assignments:

tests/runner-e2e/agent-filesystem-sandbox.test.ts

@@ -1,11 +1,15 @@
 import { execFileSync } from "node:child_process";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync, realpathSync, rmSync } from "node:fs";
 import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
 import {
+  HARNESS_PRIVATE_TEMPORARY_CHILDREN,
   agentFilesystemSandboxConfig,
+  agentTemporaryParent,
+  defaultPlaywrightCacheDirectory,
+  playwrightCacheDirectory,
   isRunnerdLaunch,
   parseAgentFilesystemSandboxConfig,
   probeAgentFilesystemSandbox,
@@ -48,6 +52,26 @@ describe("agent filesystem sandbox selection", () => {
     });
   });
 
+  it("hides the attempt's private children and Playwright cache when its root is outside the shared temporary directory", () => {
+    const root = "/var/tmp/paperclip-runner-e2e-abc";
+    const config = agentFilesystemSandboxConfig({ ...base, temporaryRoot: root, sharedTemporary: "/tmp" });
+    expect(config?.hidden).toEqual(expect.arrayContaining([
+      `${root}/artifacts-private`,
+      `${root}/control`,
+      `${root}/selected-private-inputs`,
+      playwrightCacheDirectory(root),
+    ]));
+    expect(config?.hidden).not.toContain(root);
+    expect(HARNESS_PRIVATE_TEMPORARY_CHILDREN).toContain("playwright-transform-cache");
+    expect(defaultPlaywrightCacheDirectory("/tmp")).toMatch(/^\/tmp\/playwright-transform-cache-/);
+  });
+
+  it("keeps a sandboxed attempt's root out of the shared temporary directory", () => {
+    const parent = agentTemporaryParent(true);
+    if (process.platform === "linux") expect(parent).toBe(realpathSync("/var/tmp"));
+    expect(agentTemporaryParent(false)).toBe(realpathSync(os.tmpdir()));
+  });
+
   it("never hides the agent's own state, the checkout root or the shared temporary directory", () => {
     const config = agentFilesystemSandboxConfig({
       ...base,
@@ -125,15 +149,24 @@ describe.skipIf(!landlock)("Landlock agent sandbox on this host", () => {
     await writeFile(path.join(repository, "tests", "runner-e2e", "agent-filesystem-sandbox.ts"), "grader");
     await writeFile(path.join(repository, "tests", "runner-e2e", "chat-scenarios", "access.yaml"), "expected_outcome: secret");
     await writeFile(path.join(repository, "packages", "paperclip-runner", "package.json"), "{}");
-    const config = { hidden: [path.join(repository, "tests")], required: true };
+    const attempt = path.join(root, "attempt");
+    await mkdir(path.join(attempt, "artifacts-private"), { recursive: true });
+    await mkdir(path.join(attempt, "workspace"));
+    await writeFile(path.join(attempt, "artifacts-private", "server.log"), "secret");
+    const config = { hidden: [path.join(repository, "tests"), path.join(attempt, "artifacts-private"), path.join(attempt, "control")], required: true };
     // The fixture lives under the shared temporary directory, so point mktemp at a granted sibling.
-    const probe = probeAgentFilesystemSandbox(config, repository, pythonExecutable(), { ...process.env, TMPDIR: path.join(root, "scratch") });
+    const probe = probeAgentFilesystemSandbox(config, repository, pythonExecutable(), { ...process.env, TMPDIR: path.join(root, "scratch") }, path.join(root, "no-playwright-cache"));
     expect(probe.failures).toEqual([]);
-    expect(probe.result).toMatchObject({ harnessFile: "denied", harnessDirectory: "denied", productFile: "readable", temporaryWrite: "allowed" });
+    expect(probe.result).toMatchObject({ playwrightDefaultCache: "denied", harnessFile: "denied", harnessDirectory: "denied", productFile: "readable", temporaryWrite: "allowed" });
     // The exact read from the reported run: grep over the checkout's scenarios.
     const invocation = sandboxedInvocation(config, pythonExecutable(), "/bin/sh", ["-c", `grep -rn expected_outcome ${repository}/tests; echo "exit=$?"`], process.env);
     const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8" });
     expect(output).not.toContain("secret");
     expect(output).toMatch(/exit=2/);
+    const attemptCheck = sandboxedInvocation(config, pythonExecutable(), "/bin/sh", ["-c", `cat ${attempt}/artifacts-private/server.log; ls ${attempt}/workspace >/dev/null && echo workspace=ok; mkdir ${attempt}/control 2>/dev/null || echo control=blocked`], process.env);
+    const attemptOutput = execFileSync(attemptCheck.command, attemptCheck.args, { encoding: "utf8" });
+    expect(attemptOutput).not.toContain("secret");
+    expect(attemptOutput).toContain("workspace=ok");
+    expect(attemptOutput).toContain("control=blocked");
   });
 });

tests/runner-e2e/agent-filesystem-sandbox.ts

@@ -1,5 +1,5 @@
 import childProcess, { execFileSync, type SpawnOptions } from "node:child_process";
-import { existsSync } from "node:fs";
+import { accessSync, constants, existsSync, realpathSync } from "node:fs";
 import { syncBuiltinESMExports } from "node:module";
 import os from "node:os";
 import path from "node:path";
@@ -28,6 +28,55 @@ function within(candidate: string, root: string) {
   return candidate === root || candidate.startsWith(`${root}${path.sep}`);
 }
 
+/** Harness-private directories inside the attempt's temporary root. */
+export const HARNESS_PRIVATE_TEMPORARY_CHILDREN = [
+  "artifacts-private",
+  "control",
+  "selected-private-inputs",
+  // Playwright's compiled copy of the harness and graders, with source maps.
+  "playwright-transform-cache",
+] as const;
+
+/** Playwright otherwise caches compiled harness code in the shared temporary directory. */
+export function playwrightCacheDirectory(temporaryRoot: string) {
+  return path.join(temporaryRoot, "playwright-transform-cache");
+}
+
+/** Playwright's default cache location for this user, which the launcher removes. */
+export function defaultPlaywrightCacheDirectory(temporaryDirectory = os.tmpdir()) {
+  return path.join(temporaryDirectory, `playwright-transform-cache-${process.geteuid?.() ?? ""}`);
+}
+
+export function agentFilesystemSandboxApplies(input: {
+  platform: NodeJS.Platform;
+  executions: readonly MatrixExecution[];
+  installedRelease: boolean;
+}) {
+  if (input.platform !== "linux" || input.installedRelease) return false;
+  if (!input.executions.some((execution) => execution.environment.id === "local" && execution.profile.generation === "native")) return false;
+  // Codex's own bubblewrap sandbox mounts, which Landlock forbids. Those cells
+  // keep their existing isolation until Codex permission profiles hide the eval.
+  return !input.executions.some(requiresCodexCiSandbox);
+}
+
+/**
+ * Landlock can only hide a path by leaving its ancestors ungranted, and an
+ * ungranted directory accepts no new entries. Hiding the attempt's private
+ * children of a root under /tmp would stop agents creating files in /tmp, so a
+ * sandboxed attempt keeps its root in /var/tmp instead.
+ */
+export function agentTemporaryParent(applies: boolean, sharedTemporary = os.tmpdir()) {
+  if (applies) {
+    try {
+      accessSync("/var/tmp", constants.W_OK | constants.X_OK);
+      return realpathSync("/var/tmp");
+    } catch {
+      // Fall back to the shared directory; its private children stay visible.
+    }
+  }
+  return realpathSync(sharedTemporary);
+}
+
 /** Selects the paths hidden from local native agents, or null when not applicable. */
 export function agentFilesystemSandboxConfig(input: {
   platform: NodeJS.Platform;
@@ -38,16 +87,14 @@ export function agentFilesystemSandboxConfig(input: {
   privateDir: string;
   evidenceRoot?: string;
   installedRelease: boolean;
+  sharedTemporary?: string;
 }): AgentFilesystemSandboxConfig | null {
-  if (input.platform !== "linux" || input.installedRelease) return null;
-  if (!input.executions.some((execution) => execution.environment.id === "local" && execution.profile.generation === "native")) return null;
-  // Codex's own bubblewrap sandbox mounts, which Landlock forbids. Those cells
-  // keep their existing isolation until Codex permission profiles hide the eval.
-  if (input.executions.some(requiresCodexCiSandbox)) return null;
-  const sharedTemporary = path.resolve(input.env.TMPDIR?.trim() || os.tmpdir());
+  if (!agentFilesystemSandboxApplies(input)) return null;
+  const sharedTemporary = path.resolve(input.sharedTemporary ?? (input.env.TMPDIR?.trim() || os.tmpdir()));
   const candidates = [
     path.join(input.repositoryRoot, "tests"),
     path.join(input.repositoryRoot, ".manual-private-trusted"),
+    ...HARNESS_PRIVATE_TEMPORARY_CHILDREN.map((child) => path.join(input.temporaryRoot, child)),
     input.privateDir,
     input.evidenceRoot,
     input.env.RUNNER_TEMP,
@@ -117,6 +164,7 @@ export function probeAgentFilesystemSandbox(
   repositoryRoot: string,
   python = pythonExecutable(),
   env: NodeJS.ProcessEnv = process.env,
+  playwrightCache = defaultPlaywrightCacheDirectory(),
 ) {
   const harness = path.join(repositoryRoot, "tests", "runner-e2e");
   const script = [
@@ -126,7 +174,8 @@ export function probeAgentFilesystemSandbox(
     'if test -e "$2"; then add harnessDirectoryStat visible; else add harnessDirectoryStat absent; fi',
     'if cat "$3" >/dev/null 2>&1; then add productFile readable; else add productFile denied; fi',
     'probe=$(mktemp) && rm -f "$probe" && add temporaryWrite allowed || add temporaryWrite denied',
-    'shift 3; for hidden in "$@"; do if ls "$hidden" >/dev/null 2>&1; then add "hidden:$hidden" listable; else add "hidden:$hidden" denied; fi; done',
+    'if ls "$4" >/dev/null 2>&1; then add playwrightDefaultCache listable; else add playwrightDefaultCache denied; fi',
+    'shift 4; for hidden in "$@"; do if ls "$hidden" >/dev/null 2>&1; then add "hidden:$hidden" listable; else add "hidden:$hidden" denied; fi; done',
     'printf "%s\\n" "$out"',
   ].join("\n");
   const invocation = sandboxedInvocation(config, python, "/bin/sh", [
@@ -134,6 +183,7 @@ export function probeAgentFilesystemSandbox(
     path.join(harness, "agent-filesystem-sandbox.ts"),
     harness,
     path.join(repositoryRoot, "packages", "paperclip-runner", "package.json"),
+    playwrightCache,
     ...config.hidden,
   ], env);
   const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8", cwd: "/", env, timeout: 30_000 });
@@ -146,6 +196,7 @@ export function probeAgentFilesystemSandbox(
     result.harnessDirectory !== "denied" && "harness directory is listable",
     result.productFile !== "readable" && "product package is unreadable",
     result.temporaryWrite !== "allowed" && "temporary directory is not writable",
+    result.playwrightDefaultCache !== "denied" && "Playwright's default transform cache is listable",
     ...config.hidden.map((hidden) => result[`hidden:${hidden}`] === "listable" && `${hidden} is listable`),
   ].filter((failure): failure is string => Boolean(failure));
   return { result, failures };

tests/runner-e2e/launch.ts

@@ -31,7 +31,10 @@ import { createRunnerE2ETemporaryRoot } from "./server-config.js";
 import { createExternalControllerDatabase } from "./external-controller-database.js";
 import { createHash, randomBytes } from "node:crypto";
 import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js";
-import { AGENT_FILESYSTEM_SANDBOX_ENV, agentFilesystemSandboxConfig } from "./agent-filesystem-sandbox.js";
+import {
+  AGENT_FILESYSTEM_SANDBOX_ENV, agentFilesystemSandboxApplies, agentFilesystemSandboxConfig, agentTemporaryParent,
+  defaultPlaywrightCacheDirectory, playwrightCacheDirectory,
+} from "./agent-filesystem-sandbox.js";
 import { spawn } from "node:child_process";
 import { createWriteStream } from "node:fs";
 import { createRequire } from "node:module";
@@ -509,7 +512,8 @@ async function runAttempt(input: {
   const installedPlugin = await verifyInstalledDaytonaPlugin(process.env, executions);
   const startedAtMs = Date.now();
   const sharedMemoryBaseline = snapshotDarwinSharedMemory();
-  const temporaryParent = await realpath(os.tmpdir());
+  const sandboxedAgents = agentFilesystemSandboxApplies({ platform: process.platform, executions, installedRelease: Boolean(installedCli) });
+  const temporaryParent = agentTemporaryParent(sandboxedAgents);
   const temporaryRoot = await createRunnerE2ETemporaryRoot(temporaryParent);
   const selectedPrivateInputsDirectory = path.join(temporaryRoot, "selected-private-inputs");
   const privateDir = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "capture") : path.join(temporaryRoot, "artifacts-private");
@@ -594,6 +598,8 @@ async function runAttempt(input: {
       platform: process.platform, env: process.env, executions, repositoryRoot, temporaryRoot, privateDir,
       evidenceRoot: manualPrivateHandoff?.root, installedRelease: Boolean(installedCli),
     });
+    // Playwright's default cache holds compiled graders in the shared /tmp.
+    if (sandboxedAgents) await rm(defaultPlaywrightCacheDirectory(), { recursive: true, force: true });
     const childEnv: NodeJS.ProcessEnv = {
       ...buildRunnerE2EProcessEnvironment(sourceEnvironment, executions),
       [CHAT_PRIVATE_INPUTS_ENV]: privateInputs?.filename,
@@ -615,6 +621,7 @@ async function runAttempt(input: {
       PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot,
       PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir,
       [AGENT_FILESYSTEM_SANDBOX_ENV]: agentFilesystemSandbox ? JSON.stringify(agentFilesystemSandbox) : undefined,
+      PWTEST_CACHE_DIR: playwrightCacheDirectory(temporaryRoot),
       PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace,
       PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(temporaryRoot, "artifacts-private", "server.log"),
       PAPERCLIP_RUNNER_BINARY: runnerBinary,