test(runner-e2e): hide the checkout's .git, fail closed on unhidable paths, salt fixture names

Greptile and review findings on #20044:
- git show HEAD:tests/... read the graders through the checkout's .git; hide it.
- Harness material under the shared temp dir or around agent state was
  dropped silently; a required (CI) sandbox now refuses to start.
- Unsalted name digests were reversible from public catalog ids.
- Unit-test the runnerd spawn wrapper directly.
- Document that the boundary is filesystem only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/SECURITY.md

@@ -331,7 +331,8 @@ file modes cannot separate them. OpenCode's `external_directory` policy guards
 its own read and grep tools, but not arbitrary shell commands.
 
 `agent-filesystem-sandbox.ts` closes that gap for local native cells on Linux.
-The launcher selects the hidden paths: `tests/`, the trusted
+The launcher selects the hidden paths: `tests/`, the checkout's `.git/` (which
+holds the committed scenarios and graders), the trusted
 `.manual-private-trusted/` copy, the private evidence directory, `RUNNER_TEMP`,
 and the attempt root's `artifacts-private/`, `control/`,
 `selected-private-inputs/` and `playwright-transform-cache/` children.
@@ -343,7 +344,9 @@ directory.
 Landlock hides a path by leaving its ancestors ungranted, and an ungranted
 directory accepts no new entries. Agents must keep creating files in `/tmp`, so
 a sandboxed attempt creates its root in `/var/tmp` instead of `/tmp`. The
-agent's workspace, home and Paperclip home stay readable inside that root. The E2E server entry starts each `paperclip-runnerd` through
+agent's workspace, home and Paperclip home stay readable inside that root. If
+harness material would stay under the shared temporary directory or contain the
+agent's state, the launcher fails the attempt in CI and warns elsewhere. The E2E server entry starts each `paperclip-runnerd` through
 `agent-filesystem-sandbox.py`, which applies a Landlock domain and then execs
 runnerd with its original PID, argv, cwd and environment. Runnerd, the provider,
 its tools and every shell command inherit the domain. The domain also sets
@@ -362,12 +365,15 @@ inside Landlock. Legacy adapters, installed-release launches and macOS run
 without this sandbox. A process that can reach the Docker daemon can still
 mount the checkout into a container, and `ps` shows the launcher's command
 line. Case fixtures that create their own private directories directly in the
-shared temporary directory stay readable. Daytona cells run the agent in a remote sandbox that holds only the
+shared temporary directory stay readable. The boundary is filesystem only: the
+scenarios and graders are public on GitHub, and agents have network access for
+their model provider. Daytona cells run the agent in a remote sandbox that holds only the
 staged runnerd and provider pack, not the checkout.
 
 Fixture names that an agent can read (company, agent, environment, secret,
 connection and project) use a neutral word and a digest suffix. They do not
-contain case ids, suite names, "E2E" or "test".
+contain case ids, suite names, "E2E" or "test". The digest is salted per
+process, so public catalog ids cannot be hashed back to a case.
 
 ## AWS OIDC and S3
 

tests/runner-e2e/agent-filesystem-sandbox.test.ts

@@ -16,11 +16,12 @@ import {
   pythonExecutable,
   sandboxedInvocation,
   agentFilesystemSandboxHelper,
+  wrapRunnerdSpawn,
 } from "./agent-filesystem-sandbox.js";
 import { runnerMatrix } from "./catalog.js";
 
 const repositoryRoot = "/work/paperclip";
-const temporaryRoot = path.join(os.tmpdir(), "paperclip-runner-e2e-abc");
+const temporaryRoot = "/var/tmp/paperclip-runner-e2e-abc";
 const cell = (id: string) => {
   const execution = runnerMatrix.find((candidate) => candidate.id === id);
   if (!execution) throw new Error(`missing ${id}`);
@@ -36,6 +37,7 @@ const base = {
   privateDir: "/work/_temp/manual-private-evidence-x/capture",
   evidenceRoot: "/work/_temp/manual-private-evidence-x",
   installedRelease: false,
+  sharedTemporary: "/tmp",
 };
 
 describe("agent filesystem sandbox selection", () => {
@@ -43,7 +45,12 @@ describe("agent filesystem sandbox selection", () => {
     expect(agentFilesystemSandboxConfig(base)).toEqual({
       hidden: [
         "/work/paperclip/tests",
+        "/work/paperclip/.git",
         "/work/paperclip/.manual-private-trusted",
+        `${temporaryRoot}/artifacts-private`,
+        `${temporaryRoot}/control`,
+        `${temporaryRoot}/selected-private-inputs`,
+        `${temporaryRoot}/playwright-transform-cache`,
         "/work/_temp/manual-private-evidence-x/capture",
         "/work/_temp/manual-private-evidence-x",
         "/work/_temp",
@@ -53,33 +60,34 @@ describe("agent filesystem sandbox selection", () => {
   });
 
   it("hides the attempt's private children and Playwright cache when its root is outside the shared temporary directory", () => {
-    const root = "/var/tmp/paperclip-runner-e2e-abc";
-    const config = agentFilesystemSandboxConfig({ ...base, temporaryRoot: root, sharedTemporary: "/tmp" });
-    expect(config?.hidden).toEqual(expect.arrayContaining([
-      `${root}/artifacts-private`,
-      `${root}/control`,
-      `${root}/selected-private-inputs`,
-      playwrightCacheDirectory(root),
-    ]));
-    expect(config?.hidden).not.toContain(root);
+    const config = agentFilesystemSandboxConfig(base);
+    expect(config?.hidden).toContain(playwrightCacheDirectory(temporaryRoot));
+    expect(config?.hidden).not.toContain(temporaryRoot);
     expect(HARNESS_PRIVATE_TEMPORARY_CHILDREN).toContain("playwright-transform-cache");
     expect(defaultPlaywrightCacheDirectory("/tmp")).toMatch(/^\/tmp\/playwright-transform-cache-/);
   });
 
+  it("fails closed in CI when harness material sits where it cannot be hidden", () => {
+    const underShared = { ...base, temporaryRoot: "/tmp/paperclip-runner-e2e-abc", privateDir: "/tmp/paperclip-runner-e2e-abc/artifacts-private" };
+    expect(() => agentFilesystemSandboxConfig(underShared)).toThrow(/agent_filesystem_sandbox_incomplete: .*artifacts-private/);
+    expect(() => agentFilesystemSandboxConfig({ ...base, repositoryRoot: "/tmp/checkout" })).toThrow(/\/tmp\/checkout\/tests/);
+  });
+
   it("keeps a sandboxed attempt's root out of the shared temporary directory", () => {
     const parent = agentTemporaryParent(true);
     if (process.platform === "linux") expect(parent).toBe(realpathSync("/var/tmp"));
     expect(agentTemporaryParent(false)).toBe(realpathSync(os.tmpdir()));
   });
 
-  it("never hides the agent's own state, the checkout root or the shared temporary directory", () => {
+  it("never hides the agent's own state or the checkout root, and only warns outside CI", () => {
     const config = agentFilesystemSandboxConfig({
       ...base,
-      env: { RUNNER_TEMP: os.tmpdir() },
-      privateDir: path.join(temporaryRoot, "artifacts-private"),
+      env: { RUNNER_TEMP: "/tmp" },
+      temporaryRoot: "/tmp/paperclip-runner-e2e-abc",
+      privateDir: "/tmp/paperclip-runner-e2e-abc/artifacts-private",
       evidenceRoot: "/work",
     });
-    expect(config).toEqual({ hidden: ["/work/paperclip/tests", "/work/paperclip/.manual-private-trusted"], required: false });
+    expect(config).toEqual({ hidden: ["/work/paperclip/tests", "/work/paperclip/.git", "/work/paperclip/.manual-private-trusted"], required: false });
   });
 
   it.each([
@@ -115,6 +123,18 @@ describe("runnerd launch rewriting", () => {
     expect(sandboxedInvocation({ hidden: [], required: false }, "p", runnerd, [], { LC_CTYPE: "C" }).args).toContain("LC_CTYPE=C");
   });
 
+  it("rewrites runnerd launches through the wrapped spawn and passes other launches through", () => {
+    const calls: unknown[][] = [];
+    const fake = ((...args: unknown[]) => { calls.push(args); return { pid: 7 }; }) as unknown as typeof import("node:child_process").spawn;
+    const spawn = wrapRunnerdSpawn(fake, { hidden: ["/work/paperclip/tests"], required: true }, "/usr/bin/python3");
+    const runnerd = "/work/paperclip/packages/paperclip-runner/runner/target/debug/paperclip-runnerd";
+    spawn(runnerd, ["--run-id", "r"], { cwd: "/work", env: { LC_CTYPE: "C.UTF-8" }, detached: true });
+    spawn("/usr/bin/git", ["status"], { cwd: "/work" });
+    expect(calls[0]).toEqual(["/usr/bin/python3", expect.arrayContaining(["--hide", "/work/paperclip/tests", "--", runnerd, "--run-id", "r"]),
+      { cwd: "/work", env: { LC_CTYPE: "C.UTF-8" }, detached: true }]);
+    expect(calls[1]).toEqual(["/usr/bin/git", ["status"], { cwd: "/work" }]);
+  });
+
   it("installs the sandbox in the E2E server entry before the product starts", async () => {
     const entry = await readFile(path.join(import.meta.dirname, "server-entry.ts"), "utf8");
     const install = entry.indexOf("installAgentFilesystemSandbox(");

tests/runner-e2e/agent-filesystem-sandbox.ts

@@ -91,26 +91,39 @@ export function agentFilesystemSandboxConfig(input: {
 }): AgentFilesystemSandboxConfig | null {
   if (!agentFilesystemSandboxApplies(input)) return null;
   const sharedTemporary = path.resolve(input.sharedTemporary ?? (input.env.TMPDIR?.trim() || os.tmpdir()));
-  const candidates = [
+  const required = input.env.GITHUB_ACTIONS === "true";
+  // Harness material that must be hidden. Git holds the committed scenarios
+  // and graders, so `git show HEAD:tests/...` would otherwise bypass `tests/`.
+  const essential = [
     path.join(input.repositoryRoot, "tests"),
+    path.join(input.repositoryRoot, ".git"),
     path.join(input.repositoryRoot, ".manual-private-trusted"),
     ...HARNESS_PRIVATE_TEMPORARY_CHILDREN.map((child) => path.join(input.temporaryRoot, child)),
     input.privateDir,
-    input.evidenceRoot,
-    input.env.RUNNER_TEMP,
   ];
+  const optional = [input.evidenceRoot, input.env.RUNNER_TEMP];
   const hidden: string[] = [];
-  for (const candidate of candidates) {
+  const exposed: string[] = [];
+  for (const [candidate, mustHide] of [...essential.map((value) => [value, true] as const), ...optional.map((value) => [value, false] as const)]) {
     const value = candidate?.trim();
     if (!value || !path.isAbsolute(value)) continue;
     const resolved = path.resolve(value);
     // Never hide a path that holds the agent's own state, or one beneath the
     // shared temporary directory: its ancestors would lose new entries.
-    if (within(input.temporaryRoot, resolved) || within(input.repositoryRoot, resolved)) continue;
-    if (within(resolved, sharedTemporary) || within(sharedTemporary, resolved)) continue;
+    if (within(input.temporaryRoot, resolved) || within(input.repositoryRoot, resolved)
+      || within(resolved, sharedTemporary) || within(sharedTemporary, resolved)) {
+      if (mustHide) exposed.push(resolved);
+      continue;
+    }
     if (!hidden.includes(resolved)) hidden.push(resolved);
   }
-  return hidden.length ? { hidden, required: input.env.GITHUB_ACTIONS === "true" } : null;
+  if (exposed.length) {
+    const message = `agent_filesystem_sandbox_incomplete: cannot hide ${exposed.join(", ")}; keep the checkout and attempt root outside ${sharedTemporary}`;
+    // A required sandbox fails closed rather than run an eval agents can read.
+    if (required) throw new Error(message);
+    console.warn(`[agent-filesystem-sandbox] ${message}`);
+  }
+  return hidden.length ? { hidden, required } : null;
 }
 
 export function parseAgentFilesystemSandboxConfig(value: string | undefined): AgentFilesystemSandboxConfig | null {
@@ -227,8 +240,14 @@ export function installAgentFilesystemSandbox(repositoryRoot: string, env: NodeJ
     console.warn(`[agent-filesystem-sandbox] disabled: ${probe.failures.join("; ")}`);
     return null;
   }
-  const original = childProcess.spawn;
-  const spawn = function (this: unknown, command: string, ...rest: unknown[]) {
+  childProcess.spawn = wrapRunnerdSpawn(childProcess.spawn, config, python);
+  syncBuiltinESMExports();
+  return config;
+}
+
+/** Returns a `spawn` that starts runnerd inside the sandbox and passes other launches through. */
+export function wrapRunnerdSpawn(original: typeof childProcess.spawn, config: AgentFilesystemSandboxConfig, python: string) {
+  return function (this: unknown, command: string, ...rest: unknown[]) {
     const args = Array.isArray(rest[0]) ? rest[0] as string[] : [];
     const options = (Array.isArray(rest[0]) ? rest[1] : rest[0]) as SpawnOptions | undefined;
     if (isRunnerdLaunch(command, options)) {
@@ -239,7 +258,4 @@ export function installAgentFilesystemSandbox(repositoryRoot: string, env: NodeJ
     }
     return Reflect.apply(original, this, [command, ...rest]);
   } as typeof childProcess.spawn;
-  childProcess.spawn = spawn;
-  syncBuiltinESMExports();
-  return config;
 }

tests/runner-e2e/fixture-names.ts

@@ -1,9 +1,13 @@
-import { createHash } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
+
+// Per-process salt: the public catalog ids cannot be hashed back to a case.
+const salt = randomBytes(16);
 
 /**
  * Names and descriptions an agent can read must not reveal the eval: no case
- * ids, suite names, "E2E" or "test". A short digest keeps them unique.
+ * ids, suite names, "E2E" or "test". A short salted digest keeps them unique
+ * and stable within one attempt, without being reversible from public ids.
  */
 export function neutralFixtureSuffix(...parts: readonly string[]): string {
-  return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 8);
+  return createHash("sha256").update(salt).update(parts.join("\0")).digest("hex").slice(0, 8);
 }