diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 151a024ed47..60e38958994 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -332,8 +332,18 @@ its own read and grep tools, but not arbitrary shell commands. `agent-filesystem-sandbox.ts` closes that gap for local native cells on Linux. The launcher selects the hidden paths: `tests/`, the trusted -`.manual-private-trusted/` copy, the private evidence directory and -`RUNNER_TEMP`. The E2E server entry starts each `paperclip-runnerd` through +`.manual-private-trusted/` copy, the private evidence directory, `RUNNER_TEMP`, +and the attempt root's `artifacts-private/`, `control/`, +`selected-private-inputs/` and `playwright-transform-cache/` children. +Playwright compiles the harness and graders (with source maps) into +`PWTEST_CACHE_DIR`, which the launcher points at that hidden child. The +launcher also removes Playwright's default cache in the shared temporary +directory. + +Landlock hides a path by leaving its ancestors ungranted, and an ungranted +directory accepts no new entries. Agents must keep creating files in `/tmp`, so +a sandboxed attempt creates its root in `/var/tmp` instead of `/tmp`. The +agent's workspace, home and Paperclip home stay readable inside that root. The E2E server entry starts each `paperclip-runnerd` through `agent-filesystem-sandbox.py`, which applies a Landlock domain and then execs runnerd with its original PID, argv, cwd and environment. Runnerd, the provider, its tools and every shell command inherit the domain. The domain also sets @@ -341,7 +351,7 @@ its tools and every shell command inherit the domain. The domain also sets unrestricted because it loads its E2E entry from the harness. Before the product starts an agent, a probe runs a sandboxed shell. The probe -must be denied the harness and every hidden path, and it must read the product +must be denied the harness, every hidden path and Playwright's default cache, and it must read the product package and write a temporary file. The server log records the probe result and each wrapped runnerd launch. In CI a missing or ineffective sandbox stops the attempt. Landlock does not hide that a path exists (`stat` still works); it @@ -351,7 +361,8 @@ Limits: Codex cells keep Codex's own bubblewrap sandbox, which cannot mount inside Landlock. Legacy adapters, installed-release launches and macOS run without this sandbox. A process that can reach the Docker daemon can still mount the checkout into a container, and `ps` shows the launcher's command -line. Daytona cells run the agent in a remote sandbox that holds only the +line. Case fixtures that create their own private directories directly in the +shared temporary directory stay readable. Daytona cells run the agent in a remote sandbox that holds only the staged runnerd and provider pack, not the checkout. Fixture names that an agent can read (company, agent, environment, secret, diff --git a/tests/runner-e2e/agent-filesystem-sandbox.py b/tests/runner-e2e/agent-filesystem-sandbox.py index 46ad5175a7d..796db1be69d 100644 --- a/tests/runner-e2e/agent-filesystem-sandbox.py +++ b/tests/runner-e2e/agent-filesystem-sandbox.py @@ -188,11 +188,12 @@ def main(): required, hidden_inputs, unset, assignments, command = parse(sys.argv[1:]) hidden = [] for path in hidden_inputs: - if os.path.lexists(path): - real = os.path.realpath(path) - if real == "/": - fail("refusing to hide the filesystem root", 2) - hidden.append(real) + # Keep paths that do not exist yet: their ancestors stay ungranted, so + # a later mkdir there is unreachable too. + real = os.path.realpath(path) + if real == "/": + fail("refusing to hide the filesystem root", 2) + hidden.append(real) for name in unset: os.environ.pop(name, None) for name, value in assignments: diff --git a/tests/runner-e2e/agent-filesystem-sandbox.test.ts b/tests/runner-e2e/agent-filesystem-sandbox.test.ts index f7942778275..0000075b210 100644 --- a/tests/runner-e2e/agent-filesystem-sandbox.test.ts +++ b/tests/runner-e2e/agent-filesystem-sandbox.test.ts @@ -1,11 +1,15 @@ import { execFileSync } from "node:child_process"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, realpathSync, rmSync } from "node:fs"; import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { + HARNESS_PRIVATE_TEMPORARY_CHILDREN, agentFilesystemSandboxConfig, + agentTemporaryParent, + defaultPlaywrightCacheDirectory, + playwrightCacheDirectory, isRunnerdLaunch, parseAgentFilesystemSandboxConfig, probeAgentFilesystemSandbox, @@ -48,6 +52,26 @@ describe("agent filesystem sandbox selection", () => { }); }); + it("hides the attempt's private children and Playwright cache when its root is outside the shared temporary directory", () => { + const root = "/var/tmp/paperclip-runner-e2e-abc"; + const config = agentFilesystemSandboxConfig({ ...base, temporaryRoot: root, sharedTemporary: "/tmp" }); + expect(config?.hidden).toEqual(expect.arrayContaining([ + `${root}/artifacts-private`, + `${root}/control`, + `${root}/selected-private-inputs`, + playwrightCacheDirectory(root), + ])); + expect(config?.hidden).not.toContain(root); + expect(HARNESS_PRIVATE_TEMPORARY_CHILDREN).toContain("playwright-transform-cache"); + expect(defaultPlaywrightCacheDirectory("/tmp")).toMatch(/^\/tmp\/playwright-transform-cache-/); + }); + + it("keeps a sandboxed attempt's root out of the shared temporary directory", () => { + const parent = agentTemporaryParent(true); + if (process.platform === "linux") expect(parent).toBe(realpathSync("/var/tmp")); + expect(agentTemporaryParent(false)).toBe(realpathSync(os.tmpdir())); + }); + it("never hides the agent's own state, the checkout root or the shared temporary directory", () => { const config = agentFilesystemSandboxConfig({ ...base, @@ -125,15 +149,24 @@ describe.skipIf(!landlock)("Landlock agent sandbox on this host", () => { await writeFile(path.join(repository, "tests", "runner-e2e", "agent-filesystem-sandbox.ts"), "grader"); await writeFile(path.join(repository, "tests", "runner-e2e", "chat-scenarios", "access.yaml"), "expected_outcome: secret"); await writeFile(path.join(repository, "packages", "paperclip-runner", "package.json"), "{}"); - const config = { hidden: [path.join(repository, "tests")], required: true }; + const attempt = path.join(root, "attempt"); + await mkdir(path.join(attempt, "artifacts-private"), { recursive: true }); + await mkdir(path.join(attempt, "workspace")); + await writeFile(path.join(attempt, "artifacts-private", "server.log"), "secret"); + const config = { hidden: [path.join(repository, "tests"), path.join(attempt, "artifacts-private"), path.join(attempt, "control")], required: true }; // The fixture lives under the shared temporary directory, so point mktemp at a granted sibling. - const probe = probeAgentFilesystemSandbox(config, repository, pythonExecutable(), { ...process.env, TMPDIR: path.join(root, "scratch") }); + const probe = probeAgentFilesystemSandbox(config, repository, pythonExecutable(), { ...process.env, TMPDIR: path.join(root, "scratch") }, path.join(root, "no-playwright-cache")); expect(probe.failures).toEqual([]); - expect(probe.result).toMatchObject({ harnessFile: "denied", harnessDirectory: "denied", productFile: "readable", temporaryWrite: "allowed" }); + expect(probe.result).toMatchObject({ playwrightDefaultCache: "denied", harnessFile: "denied", harnessDirectory: "denied", productFile: "readable", temporaryWrite: "allowed" }); // The exact read from the reported run: grep over the checkout's scenarios. const invocation = sandboxedInvocation(config, pythonExecutable(), "/bin/sh", ["-c", `grep -rn expected_outcome ${repository}/tests; echo "exit=$?"`], process.env); const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8" }); expect(output).not.toContain("secret"); expect(output).toMatch(/exit=2/); + const attemptCheck = sandboxedInvocation(config, pythonExecutable(), "/bin/sh", ["-c", `cat ${attempt}/artifacts-private/server.log; ls ${attempt}/workspace >/dev/null && echo workspace=ok; mkdir ${attempt}/control 2>/dev/null || echo control=blocked`], process.env); + const attemptOutput = execFileSync(attemptCheck.command, attemptCheck.args, { encoding: "utf8" }); + expect(attemptOutput).not.toContain("secret"); + expect(attemptOutput).toContain("workspace=ok"); + expect(attemptOutput).toContain("control=blocked"); }); }); diff --git a/tests/runner-e2e/agent-filesystem-sandbox.ts b/tests/runner-e2e/agent-filesystem-sandbox.ts index 797e7462b64..33c42261681 100644 --- a/tests/runner-e2e/agent-filesystem-sandbox.ts +++ b/tests/runner-e2e/agent-filesystem-sandbox.ts @@ -1,5 +1,5 @@ import childProcess, { execFileSync, type SpawnOptions } from "node:child_process"; -import { existsSync } from "node:fs"; +import { accessSync, constants, existsSync, realpathSync } from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; @@ -28,6 +28,55 @@ function within(candidate: string, root: string) { return candidate === root || candidate.startsWith(`${root}${path.sep}`); } +/** Harness-private directories inside the attempt's temporary root. */ +export const HARNESS_PRIVATE_TEMPORARY_CHILDREN = [ + "artifacts-private", + "control", + "selected-private-inputs", + // Playwright's compiled copy of the harness and graders, with source maps. + "playwright-transform-cache", +] as const; + +/** Playwright otherwise caches compiled harness code in the shared temporary directory. */ +export function playwrightCacheDirectory(temporaryRoot: string) { + return path.join(temporaryRoot, "playwright-transform-cache"); +} + +/** Playwright's default cache location for this user, which the launcher removes. */ +export function defaultPlaywrightCacheDirectory(temporaryDirectory = os.tmpdir()) { + return path.join(temporaryDirectory, `playwright-transform-cache-${process.geteuid?.() ?? ""}`); +} + +export function agentFilesystemSandboxApplies(input: { + platform: NodeJS.Platform; + executions: readonly MatrixExecution[]; + installedRelease: boolean; +}) { + if (input.platform !== "linux" || input.installedRelease) return false; + if (!input.executions.some((execution) => execution.environment.id === "local" && execution.profile.generation === "native")) return false; + // Codex's own bubblewrap sandbox mounts, which Landlock forbids. Those cells + // keep their existing isolation until Codex permission profiles hide the eval. + return !input.executions.some(requiresCodexCiSandbox); +} + +/** + * Landlock can only hide a path by leaving its ancestors ungranted, and an + * ungranted directory accepts no new entries. Hiding the attempt's private + * children of a root under /tmp would stop agents creating files in /tmp, so a + * sandboxed attempt keeps its root in /var/tmp instead. + */ +export function agentTemporaryParent(applies: boolean, sharedTemporary = os.tmpdir()) { + if (applies) { + try { + accessSync("/var/tmp", constants.W_OK | constants.X_OK); + return realpathSync("/var/tmp"); + } catch { + // Fall back to the shared directory; its private children stay visible. + } + } + return realpathSync(sharedTemporary); +} + /** Selects the paths hidden from local native agents, or null when not applicable. */ export function agentFilesystemSandboxConfig(input: { platform: NodeJS.Platform; @@ -38,16 +87,14 @@ export function agentFilesystemSandboxConfig(input: { privateDir: string; evidenceRoot?: string; installedRelease: boolean; + sharedTemporary?: string; }): AgentFilesystemSandboxConfig | null { - if (input.platform !== "linux" || input.installedRelease) return null; - if (!input.executions.some((execution) => execution.environment.id === "local" && execution.profile.generation === "native")) return null; - // Codex's own bubblewrap sandbox mounts, which Landlock forbids. Those cells - // keep their existing isolation until Codex permission profiles hide the eval. - if (input.executions.some(requiresCodexCiSandbox)) return null; - const sharedTemporary = path.resolve(input.env.TMPDIR?.trim() || os.tmpdir()); + if (!agentFilesystemSandboxApplies(input)) return null; + const sharedTemporary = path.resolve(input.sharedTemporary ?? (input.env.TMPDIR?.trim() || os.tmpdir())); const candidates = [ path.join(input.repositoryRoot, "tests"), path.join(input.repositoryRoot, ".manual-private-trusted"), + ...HARNESS_PRIVATE_TEMPORARY_CHILDREN.map((child) => path.join(input.temporaryRoot, child)), input.privateDir, input.evidenceRoot, input.env.RUNNER_TEMP, @@ -117,6 +164,7 @@ export function probeAgentFilesystemSandbox( repositoryRoot: string, python = pythonExecutable(), env: NodeJS.ProcessEnv = process.env, + playwrightCache = defaultPlaywrightCacheDirectory(), ) { const harness = path.join(repositoryRoot, "tests", "runner-e2e"); const script = [ @@ -126,7 +174,8 @@ export function probeAgentFilesystemSandbox( 'if test -e "$2"; then add harnessDirectoryStat visible; else add harnessDirectoryStat absent; fi', 'if cat "$3" >/dev/null 2>&1; then add productFile readable; else add productFile denied; fi', 'probe=$(mktemp) && rm -f "$probe" && add temporaryWrite allowed || add temporaryWrite denied', - 'shift 3; for hidden in "$@"; do if ls "$hidden" >/dev/null 2>&1; then add "hidden:$hidden" listable; else add "hidden:$hidden" denied; fi; done', + 'if ls "$4" >/dev/null 2>&1; then add playwrightDefaultCache listable; else add playwrightDefaultCache denied; fi', + 'shift 4; for hidden in "$@"; do if ls "$hidden" >/dev/null 2>&1; then add "hidden:$hidden" listable; else add "hidden:$hidden" denied; fi; done', 'printf "%s\\n" "$out"', ].join("\n"); const invocation = sandboxedInvocation(config, python, "/bin/sh", [ @@ -134,6 +183,7 @@ export function probeAgentFilesystemSandbox( path.join(harness, "agent-filesystem-sandbox.ts"), harness, path.join(repositoryRoot, "packages", "paperclip-runner", "package.json"), + playwrightCache, ...config.hidden, ], env); const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8", cwd: "/", env, timeout: 30_000 }); @@ -146,6 +196,7 @@ export function probeAgentFilesystemSandbox( result.harnessDirectory !== "denied" && "harness directory is listable", result.productFile !== "readable" && "product package is unreadable", result.temporaryWrite !== "allowed" && "temporary directory is not writable", + result.playwrightDefaultCache !== "denied" && "Playwright's default transform cache is listable", ...config.hidden.map((hidden) => result[`hidden:${hidden}`] === "listable" && `${hidden} is listable`), ].filter((failure): failure is string => Boolean(failure)); return { result, failures }; diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index ac1c0b89bc6..7737de4a631 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -31,7 +31,10 @@ import { createRunnerE2ETemporaryRoot } from "./server-config.js"; import { createExternalControllerDatabase } from "./external-controller-database.js"; import { createHash, randomBytes } from "node:crypto"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; -import { AGENT_FILESYSTEM_SANDBOX_ENV, agentFilesystemSandboxConfig } from "./agent-filesystem-sandbox.js"; +import { + AGENT_FILESYSTEM_SANDBOX_ENV, agentFilesystemSandboxApplies, agentFilesystemSandboxConfig, agentTemporaryParent, + defaultPlaywrightCacheDirectory, playwrightCacheDirectory, +} from "./agent-filesystem-sandbox.js"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; import { createRequire } from "node:module"; @@ -509,7 +512,8 @@ async function runAttempt(input: { const installedPlugin = await verifyInstalledDaytonaPlugin(process.env, executions); const startedAtMs = Date.now(); const sharedMemoryBaseline = snapshotDarwinSharedMemory(); - const temporaryParent = await realpath(os.tmpdir()); + const sandboxedAgents = agentFilesystemSandboxApplies({ platform: process.platform, executions, installedRelease: Boolean(installedCli) }); + const temporaryParent = agentTemporaryParent(sandboxedAgents); const temporaryRoot = await createRunnerE2ETemporaryRoot(temporaryParent); const selectedPrivateInputsDirectory = path.join(temporaryRoot, "selected-private-inputs"); const privateDir = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "capture") : path.join(temporaryRoot, "artifacts-private"); @@ -594,6 +598,8 @@ async function runAttempt(input: { platform: process.platform, env: process.env, executions, repositoryRoot, temporaryRoot, privateDir, evidenceRoot: manualPrivateHandoff?.root, installedRelease: Boolean(installedCli), }); + // Playwright's default cache holds compiled graders in the shared /tmp. + if (sandboxedAgents) await rm(defaultPlaywrightCacheDirectory(), { recursive: true, force: true }); const childEnv: NodeJS.ProcessEnv = { ...buildRunnerE2EProcessEnvironment(sourceEnvironment, executions), [CHAT_PRIVATE_INPUTS_ENV]: privateInputs?.filename, @@ -615,6 +621,7 @@ async function runAttempt(input: { PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot, PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir, [AGENT_FILESYSTEM_SANDBOX_ENV]: agentFilesystemSandbox ? JSON.stringify(agentFilesystemSandbox) : undefined, + PWTEST_CACHE_DIR: playwrightCacheDirectory(temporaryRoot), PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace, PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(temporaryRoot, "artifacts-private", "server.log"), PAPERCLIP_RUNNER_BINARY: runnerBinary,