diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 60e38958994..d0cc06b16cd 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -331,7 +331,8 @@ file modes cannot separate them. OpenCode's `external_directory` policy guards its own read and grep tools, but not arbitrary shell commands. `agent-filesystem-sandbox.ts` closes that gap for local native cells on Linux. -The launcher selects the hidden paths: `tests/`, the trusted +The launcher selects the hidden paths: `tests/`, the checkout's `.git/` (which +holds the committed scenarios and graders), the trusted `.manual-private-trusted/` copy, the private evidence directory, `RUNNER_TEMP`, and the attempt root's `artifacts-private/`, `control/`, `selected-private-inputs/` and `playwright-transform-cache/` children. @@ -343,7 +344,9 @@ directory. Landlock hides a path by leaving its ancestors ungranted, and an ungranted directory accepts no new entries. Agents must keep creating files in `/tmp`, so a sandboxed attempt creates its root in `/var/tmp` instead of `/tmp`. The -agent's workspace, home and Paperclip home stay readable inside that root. The E2E server entry starts each `paperclip-runnerd` through +agent's workspace, home and Paperclip home stay readable inside that root. If +harness material would stay under the shared temporary directory or contain the +agent's state, the launcher fails the attempt in CI and warns elsewhere. The E2E server entry starts each `paperclip-runnerd` through `agent-filesystem-sandbox.py`, which applies a Landlock domain and then execs runnerd with its original PID, argv, cwd and environment. Runnerd, the provider, its tools and every shell command inherit the domain. The domain also sets @@ -362,12 +365,15 @@ inside Landlock. Legacy adapters, installed-release launches and macOS run without this sandbox. A process that can reach the Docker daemon can still mount the checkout into a container, and `ps` shows the launcher's command line. Case fixtures that create their own private directories directly in the -shared temporary directory stay readable. Daytona cells run the agent in a remote sandbox that holds only the +shared temporary directory stay readable. The boundary is filesystem only: the +scenarios and graders are public on GitHub, and agents have network access for +their model provider. Daytona cells run the agent in a remote sandbox that holds only the staged runnerd and provider pack, not the checkout. Fixture names that an agent can read (company, agent, environment, secret, connection and project) use a neutral word and a digest suffix. They do not -contain case ids, suite names, "E2E" or "test". +contain case ids, suite names, "E2E" or "test". The digest is salted per +process, so public catalog ids cannot be hashed back to a case. ## AWS OIDC and S3 diff --git a/tests/runner-e2e/agent-filesystem-sandbox.test.ts b/tests/runner-e2e/agent-filesystem-sandbox.test.ts index 0000075b210..b9b4e4d94b6 100644 --- a/tests/runner-e2e/agent-filesystem-sandbox.test.ts +++ b/tests/runner-e2e/agent-filesystem-sandbox.test.ts @@ -16,11 +16,12 @@ import { pythonExecutable, sandboxedInvocation, agentFilesystemSandboxHelper, + wrapRunnerdSpawn, } from "./agent-filesystem-sandbox.js"; import { runnerMatrix } from "./catalog.js"; const repositoryRoot = "/work/paperclip"; -const temporaryRoot = path.join(os.tmpdir(), "paperclip-runner-e2e-abc"); +const temporaryRoot = "/var/tmp/paperclip-runner-e2e-abc"; const cell = (id: string) => { const execution = runnerMatrix.find((candidate) => candidate.id === id); if (!execution) throw new Error(`missing ${id}`); @@ -36,6 +37,7 @@ const base = { privateDir: "/work/_temp/manual-private-evidence-x/capture", evidenceRoot: "/work/_temp/manual-private-evidence-x", installedRelease: false, + sharedTemporary: "/tmp", }; describe("agent filesystem sandbox selection", () => { @@ -43,7 +45,12 @@ describe("agent filesystem sandbox selection", () => { expect(agentFilesystemSandboxConfig(base)).toEqual({ hidden: [ "/work/paperclip/tests", + "/work/paperclip/.git", "/work/paperclip/.manual-private-trusted", + `${temporaryRoot}/artifacts-private`, + `${temporaryRoot}/control`, + `${temporaryRoot}/selected-private-inputs`, + `${temporaryRoot}/playwright-transform-cache`, "/work/_temp/manual-private-evidence-x/capture", "/work/_temp/manual-private-evidence-x", "/work/_temp", @@ -53,33 +60,34 @@ describe("agent filesystem sandbox selection", () => { }); it("hides the attempt's private children and Playwright cache when its root is outside the shared temporary directory", () => { - const root = "/var/tmp/paperclip-runner-e2e-abc"; - const config = agentFilesystemSandboxConfig({ ...base, temporaryRoot: root, sharedTemporary: "/tmp" }); - expect(config?.hidden).toEqual(expect.arrayContaining([ - `${root}/artifacts-private`, - `${root}/control`, - `${root}/selected-private-inputs`, - playwrightCacheDirectory(root), - ])); - expect(config?.hidden).not.toContain(root); + const config = agentFilesystemSandboxConfig(base); + expect(config?.hidden).toContain(playwrightCacheDirectory(temporaryRoot)); + expect(config?.hidden).not.toContain(temporaryRoot); expect(HARNESS_PRIVATE_TEMPORARY_CHILDREN).toContain("playwright-transform-cache"); expect(defaultPlaywrightCacheDirectory("/tmp")).toMatch(/^\/tmp\/playwright-transform-cache-/); }); + it("fails closed in CI when harness material sits where it cannot be hidden", () => { + const underShared = { ...base, temporaryRoot: "/tmp/paperclip-runner-e2e-abc", privateDir: "/tmp/paperclip-runner-e2e-abc/artifacts-private" }; + expect(() => agentFilesystemSandboxConfig(underShared)).toThrow(/agent_filesystem_sandbox_incomplete: .*artifacts-private/); + expect(() => agentFilesystemSandboxConfig({ ...base, repositoryRoot: "/tmp/checkout" })).toThrow(/\/tmp\/checkout\/tests/); + }); + it("keeps a sandboxed attempt's root out of the shared temporary directory", () => { const parent = agentTemporaryParent(true); if (process.platform === "linux") expect(parent).toBe(realpathSync("/var/tmp")); expect(agentTemporaryParent(false)).toBe(realpathSync(os.tmpdir())); }); - it("never hides the agent's own state, the checkout root or the shared temporary directory", () => { + it("never hides the agent's own state or the checkout root, and only warns outside CI", () => { const config = agentFilesystemSandboxConfig({ ...base, - env: { RUNNER_TEMP: os.tmpdir() }, - privateDir: path.join(temporaryRoot, "artifacts-private"), + env: { RUNNER_TEMP: "/tmp" }, + temporaryRoot: "/tmp/paperclip-runner-e2e-abc", + privateDir: "/tmp/paperclip-runner-e2e-abc/artifacts-private", evidenceRoot: "/work", }); - expect(config).toEqual({ hidden: ["/work/paperclip/tests", "/work/paperclip/.manual-private-trusted"], required: false }); + expect(config).toEqual({ hidden: ["/work/paperclip/tests", "/work/paperclip/.git", "/work/paperclip/.manual-private-trusted"], required: false }); }); it.each([ @@ -115,6 +123,18 @@ describe("runnerd launch rewriting", () => { expect(sandboxedInvocation({ hidden: [], required: false }, "p", runnerd, [], { LC_CTYPE: "C" }).args).toContain("LC_CTYPE=C"); }); + it("rewrites runnerd launches through the wrapped spawn and passes other launches through", () => { + const calls: unknown[][] = []; + const fake = ((...args: unknown[]) => { calls.push(args); return { pid: 7 }; }) as unknown as typeof import("node:child_process").spawn; + const spawn = wrapRunnerdSpawn(fake, { hidden: ["/work/paperclip/tests"], required: true }, "/usr/bin/python3"); + const runnerd = "/work/paperclip/packages/paperclip-runner/runner/target/debug/paperclip-runnerd"; + spawn(runnerd, ["--run-id", "r"], { cwd: "/work", env: { LC_CTYPE: "C.UTF-8" }, detached: true }); + spawn("/usr/bin/git", ["status"], { cwd: "/work" }); + expect(calls[0]).toEqual(["/usr/bin/python3", expect.arrayContaining(["--hide", "/work/paperclip/tests", "--", runnerd, "--run-id", "r"]), + { cwd: "/work", env: { LC_CTYPE: "C.UTF-8" }, detached: true }]); + expect(calls[1]).toEqual(["/usr/bin/git", ["status"], { cwd: "/work" }]); + }); + it("installs the sandbox in the E2E server entry before the product starts", async () => { const entry = await readFile(path.join(import.meta.dirname, "server-entry.ts"), "utf8"); const install = entry.indexOf("installAgentFilesystemSandbox("); diff --git a/tests/runner-e2e/agent-filesystem-sandbox.ts b/tests/runner-e2e/agent-filesystem-sandbox.ts index 33c42261681..a2d7a7d6397 100644 --- a/tests/runner-e2e/agent-filesystem-sandbox.ts +++ b/tests/runner-e2e/agent-filesystem-sandbox.ts @@ -91,26 +91,39 @@ export function agentFilesystemSandboxConfig(input: { }): AgentFilesystemSandboxConfig | null { if (!agentFilesystemSandboxApplies(input)) return null; const sharedTemporary = path.resolve(input.sharedTemporary ?? (input.env.TMPDIR?.trim() || os.tmpdir())); - const candidates = [ + const required = input.env.GITHUB_ACTIONS === "true"; + // Harness material that must be hidden. Git holds the committed scenarios + // and graders, so `git show HEAD:tests/...` would otherwise bypass `tests/`. + const essential = [ path.join(input.repositoryRoot, "tests"), + path.join(input.repositoryRoot, ".git"), path.join(input.repositoryRoot, ".manual-private-trusted"), ...HARNESS_PRIVATE_TEMPORARY_CHILDREN.map((child) => path.join(input.temporaryRoot, child)), input.privateDir, - input.evidenceRoot, - input.env.RUNNER_TEMP, ]; + const optional = [input.evidenceRoot, input.env.RUNNER_TEMP]; const hidden: string[] = []; - for (const candidate of candidates) { + const exposed: string[] = []; + for (const [candidate, mustHide] of [...essential.map((value) => [value, true] as const), ...optional.map((value) => [value, false] as const)]) { const value = candidate?.trim(); if (!value || !path.isAbsolute(value)) continue; const resolved = path.resolve(value); // Never hide a path that holds the agent's own state, or one beneath the // shared temporary directory: its ancestors would lose new entries. - if (within(input.temporaryRoot, resolved) || within(input.repositoryRoot, resolved)) continue; - if (within(resolved, sharedTemporary) || within(sharedTemporary, resolved)) continue; + if (within(input.temporaryRoot, resolved) || within(input.repositoryRoot, resolved) + || within(resolved, sharedTemporary) || within(sharedTemporary, resolved)) { + if (mustHide) exposed.push(resolved); + continue; + } if (!hidden.includes(resolved)) hidden.push(resolved); } - return hidden.length ? { hidden, required: input.env.GITHUB_ACTIONS === "true" } : null; + if (exposed.length) { + const message = `agent_filesystem_sandbox_incomplete: cannot hide ${exposed.join(", ")}; keep the checkout and attempt root outside ${sharedTemporary}`; + // A required sandbox fails closed rather than run an eval agents can read. + if (required) throw new Error(message); + console.warn(`[agent-filesystem-sandbox] ${message}`); + } + return hidden.length ? { hidden, required } : null; } export function parseAgentFilesystemSandboxConfig(value: string | undefined): AgentFilesystemSandboxConfig | null { @@ -227,8 +240,14 @@ export function installAgentFilesystemSandbox(repositoryRoot: string, env: NodeJ console.warn(`[agent-filesystem-sandbox] disabled: ${probe.failures.join("; ")}`); return null; } - const original = childProcess.spawn; - const spawn = function (this: unknown, command: string, ...rest: unknown[]) { + childProcess.spawn = wrapRunnerdSpawn(childProcess.spawn, config, python); + syncBuiltinESMExports(); + return config; +} + +/** Returns a `spawn` that starts runnerd inside the sandbox and passes other launches through. */ +export function wrapRunnerdSpawn(original: typeof childProcess.spawn, config: AgentFilesystemSandboxConfig, python: string) { + return function (this: unknown, command: string, ...rest: unknown[]) { const args = Array.isArray(rest[0]) ? rest[0] as string[] : []; const options = (Array.isArray(rest[0]) ? rest[1] : rest[0]) as SpawnOptions | undefined; if (isRunnerdLaunch(command, options)) { @@ -239,7 +258,4 @@ export function installAgentFilesystemSandbox(repositoryRoot: string, env: NodeJ } return Reflect.apply(original, this, [command, ...rest]); } as typeof childProcess.spawn; - childProcess.spawn = spawn; - syncBuiltinESMExports(); - return config; } diff --git a/tests/runner-e2e/fixture-names.ts b/tests/runner-e2e/fixture-names.ts index 7b7960b8575..b66b9a3f660 100644 --- a/tests/runner-e2e/fixture-names.ts +++ b/tests/runner-e2e/fixture-names.ts @@ -1,9 +1,13 @@ -import { createHash } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; + +// Per-process salt: the public catalog ids cannot be hashed back to a case. +const salt = randomBytes(16); /** * Names and descriptions an agent can read must not reveal the eval: no case - * ids, suite names, "E2E" or "test". A short digest keeps them unique. + * ids, suite names, "E2E" or "test". A short salted digest keeps them unique + * and stable within one attempt, without being reversible from public ids. */ export function neutralFixtureSuffix(...parts: readonly string[]): string { - return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 8); + return createHash("sha256").update(salt).update(parts.join("\0")).digest("hex").slice(0, 8); }