diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 0922095aad6..7fcd338c551 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -323,6 +323,61 @@ checks it out only after authorization. Target-controlled code cannot replace the report sanitizer or the AWS history publisher. Fork refs and target-controlled workflow definitions do not enter this path. +## Agent view of the eval harness + +The agent under test must not read the eval. A local cell runs the product from +the checkout that also holds `tests/`, as the same OS user as the harness, so +file modes cannot separate them. OpenCode's `external_directory` policy guards +its own read and grep tools, but not arbitrary shell commands. + +`agent-filesystem-sandbox.ts` closes that gap for local native cells on Linux. +The launcher selects the hidden paths: `tests/`, the checkout's Git store (which +holds the committed scenarios and graders; for a linked worktree, its gitdir +and the shared common directory), the trusted +`.manual-private-trusted/` copy, the private evidence directory, `RUNNER_TEMP`, +and the attempt root's `artifacts-private/`, `control/`, +`selected-private-inputs/` and `playwright-transform-cache/` children. +Playwright compiles the harness and graders (with source maps) into +`PWTEST_CACHE_DIR`, which the launcher points at that hidden child. The +launcher also removes Playwright's default cache in the shared temporary +directory. + +Landlock hides a path by leaving its ancestors ungranted, and an ungranted +directory accepts no new entries. Agents must keep creating files in `/tmp`, so +a sandboxed attempt creates its root in `/var/tmp` instead of `/tmp`. The +agent's workspace, home and Paperclip home stay readable inside that root. If +harness material would stay under the shared temporary directory or contain the +agent's state, the launcher fails the attempt in CI and warns elsewhere. The E2E server entry starts each `paperclip-runnerd` (through `spawn`, +`spawnSync`, `execFile` or `execFileSync`, by absolute path or PATH; a shell +launch is refused) through +`agent-filesystem-sandbox.py`, which applies a Landlock domain and then execs +runnerd with its original PID, argv, cwd and environment. Runnerd, the provider, +its tools and every shell command inherit the domain. The domain also sets +`no_new_privs`, so `sudo` cannot widen it. The product server itself stays +unrestricted because it loads its E2E entry from the harness. + +Before the product starts an agent, a probe runs a sandboxed shell. The probe +must be denied the harness, every hidden path and Playwright's default cache, and it must read the product +package and write a temporary file. The server log records the probe result and +each wrapped runnerd launch. In CI a missing or ineffective sandbox stops the +attempt. Landlock does not hide that a path exists (`stat` still works); it +denies reading and listing. + +Limits: Codex cells keep Codex's own bubblewrap sandbox, which cannot mount +inside Landlock. Legacy adapters, installed-release launches and macOS run +without this sandbox. A process that can reach the Docker daemon can still +mount the checkout into a container, and `ps` shows the launcher's command +line. Case fixtures that create their own private directories directly in the +shared temporary directory stay readable. The boundary is filesystem only: the +scenarios and graders are public on GitHub, and agents have network access for +their model provider. Daytona cells run the agent in a remote sandbox that holds only the +staged runnerd and provider pack, not the checkout. + +Fixture names that an agent can read (company, agent, environment, secret, +connection and project) use a neutral word and a digest suffix. They do not +contain case ids, suite names, "E2E" or "test". The digest is salted per +process, so public catalog ids cannot be hashed back to a case. + ## AWS OIDC and S3 The AWS role trust policy should accept only GitHub's OIDC audience and the diff --git a/tests/runner-e2e/agent-filesystem-sandbox.py b/tests/runner-e2e/agent-filesystem-sandbox.py new file mode 100644 index 00000000000..121a72f2b21 --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.py @@ -0,0 +1,221 @@ +"""Execute a command whose process tree cannot read the eval harness. + +Runner E2E starts the product from a repository checkout that also holds the +scenarios, fixtures and graders. This helper applies a Linux Landlock domain +that grants filesystem access everywhere except the given hidden paths, then +replaces itself with the command (same PID, argv and environment). Every +descendant inherits the domain, including an agent's shell commands. + +Usage: + agent-filesystem-sandbox.py [--required] [--unset NAME]... [--set NAME=VALUE]... + --hide PATH [--hide PATH]... -- COMMAND [ARG]... + +Without Landlock, --required exits 125 instead of running the command. +""" + +import ctypes +import os +import stat +import struct +import sys + +SYS_LANDLOCK_CREATE_RULESET = 444 +SYS_LANDLOCK_ADD_RULE = 445 +SYS_LANDLOCK_RESTRICT_SELF = 446 +LANDLOCK_CREATE_RULESET_VERSION = 1 +LANDLOCK_RULE_PATH_BENEATH = 1 +PR_SET_NO_NEW_PRIVS = 38 + +FILE_RIGHTS = (1 << 0) | (1 << 1) | (1 << 2) # execute, write, read +ABI_1_RIGHTS = (1 << 13) - 1 +REFER = 1 << 13 +TRUNCATE = 1 << 14 +IOCTL_DEV = 1 << 15 + + +class RulesetAttr(ctypes.Structure): + _fields_ = [("handled_access_fs", ctypes.c_uint64)] + + +def fail(message, code=125): + sys.stderr.write("agent_filesystem_sandbox: %s\n" % message) + sys.stderr.flush() + os._exit(code) + + +def parse(argv): + required = False + hidden = [] + unset = [] + assignments = [] + index = 0 + while index < len(argv): + token = argv[index] + if token == "--": + command = argv[index + 1:] + if not command: + fail("missing command", 2) + return required, hidden, unset, assignments, command + if token == "--required": + required = True + elif token in ("--hide", "--unset", "--set") and index + 1 < len(argv): + index += 1 + value = argv[index] + if token == "--hide": + if not os.path.isabs(value): + fail("hidden path must be absolute: %s" % value, 2) + hidden.append(value) + elif token == "--unset": + unset.append(value) + else: + name, separator, assigned = value.partition("=") + if not separator or not name: + fail("--set requires NAME=VALUE", 2) + assignments.append((name, assigned)) + else: + fail("unexpected argument: %s" % token, 2) + index += 1 + fail("missing -- before the command", 2) + + +def within(candidate, root): + return candidate == root or candidate.startswith(root.rstrip("/") + "/") + + +def landlock(): + libc = ctypes.CDLL(None, use_errno=True) + libc.syscall.restype = ctypes.c_long + + def syscall(*args): + result = libc.syscall(*args) + if result < 0: + raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno())) + return result + + return libc, syscall + + +def restrict(hidden): + libc, syscall = landlock() + try: + abi = syscall( + ctypes.c_long(SYS_LANDLOCK_CREATE_RULESET), + ctypes.c_void_p(None), + ctypes.c_long(0), + ctypes.c_long(LANDLOCK_CREATE_RULESET_VERSION), + ) + except OSError as error: + raise RuntimeError("Landlock is unavailable on this kernel (%s)" % error) + handled = ABI_1_RIGHTS + if abi >= 2: + handled |= REFER + if abi >= 3: + handled |= TRUNCATE + if abi >= 5: + handled |= IOCTL_DEV + attr = RulesetAttr(handled) + ruleset = syscall( + ctypes.c_long(SYS_LANDLOCK_CREATE_RULESET), + ctypes.byref(attr), + ctypes.c_long(ctypes.sizeof(attr)), + ctypes.c_long(0), + ) + + def allow(path): + try: + descriptor = os.open(path, os.O_PATH | os.O_CLOEXEC) + except OSError: + return + try: + mode = os.fstat(descriptor).st_mode + access = handled if stat.S_ISDIR(mode) else handled & (FILE_RIGHTS | TRUNCATE | IOCTL_DEV) + # struct landlock_path_beneath_attr is packed: u64 allowed_access, s32 parent_fd. + rule = ctypes.create_string_buffer(struct.pack("=Qi", access, descriptor), 12) + syscall( + ctypes.c_long(SYS_LANDLOCK_ADD_RULE), + ctypes.c_long(ruleset), + ctypes.c_long(LANDLOCK_RULE_PATH_BENEATH), + rule, + ctypes.c_long(0), + ) + finally: + os.close(descriptor) + + # Landlock only grants. Grant every entry except the hidden paths, walking + # down only through their ancestors; ancestors themselves get no rule. + def grant(directory): + try: + names = sorted(os.listdir(directory)) + except OSError: + return + for name in names: + child = os.path.join(directory, name) + if any(within(child, path) for path in hidden): + continue + try: + is_link = os.path.islink(child) + real = os.path.realpath(child) + except OSError: + continue + if any(within(real, path) or within(path, real) for path in hidden): + if not is_link and any(within(path, child) for path in hidden): + grant(child) + # A link into, or onto an ancestor of, a hidden path stays ungranted. + continue + allow(child) + + grant("/") + if libc.prctl(ctypes.c_int(PR_SET_NO_NEW_PRIVS), ctypes.c_ulong(1), ctypes.c_ulong(0), ctypes.c_ulong(0), ctypes.c_ulong(0)) != 0: + raise RuntimeError("prctl(PR_SET_NO_NEW_PRIVS) failed") + syscall(ctypes.c_long(SYS_LANDLOCK_RESTRICT_SELF), ctypes.c_long(ruleset), ctypes.c_long(0)) + os.close(ruleset) + # Fail closed unless this process (which becomes the command) is denied. + for path in hidden: + try: + if os.path.isdir(path): + os.listdir(path) + else: + os.close(os.open(path, os.O_RDONLY | os.O_CLOEXEC)) + except PermissionError: + continue + except OSError: + continue + raise RuntimeError("sandbox ineffective: %s is still readable" % path) + return abi + + +def main(): + required, hidden_inputs, unset, assignments, command = parse(sys.argv[1:]) + hidden = [] + for path in hidden_inputs: + # Keep paths that do not exist yet: their ancestors stay ungranted, so + # a later mkdir there is unreachable too. + real = os.path.realpath(path) + if real == "/": + fail("refusing to hide the filesystem root", 2) + hidden.append(real) + for name in unset: + os.environ.pop(name, None) + for name, value in assignments: + os.environ[name] = value + if hidden: + if not sys.platform.startswith("linux"): + if required: + fail("Landlock requires Linux") + else: + try: + restrict(hidden) + except (OSError, RuntimeError) as error: + if required: + fail(str(error)) + sys.stderr.write("agent_filesystem_sandbox: continuing without isolation: %s\n" % error) + try: + if os.path.isabs(command[0]): + os.execv(command[0], command) + os.execvp(command[0], command) + except OSError as error: + fail("cannot execute %s: %s" % (command[0], error), 127) + + +if __name__ == "__main__": + main() diff --git a/tests/runner-e2e/agent-filesystem-sandbox.test.ts b/tests/runner-e2e/agent-filesystem-sandbox.test.ts new file mode 100644 index 00000000000..233766681c2 --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.test.ts @@ -0,0 +1,234 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + HARNESS_PRIVATE_TEMPORARY_CHILDREN, + agentFilesystemSandboxConfig, + agentTemporaryParent, + defaultPlaywrightCacheDirectory, + playwrightCacheDirectory, + isRunnerdLaunch, + parseAgentFilesystemSandboxConfig, + probeAgentFilesystemSandbox, + pythonExecutable, + sandboxedInvocation, + agentFilesystemSandboxHelper, + wrapRunnerdSpawn, + wrapRunnerdLaunch, + gitStorePaths, + RUNNERD_LAUNCH_APIS, +} from "./agent-filesystem-sandbox.js"; +import { runnerMatrix } from "./catalog.js"; + +const repositoryRoot = "/work/paperclip"; +const temporaryRoot = "/var/tmp/paperclip-runner-e2e-abc"; +const cell = (id: string) => { + const execution = runnerMatrix.find((candidate) => candidate.id === id); + if (!execution) throw new Error(`missing ${id}`); + return execution; +}; +const gitStores = ["/work/paperclip/.git"]; +const opencodeLocal = cell("task-workspaces.runner-opencode.local.task-directory-repository-resume"); +const base = { + platform: "linux" as const, + env: { GITHUB_ACTIONS: "true", RUNNER_TEMP: "/work/_temp" }, + executions: [opencodeLocal], + repositoryRoot, + temporaryRoot, + privateDir: "/work/_temp/manual-private-evidence-x/capture", + evidenceRoot: "/work/_temp/manual-private-evidence-x", + installedRelease: false, + sharedTemporary: "/tmp", + gitStores, +}; + +describe("agent filesystem sandbox selection", () => { + it("hides the harness, the trusted copy and captured evidence from local native agents", () => { + expect(agentFilesystemSandboxConfig(base)).toEqual({ + hidden: [ + "/work/paperclip/tests", + "/work/paperclip/.git", + "/work/paperclip/.manual-private-trusted", + `${temporaryRoot}/artifacts-private`, + `${temporaryRoot}/control`, + `${temporaryRoot}/selected-private-inputs`, + `${temporaryRoot}/playwright-transform-cache`, + "/work/_temp/manual-private-evidence-x/capture", + "/work/_temp/manual-private-evidence-x", + "/work/_temp", + ], + required: true, + }); + }); + + it("hides the attempt's private children and Playwright cache when its root is outside the shared temporary directory", () => { + const config = agentFilesystemSandboxConfig(base); + expect(config?.hidden).toContain(playwrightCacheDirectory(temporaryRoot)); + expect(config?.hidden).not.toContain(temporaryRoot); + expect(HARNESS_PRIVATE_TEMPORARY_CHILDREN).toContain("playwright-transform-cache"); + expect(defaultPlaywrightCacheDirectory("/tmp")).toMatch(/^\/tmp\/playwright-transform-cache-/); + }); + + it("fails closed in CI when harness material sits where it cannot be hidden", () => { + const underShared = { ...base, temporaryRoot: "/tmp/paperclip-runner-e2e-abc", privateDir: "/tmp/paperclip-runner-e2e-abc/artifacts-private" }; + expect(() => agentFilesystemSandboxConfig(underShared)).toThrow(/agent_filesystem_sandbox_incomplete: .*artifacts-private/); + expect(() => agentFilesystemSandboxConfig({ ...base, repositoryRoot: "/tmp/checkout" })).toThrow(/\/tmp\/checkout\/tests/); + }); + + it("keeps a sandboxed attempt's root out of the shared temporary directory", () => { + const parent = agentTemporaryParent(true); + if (process.platform === "linux") expect(parent).toBe(realpathSync("/var/tmp")); + expect(agentTemporaryParent(false)).toBe(realpathSync(os.tmpdir())); + }); + + it("never hides the agent's own state or the checkout root, and only warns outside CI", () => { + const config = agentFilesystemSandboxConfig({ + ...base, + env: { RUNNER_TEMP: "/tmp" }, + temporaryRoot: "/tmp/paperclip-runner-e2e-abc", + privateDir: "/tmp/paperclip-runner-e2e-abc/artifacts-private", + evidenceRoot: "/work", + }); + expect(config).toEqual({ hidden: ["/work/paperclip/tests", "/work/paperclip/.git", "/work/paperclip/.manual-private-trusted"], required: false }); + }); + + it.each([ + ["macOS", { platform: "darwin" as const }], + ["an installed release", { installedRelease: true }], + ["remote-only cells", { executions: [cell("rolling-web-deployment.runner-opencode.daytona.recover-controller")] }], + ["legacy adapters", { executions: [cell("blocker-guidance.legacy-codex.local.human-authority")] }], + ["Codex's own mount sandbox", { executions: [opencodeLocal, cell("necessary-question.runner-codex.local.necessary-free-text")] }], + ])("does not apply to %s", (_label, override) => { + expect(agentFilesystemSandboxConfig({ ...base, ...override })).toBeNull(); + }); + + it("round-trips through the server environment and rejects malformed values", () => { + const config = agentFilesystemSandboxConfig(base)!; + expect(parseAgentFilesystemSandboxConfig(JSON.stringify(config))).toEqual(config); + expect(parseAgentFilesystemSandboxConfig(undefined)).toBeNull(); + expect(() => parseAgentFilesystemSandboxConfig(JSON.stringify({ hidden: ["relative"], required: true }))).toThrow(/malformed/); + }); +}); + +describe("runnerd launch rewriting", () => { + it("wraps only the absolute runnerd executable and execs it with the original argv", () => { + const runnerd = "/work/paperclip/packages/paperclip-runner/runner/target/debug/paperclip-runnerd"; + expect(isRunnerdLaunch(runnerd)).toBe(true); + expect(isRunnerdLaunch("paperclip-runnerd")).toBe(true); + expect(isRunnerdLaunch("/usr/bin/git")).toBe(false); + const invocation = sandboxedInvocation({ hidden: ["/work/paperclip/tests"], required: true }, "/usr/bin/python3", runnerd, ["--run-id", "r"], {}); + expect(invocation).toEqual({ + command: "/usr/bin/python3", + args: ["-I", "-S", agentFilesystemSandboxHelper, "--required", "--unset", "LC_CTYPE", "--hide", "/work/paperclip/tests", "--", runnerd, "--run-id", "r"], + }); + expect(sandboxedInvocation({ hidden: [], required: false }, "p", runnerd, [], { LC_CTYPE: "C" }).args).toContain("LC_CTYPE=C"); + }); + + it("rewrites runnerd launches through the wrapped spawn and passes other launches through", () => { + const calls: unknown[][] = []; + const fake = ((...args: unknown[]) => { calls.push(args); return { pid: 7 }; }) as unknown as typeof import("node:child_process").spawn; + const spawn = wrapRunnerdSpawn(fake, { hidden: ["/work/paperclip/tests"], required: true }, "/usr/bin/python3"); + const runnerd = "/work/paperclip/packages/paperclip-runner/runner/target/debug/paperclip-runnerd"; + spawn(runnerd, ["--run-id", "r"], { cwd: "/work", env: { LC_CTYPE: "C.UTF-8" }, detached: true }); + spawn("/usr/bin/git", ["status"], { cwd: "/work" }); + expect(calls[0]).toEqual(["/usr/bin/python3", expect.arrayContaining(["--hide", "/work/paperclip/tests", "--", runnerd, "--run-id", "r"]), + { cwd: "/work", env: { LC_CTYPE: "C.UTF-8" }, detached: true }]); + expect(calls[1]).toEqual(["/usr/bin/git", ["status"], { cwd: "/work" }]); + }); + + it("rewrites runnerd for every direct launch API and refuses a shell launch", () => { + const calls: unknown[][] = []; + const fake = (...args: unknown[]) => { calls.push(args); return { pid: 9 }; }; + const config = { hidden: ["/work/paperclip/tests"], required: true }; + const callback = () => undefined; + wrapRunnerdLaunch(fake, config, "/usr/bin/python3", "execFile")("paperclip-runnerd", ["--x"], { cwd: "/w" }, callback); + wrapRunnerdLaunch(fake, config, "/usr/bin/python3", "spawnSync")("/r/paperclip-runnerd", { cwd: "/w" }); + expect(calls[0]).toEqual(["/usr/bin/python3", expect.arrayContaining(["--", "paperclip-runnerd", "--x"]), { cwd: "/w" }, callback]); + expect(calls[1]).toEqual(["/usr/bin/python3", expect.arrayContaining(["--", "/r/paperclip-runnerd"]), { cwd: "/w" }]); + expect(() => wrapRunnerdLaunch(fake, config, "/usr/bin/python3")("/r/paperclip-runnerd", [], { shell: true })).toThrow(/refusing/); + expect(RUNNERD_LAUNCH_APIS).toEqual(["spawn", "spawnSync", "execFile", "execFileSync"]); + }); + + it("keeps promisify(execFile) resolving { stdout, stderr } after wrapping", async () => { + const { execFile } = await import("node:child_process"); + const { promisify } = await import("node:util"); + const wrapped = wrapRunnerdLaunch(execFile, { hidden: ["/nowhere"], required: false }, "/usr/bin/python3", "execFile"); + const result = await promisify(wrapped)(process.execPath, ["-e", "process.stdout.write('ok')"]); + expect(result).toEqual({ stdout: "ok", stderr: "" }); + }); + + it("hides a linked worktree's gitdir and the shared object store", async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "agent-sandbox-git-"))); + try { + const common = path.join(root, "main", ".git"); + const gitDir = path.join(common, "worktrees", "wt"); + await mkdir(gitDir, { recursive: true }); + await writeFile(path.join(gitDir, "commondir"), "../..\n"); + const checkout = path.join(root, "wt"); + await mkdir(checkout); + await writeFile(path.join(checkout, ".git"), `gitdir: ${gitDir}\n`); + expect(gitStorePaths(checkout)).toEqual([path.join(checkout, ".git"), gitDir, common]); + expect(gitStorePaths(path.join(root, "main"))).toEqual([common]); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("installs the sandbox in the E2E server entry before the product starts", async () => { + const entry = await readFile(path.join(import.meta.dirname, "server-entry.ts"), "utf8"); + const install = entry.indexOf("installAgentFilesystemSandbox("); + expect(install).toBeGreaterThan(0); + expect(install).toBeLessThan(entry.indexOf('await import("../../cli/src/index.js")')); + }); +}); + +const landlock = process.platform === "linux" && (() => { + const probe = mkdtempSync(path.join(os.tmpdir(), "agent-sandbox-abi-")); + try { + execFileSync(pythonExecutable(), ["-I", "-S", agentFilesystemSandboxHelper, "--required", "--hide", probe, "--", "/bin/true"]); + return true; + } catch { + return false; + } finally { + rmSync(probe, { recursive: true, force: true }); + } +})(); + +describe.skipIf(!landlock)("Landlock agent sandbox on this host", () => { + const roots: string[] = []; + afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); + + it("denies a sandboxed process the harness while keeping the product and temporary files usable", async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "agent-sandbox-"))); + roots.push(root); + const repository = path.join(root, "checkout"); + await mkdir(path.join(repository, "tests", "runner-e2e", "chat-scenarios"), { recursive: true }); + await mkdir(path.join(repository, "packages", "paperclip-runner"), { recursive: true }); + await mkdir(path.join(root, "scratch")); + await writeFile(path.join(repository, "tests", "runner-e2e", "agent-filesystem-sandbox.ts"), "grader"); + await writeFile(path.join(repository, "tests", "runner-e2e", "chat-scenarios", "access.yaml"), "expected_outcome: secret"); + await writeFile(path.join(repository, "packages", "paperclip-runner", "package.json"), "{}"); + const attempt = path.join(root, "attempt"); + await mkdir(path.join(attempt, "artifacts-private"), { recursive: true }); + await mkdir(path.join(attempt, "workspace")); + await writeFile(path.join(attempt, "artifacts-private", "server.log"), "secret"); + const config = { hidden: [path.join(repository, "tests"), path.join(attempt, "artifacts-private"), path.join(attempt, "control")], required: true }; + // The fixture lives under the shared temporary directory, so point mktemp at a granted sibling. + const probe = probeAgentFilesystemSandbox(config, repository, pythonExecutable(), { ...process.env, TMPDIR: path.join(root, "scratch") }, path.join(root, "no-playwright-cache")); + expect(probe.failures).toEqual([]); + expect(probe.result).toMatchObject({ playwrightDefaultCache: "denied", harnessFile: "denied", harnessDirectory: "denied", productFile: "readable", temporaryWrite: "allowed" }); + // The exact read from the reported run: grep over the checkout's scenarios. + const invocation = sandboxedInvocation(config, pythonExecutable(), "/bin/sh", ["-c", `grep -rn expected_outcome ${repository}/tests; echo "exit=$?"`], process.env); + const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8" }); + expect(output).not.toContain("secret"); + expect(output).toMatch(/exit=2/); + const attemptCheck = sandboxedInvocation(config, pythonExecutable(), "/bin/sh", ["-c", `cat ${attempt}/artifacts-private/server.log; ls ${attempt}/workspace >/dev/null && echo workspace=ok; mkdir ${attempt}/control 2>/dev/null || echo control=blocked`], process.env); + const attemptOutput = execFileSync(attemptCheck.command, attemptCheck.args, { encoding: "utf8" }); + expect(attemptOutput).not.toContain("secret"); + expect(attemptOutput).toContain("workspace=ok"); + expect(attemptOutput).toContain("control=blocked"); + }); +}); diff --git a/tests/runner-e2e/agent-filesystem-sandbox.ts b/tests/runner-e2e/agent-filesystem-sandbox.ts new file mode 100644 index 00000000000..0822890d495 --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.ts @@ -0,0 +1,310 @@ +import childProcess, { execFileSync, type SpawnOptions } from "node:child_process"; +import { accessSync, constants, existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import os from "node:os"; +import { promisify } from "node:util"; +import path from "node:path"; +import { requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; +import type { MatrixExecution } from "./types.js"; + +/** + * The agent under test must not read the eval: its scenarios, fixtures, + * graders or captured evidence. A local cell runs the product from the same + * checkout and as the same OS user as the harness, so file modes cannot + * separate them. The server launches every native agent through runnerd; the + * E2E server entry starts runnerd inside a Linux Landlock domain that hides + * these paths from runnerd and all of its descendants (provider, tools, shell). + * The product server itself stays unrestricted because it loads its E2E entry + * from the harness. + */ +export const AGENT_FILESYSTEM_SANDBOX_ENV = "PAPERCLIP_RUNNER_E2E_AGENT_FS_SANDBOX"; +export const RUNNERD_EXECUTABLE_NAME = "paperclip-runnerd"; + +export interface AgentFilesystemSandboxConfig { + hidden: string[]; + required: boolean; +} + +function within(candidate: string, root: string) { + return candidate === root || candidate.startsWith(`${root}${path.sep}`); +} + +/** Harness-private directories inside the attempt's temporary root. */ +export const HARNESS_PRIVATE_TEMPORARY_CHILDREN = [ + "artifacts-private", + "control", + "selected-private-inputs", + // Playwright's compiled copy of the harness and graders, with source maps. + "playwright-transform-cache", +] as const; + +/** Playwright otherwise caches compiled harness code in the shared temporary directory. */ +export function playwrightCacheDirectory(temporaryRoot: string) { + return path.join(temporaryRoot, "playwright-transform-cache"); +} + +/** Playwright's default cache location for this user, which the launcher removes. */ +export function defaultPlaywrightCacheDirectory(temporaryDirectory = os.tmpdir()) { + return path.join(temporaryDirectory, `playwright-transform-cache-${process.geteuid?.() ?? ""}`); +} + +export function agentFilesystemSandboxApplies(input: { + platform: NodeJS.Platform; + executions: readonly MatrixExecution[]; + installedRelease: boolean; +}) { + if (input.platform !== "linux" || input.installedRelease) return false; + if (!input.executions.some((execution) => execution.environment.id === "local" && execution.profile.generation === "native")) return false; + // Codex's own bubblewrap sandbox mounts, which Landlock forbids. Those cells + // keep their existing isolation until Codex permission profiles hide the eval. + return !input.executions.some(requiresCodexCiSandbox); +} + +/** + * Landlock can only hide a path by leaving its ancestors ungranted, and an + * ungranted directory accepts no new entries. Hiding the attempt's private + * children of a root under /tmp would stop agents creating files in /tmp, so a + * sandboxed attempt keeps its root in /var/tmp instead. + */ +export function agentTemporaryParent(applies: boolean, sharedTemporary = os.tmpdir()) { + if (applies) { + try { + accessSync("/var/tmp", constants.W_OK | constants.X_OK); + return realpathSync("/var/tmp"); + } catch { + // Fall back to the shared directory; its private children stay visible. + } + } + return realpathSync(sharedTemporary); +} + +/** Selects the paths hidden from local native agents, or null when not applicable. */ +export function agentFilesystemSandboxConfig(input: { + platform: NodeJS.Platform; + env: NodeJS.ProcessEnv; + executions: readonly MatrixExecution[]; + repositoryRoot: string; + temporaryRoot: string; + privateDir: string; + evidenceRoot?: string; + installedRelease: boolean; + sharedTemporary?: string; + gitStores?: readonly string[]; +}): AgentFilesystemSandboxConfig | null { + if (!agentFilesystemSandboxApplies(input)) return null; + const sharedTemporary = path.resolve(input.sharedTemporary ?? (input.env.TMPDIR?.trim() || os.tmpdir())); + const required = input.env.GITHUB_ACTIONS === "true"; + // Harness material that must be hidden. Git holds the committed scenarios + // and graders, so `git show HEAD:tests/...` would otherwise bypass `tests/`. + const essential = [ + path.join(input.repositoryRoot, "tests"), + ...(input.gitStores ?? gitStorePaths(input.repositoryRoot)), + path.join(input.repositoryRoot, ".manual-private-trusted"), + ...HARNESS_PRIVATE_TEMPORARY_CHILDREN.map((child) => path.join(input.temporaryRoot, child)), + input.privateDir, + ]; + const optional = [input.evidenceRoot, input.env.RUNNER_TEMP]; + const hidden: string[] = []; + const exposed: string[] = []; + for (const [candidate, mustHide] of [...essential.map((value) => [value, true] as const), ...optional.map((value) => [value, false] as const)]) { + const value = candidate?.trim(); + if (!value || !path.isAbsolute(value)) continue; + const resolved = path.resolve(value); + // Never hide a path that holds the agent's own state, or one beneath the + // shared temporary directory: its ancestors would lose new entries. + if (within(input.temporaryRoot, resolved) || within(input.repositoryRoot, resolved) + || within(resolved, sharedTemporary) || within(sharedTemporary, resolved)) { + if (mustHide) exposed.push(resolved); + continue; + } + if (!hidden.includes(resolved)) hidden.push(resolved); + } + if (exposed.length) { + const message = `agent_filesystem_sandbox_incomplete: cannot hide ${exposed.join(", ")}; keep the checkout and attempt root outside ${sharedTemporary}`; + // A required sandbox fails closed rather than run an eval agents can read. + if (required) throw new Error(message); + console.warn(`[agent-filesystem-sandbox] ${message}`); + } + return hidden.length ? { hidden, required } : null; +} + +export function parseAgentFilesystemSandboxConfig(value: string | undefined): AgentFilesystemSandboxConfig | null { + if (!value?.trim()) return null; + const parsed = JSON.parse(value) as Partial; + if (!Array.isArray(parsed.hidden) || parsed.hidden.some((entry) => typeof entry !== "string" || !path.isAbsolute(entry)) || typeof parsed.required !== "boolean") { + throw new Error(`${AGENT_FILESYSTEM_SANDBOX_ENV} is malformed`); + } + return { hidden: parsed.hidden, required: parsed.required }; +} + +export const agentFilesystemSandboxHelper = path.join(import.meta.dirname, "agent-filesystem-sandbox.py"); + +/** Rewrites one launch so the helper applies the domain and then execs it unchanged. */ +export function sandboxedInvocation( + config: AgentFilesystemSandboxConfig, + python: string, + command: string, + args: readonly string[], + env: NodeJS.ProcessEnv, +): { command: string; args: string[] } { + // Python may coerce a C locale by exporting LC_CTYPE; restore the caller's value. + const locale = env.LC_CTYPE === undefined ? ["--unset", "LC_CTYPE"] : ["--set", `LC_CTYPE=${env.LC_CTYPE}`]; + return { + command: python, + args: [ + "-I", "-S", agentFilesystemSandboxHelper, + ...(config.required ? ["--required"] : []), + ...locale, + ...config.hidden.flatMap((hidden) => ["--hide", hidden]), + "--", command, ...args, + ], + }; +} + +/** Matches runnerd by executable name, whether absolute or resolved through PATH. */ +export function isRunnerdLaunch(command: unknown) { + return typeof command === "string" && path.basename(command) === RUNNERD_EXECUTABLE_NAME; +} + +/** + * The checkout's Git store holds the committed scenarios and graders. A linked + * worktree's `.git` is a file that points into the main repository, so hide the + * worktree's gitdir and the shared common directory as well. + */ +export function gitStorePaths(repositoryRoot: string): string[] { + const dotGit = path.join(repositoryRoot, ".git"); + const stores = [dotGit]; + try { + if (!statSync(dotGit).isFile()) return stores; + const pointer = /^gitdir:\s*(.+)$/m.exec(readFileSync(dotGit, "utf8"))?.[1]?.trim(); + if (!pointer) return stores; + const gitDir = path.resolve(repositoryRoot, pointer); + stores.push(gitDir); + const common = readFileSync(path.join(gitDir, "commondir"), "utf8").trim(); + if (common) stores.push(path.resolve(gitDir, common)); + } catch { + // A missing .git or commondir leaves only the paths found so far. + } + return stores; +} + +export function pythonExecutable() { + for (const candidate of ["/usr/bin/python3", "/usr/local/bin/python3"]) { + if (existsSync(candidate)) return candidate; + } + return "python3"; +} + +/** Proves, before any agent starts, that a sandboxed process cannot read the eval. */ +export function probeAgentFilesystemSandbox( + config: AgentFilesystemSandboxConfig, + repositoryRoot: string, + python = pythonExecutable(), + env: NodeJS.ProcessEnv = process.env, + playwrightCache = defaultPlaywrightCacheDirectory(), +) { + const harness = path.join(repositoryRoot, "tests", "runner-e2e"); + const script = [ + 'out=""; add() { out="$out$1=$2;"; }', + 'if cat "$1" >/dev/null 2>&1; then add harnessFile readable; else add harnessFile denied; fi', + 'if ls "$2" >/dev/null 2>&1; then add harnessDirectory listable; else add harnessDirectory denied; fi', + 'if test -e "$2"; then add harnessDirectoryStat visible; else add harnessDirectoryStat absent; fi', + 'if cat "$3" >/dev/null 2>&1; then add productFile readable; else add productFile denied; fi', + 'probe=$(mktemp) && rm -f "$probe" && add temporaryWrite allowed || add temporaryWrite denied', + 'if ls "$4" >/dev/null 2>&1; then add playwrightDefaultCache listable; else add playwrightDefaultCache denied; fi', + 'shift 4; for hidden in "$@"; do if ls "$hidden" >/dev/null 2>&1; then add "hidden:$hidden" listable; else add "hidden:$hidden" denied; fi; done', + 'printf "%s\\n" "$out"', + ].join("\n"); + const invocation = sandboxedInvocation(config, python, "/bin/sh", [ + "-c", script, "sh", + path.join(harness, "agent-filesystem-sandbox.ts"), + harness, + path.join(repositoryRoot, "packages", "paperclip-runner", "package.json"), + playwrightCache, + ...config.hidden, + ], env); + const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8", cwd: "/", env, timeout: 30_000 }); + const result = Object.fromEntries(output.trim().split(";").filter(Boolean).map((entry) => { + const separator = entry.lastIndexOf("="); + return [entry.slice(0, separator), entry.slice(separator + 1)]; + })); + const failures = [ + result.harnessFile !== "denied" && "harness file is readable", + result.harnessDirectory !== "denied" && "harness directory is listable", + result.productFile !== "readable" && "product package is unreadable", + result.temporaryWrite !== "allowed" && "temporary directory is not writable", + result.playwrightDefaultCache !== "denied" && "Playwright's default transform cache is listable", + ...config.hidden.map((hidden) => result[`hidden:${hidden}`] === "listable" && `${hidden} is listable`), + ].filter((failure): failure is string => Boolean(failure)); + return { result, failures }; +} + +/** + * Wraps runnerd launches made by this process. Runs only in the E2E server + * entry; the wrapper execs runnerd, so its PID, argv, cwd and environment are + * the ones the product requested. + */ +export function installAgentFilesystemSandbox(repositoryRoot: string, env: NodeJS.ProcessEnv = process.env) { + const config = parseAgentFilesystemSandboxConfig(env[AGENT_FILESYSTEM_SANDBOX_ENV]); + delete env[AGENT_FILESYSTEM_SANDBOX_ENV]; + if (!config) return null; + const python = pythonExecutable(); + let probe: ReturnType; + try { + probe = probeAgentFilesystemSandbox(config, repositoryRoot, python); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (config.required) throw new Error(`agent_filesystem_sandbox_unavailable: ${message}`); + console.warn(`[agent-filesystem-sandbox] disabled: ${message}`); + return null; + } + console.log(`[agent-filesystem-sandbox] probe ${JSON.stringify({ hidden: config.hidden, required: config.required, ...probe })}`); + if (probe.failures.length) { + if (config.required) throw new Error(`agent_filesystem_sandbox_ineffective: ${probe.failures.join("; ")}`); + console.warn(`[agent-filesystem-sandbox] disabled: ${probe.failures.join("; ")}`); + return null; + } + const launchers = childProcess as unknown as Record unknown>; + for (const api of RUNNERD_LAUNCH_APIS) launchers[api] = wrapRunnerdLaunch(launchers[api], config, python, api); + syncBuiltinESMExports(); + return config; +} + +/** Every child_process API that can start an executable directly. */ +export const RUNNERD_LAUNCH_APIS = ["spawn", "spawnSync", "execFile", "execFileSync"] as const; +export type RunnerdLaunchApi = typeof RUNNERD_LAUNCH_APIS[number]; + +/** + * Wraps one launch API so runnerd starts inside the sandbox and other launches + * pass through. A runnerd launch the wrapper cannot rewrite fails closed. + */ +export function wrapRunnerdLaunch unknown>(original: T, config: AgentFilesystemSandboxConfig, python: string, api: RunnerdLaunchApi = "spawn", wrapCustom = true): T { + const wrapped = function (this: unknown, command: unknown, ...rest: unknown[]) { + if (!isRunnerdLaunch(command)) return Reflect.apply(original, this, [command, ...rest]); + const hasArgs = Array.isArray(rest[0]); + const args = hasArgs ? rest[0] as string[] : []; + const afterArgs = hasArgs ? rest.slice(1) : rest; + const options = afterArgs[0] && typeof afterArgs[0] === "object" ? afterArgs[0] as SpawnOptions : undefined; + if (options?.shell) throw new Error("agent_filesystem_sandbox: refusing to start runnerd through a shell, outside the sandbox"); + const invocation = sandboxedInvocation(config, python, command as string, args, options?.env ?? process.env); + const result = Reflect.apply(original, this, [invocation.command, invocation.args, ...afterArgs]); + const pid = (result as { pid?: number } | undefined)?.pid; + console.log(`[agent-filesystem-sandbox] runnerd launched inside the sandbox via ${api}${pid ? ` (pid ${pid})` : ""}`); + return result; + } as T; + // execFile's promisified form resolves { stdout, stderr } through this hook; + // keep it, wrapped the same way, so promisify(execFile) keeps its shape. + const custom = (original as unknown as Record)[promisify.custom]; + if (wrapCustom && typeof custom === "function") { + Object.defineProperty(wrapped, promisify.custom, { + value: wrapRunnerdLaunch(custom as (...args: any[]) => unknown, config, python, api, false), + configurable: true, + }); + } + return wrapped; +} + +/** Back-compatible name for the spawn wrapper. */ +export function wrapRunnerdSpawn(original: typeof childProcess.spawn, config: AgentFilesystemSandboxConfig, python: string) { + return wrapRunnerdLaunch(original, config, python, "spawn"); +} diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 09b6a158a38..523100dcfed 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -49,6 +49,7 @@ import { firstTaskTasks, onboardingApprovalHistoryTask } from "./first-task-case import { APPROVAL_HISTORY_SUITE, approvalHistoryDefinitionDigest } from "./first-task-approval-history.js"; import { chatTasks, chatHardeningTasks, chatStoryTasks, chatQualificationTasks, chatCompletionTasks } from "./chat-cases.js"; import { createHash } from "node:crypto"; +import { neutralFixtureSuffix } from "./fixture-names.js"; import { fileURLToPath } from "node:url"; import { createAgentSchema } from "../../packages/shared/src/validators/agent.js"; import { createEnvironmentSchema } from "../../packages/shared/src/validators/environment.js"; @@ -117,11 +118,11 @@ function commonAgent( additionalInstructions: readonly string[] = [], ) { return { - name: `Runner E2E ${fixtureId} ${input.executionId}`, + name: `Agent ${neutralFixtureSuffix(fixtureId, input.executionId)}`, role: "qa", - title: "Paid full-stack runner acceptance fixture", + title: "Operations agent", capabilities: - "Completes deterministic standard, planning, and ask-mode runner acceptance tasks.", + "Completes standard, planning, and ask-mode tasks for the company.", adapterType, ...(adapterType === "codex_local" ? { runner: "legacy" as const } : {}), adapterConfig, @@ -504,8 +505,8 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ expectedExecutionTarget: { kind: "local" }, buildEnvironment(input) { return { - name: `Runner E2E local ${input.executionId}`, - description: "Ephemeral local runner E2E environment", + name: `Local ${neutralFixtureSuffix(input.executionId)}`, + description: "Local execution environment", driver: "local", config: {}, envVars: {}, @@ -532,8 +533,8 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ ); } return { - name: `Runner E2E Daytona ${input.executionId}`, - description: "Ephemeral Daytona runner E2E environment", + name: `Daytona ${neutralFixtureSuffix(input.executionId)}`, + description: "Daytona sandbox environment", driver: "sandbox", config: { provider: "daytona", @@ -580,8 +581,8 @@ export const daytonaWarmEnvironment: EnvironmentFixture = { ); } return { - name: `Runner E2E Daytona warm ${input.executionId}`, - description: "Ephemeral reusable Daytona runner E2E environment", + name: `Daytona warm ${neutralFixtureSuffix(input.executionId)}`, + description: "Reusable Daytona sandbox environment", driver: "sandbox", config: { provider: "daytona", diff --git a/tests/runner-e2e/fixture-names.ts b/tests/runner-e2e/fixture-names.ts new file mode 100644 index 00000000000..b66b9a3f660 --- /dev/null +++ b/tests/runner-e2e/fixture-names.ts @@ -0,0 +1,13 @@ +import { createHash, randomBytes } from "node:crypto"; + +// Per-process salt: the public catalog ids cannot be hashed back to a case. +const salt = randomBytes(16); + +/** + * Names and descriptions an agent can read must not reveal the eval: no case + * ids, suite names, "E2E" or "test". A short salted digest keeps them unique + * and stable within one attempt, without being reversible from public ids. + */ +export function neutralFixtureSuffix(...parts: readonly string[]): string { + return createHash("sha256").update(salt).update(parts.join("\0")).digest("hex").slice(0, 8); +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 6f5f7d2ffef..457d7ecb9e4 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -14,6 +14,10 @@ import { createRunnerE2ETemporaryRoot } from "./server-config.js"; import { createExternalControllerDatabase } from "./external-controller-database.js"; import { createHash, randomBytes } from "node:crypto"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; +import { + AGENT_FILESYSTEM_SANDBOX_ENV, agentFilesystemSandboxApplies, agentFilesystemSandboxConfig, agentTemporaryParent, + defaultPlaywrightCacheDirectory, playwrightCacheDirectory, +} from "./agent-filesystem-sandbox.js"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; import { createRequire } from "node:module"; @@ -492,7 +496,8 @@ async function runAttempt(input: { const installedPlugin = await verifyInstalledDaytonaPlugin(process.env, executions); const startedAtMs = Date.now(); const sharedMemoryBaseline = snapshotDarwinSharedMemory(); - const temporaryParent = await realpath(os.tmpdir()); + const sandboxedAgents = agentFilesystemSandboxApplies({ platform: process.platform, executions, installedRelease: Boolean(installedCli) }); + const temporaryParent = agentTemporaryParent(sandboxedAgents); const temporaryRoot = await createRunnerE2ETemporaryRoot(temporaryParent); const privateDir = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "capture") : path.join(temporaryRoot, "artifacts-private"); const publishedResults: RunnerE2EResult[] = []; @@ -554,6 +559,12 @@ async function runAttempt(input: { executions, repositoryRoot, ); + const agentFilesystemSandbox = agentFilesystemSandboxConfig({ + platform: process.platform, env: process.env, executions, repositoryRoot, temporaryRoot, privateDir, + evidenceRoot: manualPrivateHandoff?.root, installedRelease: Boolean(installedCli), + }); + // Playwright's default cache holds compiled graders in the shared /tmp. + if (sandboxedAgents) await rm(defaultPlaywrightCacheDirectory(), { recursive: true, force: true }); const childEnv: NodeJS.ProcessEnv = { ...buildRunnerE2EProcessEnvironment(process.env, executions), PATH: providerPath, @@ -568,6 +579,8 @@ async function runAttempt(input: { PAPERCLIP_RUNNER_E2E_PORT: String(port), PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot, PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir, + [AGENT_FILESYSTEM_SANDBOX_ENV]: agentFilesystemSandbox ? JSON.stringify(agentFilesystemSandbox) : undefined, + PWTEST_CACHE_DIR: playwrightCacheDirectory(temporaryRoot), PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace, PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(privateDir, "server.log"), PAPERCLIP_RUNNER_BINARY: runnerBinary, diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 306da7fe9b1..feab9264cb2 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,4 +1,5 @@ import { isEvalRemoteOrigin } from "./eval-remote-gateway.js"; +import { neutralFixtureSuffix } from "./fixture-names.js"; import { isEverydayReliabilityCase } from "./everyday-reliability-cases.js"; import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js"; import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; @@ -217,8 +218,8 @@ export async function setupLiveFixtures(input: { id: "company", async setup() { const company = await api.post("/api/companies", { - name: `Runner E2E ${execution.id} ${input.executionNonce}`, - description: "Ephemeral paid full-stack runner acceptance fixture", + name: `Company ${neutralFixtureSuffix(execution.id, input.executionNonce)}`, + description: "Operations company", budgetMonthlyCents: execution.suite.id === CHAT_FEATURE_SUITE ? CHAT_FEATURE_BUDGET_CENTS : receiptSmoke ? CASE_RECEIPT_BUDGET_CENTS : ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id) @@ -250,10 +251,10 @@ export async function setupLiveFixtures(input: { const secret = await api.postSensitive( `/api/companies/${company.id}/secrets`, { - name: `Runner E2E ${credentialName} ${input.executionNonce}`, + name: `${credentialName} ${neutralFixtureSuffix(input.executionNonce)}`, key: credentialName, value: rawValue, - description: `Ephemeral credential for ${execution.id}`, + description: "Model provider credential", }, ); refs[credentialName] = { @@ -353,7 +354,7 @@ export async function setupLiveFixtures(input: { { provider, method: "api_key", - name: `Runner E2E account ${input.executionNonce}`, + name: `Account ${neutralFixtureSuffix(input.executionNonce)}`, ownership: budgetedGateway ? "shared" : "personal", ...(budgetedGateway ? { routing: { kind: "gateway", protocol: provider === "anthropic" ? "messages" : "responses", baseUrl: provider === "anthropic" ? gatewayUrl : `${gatewayUrl}/v1`, auth: "bearer", models: [{ id: execution.profile.model }] } } : {}), @@ -473,9 +474,9 @@ export async function setupLiveFixtures(input: { return api.post( `/api/companies/${company.id}/projects`, { - name: `Runner E2E workspace project ${input.executionNonce}`, + name: `Workspace project ${neutralFixtureSuffix(input.executionNonce)}`, description: - "Ephemeral project anchoring the fixture execution workspace and file copy-back", + "Project for the execution workspace and file copy-back", executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", diff --git a/tests/runner-e2e/server-entry.ts b/tests/runner-e2e/server-entry.ts index 46c68c88bce..1e206fd6c12 100644 --- a/tests/runner-e2e/server-entry.ts +++ b/tests/runner-e2e/server-entry.ts @@ -1,6 +1,8 @@ // This entrypoint is used only by isolated Runner E2E instances. Production // service code has no test flag, delay, altered prompt, or private test API. import { ServerResponse } from "node:http"; +import path from "node:path"; +import { installAgentFilesystemSandbox } from "./agent-filesystem-sandbox.js"; import { PaperclipRunnerToolAuthority } from "../../server/src/services/native-runtime/paperclip-runner-tool-authority.js"; import { canonicalDocumentIssueId, contextCommentGateSelected, holdCommittedDocumentResponse, isArmed } from "./context-comment-gate.js"; import { holdInteractionResponse } from "./interaction-response-gate.js"; @@ -81,4 +83,6 @@ if (contextCommentGate || completionBusyGate) { return Reflect.apply(end, this, args); } as typeof end; } +// Before the product starts any agent, hide the eval harness from runnerd and its descendants. +installAgentFilesystemSandbox(path.resolve(import.meta.dirname, "../..")); await import("../../cli/src/index.js");