From b8ccd434ac2980052cbdc601f20c8d1ff350943e Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 06:26:13 -0600 Subject: [PATCH] test(runner-e2e): hide the eval harness from local native agents A local cell runs the product from the same checkout and OS user as the harness. In run 42472276601 the OpenCode agent found the checkout through its PATH, then used bash ls/sed/grep (which OpenCode's external_directory policy does not gate) to read tests/runner-e2e/chat-scenarios/access.yaml and the case's grader. The E2E server entry now launches runnerd inside a Linux Landlock domain that hides tests/, the trusted manual-private copy, the private evidence directory and RUNNER_TEMP from runnerd and every descendant. A setup probe proves the denial before any agent starts and fails closed in CI. Codex cells keep their own bubblewrap sandbox, which cannot run under Landlock. Fixture companies, agents, environments, secrets and projects get neutral names with a digest suffix instead of case ids and 'Runner E2E'. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/agent-filesystem-sandbox.py | 218 ++++++++++++++++++ .../agent-filesystem-sandbox.test.ts | 139 +++++++++++ tests/runner-e2e/agent-filesystem-sandbox.ts | 194 ++++++++++++++++ tests/runner-e2e/catalog.ts | 19 +- tests/runner-e2e/fixture-names.ts | 9 + tests/runner-e2e/launch.ts | 6 + tests/runner-e2e/live-fixtures.ts | 15 +- tests/runner-e2e/server-entry.ts | 4 + 8 files changed, 588 insertions(+), 16 deletions(-) create mode 100644 tests/runner-e2e/agent-filesystem-sandbox.py create mode 100644 tests/runner-e2e/agent-filesystem-sandbox.test.ts create mode 100644 tests/runner-e2e/agent-filesystem-sandbox.ts create mode 100644 tests/runner-e2e/fixture-names.ts diff --git a/tests/runner-e2e/agent-filesystem-sandbox.py b/tests/runner-e2e/agent-filesystem-sandbox.py new file mode 100644 index 00000000000..46ad5175a7d --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.py @@ -0,0 +1,218 @@ +"""Execute a command whose process tree cannot read the eval harness. + +Runner E2E starts the product from a repository checkout that also holds the +scenarios, fixtures and graders. This helper applies a Linux Landlock domain +that grants filesystem access everywhere except the given hidden paths, then +replaces itself with the command (same PID, argv and environment). Every +descendant inherits the domain, including an agent's shell commands. + +Usage: + agent-filesystem-sandbox.py [--required] [--unset NAME]... [--set NAME=VALUE]... + --hide PATH [--hide PATH]... -- COMMAND [ARG]... + +Without Landlock, --required exits 125 instead of running the command. +""" + +import ctypes +import os +import stat +import struct +import sys + +SYS_LANDLOCK_CREATE_RULESET = 444 +SYS_LANDLOCK_ADD_RULE = 445 +SYS_LANDLOCK_RESTRICT_SELF = 446 +LANDLOCK_CREATE_RULESET_VERSION = 1 +LANDLOCK_RULE_PATH_BENEATH = 1 +PR_SET_NO_NEW_PRIVS = 38 + +FILE_RIGHTS = (1 << 0) | (1 << 1) | (1 << 2) # execute, write, read +ABI_1_RIGHTS = (1 << 13) - 1 +REFER = 1 << 13 +TRUNCATE = 1 << 14 +IOCTL_DEV = 1 << 15 + + +class RulesetAttr(ctypes.Structure): + _fields_ = [("handled_access_fs", ctypes.c_uint64)] + + +def fail(message, code=125): + sys.stderr.write("agent_filesystem_sandbox: %s\n" % message) + sys.stderr.flush() + os._exit(code) + + +def parse(argv): + required = False + hidden = [] + unset = [] + assignments = [] + index = 0 + while index < len(argv): + token = argv[index] + if token == "--": + command = argv[index + 1:] + if not command: + fail("missing command", 2) + return required, hidden, unset, assignments, command + if token == "--required": + required = True + elif token in ("--hide", "--unset", "--set") and index + 1 < len(argv): + index += 1 + value = argv[index] + if token == "--hide": + if not os.path.isabs(value): + fail("hidden path must be absolute: %s" % value, 2) + hidden.append(value) + elif token == "--unset": + unset.append(value) + else: + name, separator, assigned = value.partition("=") + if not separator or not name: + fail("--set requires NAME=VALUE", 2) + assignments.append((name, assigned)) + else: + fail("unexpected argument: %s" % token, 2) + index += 1 + fail("missing -- before the command", 2) + + +def within(candidate, root): + return candidate == root or candidate.startswith(root.rstrip("/") + "/") + + +def landlock(): + libc = ctypes.CDLL(None, use_errno=True) + libc.syscall.restype = ctypes.c_long + + def syscall(*args): + result = libc.syscall(*args) + if result < 0: + raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno())) + return result + + return libc, syscall + + +def restrict(hidden): + libc, syscall = landlock() + try: + abi = syscall( + ctypes.c_long(SYS_LANDLOCK_CREATE_RULESET), + ctypes.c_void_p(None), + ctypes.c_long(0), + ctypes.c_long(LANDLOCK_CREATE_RULESET_VERSION), + ) + except OSError as error: + raise RuntimeError("Landlock is unavailable on this kernel (%s)" % error) + handled = ABI_1_RIGHTS + if abi >= 2: + handled |= REFER + if abi >= 3: + handled |= TRUNCATE + if abi >= 5: + handled |= IOCTL_DEV + attr = RulesetAttr(handled) + ruleset = syscall( + ctypes.c_long(SYS_LANDLOCK_CREATE_RULESET), + ctypes.byref(attr), + ctypes.c_long(ctypes.sizeof(attr)), + ctypes.c_long(0), + ) + + def allow(path): + try: + descriptor = os.open(path, os.O_PATH | os.O_CLOEXEC) + except OSError: + return + try: + mode = os.fstat(descriptor).st_mode + access = handled if stat.S_ISDIR(mode) else handled & (FILE_RIGHTS | TRUNCATE | IOCTL_DEV) + # struct landlock_path_beneath_attr is packed: u64 allowed_access, s32 parent_fd. + rule = ctypes.create_string_buffer(struct.pack("=Qi", access, descriptor), 12) + syscall( + ctypes.c_long(SYS_LANDLOCK_ADD_RULE), + ctypes.c_long(ruleset), + ctypes.c_long(LANDLOCK_RULE_PATH_BENEATH), + rule, + ctypes.c_long(0), + ) + finally: + os.close(descriptor) + + # Landlock only grants. Grant every entry except the hidden paths, walking + # down only through their ancestors; ancestors themselves get no rule. + def grant(directory): + try: + names = sorted(os.listdir(directory)) + except OSError: + return + for name in names: + child = os.path.join(directory, name) + if any(within(child, path) for path in hidden): + continue + try: + is_link = os.path.islink(child) + real = os.path.realpath(child) + except OSError: + continue + if any(within(real, path) or within(path, real) for path in hidden): + if not is_link and any(within(path, child) for path in hidden): + grant(child) + # A link into, or onto an ancestor of, a hidden path stays ungranted. + continue + allow(child) + + grant("/") + if libc.prctl(ctypes.c_int(PR_SET_NO_NEW_PRIVS), ctypes.c_ulong(1), ctypes.c_ulong(0), ctypes.c_ulong(0), ctypes.c_ulong(0)) != 0: + raise RuntimeError("prctl(PR_SET_NO_NEW_PRIVS) failed") + syscall(ctypes.c_long(SYS_LANDLOCK_RESTRICT_SELF), ctypes.c_long(ruleset), ctypes.c_long(0)) + os.close(ruleset) + # Fail closed unless this process (which becomes the command) is denied. + for path in hidden: + try: + if os.path.isdir(path): + os.listdir(path) + else: + os.close(os.open(path, os.O_RDONLY | os.O_CLOEXEC)) + except PermissionError: + continue + except OSError: + continue + raise RuntimeError("sandbox ineffective: %s is still readable" % path) + return abi + + +def main(): + required, hidden_inputs, unset, assignments, command = parse(sys.argv[1:]) + hidden = [] + for path in hidden_inputs: + if os.path.lexists(path): + real = os.path.realpath(path) + if real == "/": + fail("refusing to hide the filesystem root", 2) + hidden.append(real) + for name in unset: + os.environ.pop(name, None) + for name, value in assignments: + os.environ[name] = value + if hidden: + if not sys.platform.startswith("linux"): + if required: + fail("Landlock requires Linux") + else: + try: + restrict(hidden) + except (OSError, RuntimeError) as error: + if required: + fail(str(error)) + sys.stderr.write("agent_filesystem_sandbox: continuing without isolation: %s\n" % error) + try: + os.execv(command[0], command) + except OSError as error: + fail("cannot execute %s: %s" % (command[0], error), 127) + + +if __name__ == "__main__": + main() diff --git a/tests/runner-e2e/agent-filesystem-sandbox.test.ts b/tests/runner-e2e/agent-filesystem-sandbox.test.ts new file mode 100644 index 00000000000..f7942778275 --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.test.ts @@ -0,0 +1,139 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + agentFilesystemSandboxConfig, + isRunnerdLaunch, + parseAgentFilesystemSandboxConfig, + probeAgentFilesystemSandbox, + pythonExecutable, + sandboxedInvocation, + agentFilesystemSandboxHelper, +} from "./agent-filesystem-sandbox.js"; +import { runnerMatrix } from "./catalog.js"; + +const repositoryRoot = "/work/paperclip"; +const temporaryRoot = path.join(os.tmpdir(), "paperclip-runner-e2e-abc"); +const cell = (id: string) => { + const execution = runnerMatrix.find((candidate) => candidate.id === id); + if (!execution) throw new Error(`missing ${id}`); + return execution; +}; +const opencodeLocal = cell("task-workspaces.runner-opencode.local.task-directory-repository-resume"); +const base = { + platform: "linux" as const, + env: { GITHUB_ACTIONS: "true", RUNNER_TEMP: "/work/_temp" }, + executions: [opencodeLocal], + repositoryRoot, + temporaryRoot, + privateDir: "/work/_temp/manual-private-evidence-x/capture", + evidenceRoot: "/work/_temp/manual-private-evidence-x", + installedRelease: false, +}; + +describe("agent filesystem sandbox selection", () => { + it("hides the harness, the trusted copy and captured evidence from local native agents", () => { + expect(agentFilesystemSandboxConfig(base)).toEqual({ + hidden: [ + "/work/paperclip/tests", + "/work/paperclip/.manual-private-trusted", + "/work/_temp/manual-private-evidence-x/capture", + "/work/_temp/manual-private-evidence-x", + "/work/_temp", + ], + required: true, + }); + }); + + it("never hides the agent's own state, the checkout root or the shared temporary directory", () => { + const config = agentFilesystemSandboxConfig({ + ...base, + env: { RUNNER_TEMP: os.tmpdir() }, + privateDir: path.join(temporaryRoot, "artifacts-private"), + evidenceRoot: "/work", + }); + expect(config).toEqual({ hidden: ["/work/paperclip/tests", "/work/paperclip/.manual-private-trusted"], required: false }); + }); + + it.each([ + ["macOS", { platform: "darwin" as const }], + ["an installed release", { installedRelease: true }], + ["remote-only cells", { executions: [cell("rolling-web-deployment.runner-opencode.daytona.recover-controller")] }], + ["legacy adapters", { executions: [cell("blocker-guidance.legacy-codex.local.human-authority")] }], + ["Codex's own mount sandbox", { executions: [opencodeLocal, cell("necessary-question.runner-codex.local.necessary-free-text")] }], + ])("does not apply to %s", (_label, override) => { + expect(agentFilesystemSandboxConfig({ ...base, ...override })).toBeNull(); + }); + + it("round-trips through the server environment and rejects malformed values", () => { + const config = agentFilesystemSandboxConfig(base)!; + expect(parseAgentFilesystemSandboxConfig(JSON.stringify(config))).toEqual(config); + expect(parseAgentFilesystemSandboxConfig(undefined)).toBeNull(); + expect(() => parseAgentFilesystemSandboxConfig(JSON.stringify({ hidden: ["relative"], required: true }))).toThrow(/malformed/); + }); +}); + +describe("runnerd launch rewriting", () => { + it("wraps only the absolute runnerd executable and execs it with the original argv", () => { + const runnerd = "/work/paperclip/packages/paperclip-runner/runner/target/debug/paperclip-runnerd"; + expect(isRunnerdLaunch(runnerd, { cwd: "/x" })).toBe(true); + expect(isRunnerdLaunch(runnerd, { shell: true })).toBe(false); + expect(isRunnerdLaunch("paperclip-runnerd", {})).toBe(false); + expect(isRunnerdLaunch("/usr/bin/git", {})).toBe(false); + const invocation = sandboxedInvocation({ hidden: ["/work/paperclip/tests"], required: true }, "/usr/bin/python3", runnerd, ["--run-id", "r"], {}); + expect(invocation).toEqual({ + command: "/usr/bin/python3", + args: ["-I", "-S", agentFilesystemSandboxHelper, "--required", "--unset", "LC_CTYPE", "--hide", "/work/paperclip/tests", "--", runnerd, "--run-id", "r"], + }); + expect(sandboxedInvocation({ hidden: [], required: false }, "p", runnerd, [], { LC_CTYPE: "C" }).args).toContain("LC_CTYPE=C"); + }); + + it("installs the sandbox in the E2E server entry before the product starts", async () => { + const entry = await readFile(path.join(import.meta.dirname, "server-entry.ts"), "utf8"); + const install = entry.indexOf("installAgentFilesystemSandbox("); + expect(install).toBeGreaterThan(0); + expect(install).toBeLessThan(entry.indexOf('await import("../../cli/src/index.js")')); + }); +}); + +const landlock = process.platform === "linux" && (() => { + const probe = mkdtempSync(path.join(os.tmpdir(), "agent-sandbox-abi-")); + try { + execFileSync(pythonExecutable(), ["-I", "-S", agentFilesystemSandboxHelper, "--required", "--hide", probe, "--", "/bin/true"]); + return true; + } catch { + return false; + } finally { + rmSync(probe, { recursive: true, force: true }); + } +})(); + +describe.skipIf(!landlock)("Landlock agent sandbox on this host", () => { + const roots: string[] = []; + afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); + + it("denies a sandboxed process the harness while keeping the product and temporary files usable", async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "agent-sandbox-"))); + roots.push(root); + const repository = path.join(root, "checkout"); + await mkdir(path.join(repository, "tests", "runner-e2e", "chat-scenarios"), { recursive: true }); + await mkdir(path.join(repository, "packages", "paperclip-runner"), { recursive: true }); + await mkdir(path.join(root, "scratch")); + await writeFile(path.join(repository, "tests", "runner-e2e", "agent-filesystem-sandbox.ts"), "grader"); + await writeFile(path.join(repository, "tests", "runner-e2e", "chat-scenarios", "access.yaml"), "expected_outcome: secret"); + await writeFile(path.join(repository, "packages", "paperclip-runner", "package.json"), "{}"); + const config = { hidden: [path.join(repository, "tests")], required: true }; + // The fixture lives under the shared temporary directory, so point mktemp at a granted sibling. + const probe = probeAgentFilesystemSandbox(config, repository, pythonExecutable(), { ...process.env, TMPDIR: path.join(root, "scratch") }); + expect(probe.failures).toEqual([]); + expect(probe.result).toMatchObject({ harnessFile: "denied", harnessDirectory: "denied", productFile: "readable", temporaryWrite: "allowed" }); + // The exact read from the reported run: grep over the checkout's scenarios. + const invocation = sandboxedInvocation(config, pythonExecutable(), "/bin/sh", ["-c", `grep -rn expected_outcome ${repository}/tests; echo "exit=$?"`], process.env); + const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8" }); + expect(output).not.toContain("secret"); + expect(output).toMatch(/exit=2/); + }); +}); diff --git a/tests/runner-e2e/agent-filesystem-sandbox.ts b/tests/runner-e2e/agent-filesystem-sandbox.ts new file mode 100644 index 00000000000..797e7462b64 --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.ts @@ -0,0 +1,194 @@ +import childProcess, { execFileSync, type SpawnOptions } from "node:child_process"; +import { existsSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; +import type { MatrixExecution } from "./types.js"; + +/** + * The agent under test must not read the eval: its scenarios, fixtures, + * graders or captured evidence. A local cell runs the product from the same + * checkout and as the same OS user as the harness, so file modes cannot + * separate them. The server launches every native agent through runnerd; the + * E2E server entry starts runnerd inside a Linux Landlock domain that hides + * these paths from runnerd and all of its descendants (provider, tools, shell). + * The product server itself stays unrestricted because it loads its E2E entry + * from the harness. + */ +export const AGENT_FILESYSTEM_SANDBOX_ENV = "PAPERCLIP_RUNNER_E2E_AGENT_FS_SANDBOX"; +export const RUNNERD_EXECUTABLE_NAME = "paperclip-runnerd"; + +export interface AgentFilesystemSandboxConfig { + hidden: string[]; + required: boolean; +} + +function within(candidate: string, root: string) { + return candidate === root || candidate.startsWith(`${root}${path.sep}`); +} + +/** Selects the paths hidden from local native agents, or null when not applicable. */ +export function agentFilesystemSandboxConfig(input: { + platform: NodeJS.Platform; + env: NodeJS.ProcessEnv; + executions: readonly MatrixExecution[]; + repositoryRoot: string; + temporaryRoot: string; + privateDir: string; + evidenceRoot?: string; + installedRelease: boolean; +}): AgentFilesystemSandboxConfig | null { + if (input.platform !== "linux" || input.installedRelease) return null; + if (!input.executions.some((execution) => execution.environment.id === "local" && execution.profile.generation === "native")) return null; + // Codex's own bubblewrap sandbox mounts, which Landlock forbids. Those cells + // keep their existing isolation until Codex permission profiles hide the eval. + if (input.executions.some(requiresCodexCiSandbox)) return null; + const sharedTemporary = path.resolve(input.env.TMPDIR?.trim() || os.tmpdir()); + const candidates = [ + path.join(input.repositoryRoot, "tests"), + path.join(input.repositoryRoot, ".manual-private-trusted"), + input.privateDir, + input.evidenceRoot, + input.env.RUNNER_TEMP, + ]; + const hidden: string[] = []; + for (const candidate of candidates) { + const value = candidate?.trim(); + if (!value || !path.isAbsolute(value)) continue; + const resolved = path.resolve(value); + // Never hide a path that holds the agent's own state, or one beneath the + // shared temporary directory: its ancestors would lose new entries. + if (within(input.temporaryRoot, resolved) || within(input.repositoryRoot, resolved)) continue; + if (within(resolved, sharedTemporary) || within(sharedTemporary, resolved)) continue; + if (!hidden.includes(resolved)) hidden.push(resolved); + } + return hidden.length ? { hidden, required: input.env.GITHUB_ACTIONS === "true" } : null; +} + +export function parseAgentFilesystemSandboxConfig(value: string | undefined): AgentFilesystemSandboxConfig | null { + if (!value?.trim()) return null; + const parsed = JSON.parse(value) as Partial; + if (!Array.isArray(parsed.hidden) || parsed.hidden.some((entry) => typeof entry !== "string" || !path.isAbsolute(entry)) || typeof parsed.required !== "boolean") { + throw new Error(`${AGENT_FILESYSTEM_SANDBOX_ENV} is malformed`); + } + return { hidden: parsed.hidden, required: parsed.required }; +} + +export const agentFilesystemSandboxHelper = path.join(import.meta.dirname, "agent-filesystem-sandbox.py"); + +/** Rewrites one launch so the helper applies the domain and then execs it unchanged. */ +export function sandboxedInvocation( + config: AgentFilesystemSandboxConfig, + python: string, + command: string, + args: readonly string[], + env: NodeJS.ProcessEnv, +): { command: string; args: string[] } { + // Python may coerce a C locale by exporting LC_CTYPE; restore the caller's value. + const locale = env.LC_CTYPE === undefined ? ["--unset", "LC_CTYPE"] : ["--set", `LC_CTYPE=${env.LC_CTYPE}`]; + return { + command: python, + args: [ + "-I", "-S", agentFilesystemSandboxHelper, + ...(config.required ? ["--required"] : []), + ...locale, + ...config.hidden.flatMap((hidden) => ["--hide", hidden]), + "--", command, ...args, + ], + }; +} + +export function isRunnerdLaunch(command: unknown, options: SpawnOptions | undefined) { + return typeof command === "string" && path.isAbsolute(command) && !options?.shell && + path.basename(command) === RUNNERD_EXECUTABLE_NAME; +} + +export function pythonExecutable() { + for (const candidate of ["/usr/bin/python3", "/usr/local/bin/python3"]) { + if (existsSync(candidate)) return candidate; + } + return "python3"; +} + +/** Proves, before any agent starts, that a sandboxed process cannot read the eval. */ +export function probeAgentFilesystemSandbox( + config: AgentFilesystemSandboxConfig, + repositoryRoot: string, + python = pythonExecutable(), + env: NodeJS.ProcessEnv = process.env, +) { + const harness = path.join(repositoryRoot, "tests", "runner-e2e"); + const script = [ + 'out=""; add() { out="$out$1=$2;"; }', + 'if cat "$1" >/dev/null 2>&1; then add harnessFile readable; else add harnessFile denied; fi', + 'if ls "$2" >/dev/null 2>&1; then add harnessDirectory listable; else add harnessDirectory denied; fi', + 'if test -e "$2"; then add harnessDirectoryStat visible; else add harnessDirectoryStat absent; fi', + 'if cat "$3" >/dev/null 2>&1; then add productFile readable; else add productFile denied; fi', + 'probe=$(mktemp) && rm -f "$probe" && add temporaryWrite allowed || add temporaryWrite denied', + 'shift 3; for hidden in "$@"; do if ls "$hidden" >/dev/null 2>&1; then add "hidden:$hidden" listable; else add "hidden:$hidden" denied; fi; done', + 'printf "%s\\n" "$out"', + ].join("\n"); + const invocation = sandboxedInvocation(config, python, "/bin/sh", [ + "-c", script, "sh", + path.join(harness, "agent-filesystem-sandbox.ts"), + harness, + path.join(repositoryRoot, "packages", "paperclip-runner", "package.json"), + ...config.hidden, + ], env); + const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8", cwd: "/", env, timeout: 30_000 }); + const result = Object.fromEntries(output.trim().split(";").filter(Boolean).map((entry) => { + const separator = entry.lastIndexOf("="); + return [entry.slice(0, separator), entry.slice(separator + 1)]; + })); + const failures = [ + result.harnessFile !== "denied" && "harness file is readable", + result.harnessDirectory !== "denied" && "harness directory is listable", + result.productFile !== "readable" && "product package is unreadable", + result.temporaryWrite !== "allowed" && "temporary directory is not writable", + ...config.hidden.map((hidden) => result[`hidden:${hidden}`] === "listable" && `${hidden} is listable`), + ].filter((failure): failure is string => Boolean(failure)); + return { result, failures }; +} + +/** + * Wraps runnerd launches made by this process. Runs only in the E2E server + * entry; the wrapper execs runnerd, so its PID, argv, cwd and environment are + * the ones the product requested. + */ +export function installAgentFilesystemSandbox(repositoryRoot: string, env: NodeJS.ProcessEnv = process.env) { + const config = parseAgentFilesystemSandboxConfig(env[AGENT_FILESYSTEM_SANDBOX_ENV]); + delete env[AGENT_FILESYSTEM_SANDBOX_ENV]; + if (!config) return null; + const python = pythonExecutable(); + let probe: ReturnType; + try { + probe = probeAgentFilesystemSandbox(config, repositoryRoot, python); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (config.required) throw new Error(`agent_filesystem_sandbox_unavailable: ${message}`); + console.warn(`[agent-filesystem-sandbox] disabled: ${message}`); + return null; + } + console.log(`[agent-filesystem-sandbox] probe ${JSON.stringify({ hidden: config.hidden, required: config.required, ...probe })}`); + if (probe.failures.length) { + if (config.required) throw new Error(`agent_filesystem_sandbox_ineffective: ${probe.failures.join("; ")}`); + console.warn(`[agent-filesystem-sandbox] disabled: ${probe.failures.join("; ")}`); + return null; + } + const original = childProcess.spawn; + const spawn = function (this: unknown, command: string, ...rest: unknown[]) { + const args = Array.isArray(rest[0]) ? rest[0] as string[] : []; + const options = (Array.isArray(rest[0]) ? rest[1] : rest[0]) as SpawnOptions | undefined; + if (isRunnerdLaunch(command, options)) { + const invocation = sandboxedInvocation(config, python, command, args, options?.env ?? process.env); + const child = Reflect.apply(original, this, [invocation.command, invocation.args, options ?? {}]) as ReturnType; + console.log(`[agent-filesystem-sandbox] runnerd launched inside the sandbox (pid ${child.pid ?? "unknown"})`); + return child; + } + return Reflect.apply(original, this, [command, ...rest]); + } as typeof childProcess.spawn; + childProcess.spawn = spawn; + syncBuiltinESMExports(); + return config; +} diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index dd9adba57d9..f491975ffce 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -53,6 +53,7 @@ import { firstTaskTasks, onboardingApprovalHistoryTask } from "./first-task-case import { APPROVAL_HISTORY_SUITE, approvalHistoryDefinitionDigest } from "./first-task-approval-history.js"; import { chatTasks, chatHardeningTasks, chatStoryTasks, chatQualificationTasks, chatCompletionTasks } from "./chat-cases.js"; import { createHash } from "node:crypto"; +import { neutralFixtureSuffix } from "./fixture-names.js"; import { fileURLToPath } from "node:url"; import { createAgentSchema } from "../../packages/shared/src/validators/agent.js"; import { createEnvironmentSchema } from "../../packages/shared/src/validators/environment.js"; @@ -121,11 +122,11 @@ function commonAgent( additionalInstructions: readonly string[] = [], ) { return { - name: `Runner E2E ${fixtureId} ${input.executionId}`, + name: `Agent ${neutralFixtureSuffix(fixtureId, input.executionId)}`, role: "qa", - title: "Paid full-stack runner acceptance fixture", + title: "Operations agent", capabilities: - "Completes deterministic standard, planning, and ask-mode runner acceptance tasks.", + "Completes standard, planning, and ask-mode tasks for the company.", adapterType, ...(adapterType === "codex_local" ? { runner: "legacy" as const } : {}), adapterConfig, @@ -508,8 +509,8 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ expectedExecutionTarget: { kind: "local" }, buildEnvironment(input) { return { - name: `Runner E2E local ${input.executionId}`, - description: "Ephemeral local runner E2E environment", + name: `Local ${neutralFixtureSuffix(input.executionId)}`, + description: "Local execution environment", driver: "local", config: {}, envVars: {}, @@ -536,8 +537,8 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ ); } return { - name: `Runner E2E Daytona ${input.executionId}`, - description: "Ephemeral Daytona runner E2E environment", + name: `Daytona ${neutralFixtureSuffix(input.executionId)}`, + description: "Daytona sandbox environment", driver: "sandbox", config: { provider: "daytona", @@ -584,8 +585,8 @@ export const daytonaWarmEnvironment: EnvironmentFixture = { ); } return { - name: `Runner E2E Daytona warm ${input.executionId}`, - description: "Ephemeral reusable Daytona runner E2E environment", + name: `Daytona warm ${neutralFixtureSuffix(input.executionId)}`, + description: "Reusable Daytona sandbox environment", driver: "sandbox", config: { provider: "daytona", diff --git a/tests/runner-e2e/fixture-names.ts b/tests/runner-e2e/fixture-names.ts new file mode 100644 index 00000000000..7b7960b8575 --- /dev/null +++ b/tests/runner-e2e/fixture-names.ts @@ -0,0 +1,9 @@ +import { createHash } from "node:crypto"; + +/** + * Names and descriptions an agent can read must not reveal the eval: no case + * ids, suite names, "E2E" or "test". A short digest keeps them unique. + */ +export function neutralFixtureSuffix(...parts: readonly string[]): string { + return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 8); +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 96b11e53866..ac1c0b89bc6 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -31,6 +31,7 @@ import { createRunnerE2ETemporaryRoot } from "./server-config.js"; import { createExternalControllerDatabase } from "./external-controller-database.js"; import { createHash, randomBytes } from "node:crypto"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; +import { AGENT_FILESYSTEM_SANDBOX_ENV, agentFilesystemSandboxConfig } from "./agent-filesystem-sandbox.js"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; import { createRequire } from "node:module"; @@ -589,6 +590,10 @@ async function runAttempt(input: { executions, repositoryRoot, ); + const agentFilesystemSandbox = agentFilesystemSandboxConfig({ + platform: process.platform, env: process.env, executions, repositoryRoot, temporaryRoot, privateDir, + evidenceRoot: manualPrivateHandoff?.root, installedRelease: Boolean(installedCli), + }); const childEnv: NodeJS.ProcessEnv = { ...buildRunnerE2EProcessEnvironment(sourceEnvironment, executions), [CHAT_PRIVATE_INPUTS_ENV]: privateInputs?.filename, @@ -609,6 +614,7 @@ async function runAttempt(input: { PAPERCLIP_RUNNER_E2E_PORT: String(port), PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot, PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir, + [AGENT_FILESYSTEM_SANDBOX_ENV]: agentFilesystemSandbox ? JSON.stringify(agentFilesystemSandbox) : undefined, PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace, PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(temporaryRoot, "artifacts-private", "server.log"), PAPERCLIP_RUNNER_BINARY: runnerBinary, diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 52517db05a3..7795f356094 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,5 +1,6 @@ import { CHAT_SETUP_HEALTH_SUITE } from "./chat-setup-health.js"; import { isEvalRemoteOrigin } from "./eval-remote-gateway.js"; +import { neutralFixtureSuffix } from "./fixture-names.js"; import { isEverydayReliabilityCase } from "./everyday-reliability-cases.js"; import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js"; import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; @@ -218,8 +219,8 @@ export async function setupLiveFixtures(input: { id: "company", async setup() { const company = await api.post("/api/companies", { - name: `Runner E2E ${execution.id} ${input.executionNonce}`, - description: "Ephemeral paid full-stack runner acceptance fixture", + name: `Company ${neutralFixtureSuffix(execution.id, input.executionNonce)}`, + description: "Operations company", budgetMonthlyCents: [CHAT_FEATURE_SUITE, CHAT_SETUP_HEALTH_SUITE].includes(execution.suite.id) ? CHAT_FEATURE_BUDGET_CENTS : receiptSmoke ? CASE_RECEIPT_BUDGET_CENTS : ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id) @@ -253,10 +254,10 @@ export async function setupLiveFixtures(input: { const secret = await api.postSensitive( `/api/companies/${company.id}/secrets`, { - name: `Runner E2E ${credentialName} ${input.executionNonce}`, + name: `${credentialName} ${neutralFixtureSuffix(input.executionNonce)}`, key: credentialName, value: rawValue, - description: `Ephemeral credential for ${execution.id}`, + description: "Model provider credential", }, ); refs[credentialName] = { @@ -356,7 +357,7 @@ export async function setupLiveFixtures(input: { { provider, method: "api_key", - name: `Runner E2E account ${input.executionNonce}`, + name: `Account ${neutralFixtureSuffix(input.executionNonce)}`, ownership: budgetedGateway ? "shared" : "personal", ...(budgetedGateway ? { routing: { kind: "gateway", protocol: provider === "anthropic" ? "messages" : "responses", baseUrl: provider === "anthropic" ? gatewayUrl : `${gatewayUrl}/v1`, auth: "bearer", models: [{ id: execution.profile.model }] } } : {}), @@ -476,9 +477,9 @@ export async function setupLiveFixtures(input: { return api.post( `/api/companies/${company.id}/projects`, { - name: `Runner E2E workspace project ${input.executionNonce}`, + name: `Workspace project ${neutralFixtureSuffix(input.executionNonce)}`, description: - "Ephemeral project anchoring the fixture execution workspace and file copy-back", + "Project for the execution workspace and file copy-back", executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", diff --git a/tests/runner-e2e/server-entry.ts b/tests/runner-e2e/server-entry.ts index e2175485d66..808e515d424 100644 --- a/tests/runner-e2e/server-entry.ts +++ b/tests/runner-e2e/server-entry.ts @@ -3,6 +3,8 @@ import { installChatReadinessObserver } from "./chat-readiness-observer.js"; // This entrypoint is used only by isolated Runner E2E instances. Production // service code has no test flag, delay, altered prompt, or private test API. import { ServerResponse } from "node:http"; +import path from "node:path"; +import { installAgentFilesystemSandbox } from "./agent-filesystem-sandbox.js"; import { PaperclipRunnerToolAuthority } from "../../server/src/services/native-runtime/paperclip-runner-tool-authority.js"; import { canonicalDocumentIssueId, contextCommentGateSelected, holdCommittedDocumentResponse, isArmed } from "./context-comment-gate.js"; import { holdInteractionResponse } from "./interaction-response-gate.js"; @@ -92,4 +94,6 @@ if (contextCommentGate || completionBusyGate) { return Reflect.apply(end, this, args); } as typeof end; } +// Before the product starts any agent, hide the eval harness from runnerd and its descendants. +installAgentFilesystemSandbox(path.resolve(import.meta.dirname, "../..")); await import("../../cli/src/index.js");