diff --git a/tests/runner-e2e/agent-filesystem-sandbox.py b/tests/runner-e2e/agent-filesystem-sandbox.py new file mode 100644 index 00000000000..46ad5175a7d --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.py @@ -0,0 +1,218 @@ +"""Execute a command whose process tree cannot read the eval harness. + +Runner E2E starts the product from a repository checkout that also holds the +scenarios, fixtures and graders. This helper applies a Linux Landlock domain +that grants filesystem access everywhere except the given hidden paths, then +replaces itself with the command (same PID, argv and environment). Every +descendant inherits the domain, including an agent's shell commands. + +Usage: + agent-filesystem-sandbox.py [--required] [--unset NAME]... [--set NAME=VALUE]... + --hide PATH [--hide PATH]... -- COMMAND [ARG]... + +Without Landlock, --required exits 125 instead of running the command. +""" + +import ctypes +import os +import stat +import struct +import sys + +SYS_LANDLOCK_CREATE_RULESET = 444 +SYS_LANDLOCK_ADD_RULE = 445 +SYS_LANDLOCK_RESTRICT_SELF = 446 +LANDLOCK_CREATE_RULESET_VERSION = 1 +LANDLOCK_RULE_PATH_BENEATH = 1 +PR_SET_NO_NEW_PRIVS = 38 + +FILE_RIGHTS = (1 << 0) | (1 << 1) | (1 << 2) # execute, write, read +ABI_1_RIGHTS = (1 << 13) - 1 +REFER = 1 << 13 +TRUNCATE = 1 << 14 +IOCTL_DEV = 1 << 15 + + +class RulesetAttr(ctypes.Structure): + _fields_ = [("handled_access_fs", ctypes.c_uint64)] + + +def fail(message, code=125): + sys.stderr.write("agent_filesystem_sandbox: %s\n" % message) + sys.stderr.flush() + os._exit(code) + + +def parse(argv): + required = False + hidden = [] + unset = [] + assignments = [] + index = 0 + while index < len(argv): + token = argv[index] + if token == "--": + command = argv[index + 1:] + if not command: + fail("missing command", 2) + return required, hidden, unset, assignments, command + if token == "--required": + required = True + elif token in ("--hide", "--unset", "--set") and index + 1 < len(argv): + index += 1 + value = argv[index] + if token == "--hide": + if not os.path.isabs(value): + fail("hidden path must be absolute: %s" % value, 2) + hidden.append(value) + elif token == "--unset": + unset.append(value) + else: + name, separator, assigned = value.partition("=") + if not separator or not name: + fail("--set requires NAME=VALUE", 2) + assignments.append((name, assigned)) + else: + fail("unexpected argument: %s" % token, 2) + index += 1 + fail("missing -- before the command", 2) + + +def within(candidate, root): + return candidate == root or candidate.startswith(root.rstrip("/") + "/") + + +def landlock(): + libc = ctypes.CDLL(None, use_errno=True) + libc.syscall.restype = ctypes.c_long + + def syscall(*args): + result = libc.syscall(*args) + if result < 0: + raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno())) + return result + + return libc, syscall + + +def restrict(hidden): + libc, syscall = landlock() + try: + abi = syscall( + ctypes.c_long(SYS_LANDLOCK_CREATE_RULESET), + ctypes.c_void_p(None), + ctypes.c_long(0), + ctypes.c_long(LANDLOCK_CREATE_RULESET_VERSION), + ) + except OSError as error: + raise RuntimeError("Landlock is unavailable on this kernel (%s)" % error) + handled = ABI_1_RIGHTS + if abi >= 2: + handled |= REFER + if abi >= 3: + handled |= TRUNCATE + if abi >= 5: + handled |= IOCTL_DEV + attr = RulesetAttr(handled) + ruleset = syscall( + ctypes.c_long(SYS_LANDLOCK_CREATE_RULESET), + ctypes.byref(attr), + ctypes.c_long(ctypes.sizeof(attr)), + ctypes.c_long(0), + ) + + def allow(path): + try: + descriptor = os.open(path, os.O_PATH | os.O_CLOEXEC) + except OSError: + return + try: + mode = os.fstat(descriptor).st_mode + access = handled if stat.S_ISDIR(mode) else handled & (FILE_RIGHTS | TRUNCATE | IOCTL_DEV) + # struct landlock_path_beneath_attr is packed: u64 allowed_access, s32 parent_fd. + rule = ctypes.create_string_buffer(struct.pack("=Qi", access, descriptor), 12) + syscall( + ctypes.c_long(SYS_LANDLOCK_ADD_RULE), + ctypes.c_long(ruleset), + ctypes.c_long(LANDLOCK_RULE_PATH_BENEATH), + rule, + ctypes.c_long(0), + ) + finally: + os.close(descriptor) + + # Landlock only grants. Grant every entry except the hidden paths, walking + # down only through their ancestors; ancestors themselves get no rule. + def grant(directory): + try: + names = sorted(os.listdir(directory)) + except OSError: + return + for name in names: + child = os.path.join(directory, name) + if any(within(child, path) for path in hidden): + continue + try: + is_link = os.path.islink(child) + real = os.path.realpath(child) + except OSError: + continue + if any(within(real, path) or within(path, real) for path in hidden): + if not is_link and any(within(path, child) for path in hidden): + grant(child) + # A link into, or onto an ancestor of, a hidden path stays ungranted. + continue + allow(child) + + grant("/") + if libc.prctl(ctypes.c_int(PR_SET_NO_NEW_PRIVS), ctypes.c_ulong(1), ctypes.c_ulong(0), ctypes.c_ulong(0), ctypes.c_ulong(0)) != 0: + raise RuntimeError("prctl(PR_SET_NO_NEW_PRIVS) failed") + syscall(ctypes.c_long(SYS_LANDLOCK_RESTRICT_SELF), ctypes.c_long(ruleset), ctypes.c_long(0)) + os.close(ruleset) + # Fail closed unless this process (which becomes the command) is denied. + for path in hidden: + try: + if os.path.isdir(path): + os.listdir(path) + else: + os.close(os.open(path, os.O_RDONLY | os.O_CLOEXEC)) + except PermissionError: + continue + except OSError: + continue + raise RuntimeError("sandbox ineffective: %s is still readable" % path) + return abi + + +def main(): + required, hidden_inputs, unset, assignments, command = parse(sys.argv[1:]) + hidden = [] + for path in hidden_inputs: + if os.path.lexists(path): + real = os.path.realpath(path) + if real == "/": + fail("refusing to hide the filesystem root", 2) + hidden.append(real) + for name in unset: + os.environ.pop(name, None) + for name, value in assignments: + os.environ[name] = value + if hidden: + if not sys.platform.startswith("linux"): + if required: + fail("Landlock requires Linux") + else: + try: + restrict(hidden) + except (OSError, RuntimeError) as error: + if required: + fail(str(error)) + sys.stderr.write("agent_filesystem_sandbox: continuing without isolation: %s\n" % error) + try: + os.execv(command[0], command) + except OSError as error: + fail("cannot execute %s: %s" % (command[0], error), 127) + + +if __name__ == "__main__": + main() diff --git a/tests/runner-e2e/agent-filesystem-sandbox.test.ts b/tests/runner-e2e/agent-filesystem-sandbox.test.ts new file mode 100644 index 00000000000..f7942778275 --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.test.ts @@ -0,0 +1,139 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + agentFilesystemSandboxConfig, + isRunnerdLaunch, + parseAgentFilesystemSandboxConfig, + probeAgentFilesystemSandbox, + pythonExecutable, + sandboxedInvocation, + agentFilesystemSandboxHelper, +} from "./agent-filesystem-sandbox.js"; +import { runnerMatrix } from "./catalog.js"; + +const repositoryRoot = "/work/paperclip"; +const temporaryRoot = path.join(os.tmpdir(), "paperclip-runner-e2e-abc"); +const cell = (id: string) => { + const execution = runnerMatrix.find((candidate) => candidate.id === id); + if (!execution) throw new Error(`missing ${id}`); + return execution; +}; +const opencodeLocal = cell("task-workspaces.runner-opencode.local.task-directory-repository-resume"); +const base = { + platform: "linux" as const, + env: { GITHUB_ACTIONS: "true", RUNNER_TEMP: "/work/_temp" }, + executions: [opencodeLocal], + repositoryRoot, + temporaryRoot, + privateDir: "/work/_temp/manual-private-evidence-x/capture", + evidenceRoot: "/work/_temp/manual-private-evidence-x", + installedRelease: false, +}; + +describe("agent filesystem sandbox selection", () => { + it("hides the harness, the trusted copy and captured evidence from local native agents", () => { + expect(agentFilesystemSandboxConfig(base)).toEqual({ + hidden: [ + "/work/paperclip/tests", + "/work/paperclip/.manual-private-trusted", + "/work/_temp/manual-private-evidence-x/capture", + "/work/_temp/manual-private-evidence-x", + "/work/_temp", + ], + required: true, + }); + }); + + it("never hides the agent's own state, the checkout root or the shared temporary directory", () => { + const config = agentFilesystemSandboxConfig({ + ...base, + env: { RUNNER_TEMP: os.tmpdir() }, + privateDir: path.join(temporaryRoot, "artifacts-private"), + evidenceRoot: "/work", + }); + expect(config).toEqual({ hidden: ["/work/paperclip/tests", "/work/paperclip/.manual-private-trusted"], required: false }); + }); + + it.each([ + ["macOS", { platform: "darwin" as const }], + ["an installed release", { installedRelease: true }], + ["remote-only cells", { executions: [cell("rolling-web-deployment.runner-opencode.daytona.recover-controller")] }], + ["legacy adapters", { executions: [cell("blocker-guidance.legacy-codex.local.human-authority")] }], + ["Codex's own mount sandbox", { executions: [opencodeLocal, cell("necessary-question.runner-codex.local.necessary-free-text")] }], + ])("does not apply to %s", (_label, override) => { + expect(agentFilesystemSandboxConfig({ ...base, ...override })).toBeNull(); + }); + + it("round-trips through the server environment and rejects malformed values", () => { + const config = agentFilesystemSandboxConfig(base)!; + expect(parseAgentFilesystemSandboxConfig(JSON.stringify(config))).toEqual(config); + expect(parseAgentFilesystemSandboxConfig(undefined)).toBeNull(); + expect(() => parseAgentFilesystemSandboxConfig(JSON.stringify({ hidden: ["relative"], required: true }))).toThrow(/malformed/); + }); +}); + +describe("runnerd launch rewriting", () => { + it("wraps only the absolute runnerd executable and execs it with the original argv", () => { + const runnerd = "/work/paperclip/packages/paperclip-runner/runner/target/debug/paperclip-runnerd"; + expect(isRunnerdLaunch(runnerd, { cwd: "/x" })).toBe(true); + expect(isRunnerdLaunch(runnerd, { shell: true })).toBe(false); + expect(isRunnerdLaunch("paperclip-runnerd", {})).toBe(false); + expect(isRunnerdLaunch("/usr/bin/git", {})).toBe(false); + const invocation = sandboxedInvocation({ hidden: ["/work/paperclip/tests"], required: true }, "/usr/bin/python3", runnerd, ["--run-id", "r"], {}); + expect(invocation).toEqual({ + command: "/usr/bin/python3", + args: ["-I", "-S", agentFilesystemSandboxHelper, "--required", "--unset", "LC_CTYPE", "--hide", "/work/paperclip/tests", "--", runnerd, "--run-id", "r"], + }); + expect(sandboxedInvocation({ hidden: [], required: false }, "p", runnerd, [], { LC_CTYPE: "C" }).args).toContain("LC_CTYPE=C"); + }); + + it("installs the sandbox in the E2E server entry before the product starts", async () => { + const entry = await readFile(path.join(import.meta.dirname, "server-entry.ts"), "utf8"); + const install = entry.indexOf("installAgentFilesystemSandbox("); + expect(install).toBeGreaterThan(0); + expect(install).toBeLessThan(entry.indexOf('await import("../../cli/src/index.js")')); + }); +}); + +const landlock = process.platform === "linux" && (() => { + const probe = mkdtempSync(path.join(os.tmpdir(), "agent-sandbox-abi-")); + try { + execFileSync(pythonExecutable(), ["-I", "-S", agentFilesystemSandboxHelper, "--required", "--hide", probe, "--", "/bin/true"]); + return true; + } catch { + return false; + } finally { + rmSync(probe, { recursive: true, force: true }); + } +})(); + +describe.skipIf(!landlock)("Landlock agent sandbox on this host", () => { + const roots: string[] = []; + afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); + + it("denies a sandboxed process the harness while keeping the product and temporary files usable", async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "agent-sandbox-"))); + roots.push(root); + const repository = path.join(root, "checkout"); + await mkdir(path.join(repository, "tests", "runner-e2e", "chat-scenarios"), { recursive: true }); + await mkdir(path.join(repository, "packages", "paperclip-runner"), { recursive: true }); + await mkdir(path.join(root, "scratch")); + await writeFile(path.join(repository, "tests", "runner-e2e", "agent-filesystem-sandbox.ts"), "grader"); + await writeFile(path.join(repository, "tests", "runner-e2e", "chat-scenarios", "access.yaml"), "expected_outcome: secret"); + await writeFile(path.join(repository, "packages", "paperclip-runner", "package.json"), "{}"); + const config = { hidden: [path.join(repository, "tests")], required: true }; + // The fixture lives under the shared temporary directory, so point mktemp at a granted sibling. + const probe = probeAgentFilesystemSandbox(config, repository, pythonExecutable(), { ...process.env, TMPDIR: path.join(root, "scratch") }); + expect(probe.failures).toEqual([]); + expect(probe.result).toMatchObject({ harnessFile: "denied", harnessDirectory: "denied", productFile: "readable", temporaryWrite: "allowed" }); + // The exact read from the reported run: grep over the checkout's scenarios. + const invocation = sandboxedInvocation(config, pythonExecutable(), "/bin/sh", ["-c", `grep -rn expected_outcome ${repository}/tests; echo "exit=$?"`], process.env); + const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8" }); + expect(output).not.toContain("secret"); + expect(output).toMatch(/exit=2/); + }); +}); diff --git a/tests/runner-e2e/agent-filesystem-sandbox.ts b/tests/runner-e2e/agent-filesystem-sandbox.ts new file mode 100644 index 00000000000..797e7462b64 --- /dev/null +++ b/tests/runner-e2e/agent-filesystem-sandbox.ts @@ -0,0 +1,194 @@ +import childProcess, { execFileSync, type SpawnOptions } from "node:child_process"; +import { existsSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; +import type { MatrixExecution } from "./types.js"; + +/** + * The agent under test must not read the eval: its scenarios, fixtures, + * graders or captured evidence. A local cell runs the product from the same + * checkout and as the same OS user as the harness, so file modes cannot + * separate them. The server launches every native agent through runnerd; the + * E2E server entry starts runnerd inside a Linux Landlock domain that hides + * these paths from runnerd and all of its descendants (provider, tools, shell). + * The product server itself stays unrestricted because it loads its E2E entry + * from the harness. + */ +export const AGENT_FILESYSTEM_SANDBOX_ENV = "PAPERCLIP_RUNNER_E2E_AGENT_FS_SANDBOX"; +export const RUNNERD_EXECUTABLE_NAME = "paperclip-runnerd"; + +export interface AgentFilesystemSandboxConfig { + hidden: string[]; + required: boolean; +} + +function within(candidate: string, root: string) { + return candidate === root || candidate.startsWith(`${root}${path.sep}`); +} + +/** Selects the paths hidden from local native agents, or null when not applicable. */ +export function agentFilesystemSandboxConfig(input: { + platform: NodeJS.Platform; + env: NodeJS.ProcessEnv; + executions: readonly MatrixExecution[]; + repositoryRoot: string; + temporaryRoot: string; + privateDir: string; + evidenceRoot?: string; + installedRelease: boolean; +}): AgentFilesystemSandboxConfig | null { + if (input.platform !== "linux" || input.installedRelease) return null; + if (!input.executions.some((execution) => execution.environment.id === "local" && execution.profile.generation === "native")) return null; + // Codex's own bubblewrap sandbox mounts, which Landlock forbids. Those cells + // keep their existing isolation until Codex permission profiles hide the eval. + if (input.executions.some(requiresCodexCiSandbox)) return null; + const sharedTemporary = path.resolve(input.env.TMPDIR?.trim() || os.tmpdir()); + const candidates = [ + path.join(input.repositoryRoot, "tests"), + path.join(input.repositoryRoot, ".manual-private-trusted"), + input.privateDir, + input.evidenceRoot, + input.env.RUNNER_TEMP, + ]; + const hidden: string[] = []; + for (const candidate of candidates) { + const value = candidate?.trim(); + if (!value || !path.isAbsolute(value)) continue; + const resolved = path.resolve(value); + // Never hide a path that holds the agent's own state, or one beneath the + // shared temporary directory: its ancestors would lose new entries. + if (within(input.temporaryRoot, resolved) || within(input.repositoryRoot, resolved)) continue; + if (within(resolved, sharedTemporary) || within(sharedTemporary, resolved)) continue; + if (!hidden.includes(resolved)) hidden.push(resolved); + } + return hidden.length ? { hidden, required: input.env.GITHUB_ACTIONS === "true" } : null; +} + +export function parseAgentFilesystemSandboxConfig(value: string | undefined): AgentFilesystemSandboxConfig | null { + if (!value?.trim()) return null; + const parsed = JSON.parse(value) as Partial; + if (!Array.isArray(parsed.hidden) || parsed.hidden.some((entry) => typeof entry !== "string" || !path.isAbsolute(entry)) || typeof parsed.required !== "boolean") { + throw new Error(`${AGENT_FILESYSTEM_SANDBOX_ENV} is malformed`); + } + return { hidden: parsed.hidden, required: parsed.required }; +} + +export const agentFilesystemSandboxHelper = path.join(import.meta.dirname, "agent-filesystem-sandbox.py"); + +/** Rewrites one launch so the helper applies the domain and then execs it unchanged. */ +export function sandboxedInvocation( + config: AgentFilesystemSandboxConfig, + python: string, + command: string, + args: readonly string[], + env: NodeJS.ProcessEnv, +): { command: string; args: string[] } { + // Python may coerce a C locale by exporting LC_CTYPE; restore the caller's value. + const locale = env.LC_CTYPE === undefined ? ["--unset", "LC_CTYPE"] : ["--set", `LC_CTYPE=${env.LC_CTYPE}`]; + return { + command: python, + args: [ + "-I", "-S", agentFilesystemSandboxHelper, + ...(config.required ? ["--required"] : []), + ...locale, + ...config.hidden.flatMap((hidden) => ["--hide", hidden]), + "--", command, ...args, + ], + }; +} + +export function isRunnerdLaunch(command: unknown, options: SpawnOptions | undefined) { + return typeof command === "string" && path.isAbsolute(command) && !options?.shell && + path.basename(command) === RUNNERD_EXECUTABLE_NAME; +} + +export function pythonExecutable() { + for (const candidate of ["/usr/bin/python3", "/usr/local/bin/python3"]) { + if (existsSync(candidate)) return candidate; + } + return "python3"; +} + +/** Proves, before any agent starts, that a sandboxed process cannot read the eval. */ +export function probeAgentFilesystemSandbox( + config: AgentFilesystemSandboxConfig, + repositoryRoot: string, + python = pythonExecutable(), + env: NodeJS.ProcessEnv = process.env, +) { + const harness = path.join(repositoryRoot, "tests", "runner-e2e"); + const script = [ + 'out=""; add() { out="$out$1=$2;"; }', + 'if cat "$1" >/dev/null 2>&1; then add harnessFile readable; else add harnessFile denied; fi', + 'if ls "$2" >/dev/null 2>&1; then add harnessDirectory listable; else add harnessDirectory denied; fi', + 'if test -e "$2"; then add harnessDirectoryStat visible; else add harnessDirectoryStat absent; fi', + 'if cat "$3" >/dev/null 2>&1; then add productFile readable; else add productFile denied; fi', + 'probe=$(mktemp) && rm -f "$probe" && add temporaryWrite allowed || add temporaryWrite denied', + 'shift 3; for hidden in "$@"; do if ls "$hidden" >/dev/null 2>&1; then add "hidden:$hidden" listable; else add "hidden:$hidden" denied; fi; done', + 'printf "%s\\n" "$out"', + ].join("\n"); + const invocation = sandboxedInvocation(config, python, "/bin/sh", [ + "-c", script, "sh", + path.join(harness, "agent-filesystem-sandbox.ts"), + harness, + path.join(repositoryRoot, "packages", "paperclip-runner", "package.json"), + ...config.hidden, + ], env); + const output = execFileSync(invocation.command, invocation.args, { encoding: "utf8", cwd: "/", env, timeout: 30_000 }); + const result = Object.fromEntries(output.trim().split(";").filter(Boolean).map((entry) => { + const separator = entry.lastIndexOf("="); + return [entry.slice(0, separator), entry.slice(separator + 1)]; + })); + const failures = [ + result.harnessFile !== "denied" && "harness file is readable", + result.harnessDirectory !== "denied" && "harness directory is listable", + result.productFile !== "readable" && "product package is unreadable", + result.temporaryWrite !== "allowed" && "temporary directory is not writable", + ...config.hidden.map((hidden) => result[`hidden:${hidden}`] === "listable" && `${hidden} is listable`), + ].filter((failure): failure is string => Boolean(failure)); + return { result, failures }; +} + +/** + * Wraps runnerd launches made by this process. Runs only in the E2E server + * entry; the wrapper execs runnerd, so its PID, argv, cwd and environment are + * the ones the product requested. + */ +export function installAgentFilesystemSandbox(repositoryRoot: string, env: NodeJS.ProcessEnv = process.env) { + const config = parseAgentFilesystemSandboxConfig(env[AGENT_FILESYSTEM_SANDBOX_ENV]); + delete env[AGENT_FILESYSTEM_SANDBOX_ENV]; + if (!config) return null; + const python = pythonExecutable(); + let probe: ReturnType; + try { + probe = probeAgentFilesystemSandbox(config, repositoryRoot, python); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (config.required) throw new Error(`agent_filesystem_sandbox_unavailable: ${message}`); + console.warn(`[agent-filesystem-sandbox] disabled: ${message}`); + return null; + } + console.log(`[agent-filesystem-sandbox] probe ${JSON.stringify({ hidden: config.hidden, required: config.required, ...probe })}`); + if (probe.failures.length) { + if (config.required) throw new Error(`agent_filesystem_sandbox_ineffective: ${probe.failures.join("; ")}`); + console.warn(`[agent-filesystem-sandbox] disabled: ${probe.failures.join("; ")}`); + return null; + } + const original = childProcess.spawn; + const spawn = function (this: unknown, command: string, ...rest: unknown[]) { + const args = Array.isArray(rest[0]) ? rest[0] as string[] : []; + const options = (Array.isArray(rest[0]) ? rest[1] : rest[0]) as SpawnOptions | undefined; + if (isRunnerdLaunch(command, options)) { + const invocation = sandboxedInvocation(config, python, command, args, options?.env ?? process.env); + const child = Reflect.apply(original, this, [invocation.command, invocation.args, options ?? {}]) as ReturnType; + console.log(`[agent-filesystem-sandbox] runnerd launched inside the sandbox (pid ${child.pid ?? "unknown"})`); + return child; + } + return Reflect.apply(original, this, [command, ...rest]); + } as typeof childProcess.spawn; + childProcess.spawn = spawn; + syncBuiltinESMExports(); + return config; +} diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index dd9adba57d9..f491975ffce 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -53,6 +53,7 @@ import { firstTaskTasks, onboardingApprovalHistoryTask } from "./first-task-case import { APPROVAL_HISTORY_SUITE, approvalHistoryDefinitionDigest } from "./first-task-approval-history.js"; import { chatTasks, chatHardeningTasks, chatStoryTasks, chatQualificationTasks, chatCompletionTasks } from "./chat-cases.js"; import { createHash } from "node:crypto"; +import { neutralFixtureSuffix } from "./fixture-names.js"; import { fileURLToPath } from "node:url"; import { createAgentSchema } from "../../packages/shared/src/validators/agent.js"; import { createEnvironmentSchema } from "../../packages/shared/src/validators/environment.js"; @@ -121,11 +122,11 @@ function commonAgent( additionalInstructions: readonly string[] = [], ) { return { - name: `Runner E2E ${fixtureId} ${input.executionId}`, + name: `Agent ${neutralFixtureSuffix(fixtureId, input.executionId)}`, role: "qa", - title: "Paid full-stack runner acceptance fixture", + title: "Operations agent", capabilities: - "Completes deterministic standard, planning, and ask-mode runner acceptance tasks.", + "Completes standard, planning, and ask-mode tasks for the company.", adapterType, ...(adapterType === "codex_local" ? { runner: "legacy" as const } : {}), adapterConfig, @@ -508,8 +509,8 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ expectedExecutionTarget: { kind: "local" }, buildEnvironment(input) { return { - name: `Runner E2E local ${input.executionId}`, - description: "Ephemeral local runner E2E environment", + name: `Local ${neutralFixtureSuffix(input.executionId)}`, + description: "Local execution environment", driver: "local", config: {}, envVars: {}, @@ -536,8 +537,8 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ ); } return { - name: `Runner E2E Daytona ${input.executionId}`, - description: "Ephemeral Daytona runner E2E environment", + name: `Daytona ${neutralFixtureSuffix(input.executionId)}`, + description: "Daytona sandbox environment", driver: "sandbox", config: { provider: "daytona", @@ -584,8 +585,8 @@ export const daytonaWarmEnvironment: EnvironmentFixture = { ); } return { - name: `Runner E2E Daytona warm ${input.executionId}`, - description: "Ephemeral reusable Daytona runner E2E environment", + name: `Daytona warm ${neutralFixtureSuffix(input.executionId)}`, + description: "Reusable Daytona sandbox environment", driver: "sandbox", config: { provider: "daytona", diff --git a/tests/runner-e2e/fixture-names.ts b/tests/runner-e2e/fixture-names.ts new file mode 100644 index 00000000000..7b7960b8575 --- /dev/null +++ b/tests/runner-e2e/fixture-names.ts @@ -0,0 +1,9 @@ +import { createHash } from "node:crypto"; + +/** + * Names and descriptions an agent can read must not reveal the eval: no case + * ids, suite names, "E2E" or "test". A short digest keeps them unique. + */ +export function neutralFixtureSuffix(...parts: readonly string[]): string { + return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 8); +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 96b11e53866..ac1c0b89bc6 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -31,6 +31,7 @@ import { createRunnerE2ETemporaryRoot } from "./server-config.js"; import { createExternalControllerDatabase } from "./external-controller-database.js"; import { createHash, randomBytes } from "node:crypto"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; +import { AGENT_FILESYSTEM_SANDBOX_ENV, agentFilesystemSandboxConfig } from "./agent-filesystem-sandbox.js"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; import { createRequire } from "node:module"; @@ -589,6 +590,10 @@ async function runAttempt(input: { executions, repositoryRoot, ); + const agentFilesystemSandbox = agentFilesystemSandboxConfig({ + platform: process.platform, env: process.env, executions, repositoryRoot, temporaryRoot, privateDir, + evidenceRoot: manualPrivateHandoff?.root, installedRelease: Boolean(installedCli), + }); const childEnv: NodeJS.ProcessEnv = { ...buildRunnerE2EProcessEnvironment(sourceEnvironment, executions), [CHAT_PRIVATE_INPUTS_ENV]: privateInputs?.filename, @@ -609,6 +614,7 @@ async function runAttempt(input: { PAPERCLIP_RUNNER_E2E_PORT: String(port), PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot, PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir, + [AGENT_FILESYSTEM_SANDBOX_ENV]: agentFilesystemSandbox ? JSON.stringify(agentFilesystemSandbox) : undefined, PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace, PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(temporaryRoot, "artifacts-private", "server.log"), PAPERCLIP_RUNNER_BINARY: runnerBinary, diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 52517db05a3..7795f356094 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,5 +1,6 @@ import { CHAT_SETUP_HEALTH_SUITE } from "./chat-setup-health.js"; import { isEvalRemoteOrigin } from "./eval-remote-gateway.js"; +import { neutralFixtureSuffix } from "./fixture-names.js"; import { isEverydayReliabilityCase } from "./everyday-reliability-cases.js"; import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js"; import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; @@ -218,8 +219,8 @@ export async function setupLiveFixtures(input: { id: "company", async setup() { const company = await api.post("/api/companies", { - name: `Runner E2E ${execution.id} ${input.executionNonce}`, - description: "Ephemeral paid full-stack runner acceptance fixture", + name: `Company ${neutralFixtureSuffix(execution.id, input.executionNonce)}`, + description: "Operations company", budgetMonthlyCents: [CHAT_FEATURE_SUITE, CHAT_SETUP_HEALTH_SUITE].includes(execution.suite.id) ? CHAT_FEATURE_BUDGET_CENTS : receiptSmoke ? CASE_RECEIPT_BUDGET_CENTS : ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id) @@ -253,10 +254,10 @@ export async function setupLiveFixtures(input: { const secret = await api.postSensitive( `/api/companies/${company.id}/secrets`, { - name: `Runner E2E ${credentialName} ${input.executionNonce}`, + name: `${credentialName} ${neutralFixtureSuffix(input.executionNonce)}`, key: credentialName, value: rawValue, - description: `Ephemeral credential for ${execution.id}`, + description: "Model provider credential", }, ); refs[credentialName] = { @@ -356,7 +357,7 @@ export async function setupLiveFixtures(input: { { provider, method: "api_key", - name: `Runner E2E account ${input.executionNonce}`, + name: `Account ${neutralFixtureSuffix(input.executionNonce)}`, ownership: budgetedGateway ? "shared" : "personal", ...(budgetedGateway ? { routing: { kind: "gateway", protocol: provider === "anthropic" ? "messages" : "responses", baseUrl: provider === "anthropic" ? gatewayUrl : `${gatewayUrl}/v1`, auth: "bearer", models: [{ id: execution.profile.model }] } } : {}), @@ -476,9 +477,9 @@ export async function setupLiveFixtures(input: { return api.post( `/api/companies/${company.id}/projects`, { - name: `Runner E2E workspace project ${input.executionNonce}`, + name: `Workspace project ${neutralFixtureSuffix(input.executionNonce)}`, description: - "Ephemeral project anchoring the fixture execution workspace and file copy-back", + "Project for the execution workspace and file copy-back", executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", diff --git a/tests/runner-e2e/server-entry.ts b/tests/runner-e2e/server-entry.ts index e2175485d66..808e515d424 100644 --- a/tests/runner-e2e/server-entry.ts +++ b/tests/runner-e2e/server-entry.ts @@ -3,6 +3,8 @@ import { installChatReadinessObserver } from "./chat-readiness-observer.js"; // This entrypoint is used only by isolated Runner E2E instances. Production // service code has no test flag, delay, altered prompt, or private test API. import { ServerResponse } from "node:http"; +import path from "node:path"; +import { installAgentFilesystemSandbox } from "./agent-filesystem-sandbox.js"; import { PaperclipRunnerToolAuthority } from "../../server/src/services/native-runtime/paperclip-runner-tool-authority.js"; import { canonicalDocumentIssueId, contextCommentGateSelected, holdCommittedDocumentResponse, isArmed } from "./context-comment-gate.js"; import { holdInteractionResponse } from "./interaction-response-gate.js"; @@ -92,4 +94,6 @@ if (contextCommentGate || completionBusyGate) { return Reflect.apply(end, this, args); } as typeof end; } +// Before the product starts any agent, hide the eval harness from runnerd and its descendants. +installAgentFilesystemSandbox(path.resolve(import.meta.dirname, "../..")); await import("../../cli/src/index.js");