From d40a7494095feb1eec4e8e262f6cbb87e1a573b3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 15:54:17 -0600 Subject: [PATCH] test(chaos): allow only the node executable through the oracle sandbox; accept table-driven R1 tests Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/grader.test.ts | 3 +++ tests/restart-chaos/observer.ts | 4 ++-- tests/restart-chaos/oracle-sandbox.test.ts | 9 +++++---- tests/restart-chaos/oracle-sandbox.ts | 8 ++++---- tests/restart-chaos/workloads.ts | 4 ++-- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index fb798e11a19..ad72b13e9c4 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -243,6 +243,9 @@ describe("T-oracles", () => { slug: { exitCode: 0, stdout: "[\"hello-chaos-world\",\"-r1-paperclip-chaos-2026-\"]\n", stderr: "" } } }).verdict).toBe("fail"); expect(r1TestChecksSlug("import test from \"node:test\";")).toBe(false); expect(r1TestChecksSlug("import test from \"node:test\"; import assert from \"node:assert\"; assert.ok(chaosSlug(\"x\"));")).toBe(false); + expect(r1TestChecksSlug([ "import test from \"node:test\";", "import assert from \"node:assert/strict\";", + "for (const [input, want] of [[\"Hello Chaos World\", \"hello-chaos-world\"], [\"a b\", \"a-b\"], [\"-x-\", \"x\"]])", + " test(input, () => assert.equal(chaosSlug(input), want));"].join("\n"))).toBe(true); const argv = workloadDefinitions.R1.collect.commands.find(command => command.name === "node_test")!.argv; expect(typeof argv === "function" ? argv("abcd1234") : argv).toEqual([process.execPath, "--test", "packages/shared/src/chaos/slug-abcd1234.test.ts"]); }); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 0059b4cb6ca..61483f1ed31 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -1,6 +1,6 @@ import { spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { readFile, readdir, realpath } from "node:fs/promises"; +import { readFile, readdir } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -761,7 +761,7 @@ export class TaskObserver { for (const command of definition.collect.commands) { const argv = typeof command.argv === "function" ? command.argv(this.plan.nonce) : command.argv; commands[command.name] = await runOracleCommand(argv, workspacePath, { protectedRoots: this.protectedRoots, - readableRoots: argv[0] === process.execPath ? [path.dirname(path.dirname(await realpath(process.execPath)))] : [] }); + readableFiles: argv[0] === process.execPath ? [process.execPath] : [] }); } } return { diff --git a/tests/restart-chaos/oracle-sandbox.test.ts b/tests/restart-chaos/oracle-sandbox.test.ts index f3e3288160b..2bffdaf74e7 100644 --- a/tests/restart-chaos/oracle-sandbox.test.ts +++ b/tests/restart-chaos/oracle-sandbox.test.ts @@ -17,10 +17,11 @@ describe("oracle sandbox", () => { expect(profile.indexOf('(allow file-read* (subpath "/w")')).toBeGreaterThan(profile.indexOf("(deny file-read*")); }); - it("re-allows only the named readable roots, such as the node install, under a protected root", () => { - const profile = oracleSandboxProfile({ workspace: "/w", scratch: "/s", protectedRoots: ["/Users/me"], readableRoots: ["/Users/me/.nvm/versions/node/v26"] }); - expect(profile.indexOf('(allow file-read* (subpath "/Users/me/.nvm/versions/node/v26"))')).toBeGreaterThan(profile.indexOf('(deny file-read* (subpath "/Users/me"))')); - expect(oracleSandboxProfile({ workspace: "/w", scratch: "/s", protectedRoots: ["/Users/me"] })).not.toContain(".nvm"); + it("re-allows only the named files, such as the node executable, never a directory under a protected root", () => { + const profile = oracleSandboxProfile({ workspace: "/w", scratch: "/s", protectedRoots: ["/Users/me"], readableFiles: ["/Users/me/bin/node"] }); + expect(profile.indexOf('(allow file-read* (literal "/Users/me/bin/node"))')).toBeGreaterThan(profile.indexOf('(deny file-read* (subpath "/Users/me"))')); + expect(profile).not.toContain('(allow file-read* (subpath "/Users/me")'); + expect(profile).not.toContain('(allow file-read* (subpath "/Users/me/bin")'); }); it("refuses to run unsandboxed on other platforms unless allowed", async () => { diff --git a/tests/restart-chaos/oracle-sandbox.ts b/tests/restart-chaos/oracle-sandbox.ts index 2baea8af162..6939473f879 100644 --- a/tests/restart-chaos/oracle-sandbox.ts +++ b/tests/restart-chaos/oracle-sandbox.ts @@ -18,13 +18,13 @@ function sbplString(value: string) { return `"${value.replaceAll("\\", "\\\\").r * example ~/.secrets or ~/.ssh), write anywhere except a scratch directory, or * use the network. The workspace stays readable. */ -export function oracleSandboxProfile(input: { workspace: string; scratch: string; protectedRoots: readonly string[]; readableRoots?: readonly string[] }): string { +export function oracleSandboxProfile(input: { workspace: string; scratch: string; protectedRoots: readonly string[]; readableFiles?: readonly string[] }): string { const lines = ["(version 1)", "(allow default)", "(deny network*)", "(deny file-write*)", `(allow file-write* (subpath ${sbplString(input.scratch)}) (literal "/dev/null") (literal "/dev/tty"))`]; for (const root of input.protectedRoots) lines.push(`(deny file-read* (subpath ${sbplString(root)}))`); // Later rules win: re-allow the workspace and scratch even if they sit under a protected root. lines.push(`(allow file-read* (subpath ${sbplString(input.workspace)}) (subpath ${sbplString(input.scratch)}))`); - for (const root of input.readableRoots ?? []) lines.push(`(allow file-read* (subpath ${sbplString(root)}))`); + for (const file of input.readableFiles ?? []) lines.push(`(allow file-read* (literal ${sbplString(file)}))`); return lines.join("\n"); } @@ -35,7 +35,7 @@ export function oracleSandboxProfile(input: { workspace: string; scratch: string * temporary HOME alone does not stop absolute-path reads. */ export async function runOracleCommand(argv: readonly string[], workspace: string, options: { - protectedRoots?: readonly string[]; readableRoots?: readonly string[]; timeoutMs?: number; platform?: NodeJS.Platform; allowUnsandboxed?: boolean; + protectedRoots?: readonly string[]; readableFiles?: readonly string[]; timeoutMs?: number; platform?: NodeJS.Platform; allowUnsandboxed?: boolean; } = {}): Promise { const platform = options.platform ?? process.platform; const sandboxed = platform === "darwin"; @@ -51,7 +51,7 @@ export async function runOracleCommand(argv: readonly string[], workspace: strin const [command, args] = sandboxed ? ["/usr/bin/sandbox-exec", ["-p", oracleSandboxProfile({ workspace: await resolve(workspace), scratch, protectedRoots: await Promise.all((options.protectedRoots ?? [os.homedir()]).map(resolve)), - readableRoots: await Promise.all((options.readableRoots ?? []).map(resolve)) }), ...argv]] + readableFiles: await Promise.all((options.readableFiles ?? []).map(resolve)) }), ...argv]] : [argv[0]!, argv.slice(1)]; try { const { stdout, stderr } = await execFileAsync(command, args, { cwd: workspace, env, timeout: options.timeoutMs ?? 60_000, maxBuffer: 1024 * 1024 }); diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index 916ca949a3b..c1c837191f0 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -43,8 +43,8 @@ const r1Test = (nonce: string) => `packages/shared/src/chaos/slug-${nonce}.test. export function r1TestChecksSlug(text: string | null | undefined): boolean { if (!text) return false; - const assertions = text.match(/\bassert(\.\w+)?\s*\(/g)?.length ?? 0; - return text.includes("node:test") && text.includes("node:assert") && text.includes("chaosSlug(") && assertions >= 3; + return text.includes("node:test") && text.includes("node:assert") && /\bassert(\.\w+)?\s*\(/.test(text) + && text.includes("chaosSlug") && text.includes("hello-chaos-world"); } /** bench.sh must have actually run: at least this long between start and finish. */ export const BENCH_MIN_SECONDS = 200;