diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index fb5b348a458..a70b2d4a65f 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -102,6 +102,22 @@ Neither G1 nor H1 is in the default workloads. The board's actions and what the observer saw are in the timeline (`governance_*`) and in each task's grade (`governance`). +## Code change in a real repository (R1, `--workspace-repo`) + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex,claude --workloads R1,C1 --workspace-repo /path/to/paperclip \ + --faults K1,K2,K4 --concurrency 4 --tasks 8 --seed 22 +``` + +The harness clones the given git repository (shallow, from its committed HEAD) +as the project's workspace, in place of the chaoscli fixture. R1 asks the agent +to add a TypeScript module and a `node:test` file under +`packages/shared/src/chaos/`, named with the task's nonce so concurrent tasks +share the repository without colliding. The oracle runs the agent's test and a +hidden case against its module. B1 and B5 need the chaoscli fixture, so they do +not combine with `--workspace-repo`. Local only. + ## Upgrade mid-run (K14, `--upgrade-from`) ```sh diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 963b966c6ed..5c24f708ae4 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import path from "node:path"; import { parseArgs } from "node:util"; import { @@ -96,13 +97,14 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "crash-point": { type: "string" }, "crash-point-hold-ms": { type: "string" }, "upgrade-from": { type: "string" }, + "workspace-repo": { type: "string" }, }, }); const keep = values.keep === true; const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -138,7 +140,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC env, adapters: list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]), // The governance scenarios (G1, H1) are opt-in: H1 needs a wave of its own. - workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1")), + workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1")), // A crash-point wave schedules no fault; K1 is the restart its self-kill gets. faults: list(values.faults, FAULT_KINDS, "faults", stormMinutes ? STORM_DEFAULT_FAULTS : crashPoint ? ["K1"] : ["K1", "K2", "K4"]), triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), @@ -170,6 +172,16 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC if (crashPoint) throw new Error("--upgrade-from does not combine with --crash-point"); options.upgradeFrom = path.resolve(upgradeFrom); } + const workspaceRepo = values["workspace-repo"]; + if (options.workloads.includes("R1") !== (workspaceRepo !== undefined)) { + throw new Error("R1 (code change in a cloned repository) and --workspace-repo go together: pass both or neither"); + } + if (workspaceRepo !== undefined) { + if (env !== "local") throw new Error("--workspace-repo runs only with --env local"); + if (options.workloads.some(id => id === "B1" || id === "B5")) throw new Error("--workspace-repo replaces the chaoscli fixture that B1 and B5 need"); + options.workspaceRepo = path.resolve(workspaceRepo); + options.workspaceCommit = execFileSync("git", ["-C", options.workspaceRepo, "rev-parse", "HEAD"], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + } // H1 pauses the whole company when its budget hard stop is crossed: no other task can share its // wave, its tasks run one at a time, and the hard stop must be on. if (options.workloads.includes("H1")) { diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index f300f9567ed..ad72b13e9c4 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { crossingFaults, gradeR3, gradeR4, gradeR5, gradeTask, STALL_LIMIT_MS } from "./grader.js"; import { isResting } from "./observer.js"; import type { FaultReceipt, OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js"; -import { BENCH_EXPECTED, parseBenchRecord, workloadDefinitions } from "./workloads.js"; +import { BENCH_EXPECTED, parseBenchRecord, r1TestChecksSlug, workloadDefinitions } from "./workloads.js"; const T0 = Date.parse("2026-11-06T12:00:00Z"); const iso = (offsetMs: number) => new Date(T0 + offsetMs).toISOString(); @@ -229,6 +229,27 @@ describe("T-oracles", () => { expect(workloadDefinitions.B1.grade("abcd1234", { ...base, commands: { ...base.commands, unittest: { exitCode: 1, stdout: "", stderr: "FAILED" } } }).verdict).toBe("fail"); }); + it("grades R1 by running the agent's node test and a hidden case against its module", () => { + const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", + files: { "packages/shared/src/chaos/slug-abcd1234.ts": "export function chaosSlug() {}", + "packages/shared/src/chaos/slug-abcd1234.test.ts": [ + "import test from \"node:test\";", "import assert from \"node:assert/strict\";", "import { chaosSlug } from \"./slug-abcd1234.ts\";", + "test(\"slug\", () => {", " assert.equal(chaosSlug(\"Hello Chaos World\"), \"hello-chaos-world\");", + " assert.equal(chaosSlug(\"a b\"), \"a-b\");", " assert.equal(chaosSlug(\"--x--\"), \"x\");", "});"].join("\n") }, + commands: { node_test: { exitCode: 0, stdout: "", stderr: "" }, + slug: { exitCode: 0, stdout: "[\"hello-chaos-world\",\"r1-paperclip-chaos-2026\"]\n", stderr: "" } } }; + expect(workloadDefinitions.R1.grade("abcd1234", base).verdict).toBe("pass"); + expect(workloadDefinitions.R1.grade("abcd1234", { ...base, commands: { ...base.commands, + slug: { exitCode: 0, stdout: "[\"hello-chaos-world\",\"-r1-paperclip-chaos-2026-\"]\n", stderr: "" } } }).verdict).toBe("fail"); + expect(r1TestChecksSlug("import test from \"node:test\";")).toBe(false); + expect(r1TestChecksSlug("import test from \"node:test\"; import assert from \"node:assert\"; assert.ok(chaosSlug(\"x\"));")).toBe(false); + expect(r1TestChecksSlug([ "import test from \"node:test\";", "import assert from \"node:assert/strict\";", + "for (const [input, want] of [[\"Hello Chaos World\", \"hello-chaos-world\"], [\"a b\", \"a-b\"], [\"-x-\", \"x\"]])", + " test(input, () => assert.equal(chaosSlug(input), want));"].join("\n"))).toBe(true); + const argv = workloadDefinitions.R1.collect.commands.find(command => command.name === "node_test")!.argv; + expect(typeof argv === "function" ? argv("abcd1234") : argv).toEqual([process.execPath, "--test", "packages/shared/src/chaos/slug-abcd1234.test.ts"]); + }); + it("grades B5 against the deterministic bench result", () => { const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", commands: {}, benchProgressObserved: 6, diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index a3e77987747..98ec0309fbe 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -241,7 +241,8 @@ async function main(): Promise { adapters: [...new Set([...plan.tasks.map(task => task.adapter), ...(plan.tasks.some(task => task.workload === "M1") ? plan.options.adapters : [])])], concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, masks: plan.options.masks, - fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline, + fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), workspaceRepo: plan.options.workspaceRepo, + workspaceCommit: plan.options.workspaceCommit, waveId: plan.waveId, timeline, adapterConfigOverrides: chaosLifecycleOverrides(plan.options.lifecycle), onEnvironment: owned => { sweeper = new DaytonaWaveSweeper({ apiKey: process.env.DAYTONA_API_KEY!, companyId: owned.companyId, environmentId: owned.environmentId, api }); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index b67ddcd300a..61483f1ed31 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -1,6 +1,6 @@ import { spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { readdir, readFile } from "node:fs/promises"; +import { readFile, readdir } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -758,7 +758,11 @@ export class TaskObserver { files[relative] = await readFile(path.join(workspacePath, relative), "utf8").catch(() => null); } // Agent-editable code: run it sandboxed, away from the operator's home directory. - for (const command of definition.collect.commands) commands[command.name] = await runOracleCommand(command.argv, workspacePath, { protectedRoots: this.protectedRoots }); + for (const command of definition.collect.commands) { + const argv = typeof command.argv === "function" ? command.argv(this.plan.nonce) : command.argv; + commands[command.name] = await runOracleCommand(argv, workspacePath, { protectedRoots: this.protectedRoots, + readableFiles: argv[0] === process.execPath ? [process.execPath] : [] }); + } } return { issueStatus: issue?.status ?? this.record.finalStatus, diff --git a/tests/restart-chaos/oracle-sandbox.test.ts b/tests/restart-chaos/oracle-sandbox.test.ts index 22c2d09da02..4aef5be2a88 100644 --- a/tests/restart-chaos/oracle-sandbox.test.ts +++ b/tests/restart-chaos/oracle-sandbox.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -17,6 +17,13 @@ describe("oracle sandbox", () => { expect(profile.indexOf('(allow file-read* (subpath "/w")')).toBeGreaterThan(profile.indexOf("(deny file-read*")); }); + it("re-allows only the named files, such as the node executable, never a directory under a protected root", () => { + const profile = oracleSandboxProfile({ workspace: "/w", scratch: "/s", protectedRoots: ["/Users/me"], readableFiles: ["/Users/me/bin/node"] }); + expect(profile.indexOf('(allow file-read* (literal "/Users/me/bin/node"))')).toBeGreaterThan(profile.indexOf('(deny file-read* (subpath "/Users/me"))')); + expect(profile).not.toContain('(allow file-read* (subpath "/Users/me")'); + expect(profile).not.toContain('(allow file-read* (subpath "/Users/me/bin")'); + }); + it("refuses to run unsandboxed on other platforms unless allowed", async () => { const result = await runOracleCommand(["true"], "/", { platform: "linux" }); expect(result.exitCode).toBeNull(); @@ -34,6 +41,23 @@ describe("oracle sandbox", () => { expect(greet.stdout.trim()).toBe("hello, chaos"); }, 60_000); + it.runIf(process.platform === "darwin")("lets node resolve modules in a workspace inside a protected root without exposing its contents", async () => { + const instanceRoot = await temp("restart-chaos-instance-"); + await writeFile(path.join(instanceRoot, "master.key"), "canary-master-key\n"); + const workspace = path.join(instanceRoot, "fixtures", "workspace-repo"); + await mkdir(workspace, { recursive: true }); + await writeFile(path.join(workspace, "slug.ts"), "export const slug = (value: string): string => value.toLowerCase();\n"); + const run = await runOracleCommand([process.execPath, "--input-type=module", "-e", "import { slug } from \"./slug.ts\"; console.log(slug(\"OK\"));"], + workspace, { protectedRoots: [instanceRoot], readableFiles: [process.execPath] }); + expect(run).toMatchObject({ exitCode: 0 }); + expect(run.stdout.trim()).toBe("ok"); + const leak = await runOracleCommand([process.execPath, "-e", `console.log(require("node:fs").readFileSync(${JSON.stringify(path.join(instanceRoot, "master.key"))}, "utf8"))`], + workspace, { protectedRoots: [instanceRoot], readableFiles: [process.execPath] }); + expect(leak.stdout).not.toContain("canary-master-key"); + const listing = await runOracleCommand(["/bin/ls", instanceRoot], workspace, { protectedRoots: [instanceRoot] }); + expect(listing.stdout).not.toContain("master.key"); + }, 60_000); + it.runIf(process.platform === "darwin")("keeps agent code away from protected files but runs the fixture tests", async () => { const secretRoot = await temp("restart-chaos-secret-"); await writeFile(path.join(secretRoot, ".secrets"), "OPENAI_API_KEY=canary-value\n"); diff --git a/tests/restart-chaos/oracle-sandbox.ts b/tests/restart-chaos/oracle-sandbox.ts index 50ae362b199..c5c739ff23e 100644 --- a/tests/restart-chaos/oracle-sandbox.ts +++ b/tests/restart-chaos/oracle-sandbox.ts @@ -18,12 +18,14 @@ function sbplString(value: string) { return `"${value.replaceAll("\\", "\\\\").r * example ~/.secrets or ~/.ssh), write anywhere except a scratch directory, or * use the network. The workspace stays readable. */ -export function oracleSandboxProfile(input: { workspace: string; scratch: string; protectedRoots: readonly string[] }): string { +export function oracleSandboxProfile(input: { workspace: string; scratch: string; protectedRoots: readonly string[]; readableFiles?: readonly string[] }): string { const lines = ["(version 1)", "(allow default)", "(deny network*)", "(deny file-write*)", `(allow file-write* (subpath ${sbplString(input.scratch)}) (literal "/dev/null") (literal "/dev/tty"))`]; for (const root of input.protectedRoots) lines.push(`(deny file-read* (subpath ${sbplString(root)}))`); + for (const root of input.protectedRoots) lines.push(`(allow file-read-metadata (subpath ${sbplString(root)}))`); // Later rules win: re-allow the workspace and scratch even if they sit under a protected root. lines.push(`(allow file-read* (subpath ${sbplString(input.workspace)}) (subpath ${sbplString(input.scratch)}))`); + for (const file of input.readableFiles ?? []) lines.push(`(allow file-read* (literal ${sbplString(file)}))`); return lines.join("\n"); } @@ -34,7 +36,7 @@ export function oracleSandboxProfile(input: { workspace: string; scratch: string * temporary HOME alone does not stop absolute-path reads. */ export async function runOracleCommand(argv: readonly string[], workspace: string, options: { - protectedRoots?: readonly string[]; timeoutMs?: number; platform?: NodeJS.Platform; allowUnsandboxed?: boolean; + protectedRoots?: readonly string[]; readableFiles?: readonly string[]; timeoutMs?: number; platform?: NodeJS.Platform; allowUnsandboxed?: boolean; } = {}): Promise { const platform = options.platform ?? process.platform; const sandboxed = platform === "darwin"; @@ -49,7 +51,8 @@ export async function runOracleCommand(argv: readonly string[], workspace: strin PYTHONDONTWRITEBYTECODE: "1", PYTHONNOUSERSITE: "1", LANG: "C.UTF-8" }; const [command, args] = sandboxed ? ["/usr/bin/sandbox-exec", ["-p", oracleSandboxProfile({ workspace: await resolve(workspace), scratch, - protectedRoots: await Promise.all((options.protectedRoots ?? [os.homedir()]).map(resolve)) }), ...argv]] + protectedRoots: await Promise.all((options.protectedRoots ?? [os.homedir()]).map(resolve)), + readableFiles: await Promise.all((options.readableFiles ?? []).map(resolve)) }), ...argv]] : [argv[0]!, argv.slice(1)]; try { const { stdout, stderr } = await execFileAsync(command, args, { cwd: workspace, env, timeout: options.timeoutMs ?? 60_000, maxBuffer: 1024 * 1024 }); diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts index 525cc04228d..9a8c87fddda 100644 --- a/tests/restart-chaos/scheduler.test.ts +++ b/tests/restart-chaos/scheduler.test.ts @@ -46,6 +46,19 @@ describe("scheduler determinism", () => { expect(() => parseCommand(["--replay", "w-20261106T101112-s42/t03", "--k4-hold-max-seconds", "400"])).toThrow(/do not combine/); }); + it("pairs R1 with --workspace-repo, locally, without the chaoscli fixture workloads", () => { + const command = parseCommand(["--workloads", "R1,C1", "--workspace-repo", "."]); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options.workspaceRepo).toBe(process.cwd()); + expect(command.options.workspaceCommit).toMatch(/^[0-9a-f]{40}$/); + expect(() => parseCommand(["--workloads", "R1"])).toThrow(/go together/); + expect(() => parseCommand(["--workloads", "A1", "--workspace-repo", "x"])).toThrow(/go together/); + expect(() => parseCommand(["--workloads", "R1,B1", "--workspace-repo", "x"])).toThrow(/chaoscli fixture/); + const defaults = parseCommand([]); + if (defaults.mode !== "wave") throw new Error("expected a wave"); + expect(defaults.options.workloads).not.toContain("R1"); + }); + it("pairs K14 (upgrade mid-run) with --upgrade-from, locally, with no hold and outside storm defaults", () => { const command = parseCommand(["--faults", "K14", "--upgrade-from", "../older-checkout", "--seed", "5"]); if (command.mode !== "wave") throw new Error("expected a wave"); diff --git a/tests/restart-chaos/seed.test.ts b/tests/restart-chaos/seed.test.ts index 20c0437b59e..c00295926d0 100644 --- a/tests/restart-chaos/seed.test.ts +++ b/tests/restart-chaos/seed.test.ts @@ -1,9 +1,11 @@ +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChaosApi } from "./api.js"; -import { seedChaosLab } from "./seed.js"; +import { cloneWorkspaceRepository, createFixtureRepository, seedChaosLab } from "./seed.js"; import { Timeline } from "./timeline.js"; type Call = { method: string; route: string; body?: unknown }; @@ -104,3 +106,23 @@ describe("seedChaosLab RC-06 mask", () => { expect(timeline.entries.some(entry => entry.kind === "seed_mask_rc06")).toBe(false); }); }); + +describe("cloneWorkspaceRepository", () => { + it("clones a fresh, shallow working copy of the source repository", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "restart-chaos-clone-")); + try { + const source = await createFixtureRepository(path.join(root, "source")); + const clone = cloneWorkspaceRepository(source, path.join(root, "clone")); + expect(existsSync(path.join(clone, "chaoscli/cli.py"))).toBe(true); + expect(execFileSync("git", ["rev-parse", "--is-shallow-repository"], { cwd: clone, encoding: "utf8" }).trim()).toBe("true"); + const first = execFileSync("git", ["rev-parse", "HEAD"], { cwd: source, encoding: "utf8" }).trim(); + expect(execFileSync("git", ["rev-parse", "HEAD"], { cwd: clone, encoding: "utf8" }).trim()).toBe(first); + execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "--quiet", "--allow-empty", "-m", "later"], { cwd: source }); + const pinned = cloneWorkspaceRepository(source, path.join(root, "pinned"), first); + expect(execFileSync("git", ["rev-parse", "HEAD"], { cwd: pinned, encoding: "utf8" }).trim()).toBe(first); + expect(() => cloneWorkspaceRepository(source, path.join(root, "missing"), "0".repeat(40))).toThrow(/no longer has commit/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts index 8c1f53b1ae8..0796d7f86db 100644 --- a/tests/restart-chaos/seed.ts +++ b/tests/restart-chaos/seed.ts @@ -22,6 +22,20 @@ export interface ChaosLab { } /** Create the `chaos-sandbox` git repository: python CLI, one failing test, bench.sh. */ +export function cloneWorkspaceRepository(source: string, directory: string, commit?: string): string { + const git = (...args: string[]) => execFileSync("git", args, { cwd: directory, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + execFileSync("git", ["clone", "--quiet", "--depth", "1", "--no-hardlinks", `file://${source}`, directory], { stdio: "pipe" }); + if (commit && git("rev-parse", "HEAD") !== commit) { + try { + git("fetch", "--quiet", "--depth", "1", "origin", commit); + git("checkout", "--quiet", "--detach", commit); + } catch (error) { + throw new Error(`--workspace-repo ${source} no longer has commit ${commit} that this wave used`, { cause: error }); + } + } + return directory; +} + export async function createFixtureRepository(directory: string, benchSeconds?: number): Promise { await mkdir(directory, { recursive: true }); for (const [relative, file] of Object.entries(fixtureRepositoryFiles(benchSeconds))) { @@ -116,6 +130,8 @@ export async function seedChaosLab(input: { budgetCents: number; masks?: readonly string[]; fixtureRoot: string; + workspaceRepo?: string; + workspaceCommit?: string; waveId: string; timeline: Timeline; /** Merged into each agent's adapterConfig (for example a runner lifecycle mode). */ @@ -130,7 +146,9 @@ export async function seedChaosLab(input: { await installFixtureDaytonaProvider(runnerApiView(input.api), { environment: { id: "daytona" } } as MatrixExecution); timeline.add("seed_daytona_plugin", {}); } - const fixtureRepositoryPath = await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox")); + const fixtureRepositoryPath = input.workspaceRepo + ? cloneWorkspaceRepository(input.workspaceRepo, path.join(input.fixtureRoot, "workspace-repo"), input.workspaceCommit) + : await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox")); const company = await api.post<{ id: string }>("/api/companies", { name: "Chaos Lab", description: `Restart chaos harness wave ${input.waveId}`, diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index ea36a6305a8..427985f2b6c 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -7,7 +7,7 @@ export type FaultKind = (typeof FAULT_KINDS)[number]; export type TaskFaultKind = FaultKind | "K13"; /** G1: an approval-gated action across a restart. H1: a budget hard stop crossed mid-run with a restart (its own wave). */ -export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1"] as const; +export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1"] as const; export type WorkloadId = (typeof WORKLOAD_IDS)[number]; export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; @@ -72,6 +72,8 @@ export interface WaveOptions { * and instance directory. Later K14 faults restart the current build, like K2. */ upgradeFrom?: string; + workspaceRepo?: string; + workspaceCommit?: string; } export interface StormFault { diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index d78efa993df..c1c837191f0 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -17,7 +17,7 @@ export interface WorkloadDefinition { description(nonce: string, agents?: WorkloadAgents): string; followUp?(nonce: string): string; /** Files and commands the observer must collect for the T-oracle. */ - collect: { documents: string[]; files(nonce: string): string[]; commands: Array<{ name: string; argv: string[] }>; + collect: { documents: string[]; files(nonce: string): string[]; commands: Array<{ name: string; argv: string[] | ((nonce: string) => string[]) }>; childDocuments?: string[] }; grade(nonce: string, artifacts: OracleArtifacts, agents?: WorkloadAgents): OracleResult; } @@ -38,6 +38,14 @@ const followUpMarker = (nonce: string) => `FOLLOWUP-${nonce}`; const notesFile = (nonce: string) => `NOTES-${nonce}.md`; const benchFile = (nonce: string) => `bench-result-${nonce}.txt`; const benchRecord = (nonce: string) => `.bench/run-${nonce}.txt`; +const r1Module = (nonce: string) => `packages/shared/src/chaos/slug-${nonce}.ts`; +const r1Test = (nonce: string) => `packages/shared/src/chaos/slug-${nonce}.test.ts`; + +export function r1TestChecksSlug(text: string | null | undefined): boolean { + if (!text) return false; + return text.includes("node:test") && text.includes("node:assert") && /\bassert(\.\w+)?\s*\(/.test(text) + && text.includes("chaosSlug") && text.includes("hello-chaos-world"); +} /** bench.sh must have actually run: at least this long between start and finish. */ export const BENCH_MIN_SECONDS = 200; /** B5: distinct progress values the harness must have observed itself while polling. */ @@ -264,6 +272,45 @@ export const workloadDefinitions: Record = { ]); }, }, + R1: { + id: "R1", + label: "Code change in a Paperclip clone", + usesRepository: true, + fireWindowMs: [20_000, 90_000], + budgetMs: 15 * 60_000, + title: nonce => `Chaos R1 slug module ${nonce}`, + description: nonce => [ + "Work in this task's repository workspace, a clone of the Paperclip repository.", + `Add \`${r1Module(nonce)}\` exporting \`chaosSlug(input: string): string\`: lowercase, every run of characters`, + "other than a-z and 0-9 becomes a single `-`, and no leading or trailing `-`.", + `Add \`${r1Test(nonce)}\` that uses \`node:test\` and \`node:assert/strict\`, imports \`./slug-${nonce}.ts\`,`, + "and checks at least three cases, including `chaosSlug(\"Hello Chaos World\") === \"hello-chaos-world\"`.", + `Run \`node --test ${r1Test(nonce)}\` from the repository root until it passes.`, + "Do not install dependencies and do not change any other file. Then mark this task done.", + ].join("\n"), + collect: { + documents: [], + files: nonce => [r1Module(nonce), r1Test(nonce)], + commands: [ + { name: "node_test", argv: nonce => [process.execPath, "--test", r1Test(nonce)] }, + { name: "slug", argv: nonce => [process.execPath, "--input-type=module", "-e", + `import { chaosSlug } from "./${r1Module(nonce)}"; console.log(JSON.stringify(["Hello Chaos World", " R1 -- Paperclip__Chaos 2026! "].map(chaosSlug)));`] }, + ], + }, + grade(nonce, artifacts) { + const tests = artifacts.commands.node_test; + const slug = artifacts.commands.slug; + return result([ + statusCheck(artifacts), + { name: "workspace_located", ok: artifacts.workspacePath !== null }, + { name: "module_present", ok: typeof artifacts.files[r1Module(nonce)] === "string" }, + { name: "test_checks_chaos_slug", ok: r1TestChecksSlug(artifacts.files[r1Test(nonce)]) }, + { name: "node_test_passes", ok: tests?.exitCode === 0, detail: `exit=${tests?.exitCode ?? "not run"}` }, + { name: "required_and_hidden_cases", ok: slug?.exitCode === 0 && slug.stdout.trim() === JSON.stringify(["hello-chaos-world", "r1-paperclip-chaos-2026"]), + detail: JSON.stringify(slug?.stdout.trim() ?? null) }, + ]); + }, + }, B5: { id: "B5", label: "Long bench.sh run",