From cf38002439c247d6f3b6073ec7e3061bdfcd94ca Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 15:23:09 -0600 Subject: [PATCH] test(chaos): add R1, a code change in a fresh clone of a real repository (--workspace-repo) Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/README.md | 16 ++++++++++ tests/restart-chaos/config.ts | 14 +++++++-- tests/restart-chaos/grader.test.ts | 11 +++++++ tests/restart-chaos/main.ts | 2 +- tests/restart-chaos/observer.ts | 5 +++- tests/restart-chaos/scheduler.test.ts | 12 ++++++++ tests/restart-chaos/seed.test.ts | 20 ++++++++++++- tests/restart-chaos/seed.ts | 10 ++++++- tests/restart-chaos/types.ts | 3 +- tests/restart-chaos/workloads.ts | 42 ++++++++++++++++++++++++++- 10 files changed, 127 insertions(+), 8 deletions(-) diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index fb5b348a458..a70b2d4a65f 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -102,6 +102,22 @@ Neither G1 nor H1 is in the default workloads. The board's actions and what the observer saw are in the timeline (`governance_*`) and in each task's grade (`governance`). +## Code change in a real repository (R1, `--workspace-repo`) + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex,claude --workloads R1,C1 --workspace-repo /path/to/paperclip \ + --faults K1,K2,K4 --concurrency 4 --tasks 8 --seed 22 +``` + +The harness clones the given git repository (shallow, from its committed HEAD) +as the project's workspace, in place of the chaoscli fixture. R1 asks the agent +to add a TypeScript module and a `node:test` file under +`packages/shared/src/chaos/`, named with the task's nonce so concurrent tasks +share the repository without colliding. The oracle runs the agent's test and a +hidden case against its module. B1 and B5 need the chaoscli fixture, so they do +not combine with `--workspace-repo`. Local only. + ## Upgrade mid-run (K14, `--upgrade-from`) ```sh diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 963b966c6ed..297640475e1 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -96,13 +96,14 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "crash-point": { type: "string" }, "crash-point-hold-ms": { type: "string" }, "upgrade-from": { type: "string" }, + "workspace-repo": { type: "string" }, }, }); const keep = values.keep === true; const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -138,7 +139,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC env, adapters: list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]), // The governance scenarios (G1, H1) are opt-in: H1 needs a wave of its own. - workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1")), + workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1")), // A crash-point wave schedules no fault; K1 is the restart its self-kill gets. faults: list(values.faults, FAULT_KINDS, "faults", stormMinutes ? STORM_DEFAULT_FAULTS : crashPoint ? ["K1"] : ["K1", "K2", "K4"]), triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), @@ -170,6 +171,15 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC if (crashPoint) throw new Error("--upgrade-from does not combine with --crash-point"); options.upgradeFrom = path.resolve(upgradeFrom); } + const workspaceRepo = values["workspace-repo"]; + if (options.workloads.includes("R1") !== (workspaceRepo !== undefined)) { + throw new Error("R1 (code change in a cloned repository) and --workspace-repo go together: pass both or neither"); + } + if (workspaceRepo !== undefined) { + if (env !== "local") throw new Error("--workspace-repo runs only with --env local"); + if (options.workloads.some(id => id === "B1" || id === "B5")) throw new Error("--workspace-repo replaces the chaoscli fixture that B1 and B5 need"); + options.workspaceRepo = path.resolve(workspaceRepo); + } // H1 pauses the whole company when its budget hard stop is crossed: no other task can share its // wave, its tasks run one at a time, and the hard stop must be on. if (options.workloads.includes("H1")) { diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index f300f9567ed..e015a1fc1b6 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -229,6 +229,17 @@ describe("T-oracles", () => { expect(workloadDefinitions.B1.grade("abcd1234", { ...base, commands: { ...base.commands, unittest: { exitCode: 1, stdout: "", stderr: "FAILED" } } }).verdict).toBe("fail"); }); + it("grades R1 by running the agent's node test and a hidden case against its module", () => { + const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", + files: { "packages/shared/src/chaos/slug-abcd1234.ts": "export function chaosSlug() {}", + "packages/shared/src/chaos/slug-abcd1234.test.ts": "import test from \"node:test\";" }, + commands: { node_test: { exitCode: 0, stdout: "", stderr: "" }, slug: { exitCode: 0, stdout: "r1-paperclip-chaos-2026\n", stderr: "" } } }; + expect(workloadDefinitions.R1.grade("abcd1234", base).verdict).toBe("pass"); + expect(workloadDefinitions.R1.grade("abcd1234", { ...base, commands: { ...base.commands, slug: { exitCode: 0, stdout: "-r1-paperclip-chaos-2026-\n", stderr: "" } } }).verdict).toBe("fail"); + const argv = workloadDefinitions.R1.collect.commands.find(command => command.name === "node_test")!.argv; + expect(typeof argv === "function" ? argv("abcd1234") : argv).toEqual(["node", "--test", "packages/shared/src/chaos/slug-abcd1234.test.ts"]); + }); + it("grades B5 against the deterministic bench result", () => { const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", commands: {}, benchProgressObserved: 6, diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index a3e77987747..6e1ddd50b33 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -241,7 +241,7 @@ async function main(): Promise { adapters: [...new Set([...plan.tasks.map(task => task.adapter), ...(plan.tasks.some(task => task.workload === "M1") ? plan.options.adapters : [])])], concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, masks: plan.options.masks, - fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline, + fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), workspaceRepo: plan.options.workspaceRepo, waveId: plan.waveId, timeline, adapterConfigOverrides: chaosLifecycleOverrides(plan.options.lifecycle), onEnvironment: owned => { sweeper = new DaytonaWaveSweeper({ apiKey: process.env.DAYTONA_API_KEY!, companyId: owned.companyId, environmentId: owned.environmentId, api }); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index b67ddcd300a..de6cc48d3a8 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -758,7 +758,10 @@ export class TaskObserver { files[relative] = await readFile(path.join(workspacePath, relative), "utf8").catch(() => null); } // Agent-editable code: run it sandboxed, away from the operator's home directory. - for (const command of definition.collect.commands) commands[command.name] = await runOracleCommand(command.argv, workspacePath, { protectedRoots: this.protectedRoots }); + for (const command of definition.collect.commands) { + const argv = typeof command.argv === "function" ? command.argv(this.plan.nonce) : command.argv; + commands[command.name] = await runOracleCommand(argv, workspacePath, { protectedRoots: this.protectedRoots }); + } } return { issueStatus: issue?.status ?? this.record.finalStatus, diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts index 525cc04228d..7eac351ec81 100644 --- a/tests/restart-chaos/scheduler.test.ts +++ b/tests/restart-chaos/scheduler.test.ts @@ -46,6 +46,18 @@ describe("scheduler determinism", () => { expect(() => parseCommand(["--replay", "w-20261106T101112-s42/t03", "--k4-hold-max-seconds", "400"])).toThrow(/do not combine/); }); + it("pairs R1 with --workspace-repo, locally, without the chaoscli fixture workloads", () => { + const command = parseCommand(["--workloads", "R1,C1", "--workspace-repo", "../paperclip-clone"]); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options.workspaceRepo!.endsWith("paperclip-clone")).toBe(true); + expect(() => parseCommand(["--workloads", "R1"])).toThrow(/go together/); + expect(() => parseCommand(["--workloads", "A1", "--workspace-repo", "x"])).toThrow(/go together/); + expect(() => parseCommand(["--workloads", "R1,B1", "--workspace-repo", "x"])).toThrow(/chaoscli fixture/); + const defaults = parseCommand([]); + if (defaults.mode !== "wave") throw new Error("expected a wave"); + expect(defaults.options.workloads).not.toContain("R1"); + }); + it("pairs K14 (upgrade mid-run) with --upgrade-from, locally, with no hold and outside storm defaults", () => { const command = parseCommand(["--faults", "K14", "--upgrade-from", "../older-checkout", "--seed", "5"]); if (command.mode !== "wave") throw new Error("expected a wave"); diff --git a/tests/restart-chaos/seed.test.ts b/tests/restart-chaos/seed.test.ts index 20c0437b59e..e2fb96e81c3 100644 --- a/tests/restart-chaos/seed.test.ts +++ b/tests/restart-chaos/seed.test.ts @@ -1,9 +1,11 @@ +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChaosApi } from "./api.js"; -import { seedChaosLab } from "./seed.js"; +import { cloneWorkspaceRepository, createFixtureRepository, seedChaosLab } from "./seed.js"; import { Timeline } from "./timeline.js"; type Call = { method: string; route: string; body?: unknown }; @@ -104,3 +106,19 @@ describe("seedChaosLab RC-06 mask", () => { expect(timeline.entries.some(entry => entry.kind === "seed_mask_rc06")).toBe(false); }); }); + +describe("cloneWorkspaceRepository", () => { + it("clones a fresh, shallow working copy of the source repository", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "restart-chaos-clone-")); + try { + const source = await createFixtureRepository(path.join(root, "source")); + const clone = cloneWorkspaceRepository(source, path.join(root, "clone")); + expect(existsSync(path.join(clone, "chaoscli/cli.py"))).toBe(true); + expect(execFileSync("git", ["rev-parse", "--is-shallow-repository"], { cwd: clone, encoding: "utf8" }).trim()).toBe("true"); + expect(execFileSync("git", ["rev-parse", "HEAD"], { cwd: clone, encoding: "utf8" })) + .toBe(execFileSync("git", ["rev-parse", "HEAD"], { cwd: source, encoding: "utf8" })); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts index 8c1f53b1ae8..5ba60420085 100644 --- a/tests/restart-chaos/seed.ts +++ b/tests/restart-chaos/seed.ts @@ -22,6 +22,11 @@ export interface ChaosLab { } /** Create the `chaos-sandbox` git repository: python CLI, one failing test, bench.sh. */ +export function cloneWorkspaceRepository(source: string, directory: string): string { + execFileSync("git", ["clone", "--quiet", "--depth", "1", "--no-hardlinks", `file://${source}`, directory], { stdio: "pipe" }); + return directory; +} + export async function createFixtureRepository(directory: string, benchSeconds?: number): Promise { await mkdir(directory, { recursive: true }); for (const [relative, file] of Object.entries(fixtureRepositoryFiles(benchSeconds))) { @@ -116,6 +121,7 @@ export async function seedChaosLab(input: { budgetCents: number; masks?: readonly string[]; fixtureRoot: string; + workspaceRepo?: string; waveId: string; timeline: Timeline; /** Merged into each agent's adapterConfig (for example a runner lifecycle mode). */ @@ -130,7 +136,9 @@ export async function seedChaosLab(input: { await installFixtureDaytonaProvider(runnerApiView(input.api), { environment: { id: "daytona" } } as MatrixExecution); timeline.add("seed_daytona_plugin", {}); } - const fixtureRepositoryPath = await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox")); + const fixtureRepositoryPath = input.workspaceRepo + ? cloneWorkspaceRepository(input.workspaceRepo, path.join(input.fixtureRoot, "workspace-repo")) + : await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox")); const company = await api.post<{ id: string }>("/api/companies", { name: "Chaos Lab", description: `Restart chaos harness wave ${input.waveId}`, diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index ea36a6305a8..027f821e887 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -7,7 +7,7 @@ export type FaultKind = (typeof FAULT_KINDS)[number]; export type TaskFaultKind = FaultKind | "K13"; /** G1: an approval-gated action across a restart. H1: a budget hard stop crossed mid-run with a restart (its own wave). */ -export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1"] as const; +export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1"] as const; export type WorkloadId = (typeof WORKLOAD_IDS)[number]; export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; @@ -72,6 +72,7 @@ export interface WaveOptions { * and instance directory. Later K14 faults restart the current build, like K2. */ upgradeFrom?: string; + workspaceRepo?: string; } export interface StormFault { diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index d78efa993df..b32f8cfb99a 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -17,7 +17,7 @@ export interface WorkloadDefinition { description(nonce: string, agents?: WorkloadAgents): string; followUp?(nonce: string): string; /** Files and commands the observer must collect for the T-oracle. */ - collect: { documents: string[]; files(nonce: string): string[]; commands: Array<{ name: string; argv: string[] }>; + collect: { documents: string[]; files(nonce: string): string[]; commands: Array<{ name: string; argv: string[] | ((nonce: string) => string[]) }>; childDocuments?: string[] }; grade(nonce: string, artifacts: OracleArtifacts, agents?: WorkloadAgents): OracleResult; } @@ -38,6 +38,8 @@ const followUpMarker = (nonce: string) => `FOLLOWUP-${nonce}`; const notesFile = (nonce: string) => `NOTES-${nonce}.md`; const benchFile = (nonce: string) => `bench-result-${nonce}.txt`; const benchRecord = (nonce: string) => `.bench/run-${nonce}.txt`; +const r1Module = (nonce: string) => `packages/shared/src/chaos/slug-${nonce}.ts`; +const r1Test = (nonce: string) => `packages/shared/src/chaos/slug-${nonce}.test.ts`; /** bench.sh must have actually run: at least this long between start and finish. */ export const BENCH_MIN_SECONDS = 200; /** B5: distinct progress values the harness must have observed itself while polling. */ @@ -264,6 +266,44 @@ export const workloadDefinitions: Record = { ]); }, }, + R1: { + id: "R1", + label: "Code change in a Paperclip clone", + usesRepository: true, + fireWindowMs: [20_000, 90_000], + budgetMs: 15 * 60_000, + title: nonce => `Chaos R1 slug module ${nonce}`, + description: nonce => [ + "Work in this task's repository workspace, a clone of the Paperclip repository.", + `Add \`${r1Module(nonce)}\` exporting \`chaosSlug(input: string): string\`: lowercase, every run of characters`, + "other than a-z and 0-9 becomes a single `-`, and no leading or trailing `-`.", + `Add \`${r1Test(nonce)}\` that uses \`node:test\` and \`node:assert/strict\`, imports \`./slug-${nonce}.ts\`,`, + "and checks at least three cases, including `chaosSlug(\"Hello Chaos World\") === \"hello-chaos-world\"`.", + `Run \`node --test ${r1Test(nonce)}\` from the repository root until it passes.`, + "Do not install dependencies and do not change any other file. Then mark this task done.", + ].join("\n"), + collect: { + documents: [], + files: nonce => [r1Module(nonce), r1Test(nonce)], + commands: [ + { name: "node_test", argv: nonce => ["node", "--test", r1Test(nonce)] }, + { name: "slug", argv: nonce => ["node", "--input-type=module", "-e", + `import { chaosSlug } from "./${r1Module(nonce)}"; console.log(chaosSlug(" R1 -- Paperclip__Chaos 2026! "));`] }, + ], + }, + grade(nonce, artifacts) { + const tests = artifacts.commands.node_test; + const slug = artifacts.commands.slug; + return result([ + statusCheck(artifacts), + { name: "workspace_located", ok: artifacts.workspacePath !== null }, + { name: "module_present", ok: typeof artifacts.files[r1Module(nonce)] === "string" }, + { name: "test_present", ok: (artifacts.files[r1Test(nonce)] ?? "").includes("node:test") }, + { name: "node_test_passes", ok: tests?.exitCode === 0, detail: `exit=${tests?.exitCode ?? "not run"}` }, + { name: "hidden_case", ok: slug?.exitCode === 0 && slug.stdout.trim() === "r1-paperclip-chaos-2026", detail: JSON.stringify(slug?.stdout.trim() ?? null) }, + ]); + }, + }, B5: { id: "B5", label: "Long bench.sh run",