mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-12 06:10:44 +01:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness drives real user actions across control-plane restarts: comments, Retry, approvals and budget resumes > - The board's Stop button on a running task was not exercised. The cancel route stamps the stop as operator-initiated so that recovery stands down instead of re-waking the agent > - A restart right after a Stop is where recovery could wrongly restart work the operator stopped, or lose the operator's later "continue" direction > - This pull request adds S1: stop the running run, fire a fault right after, then comment to continue > - The benefit is a graded check that a Stop holds across a restart and that a later continue comment finishes the task ## Linked Issues or Issue Description Refs #20071 **What is missing?** The campaign's scenario list includes mid-run stops. C1 covers a mid-run comment (a steer), but no workload clicks Stop. Without one, a regression where recovery restarts operator-stopped work, or drops the continue comment, would go unseen. **Expected behavior** - After the board stops a run and a restart follows, the stopped run stays cancelled. - Nothing runs until the operator comments. - The continue comment then finishes the task. ## What Changed - S1 workload ("Stop while running, then continue"): a brief task. At its follow-up time, while a run is running, the observer posts `POST /api/heartbeat-runs/:id/cancel` (the board's Stop) and records `stop`. - New `stop` trigger, for S1 only: the planned fault fires 0–2 s after the stop. - The continue comment is posted only after the stop has settled: at least 20 s after the stop, with no run running or queued and no fault in progress. It asks the agent to continue and add a `CONTINUE-<nonce>` line. - The oracle requires: - status done; - the brief with the chaos and continue markers; - `stop_took_effect`: the stopped run ended `cancelled`; - `nothing_ran_after_stop`: no run was created between the stop and the continue comment. - S1 is not a default workload. There is a README section. ## Verification - `npx vitest run` in `tests/restart-chaos`: 205 passed. New tests cover: - the S1 grade (pass, a run restarted after the stop, the stop not taking effect, no stop); - the trigger-to-workload pairing; - an observer test that clicks Stop once and posts the continue comment only after the 20 s settle. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: clean. - A live S1 wave runs after the current chaos wave. Its result will be posted here before merge. ## Risks - If the product moves the stopped run to a status other than `cancelled`, `stop_took_effect` fails. That check is intentional. - The continue comment waits for the stop to settle, so a run stuck `queued` delays it. R5 and the wave deadline still bound the wait. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
462 lines
25 KiB
TypeScript
462 lines
25 KiB
TypeScript
import type { OracleArtifacts, OracleResult, WorkloadId } from "./types.js";
|
|
|
|
/** Deterministic bench output: sum of squares 1..240. */
|
|
export const BENCH_STEPS = 240;
|
|
export const BENCH_EXPECTED = `BENCH_RESULT=${(BENCH_STEPS * (BENCH_STEPS + 1) * (2 * BENCH_STEPS + 1)) / 6}`;
|
|
|
|
export interface WorkloadDefinition {
|
|
id: WorkloadId;
|
|
label: string;
|
|
usesRepository: boolean;
|
|
/** Fault fire offset from issue creation (seeded uniform). */
|
|
fireWindowMs: readonly [number, number];
|
|
followUpWindowMs?: readonly [number, number];
|
|
/** Time budget for the task excluding outage time (R1). */
|
|
budgetMs: number;
|
|
title(nonce: string): string;
|
|
description(nonce: string, agents?: WorkloadAgents): string;
|
|
followUp?(nonce: string): string;
|
|
stopBeforeFollowUp?: boolean;
|
|
/** Files and commands the observer must collect for the T-oracle. */
|
|
collect: { documents: string[]; files(nonce: string): string[]; commands: Array<{ name: string; argv: string[] | ((nonce: string) => string[]) }>;
|
|
childDocuments?: string[] };
|
|
grade(nonce: string, artifacts: OracleArtifacts, agents?: WorkloadAgents): OracleResult;
|
|
}
|
|
|
|
export interface WorkloadAgents { self: { id: string; name: string }; other: { id: string; name: string }; companyId?: string }
|
|
|
|
/** G1: the board's decision for this task, fixed by its nonce (about half approve, half reject). */
|
|
export function g1Decision(nonce: string): "approve" | "reject" {
|
|
return Number.parseInt(nonce.slice(0, 1), 16) % 2 === 0 ? "approve" : "reject";
|
|
}
|
|
export const g1ApprovalTitle = (nonce: string) => `Approve G1-${nonce}`;
|
|
export const g1ChildTitle = (nonce: string) => `Approved G1-${nonce}`;
|
|
|
|
const partMarker = (nonce: string, part: number) => `PART${part}-${nonce}`;
|
|
|
|
const briefMarker = (nonce: string) => `CHAOS-${nonce}`;
|
|
const followUpMarker = (nonce: string) => `FOLLOWUP-${nonce}`;
|
|
const continueMarker = (nonce: string) => `CONTINUE-${nonce}`;
|
|
const notesFile = (nonce: string) => `NOTES-${nonce}.md`;
|
|
const benchFile = (nonce: string) => `bench-result-${nonce}.txt`;
|
|
const benchRecord = (nonce: string) => `.bench/run-${nonce}.txt`;
|
|
const r1Module = (nonce: string) => `packages/shared/src/chaos/slug-${nonce}.ts`;
|
|
const r1Test = (nonce: string) => `packages/shared/src/chaos/slug-${nonce}.test.ts`;
|
|
|
|
export function r1TestChecksSlug(text: string | null | undefined): boolean {
|
|
if (!text) return false;
|
|
return text.includes("node:test") && text.includes("node:assert") && /\bassert(\.\w+)?\s*\(/.test(text)
|
|
&& text.includes("chaosSlug") && text.includes("hello-chaos-world");
|
|
}
|
|
/** bench.sh must have actually run: at least this long between start and finish. */
|
|
export const BENCH_MIN_SECONDS = 200;
|
|
/** B5: distinct progress values the harness must have observed itself while polling. */
|
|
export const BENCH_MIN_OBSERVED_PROGRESS = 3;
|
|
|
|
/** Parse bench.sh's own completion record: `started=<epoch> finished=<epoch> result=<line>`. */
|
|
export function parseBenchRecord(text: string | null | undefined) {
|
|
const fields = Object.fromEntries((text ?? "").trim().split(/\s+/).map(part => part.split("=")).filter(pair => pair.length >= 2)
|
|
.map(([key, ...rest]) => [key!, rest.join("=")]));
|
|
const started = Number(fields.started), finished = Number(fields.finished);
|
|
return Number.isFinite(started) && Number.isFinite(finished) && fields.result
|
|
? { seconds: finished - started, result: fields.result } : null;
|
|
}
|
|
|
|
export function stallWakeChecks(artifacts: OracleArtifacts): OracleResult["checks"] {
|
|
const stalls = artifacts.childStalls ?? [];
|
|
if (!stalls.length) return [];
|
|
const wakes = artifacts.stallWakes ?? [];
|
|
const matching = (stall: { childIssueId: string; recoveryActionId: string | null }) => wakes.filter(wake =>
|
|
wake.stalledIssueId === stall.childIssueId && (stall.recoveryActionId === null || wake.recoveryActionId === stall.recoveryActionId));
|
|
const unwoken = stalls.filter(stall => matching(stall).length === 0).map(stall => `${stall.childIssueId}:${stall.recoveryActionId}`);
|
|
const perPair = new Map<string, number>();
|
|
for (const wake of wakes) perPair.set(`${wake.stalledIssueId}:${wake.recoveryActionId}`, (perPair.get(`${wake.stalledIssueId}:${wake.recoveryActionId}`) ?? 0) + 1);
|
|
const repeated = [...perPair.entries()].filter(([, count]) => count > 1).map(([key]) => key);
|
|
return [{ name: "parent_woken_once_per_stall", ok: !unwoken.length && !repeated.length,
|
|
detail: `stalls=${stalls.length} wakes=${wakes.length}${unwoken.length ? ` unwoken=${unwoken.join(",")}` : ""}${repeated.length ? ` repeated=${repeated.join(",")}` : ""}` }];
|
|
}
|
|
|
|
function result(checks: OracleResult["checks"]): OracleResult {
|
|
return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks };
|
|
}
|
|
|
|
function statusCheck(artifacts: OracleArtifacts) {
|
|
return { name: "issue_done", ok: artifacts.issueStatus === "done", detail: `status=${artifacts.issueStatus ?? "unknown"}` };
|
|
}
|
|
|
|
function briefChecks(body: string | null | undefined, markers: string[]) {
|
|
const text = body ?? "";
|
|
const bullets = text.split(/\r?\n/).filter(line => /^\s*[-*] \S/.test(line)).length;
|
|
return [
|
|
{ name: "brief_document_present", ok: body != null && text.trim().length > 0 },
|
|
...markers.map(marker => ({ name: `brief_contains_${marker.split("-")[0]!.toLowerCase()}_marker`, ok: text.includes(marker), detail: marker })),
|
|
{ name: "brief_has_three_bullets", ok: bullets >= 3, detail: `bullets=${bullets}` },
|
|
];
|
|
}
|
|
|
|
const briefDescription = (nonce: string) => [
|
|
"Write a short brief on why idempotent retries matter for background job queues.",
|
|
"Save it as this issue's document with key `brief` (title \"Brief\"), in markdown.",
|
|
`The document must contain the line \`${briefMarker(nonce)}\` exactly, on its own line,`,
|
|
"and at least three bullet points that start with \"- \".",
|
|
"Do not create files or other tasks. When the document is saved, mark this task done.",
|
|
].join("\n");
|
|
|
|
const delegateDescription = (nonce: string, agents?: WorkloadAgents) => {
|
|
const self = agents?.self.name ?? "yourself";
|
|
const other = agents?.other.name ?? "yourself";
|
|
return [
|
|
"Coordinate this task by delegating it to two subtasks, then summarize.",
|
|
`1. Create subtask "Part 1 ${nonce}" with this task as its parent, assigned to ${other}. Its description:`,
|
|
` "Save this issue's document with key \`part\` (title \"Part\"), in markdown, containing the line \`${partMarker(nonce, 1)}\` exactly on its own line and one sentence on why retries need idempotency keys. Then mark this subtask done."`,
|
|
`2. Create subtask "Part 2 ${nonce}" with this task as its parent, assigned to ${self}. Same description, with the line \`${partMarker(nonce, 2)}\` instead.`,
|
|
"Do not do the subtasks' work yourself. Wait until both subtasks are done.",
|
|
`3. Then save this issue's document with key \`summary\` (title "Summary") containing both lines \`${partMarker(nonce, 1)}\` and \`${partMarker(nonce, 2)}\`, each on its own line.`,
|
|
"Then mark this task done.",
|
|
].join("\n");
|
|
};
|
|
|
|
const approvalDescription = (nonce: string, agents?: WorkloadAgents) => [
|
|
"This task needs the board's approval before you act.",
|
|
"1. Request approval exactly once with the `call_api` tool: operationId `POST /api/companies/{companyId}/approvals`,"
|
|
+ ` pathParams \`{ "companyId": "${agents?.companyId ?? "<your company id>"}" }\`, and this body:`,
|
|
` \`{ "type": "request_board_approval", "issueIds": ["<this task's id>"], "payload": { "title": "${g1ApprovalTitle(nonce)}",`
|
|
+ ` "summary": "Create the follow-up subtask for ${nonce}.", "recommendedAction": "Approve", "risks": [] } }\``,
|
|
"2. Then set this task's status to `blocked` and end your turn. Do not create anything else yet.",
|
|
`3. Only if you are woken because that approval was approved: create exactly one subtask titled "${g1ChildTitle(nonce)}"`,
|
|
" with this task as its parent and status `backlog`, unassigned. Then mark this task done.",
|
|
"If the approval is rejected, do not create the subtask.",
|
|
].join("\n");
|
|
|
|
function approvalChecks(nonce: string, artifacts: OracleArtifacts) {
|
|
const decision = g1Decision(nonce);
|
|
const governance = artifacts.governance;
|
|
const approvals = (artifacts.approvals ?? []).filter(approval => approval.title === null || approval.title.includes(nonce));
|
|
const children = (artifacts.children ?? []).filter(child => (child.title ?? "").includes(g1ChildTitle(nonce)));
|
|
return [
|
|
{ name: "issue_status", ok: artifacts.issueStatus === (decision === "approve" ? "done" : "cancelled"),
|
|
detail: `decision=${decision} status=${artifacts.issueStatus ?? "unknown"}` },
|
|
// The agent's create is not idempotent: a restart that makes it ask twice is a finding.
|
|
{ name: "one_approval_requested", ok: approvals.length === 1, detail: `approvals=${approvals.length}` },
|
|
{ name: "board_decided_after_restart", ok: governance?.decision !== undefined && governance.decision.error === null
|
|
&& governance.decision.afterFault, detail: JSON.stringify(governance?.decision ?? null) },
|
|
{ name: "approval_pending_across_restart", ok: governance?.decision?.pendingWhenDecided === true },
|
|
{ name: "action_gated_by_approval", ok: governance?.childrenBeforeDecision === 0,
|
|
detail: `subtasks_before_decision=${governance?.childrenBeforeDecision ?? "not checked"}` },
|
|
{ name: "approval_decided", ok: approvals.length === 1 && approvals[0]!.status === (decision === "approve" ? "approved" : "rejected"),
|
|
detail: approvals.map(approval => approval.status).join(",") },
|
|
{ name: decision === "approve" ? "approved_action_ran_once" : "rejected_action_never_ran",
|
|
ok: children.length === (decision === "approve" ? 1 : 0), detail: `subtasks=${children.length}` },
|
|
];
|
|
}
|
|
|
|
function budgetChecks(artifacts: OracleArtifacts) {
|
|
const governance = artifacts.governance;
|
|
return [
|
|
{ name: "budget_set", ok: typeof governance?.budgetSetCents === "number" && !governance.budgetSetError,
|
|
detail: governance?.budgetSetError ?? `budget=${governance?.budgetSetCents ?? "unset"}` },
|
|
{ name: "company_paused_for_budget", ok: typeof governance?.budgetPausedAt === "number" },
|
|
{ name: "pause_held_until_resume", ok: typeof governance?.budgetPausedAt === "number" && !governance.pauseLostAt },
|
|
{ name: "nothing_dispatched_while_paused", ok: governance?.runsStartedWhilePaused !== undefined && governance.runsStartedWhilePaused.length === 0,
|
|
detail: (governance?.runsStartedWhilePaused ?? ["not checked"]).join(",") },
|
|
{ name: "board_resumed_after_restart", ok: governance?.resume !== undefined && governance.resume.error === null && governance.resume.afterFault,
|
|
detail: JSON.stringify(governance?.resume ?? null) },
|
|
{ name: "company_active_at_rest", ok: governance?.companyActiveAtRest === true },
|
|
];
|
|
}
|
|
|
|
export const workloadDefinitions: Record<WorkloadId, WorkloadDefinition> = {
|
|
G1: {
|
|
id: "G1",
|
|
label: "Approval-gated action across a restart",
|
|
usesRepository: false,
|
|
// The `approval` trigger fires on the pending approval; a `time` fault lands around it.
|
|
fireWindowMs: [20_000, 90_000],
|
|
budgetMs: 15 * 60_000,
|
|
title: nonce => `Chaos G1 approval ${nonce}`,
|
|
description: approvalDescription,
|
|
collect: { documents: [], files: () => [], commands: [], childDocuments: [] },
|
|
grade(nonce, artifacts) {
|
|
return result(approvalChecks(nonce, artifacts));
|
|
},
|
|
},
|
|
H1: {
|
|
id: "H1",
|
|
label: "Budget hard stop crossed mid-run, across a restart",
|
|
usesRepository: false,
|
|
fireWindowMs: [15_000, 60_000],
|
|
budgetMs: 15 * 60_000,
|
|
title: nonce => `Chaos H1 budget ${nonce}`,
|
|
description: briefDescription,
|
|
collect: { documents: ["brief"], files: () => [], commands: [] },
|
|
grade(nonce, artifacts) {
|
|
return result([statusCheck(artifacts), ...briefChecks(artifacts.documents.brief, [briefMarker(nonce)]), ...budgetChecks(artifacts)]);
|
|
},
|
|
},
|
|
M1: {
|
|
id: "M1",
|
|
label: "Delegate to two subtasks",
|
|
usesRepository: false,
|
|
fireWindowMs: [20_000, 120_000],
|
|
budgetMs: 20 * 60_000,
|
|
title: nonce => `Chaos M1 delegation ${nonce}`,
|
|
description: delegateDescription,
|
|
collect: { documents: ["summary"], files: () => [], commands: [], childDocuments: ["part"] },
|
|
grade(nonce, artifacts, agents) {
|
|
const children = artifacts.children ?? [];
|
|
const holders = (n: number) => children.filter(child => (child.documents.part ?? "").includes(partMarker(nonce, n)));
|
|
const part1 = holders(1), part2 = holders(2);
|
|
const distinct = part1.length === 1 && part2.length === 1 && part1[0]!.id !== part2[0]!.id;
|
|
const summary = artifacts.documents.summary ?? "";
|
|
return result([
|
|
statusCheck(artifacts),
|
|
{ name: "two_agents", ok: Boolean(agents) && agents!.self.id !== agents!.other.id,
|
|
detail: agents ? `self=${agents.self.id} other=${agents.other.id}` : "no agents" },
|
|
{ name: "two_subtasks", ok: children.length === 2, detail: `children=${children.length}` },
|
|
{ name: "subtasks_done", ok: children.length > 0 && children.every(child => child.status === "done"),
|
|
detail: children.map(child => child.status).join(",") },
|
|
{ name: "parts_in_distinct_subtasks", ok: distinct, detail: `part1=${part1.length} part2=${part2.length}` },
|
|
{ name: "part1_delegated", ok: distinct && Boolean(agents) && part1[0]!.assigneeAgentId === agents!.other.id,
|
|
detail: `assignee=${part1[0]?.assigneeAgentId ?? "none"}` },
|
|
{ name: "part2_kept", ok: distinct && Boolean(agents) && part2[0]!.assigneeAgentId === agents!.self.id,
|
|
detail: `assignee=${part2[0]?.assigneeAgentId ?? "none"}` },
|
|
{ name: "summary_has_both_parts", ok: summary.includes(partMarker(nonce, 1)) && summary.includes(partMarker(nonce, 2)) },
|
|
...stallWakeChecks(artifacts),
|
|
]);
|
|
},
|
|
},
|
|
A1: {
|
|
id: "A1",
|
|
label: "Brief to issue document",
|
|
usesRepository: false,
|
|
fireWindowMs: [10_000, 45_000],
|
|
budgetMs: 10 * 60_000,
|
|
title: nonce => `Chaos A1 brief ${nonce}`,
|
|
description: briefDescription,
|
|
collect: { documents: ["brief"], files: () => [], commands: [] },
|
|
grade(nonce, artifacts) {
|
|
return result([statusCheck(artifacts), ...briefChecks(artifacts.documents.brief, [briefMarker(nonce)])]);
|
|
},
|
|
},
|
|
C1: {
|
|
id: "C1",
|
|
label: "Comment while running",
|
|
usesRepository: false,
|
|
fireWindowMs: [20_000, 60_000],
|
|
followUpWindowMs: [8_000, 25_000],
|
|
budgetMs: 12 * 60_000,
|
|
title: nonce => `Chaos C1 brief with follow-up ${nonce}`,
|
|
description: briefDescription,
|
|
followUp: nonce => `Additional requirement for this task: also include the line \`${followUpMarker(nonce)}\` on its own line in the \`brief\` document before you mark the task done.`,
|
|
collect: { documents: ["brief"], files: () => [], commands: [] },
|
|
grade(nonce, artifacts) {
|
|
return result([statusCheck(artifacts), ...briefChecks(artifacts.documents.brief, [briefMarker(nonce), followUpMarker(nonce)])]);
|
|
},
|
|
},
|
|
S1: {
|
|
id: "S1",
|
|
label: "Stop while running, then continue",
|
|
usesRepository: false,
|
|
fireWindowMs: [20_000, 60_000],
|
|
followUpWindowMs: [8_000, 25_000],
|
|
budgetMs: 12 * 60_000,
|
|
title: nonce => `Chaos S1 brief stopped and continued ${nonce}`,
|
|
description: briefDescription,
|
|
stopBeforeFollowUp: true,
|
|
followUp: nonce => `I stopped your previous run on purpose. Please continue the task now: finish the \`brief\` document and also include the line \`${continueMarker(nonce)}\` on its own line before you mark the task done.`,
|
|
collect: { documents: ["brief"], files: () => [], commands: [] },
|
|
grade(nonce, artifacts) {
|
|
const stop = artifacts.stop;
|
|
return result([
|
|
statusCheck(artifacts),
|
|
...briefChecks(artifacts.documents.brief, [briefMarker(nonce), continueMarker(nonce)]),
|
|
{ name: "stop_took_effect", ok: stop?.runStatus === "cancelled", detail: `run=${stop?.runId ?? "none"} status=${stop?.runStatus ?? "none"}` },
|
|
{ name: "nothing_ran_after_stop", ok: Boolean(stop) && stop!.continuePosted && stop!.runsStartedBeforeContinue.length === 0,
|
|
detail: stop ? `continue=${stop.continuePosted} runs=${stop.runsStartedBeforeContinue.join(",") || "none"}` : "not stopped" },
|
|
]);
|
|
},
|
|
},
|
|
B1: {
|
|
id: "B1",
|
|
label: "Python CLI with tests",
|
|
usesRepository: true,
|
|
fireWindowMs: [20_000, 90_000],
|
|
budgetMs: 15 * 60_000,
|
|
title: nonce => `Chaos B1 slugify command ${nonce}`,
|
|
description: nonce => [
|
|
"Work in this task's repository workspace (a small Python CLI called chaoscli).",
|
|
"Implement the `slugify` subcommand in `chaoscli/cli.py` so that",
|
|
"`python3 -m chaoscli slugify \"Hello Chaos World\"` prints `hello-chaos-world`",
|
|
"(lowercase, runs of non-alphanumeric characters become a single `-`, no leading or trailing `-`).",
|
|
"The currently failing test must pass: run `python3 -m unittest discover -s tests -v`.",
|
|
`Also create \`${notesFile(nonce)}\` in the repository root containing exactly the line \`B1-${nonce}\`.`,
|
|
"Use only the Python standard library. Do not run ./bench.sh. When the tests pass, mark this task done.",
|
|
].join("\n"),
|
|
collect: {
|
|
documents: [],
|
|
files: nonce => [notesFile(nonce), "chaoscli/cli.py"],
|
|
commands: [
|
|
{ name: "unittest", argv: ["python3", "-m", "unittest", "discover", "-s", "tests"] },
|
|
{ name: "slugify", argv: ["python3", "-m", "chaoscli", "slugify", "Hello Chaos World"] },
|
|
],
|
|
},
|
|
grade(nonce, artifacts) {
|
|
const tests = artifacts.commands.unittest;
|
|
const slug = artifacts.commands.slugify;
|
|
return result([
|
|
statusCheck(artifacts),
|
|
{ name: "workspace_located", ok: artifacts.workspacePath !== null },
|
|
{ name: "unittest_passes", ok: tests?.exitCode === 0, detail: `exit=${tests?.exitCode ?? "not run"}` },
|
|
{ name: "slugify_output", ok: slug?.exitCode === 0 && slug.stdout.trim() === "hello-chaos-world", detail: JSON.stringify(slug?.stdout.trim() ?? null) },
|
|
{ name: "notes_file", ok: artifacts.files[notesFile(nonce)]?.trim() === `B1-${nonce}` },
|
|
]);
|
|
},
|
|
},
|
|
R1: {
|
|
id: "R1",
|
|
label: "Code change in a Paperclip clone",
|
|
usesRepository: true,
|
|
fireWindowMs: [20_000, 90_000],
|
|
budgetMs: 15 * 60_000,
|
|
title: nonce => `Chaos R1 slug module ${nonce}`,
|
|
description: nonce => [
|
|
"Work in this task's repository workspace, a clone of the Paperclip repository.",
|
|
`Add \`${r1Module(nonce)}\` exporting \`chaosSlug(input: string): string\`: lowercase, every run of characters`,
|
|
"other than a-z and 0-9 becomes a single `-`, and no leading or trailing `-`.",
|
|
`Add \`${r1Test(nonce)}\` that uses \`node:test\` and \`node:assert/strict\`, imports \`./slug-${nonce}.ts\`,`,
|
|
"and checks at least three cases, including `chaosSlug(\"Hello Chaos World\") === \"hello-chaos-world\"`.",
|
|
`Run \`node --test ${r1Test(nonce)}\` from the repository root until it passes.`,
|
|
"Do not install dependencies and do not change any other file. Then mark this task done.",
|
|
].join("\n"),
|
|
collect: {
|
|
documents: [],
|
|
files: nonce => [r1Module(nonce), r1Test(nonce)],
|
|
commands: [
|
|
{ name: "node_test", argv: nonce => [process.execPath, "--test", r1Test(nonce)] },
|
|
{ name: "slug", argv: nonce => [process.execPath, "--input-type=module", "-e",
|
|
`import { chaosSlug } from "./${r1Module(nonce)}"; console.log(JSON.stringify(["Hello Chaos World", " R1 -- Paperclip__Chaos 2026! "].map(chaosSlug)));`] },
|
|
],
|
|
},
|
|
grade(nonce, artifacts) {
|
|
const tests = artifacts.commands.node_test;
|
|
const slug = artifacts.commands.slug;
|
|
return result([
|
|
statusCheck(artifacts),
|
|
{ name: "workspace_located", ok: artifacts.workspacePath !== null },
|
|
{ name: "module_present", ok: typeof artifacts.files[r1Module(nonce)] === "string" },
|
|
{ name: "test_checks_chaos_slug", ok: r1TestChecksSlug(artifacts.files[r1Test(nonce)]) },
|
|
{ name: "node_test_passes", ok: tests?.exitCode === 0, detail: `exit=${tests?.exitCode ?? "not run"}` },
|
|
{ name: "required_and_hidden_cases", ok: slug?.exitCode === 0 && slug.stdout.trim() === JSON.stringify(["hello-chaos-world", "r1-paperclip-chaos-2026"]),
|
|
detail: JSON.stringify(slug?.stdout.trim() ?? null) },
|
|
]);
|
|
},
|
|
},
|
|
B5: {
|
|
id: "B5",
|
|
label: "Long bench.sh run",
|
|
usesRepository: true,
|
|
fireWindowMs: [60_000, 180_000],
|
|
budgetMs: 20 * 60_000,
|
|
title: nonce => `Chaos B5 bench ${nonce}`,
|
|
description: nonce => [
|
|
"Work in this task's repository workspace.",
|
|
`Run \`./bench.sh ${nonce}\` from the repository root in the foreground and wait for it to finish.`,
|
|
"It takes about four minutes; if your shell tool has a timeout, set it to at least 10 minutes. Do not background or interrupt it.",
|
|
"It prints one line of the form `BENCH_RESULT=<number>`.",
|
|
`Write exactly that line into a new file \`${benchFile(nonce)}\` in the repository root.`,
|
|
"Do not compute the result yourself and do not edit bench.sh. Then mark this task done.",
|
|
].join("\n"),
|
|
collect: { documents: [], files: nonce => [benchFile(nonce), benchRecord(nonce)], commands: [] },
|
|
grade(nonce, artifacts) {
|
|
const content = artifacts.files[benchFile(nonce)]?.trim() ?? null;
|
|
const record = parseBenchRecord(artifacts.files[benchRecord(nonce)]);
|
|
return result([
|
|
statusCheck(artifacts),
|
|
{ name: "workspace_located", ok: artifacts.workspacePath !== null },
|
|
{ name: "bench_result_file", ok: content === BENCH_EXPECTED, detail: JSON.stringify(content) },
|
|
{ name: "bench_actually_ran", ok: record !== null && record.result === BENCH_EXPECTED && record.seconds >= BENCH_MIN_SECONDS,
|
|
detail: record ? `ran ${record.seconds}s, ${record.result}` : "no completion record from bench.sh" },
|
|
// The record is agent-writable; the harness's own progress observations are not.
|
|
{ name: "bench_progress_observed", ok: (artifacts.benchProgressObserved ?? 0) >= BENCH_MIN_OBSERVED_PROGRESS,
|
|
detail: `observed ${artifacts.benchProgressObserved ?? 0} progress values` },
|
|
]);
|
|
},
|
|
},
|
|
};
|
|
|
|
/** Files committed into the `chaos-sandbox` fixture repository. */
|
|
export function fixtureRepositoryFiles(benchSeconds = 240): Record<string, { content: string; mode?: number }> {
|
|
return {
|
|
"README.md": { content: "# chaos-sandbox\n\nFixture repository for the Paperclip restart chaos harness.\n" },
|
|
"chaoscli/__init__.py": { content: "" },
|
|
"chaoscli/__main__.py": { content: "from chaoscli.cli import main\n\nraise SystemExit(main())\n" },
|
|
"chaoscli/cli.py": { content: [
|
|
"import argparse",
|
|
"import sys",
|
|
"",
|
|
"",
|
|
"def greet(name: str) -> str:",
|
|
" return f\"hello, {name}\"",
|
|
"",
|
|
"",
|
|
"def slugify(text: str) -> str:",
|
|
" raise NotImplementedError(\"slugify is not implemented yet\")",
|
|
"",
|
|
"",
|
|
"def main(argv=None) -> int:",
|
|
" parser = argparse.ArgumentParser(prog=\"chaoscli\")",
|
|
" commands = parser.add_subparsers(dest=\"command\", required=True)",
|
|
" greet_parser = commands.add_parser(\"greet\")",
|
|
" greet_parser.add_argument(\"name\")",
|
|
" args = parser.parse_args(argv)",
|
|
" if args.command == \"greet\":",
|
|
" print(greet(args.name))",
|
|
" return 0",
|
|
" return 2",
|
|
"",
|
|
].join("\n") },
|
|
"tests/__init__.py": { content: "" },
|
|
"tests/test_cli.py": { content: [
|
|
"import unittest",
|
|
"",
|
|
"from chaoscli.cli import greet, slugify",
|
|
"",
|
|
"",
|
|
"class CliTest(unittest.TestCase):",
|
|
" def test_greet(self):",
|
|
" self.assertEqual(greet(\"chaos\"), \"hello, chaos\")",
|
|
"",
|
|
" def test_slugify(self):",
|
|
" self.assertEqual(slugify(\"Hello Chaos World\"), \"hello-chaos-world\")",
|
|
" self.assertEqual(slugify(\" A--b__C!! \"), \"a-b-c\")",
|
|
"",
|
|
"",
|
|
"if __name__ == \"__main__\":",
|
|
" unittest.main()",
|
|
"",
|
|
].join("\n") },
|
|
"bench.sh": { mode: 0o755, content: [
|
|
"#!/bin/sh",
|
|
"# Deterministic long-running benchmark for restart chaos boundary B6.",
|
|
"set -eu",
|
|
`steps=${BENCH_STEPS}`,
|
|
`seconds="\${BENCH_SECONDS:-${benchSeconds}}"`,
|
|
"tag=\"${1:-untagged}\"",
|
|
"mkdir -p .bench",
|
|
"started=$(date +%s)",
|
|
"total=0",
|
|
"i=1",
|
|
"while [ \"$i\" -le \"$steps\" ]; do",
|
|
" total=$((total + i * i))",
|
|
" if [ $((i % 24)) -eq 0 ]; then echo \"bench progress $i/$steps\"; echo \"$i\" > .bench/progress; fi",
|
|
" sleep \"$(awk -v s=\"$seconds\" -v n=\"$steps\" 'BEGIN { printf \"%.3f\", s / n }')\"",
|
|
" i=$((i + 1))",
|
|
"done",
|
|
"echo \"started=$started finished=$(date +%s) result=BENCH_RESULT=$total\" > \".bench/run-$tag.txt\"",
|
|
"echo \"BENCH_RESULT=$total\"",
|
|
"",
|
|
].join("\n") },
|
|
".gitignore": { content: ".bench/\n__pycache__/\n" },
|
|
};
|
|
}
|