From 68d5fb5a6ef6e2733fd4d5504d9ec4cbeee67e4b Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:06:01 -0600 Subject: [PATCH] feat(chaos): check I6 and I8 per crash point; ready the Daytona points (#19984) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The on-demand crash-point suite (#19946) kills the control plane at named durable write boundaries and checks the plan's invariants > - Its first full run measured I1–I4 and I7, but I5, I6 and I8 were reported as not measured > - #19960 (generation stamping) and #19863 (speed budgets) now record what I6 and I8 need in the run log > - The six Daytona-only points have not run yet, and three of them cannot be reached by a plain task > - This pull request checks I6 and I8 per point, names the I5 gap, and readies the Daytona points for the coming Daytona wave > - The benefit is that every invariant is either asserted or a named gap, and a Daytona run explains any point it cannot reach ## Linked Issues or Issue Description Refs #19831 (plan, Phase 5). Follows #19946 (the suite). Uses #19960 (event generations) and #19863 (speed budgets). **Problem** - The suite reported I5, I6 and I8 as "not measured". - A Daytona run would report three points as `not_reached` with no reason. - A Daytona run without its key or image failed only after a 3-minute boot. **Expected behavior** - I6 and I8 are checked at each point. - I5 is a named gap with its reason. - Daytona points say what they need, and a Daytona run checks its inputs before it boots. ## What Changed - `tests/restart-chaos/invariants.ts`: `runLogViolations` checks two invariants from the run log, scoped to the wave's company: - I6: a run that failed with an error that names a stale or earlier-generation event; - I8: each `run.performance.budget_exceeded` event, with the budget evidence (metric, measured, p95, adapter, start kind). Both are keyed by run or event id, so the per-point baseline diff charges each one to the point that produced it. I5 is reported as `notMeasured` with a reason (`I5_GAP`). - `packages/shared/src/crash-point-registry.ts`: an optional `prerequisite` on the three Daytona points a plain task cannot reach. These are the teardown claim (needs a failed release), the reusable destroy (needs a `reuse_by_environment` lease), and stop-and-retain. - `tests/restart-chaos/crash-points.ts`: - `--list` prints each point's `needs:`, and a `not_reached` result names it too; - `--env daytona` refuses to start without `DAYTONA_API_KEY` and an image. - Docs: the README and the plan describe the I5/I6/I8 coverage and the Daytona points. ## Verification - `pnpm exec vitest run --config tests/restart-chaos/vitest.config.ts tests/restart-chaos/crash-points.test.ts`: 12 passed. - `cd packages/shared && npx vitest run src/crash-points.test.ts`: 15 passed. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos` and `npx tsc --noEmit -p packages/shared` both pass. - I ran `runLogViolations` against a throwaway Postgres 18 with seeded rows, then stopped and deleted it. It returned: - I8 for the budget breach, with its evidence; - I6 for the run of this company that failed on `stale_generation_event`. It excluded the other company's stale run, the `process_lost` run and a non-budget event. - `node scripts/crash-points.mjs --list --env daytona` lists 19 points, with `needs:` on the three. `--env daytona` with no key or image fails at once with the missing names. - No harness run: the harness slot is held for other work. The next suite run exercises the new checks. ## Risks - Test harness only; no product code. - I6 is a proxy: it matches the run's error text and code. A failure that names a generation for another reason would be reported, with its evidence, for a person to judge. - I8 can flag breaches on a loaded host. Each breach carries its measured and budget values, so a load-caused breach is visible as one. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code with tool use (shell, file edits). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- ...07-durable-ownership-and-reconciliation.md | 2 +- packages/shared/src/crash-point-registry.ts | 5 ++ tests/restart-chaos/README.md | 23 +++++- tests/restart-chaos/crash-points.test.ts | 12 +++ tests/restart-chaos/crash-points.ts | 15 +++- tests/restart-chaos/invariants.ts | 76 +++++++++++++++---- 6 files changed, 114 insertions(+), 19 deletions(-) diff --git a/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md b/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md index 4465aa2f154..c8038f06820 100644 --- a/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md +++ b/doc/plans/2026-11-07-durable-ownership-and-reconciliation.md @@ -226,7 +226,7 @@ The runner stamps `generation` and `turnId` on every event. Driver and session c **Built (Phase 5).** - `crashPoint(name)` in `packages/shared/src/crash-points.ts`, with the named points in `crash-point-registry.ts`. `PAPERCLIP_CRASH_POINT=[:]` arms one point; `NODE_ENV=production` ignores it. Disarmed, a call costs about 4 ns. In-process tests use `armCrashPointForTest`. - 20 points at the boundaries in the inventory's write-ahead table, from `sandbox.create.after_provider_before_lease` to `accounting.closure.after_commit_before_spool_unlink`. `server/src/__tests__/crash-point-call-sites.test.ts` requires a call site for each one. -- `node scripts/crash-points.mjs [--points ] [--kinds ] [--adapters …]` boots one instance and arms each point in turn at runtime (`PAPERCLIP_CRASH_POINT_CONTROL`). It asserts R1–R6 plus I1–I4 and I7 per point, from the Phase 1 reconciler. I5, I6 and I8 are reported as not measured until sections F and G land. The manual CI run is `.github/workflows/crash-points.yml`. +- `node scripts/crash-points.mjs [--points ] [--kinds ] [--adapters …]` boots one instance and arms each point in turn at runtime (`PAPERCLIP_CRASH_POINT_CONTROL`). It asserts R1–R6 plus I1–I4 and I7 per point, from the Phase 1 reconciler. I6 and I8 come from the run log: a run failed by a stale-generation event, and speed-budget breaches. I5 is a known gap until restart claims record their charge decision; R3 bounds it. The manual CI run is `.github/workflows/crash-points.yml`. - First full local run (2026-11-08, 14 local points, codex A1, 11.8 min): 11 pass. Work list: - `workspace.finalize.after_start_before_copyback` and `…after_copyback_before_done` leave a `workspace_finalize` operation running after the run succeeded (I2 and R6, kind `native_workspace_finalize_attempt`). - `wake.claim.after_run_running_before_wakeup_claimed` reports I4 on a wake whose retry delivered the work: the known refusal-regex false positive, which section E removes. diff --git a/packages/shared/src/crash-point-registry.ts b/packages/shared/src/crash-point-registry.ts index a8b4a023551..76313341d8e 100644 --- a/packages/shared/src/crash-point-registry.ts +++ b/packages/shared/src/crash-point-registry.ts @@ -21,6 +21,8 @@ export type CrashPointDefinition = { environments: readonly CrashPointEnvironment[]; kinds: readonly string[]; invariants: readonly CrashPointInvariant[]; + /** What a run must go through first when one plain task does not reach the point. */ + prerequisite?: string; }; export const CRASH_POINT_NAME_PATTERN = /^[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*\.(before|after)_[a-z0-9_]+$/; @@ -97,18 +99,21 @@ export const CRASH_POINTS = { environments: ["daytona"], kinds: ["environment_teardown_claim", "environment_pending_cleanup"], invariants: ["I2", "I7"], + prerequisite: "A pending_cleanup lease: a provider release that failed or could not be verified. A plain task's release succeeds and never claims a teardown.", }, "environment.teardown.after_destroy_before_lease": { description: "A reusable sandbox was destroyed at the provider; its lease row is not released.", environments: ["daytona"], kinds: ["environment_run_lease_active", "environment_retained_hold"], invariants: ["I2"], + prerequisite: "A reuse_by_environment lease that expires or is replaced (destroyReusableSandboxLease). The chaos Daytona environment's leases are ephemeral by default.", }, "environment.stop_retain.after_claim_before_stop": { description: "A stop-and-retain intent and its claim are committed; the provider stop has not been retried.", environments: ["daytona"], kinds: ["environment_stop_and_retain_intent"], invariants: ["I2", "I7"], + prerequisite: "A stop-and-retain intent: a run whose sandbox must be stopped but kept, for a native workspace export resume.", }, "environment.lease.after_release_before_runtime_services": { description: "The run's environment leases are released; its workspace runtime services are not.", diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 5a60b99c571..f0e8a0e65fb 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -245,9 +245,16 @@ The suite runs one wave, `main.ts --crash-point :,:,...`: - two passes at the real clock for I1 (owned) and I2 (settled); - one pass with the clock moved 15 minutes ahead for I3 (admission), I4 (exactly-once intents) and I7 (visible next action). - Each violation carries its wake and run state as evidence. I5, I6 and I8 - are reported as not measured, with the reconciler's reasons; R3 and R1 - cover part of I5 and I8. + Each violation carries its wake and run state as evidence. Two run-log + checks complete the set: + - I6: a run that failed on a stale-generation event. Since #19960 an + earlier generation's event is history and must never fail the current + turn. + - I8: every `run.performance.budget_exceeded` event the speed budgets + wrote (TTFT, agent boot, restart recovery). + I5 (work-based budgets) is a known gap: charge decisions are not recorded + per restart claim, so the suite cannot tell a charged restart from an + uncharged one. R3 bounds it (no point may need a Retry). 5. Every task is graded R1–R6 and T at the end, as in any wave. The first full local run (14 points, codex A1) took 11.8 minutes: one boot, @@ -267,6 +274,16 @@ violated, or the wave exited 1 for a wave-level reason such as a Daytona sandbox left after cleanup), `not_reached` (the chosen workloads never crossed the point: pick a workload or `--nth` that does) and `harness_error`. Exit code 0, 1, 3 and 2. +**Daytona points.** Six points are reachable only with `--env daytona` +(`node scripts/crash-points.mjs --list --env daytona`). The run needs +`DAYTONA_API_KEY` and an image matching this checkout (`--daytona-image` or +`PAPERCLIP_E2E_DAYTONA_IMAGE`; see "Daytona" above), and refuses to start +without them. A plain task reaches both `sandbox.create.*` points and +`environment.lease_release.after_provider_before_lease`. The other three print +what they need (`needs:` in `--list` and in a `not_reached` result): a failed +release (teardown claim), a reusable lease (reusable destroy), or a +stop-and-retain export. + **CI.** `.github/workflows/crash-points.yml` runs the same command on demand (`workflow_dispatch`, inputs `points`, `kinds`, `adapters`, `workloads`, `nth`). It is not a required PR check. It runs only from the default branch, for diff --git a/tests/restart-chaos/crash-points.test.ts b/tests/restart-chaos/crash-points.test.ts index deaf4291158..575fa2bf0c8 100644 --- a/tests/restart-chaos/crash-points.test.ts +++ b/tests/restart-chaos/crash-points.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import type { CrashPointDefinition, CrashPointName } from "../../packages/shared/src/crash-point-registry.js"; +import { summarizeViolations } from "./invariants.js"; import { classifyPoint, globToRegExp, parseSuiteArgs, renderSummary, selectPoints, type CrashPointEntry } from "./crash-points.js"; import { parseCommand } from "./config.js"; import { readCrashPointReceipt } from "./supervisor.js"; @@ -73,6 +74,17 @@ describe("classifyPoint", () => { .toBe("harness_error"); }); + it("names what an unreached point needs when its registry entry says so", () => { + const teardown = "environment.teardown.after_claim_before_destroy" as CrashPointName; + expect(classifyPoint(teardown, 0, entry({ point: `${teardown}:1`, reached: null }))) + .toMatchObject({ outcome: "not_reached", error: expect.stringContaining("needs: A pending_cleanup lease") }); + }); + + it("counts run-log violations under their plan invariant", () => { + expect(summarizeViolations([{ rule: "I8_fast_turns" }, { rule: "I6_generation_fencing" }, { rule: "I8_fast_turns" }])) + .toEqual({ I8: 2, I6: 1 }); + }); + it("renders a summary row per point", () => { const result = { ...classifyPoint(point, 0, entry()), waveDir: null }; expect(renderSummary("cp-x", parseSuiteArgs([]), [result])).toContain("| `runner.startup.after_spawn` | pass | — | — | 2.0 |"); diff --git a/tests/restart-chaos/crash-points.ts b/tests/restart-chaos/crash-points.ts index b5481b89493..8255e258c8c 100644 --- a/tests/restart-chaos/crash-points.ts +++ b/tests/restart-chaos/crash-points.ts @@ -124,7 +124,10 @@ export function classifyPoint(point: CrashPointName, exitCode: number | null, en if (base.failedTasks.length > 0 || base.violations.length > 0) return { ...base, outcome: "fail" }; if (entry.invariants?.errors.length) return { ...base, outcome: "harness_error", error: entry.invariants.errors.join("; ") }; if (!entry.invariants) return { ...base, outcome: "harness_error", error: "no invariant check" }; - if (!entry.reached) return { ...base, outcome: "not_reached" }; + if (!entry.reached) { + const prerequisite = (CRASH_POINTS as Record)[point]?.prerequisite; + return { ...base, outcome: "not_reached", ...(prerequisite ? { error: `needs: ${prerequisite}` } : {}) }; + } return { ...base, outcome: "pass" }; } @@ -180,10 +183,18 @@ async function main(): Promise { if (options.list) { for (const name of selected) { const definition = CRASH_POINTS[name] as CrashPointDefinition; - console.info(`${name}\n ${definition.description}\n kinds ${definition.kinds.join(", ") || "—"}; invariants ${definition.invariants.join(", ")}; env ${definition.environments.join(", ")}`); + console.info(`${name}\n ${definition.description}\n kinds ${definition.kinds.join(", ") || "—"}; invariants ${definition.invariants.join(", ")}; env ${definition.environments.join(", ")}` + + (definition.prerequisite ? `\n needs: ${definition.prerequisite}` : "")); } return 0; } + if (options.env === "daytona") { + // Fail before a 3-minute boot, not after it. + const missing = [!process.env.DAYTONA_API_KEY && "DAYTONA_API_KEY", + !options.passthrough.includes("--daytona-image") && !process.env.PAPERCLIP_E2E_DAYTONA_IMAGE && "--daytona-image (or PAPERCLIP_E2E_DAYTONA_IMAGE)"] + .filter(Boolean); + if (missing.length) throw new Error(`--env daytona needs ${missing.join(" and ")}`); + } if (!selected.length) throw new Error(`No crash point matches --points ${options.points.join(",") || "*"} --kinds ${options.kinds.join(",") || "*"} --env ${options.env}`); const runId = `cp-${new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "")}`; const runDir = path.join(resultsRoot, "crash-points", runId); diff --git a/tests/restart-chaos/invariants.ts b/tests/restart-chaos/invariants.ts index 8af8fde2cb2..92066badffc 100644 --- a/tests/restart-chaos/invariants.ts +++ b/tests/restart-chaos/invariants.ts @@ -10,9 +10,15 @@ * I3, I4, I7 one more pass with the clock moved past every grace window * (accounting closure 10 min, next action 5 min). The tasks are at * rest, so anything still waiting then would wait forever. - * I5, I6, I8 not measurable by the reconciler yet; reported as coverage - * gaps with its own reasons. The chaos graders cover part of them: - * R3 (one Retry at most) bounds I5 and R1 (finished in budget) I8. + * I6 run-log proxy: a run that failed on a stale-generation event. + * Since #19960 an earlier generation's event is history and never + * fails the current turn (runner code `stale_generation_event`). + * I8 run-log check: every `run.performance.budget_exceeded` event the + * speed budgets wrote (TTFT, agent boot, restart recovery; #19863). + * I5 not measured: see I5_GAP. R3 (one Retry at most) bounds it. + * + * I6 and I8 are company-wide scans keyed by run or event id, so the per-point + * baseline diff in main.ts charges each one to the point that produced it. */ import { createDb, type Db } from "../../packages/db/src/index.js"; import { @@ -23,8 +29,23 @@ import { export type PlanInvariant = "I1" | "I2" | "I3" | "I4" | "I5" | "I6" | "I7" | "I8"; +export type SuiteRule = InvariantViolation["rule"] | "I6_generation_fencing" | "I8_fast_turns"; + /** A violation with the state of its record and run, so a failure is diagnosable after the instance is gone. */ -export type EvidencedViolation = InvariantViolation & { evidence?: Record }; +export type EvidencedViolation = Omit & { rule: SuiteRule; evidence?: Record }; + +/** + * Why I5 (work-based budgets) is not asserted. A charge is decided per restart + * claim from the attempt's event window (provider-attempt-outcome.ts, #19948) + * and recorded only indirectly in the finalization's bounded recovery + * history (progressAttempt, attemptKeptForProgress). Asserting it needs either + * an explicit per-claim charge record or reading that history per run. The + * cheap check that exists is R3, and the pre-spawn and pre-accept points + * (native.process_start.*, runner.process.*, runner.turn_start.*) recover + * without a Retry today. + */ +export const I5_GAP = "Charge decisions are not recorded per restart claim; needs an explicit charge record (section F follow-up). " + + "R3 bounds it: no point needed a Retry."; export interface InvariantCheck { violations: EvidencedViolation[]; @@ -37,9 +58,9 @@ export interface InvariantCheck { /** Grace windows the shifted pass must exceed (ownership-invariant-report.ts). */ export const GRACE_SHIFT_MS = 15 * 60_000; -export const invariantOf = (violation: Pick): PlanInvariant => violation.rule.slice(0, 2) as PlanInvariant; +export const invariantOf = (violation: Pick): PlanInvariant => violation.rule.slice(0, 2) as PlanInvariant; -export function summarizeViolations(violations: readonly InvariantViolation[]): Partial> { +export function summarizeViolations(violations: readonly Pick[]): Partial> { const counts: Partial> = {}; for (const violation of violations) counts[invariantOf(violation)] = (counts[invariantOf(violation)] ?? 0) + 1; return counts; @@ -65,12 +86,40 @@ async function withWaveInstance(instance: { paperclipHome: string; instanceId const clip = (value: unknown) => typeof value === "string" ? value.slice(0, 300) : value ?? null; -async function withEvidence(db: Db, violations: InvariantViolation[]): Promise { +type Row = Record; +const text = (value: unknown) => (typeof value === "string" ? value : ""); + +/** I6 and I8 from the run log of `companyId` (all companies when null). */ +export async function runLogViolations(db: Db, companyId: string | null): Promise { + const breaches = await db.$client.unsafe( + `select e.id::text as id, e.company_id::text as "companyId", e.run_id::text as "runId", r.issue_id::text as "issueId", e.payload + from heartbeat_run_events e join heartbeat_runs r on r.id = e.run_id + where e.event_type = 'run.performance.budget_exceeded' and ($1::text is null or e.company_id::text = $1)`, [companyId]) as Row[]; + const stale = await db.$client.unsafe( + `select id::text as id, company_id::text as "companyId", issue_id::text as "issueId", status, error_code as "errorCode", error + from heartbeat_runs + where status in ('failed', 'timed_out', 'interrupted') + and (coalesce(error_code, '') ~* 'generation' or coalesce(error, '') ~* 'stale_generation|earlier generation') + and ($1::text is null or company_id::text = $1)`, [companyId]) as Row[]; + return [ + ...breaches.map(row => { + const payload = (row.payload ?? {}) as Record; + return { rule: "I8_fast_turns" as const, kind: "speed_budget", recordId: text(row.id), companyId: text(row.companyId) || null, + runId: text(row.runId) || null, issueId: text(row.issueId) || null, reason: `${text(payload.metric) || "unknown"}_over_p95`, + evidence: { budget: { metric: payload.metric, measuredMs: payload.measuredMs, budgetP95Ms: payload.budgetP95Ms, + adapter: payload.adapter, startKind: payload.startKind } } }; + }), + ...stale.map(row => ({ rule: "I6_generation_fencing" as const, kind: "run", recordId: text(row.id), + companyId: text(row.companyId) || null, runId: text(row.id), issueId: text(row.issueId) || null, + reason: "run_failed_on_earlier_generation_event" })), + ]; +} + +async function withEvidence(db: Db, violations: EvidencedViolation[]): Promise { if (!violations.length) return violations; const wakeIds = violations.filter(violation => violation.kind === "wake_request" || violation.kind === "wake_request_claimed") .map(violation => violation.recordId); const runIds = [...new Set(violations.flatMap(violation => violation.runId ? [violation.runId] : []))]; - type Row = Record; const wakes = (wakeIds.length ? await db.$client.unsafe( "select id::text, status, reason, error, run_id::text as \"runId\" from agent_wakeup_requests where id::text = any($1)", [wakeIds]) : []) as Row[]; const runs = (runIds.length ? await db.$client.unsafe( @@ -115,16 +164,17 @@ export async function checkPlanInvariants(input: { || violation.companyId === input.companyId; const fromClock = second.violations.filter(violation => violation.rule === "I1_owned" || violation.rule === "I2_settled"); const fromShift = shifted.violations.filter(violation => violation.rule !== "I1_owned" && violation.rule !== "I2_settled"); + let runLog: EvidencedViolation[] = []; + try { runLog = await runLogViolations(db, input.companyId); } catch (error) { + errors.push(`I6/I8 run-log check: ${error instanceof Error ? error.message : String(error)}`); + } const violations = await withEvidence(db, [...fromClock, ...fromShift].filter(ownCompany)); + violations.push(...runLog); const inProgress = Object.fromEntries(second.kinds.filter(kind => kind.inProgress > 0) .map(kind => [kind.kind, { inProgress: kind.inProgress, ...kind.verdicts }])); return { violations, byInvariant: summarizeViolations(violations), inProgress, errors, - notMeasured: { - I5: second.coverageGaps.I5_work_based_budgets ?? "not measured", - I6: second.coverageGaps.I6_generation_fencing ?? "not measured", - I8: second.coverageGaps.I8_fast_turns ?? "not measured", - }, + notMeasured: { I5: I5_GAP }, }; } finally { await db.$client.end({ timeout: 5 }).catch(() => undefined);