diff --git a/packages/paperclip-runner/devtools/browser/src/App.tsx b/packages/paperclip-runner/devtools/browser/src/App.tsx index e15236e6645..c7d395043f4 100644 --- a/packages/paperclip-runner/devtools/browser/src/App.tsx +++ b/packages/paperclip-runner/devtools/browser/src/App.tsx @@ -7,7 +7,11 @@ import type { Phase2RunTrace, Phase2Scenario, } from "../../../src/contracts/phase2"; -import type { Phase3RunTrace } from "../../../src/contracts/phase3"; +import { + PHASE3_FAULTS, + type Phase3Fault, + type Phase3RunTrace, +} from "../../../src/contracts/phase3"; import { applyPhase2LiveEvent, createPhase2LiveSnapshot, @@ -49,15 +53,53 @@ const liveScenarios: Array<{ key: Phase2Scenario; label: string }> = [ { key: "duplicate-terminal", label: "Duplicate terminal guard" }, ]; +type BrowserMode = "recovery" | "live" | "replay"; +type LiveStatus = "idle" | "starting" | "running" | "terminal" | "error"; + +const phase3FaultLabels: Record = { + none: "Normal connection", + "socket-drop": "Socket drop", + "lost-ack": "Lost ACK", + "duplicate-command": "Duplicate command", + "runner-restart": "Runner restart", + "harness-restart": "Harness restart", + "malformed-input": "Malformed input", + "lease-expiry": "Lease expiry", + "storage-pressure": "Storage pressure", + drain: "Drain", + revoke: "Revoke", +}; + +const modeCopy: Record< + BrowserMode, + { eyebrow: string; title: string; description: string } +> = { + recovery: { + eyebrow: "Paperclip Runner Protocol · Phase 3", + title: "Durable recovery diagnostics", + description: + "Break the outbound transport, recover the same session, and inspect durable state.", + }, + live: { + eyebrow: "Paperclip Runner Protocol · Phase 2", + title: "Live runner diagnostics", + description: + "Run the local harness, stream validated events, and compare the live and replayed state.", + }, + replay: { + eyebrow: "Paperclip Runner Protocol · Phase 1", + title: "Static protocol replay", + description: + "Validate a protocol fixture, replay its events, and inspect the resulting session state.", + }, +}; + interface BrowserFixture { key: string; label: string; source: string; } -type BrowserMode = "recovery" | "live" | "replay"; -type LiveStatus = "idle" | "starting" | "running" | "terminal" | "error"; - type Phase2StreamRecord = | { kind: "event"; event: unknown } | { kind: "diagnostic"; message: string } @@ -86,11 +128,22 @@ function terminalTone(snapshot: SessionSnapshot) { } function humanizeProtocolLabel(value: string): string { - const words = value.replaceAll("_", " "); + const words = value.replaceAll(/[_-]/g, " "); return words.length === 0 ? words : `${words[0]?.toUpperCase()}${words.slice(1)}`; } +function recoveryOutcomeTone(outcome: Phase3RunTrace["diagnostics"]["recovery"]["outcome"]) { + if (outcome === "recovered") { + return "success" as const; + } + if (outcome === "unrecoverable") { + return "danger" as const; + } + return "warning" as const; +} + function RecoveryDiagnostics() { + const [fault, setFault] = useState("lost-ack"); const [trace, setTrace] = useState(null); const [status, setStatus] = useState<"idle" | "running" | "complete" | "error">("idle"); const [error, setError] = useState(null); @@ -102,7 +155,7 @@ function RecoveryDiagnostics() { const response = await fetch("/api/phase3/recovery", { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ fault: "lost-ack" }), + body: JSON.stringify({ fault }), }); const result = (await response.json()) as Phase3RunTrace & { error?: string }; if (!response.ok) throw new Error(result.error ?? "The recovery trace failed."); @@ -124,6 +177,19 @@ function RecoveryDiagnostics() { + + @@ -151,17 +217,118 @@ function RecoveryDiagnostics() { ) : ( <>
-
Connection
{trace.diagnostics.connection.state}
-
Lease
{trace.diagnostics.connection.leaseId ?? "closed"}
-
Outbox
{trace.diagnostics.outbox.events}
-
Last ACK
{trace.diagnostics.cursors.runnerAckedSourceSeq}
-
Replayed
{trace.diagnostics.recovery.replayDeliveries}
-
Duplicate commands
{trace.diagnostics.commands.duplicateDeliveries}
-
Storage
{trace.diagnostics.outbox.bytes} bytes
-
Peak storage
{trace.diagnostics.outbox.peakBytes} bytes
-
Drain or revoke
{trace.diagnostics.recovery.outcome}
-
Secrets
{trace.diagnostics.security.secretLeakCount} leaks
+
+
Connection
+
+ {humanizeProtocolLabel(trace.diagnostics.connection.state)} +
+
+
+
Connections
+
+ {trace.diagnostics.connection.connectionCount} +
+
+
+
Reconnects
+
+ {trace.diagnostics.connection.reconnectCount} +
+
+
+
Lease
+
{trace.diagnostics.connection.leaseId ?? "closed"}
+
+
+
Outbox
+
{trace.diagnostics.outbox.events}
+
+
+
Last ACK
+
{trace.diagnostics.cursors.runnerAckedSourceSeq}
+
+
+
At-least-once redeliveries
+
{trace.diagnostics.recovery.replayDeliveries}
+
+
+
Runner restarts
+
+ {trace.diagnostics.recovery.runnerRestarts} +
+
+
+
Harness restarts
+
{trace.diagnostics.recovery.harnessRestarts}
+
+
+
Fresh bootstraps
+
+ {trace.diagnostics.recovery.freshBootstraps} +
+
+
+
Duplicate commands
+
{trace.diagnostics.commands.duplicateDeliveries}
+
+
+
Storage
+
+ + {trace.diagnostics.outbox.bytes} bytes of {trace.diagnostics.outbox.maxBytes}{" "} + max + + {trace.diagnostics.outbox.backpressure ? ( + Backpressure + ) : null} +
+
+
+
Peak storage
+
{trace.diagnostics.outbox.peakBytes} bytes
+
+
+
Outcome
+
+ + {humanizeProtocolLabel(trace.diagnostics.recovery.outcome)} + + {trace.diagnostics.recovery.outcome !== "recovered" ? ( + {humanizeProtocolLabel(trace.diagnostics.recovery.reason)} + ) : null} +
+
+
+
Secrets
+
{trace.diagnostics.security.secretLeakCount} leaks
+
+

+ At-least-once redeliveries include normal in-flight outbox resends. Use reconnects + and the per-event history to identify injected recovery. +

+
+

Recovery history

+ {trace.diagnostics.committedEvents.length} committed events +
+
    + {trace.diagnostics.committedEvents.map((event) => ( +
  1. + +
    + {event.eventType} + {event.deliveryCount > 1 ? ( + delivered {event.deliveryCount}× + ) : null} +
    +
  2. + ))} +
Same runner and session {trace.assertions.stableIdentity ? "Preserved" : "Changed"} @@ -655,16 +822,15 @@ function StaticReplay() { export function App() { const [mode, setMode] = useState("live"); + const header = modeCopy[mode]; return (
-

Paperclip Runner Protocol · Phase 3

-

Durable recovery diagnostics

-

- Break the outbound transport, recover the same session, and inspect durable state. -

+

{header.eyebrow}

+

{header.title}

+

{header.description}

Standalone · no Paperclip core
diff --git a/packages/paperclip-runner/devtools/browser/src/styles.css b/packages/paperclip-runner/devtools/browser/src/styles.css index 74d0e06f602..34a8580dc16 100644 --- a/packages/paperclip-runner/devtools/browser/src/styles.css +++ b/packages/paperclip-runner/devtools/browser/src/styles.css @@ -356,10 +356,34 @@ select { font-size: var(--font-size-sm); } -.snapshot-grid div { +.snapshot-grid div, +.recovery-grid > div { min-width: 0; } +.storage-diagnostic { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-2); + overflow: visible; + white-space: normal; +} + +.outcome-diagnostic { + display: grid; + gap: var(--space-1); + overflow: visible; + white-space: normal; +} + +.outcome-diagnostic > span:not(.ui-badge), +.delivery-marker { + color: var(--muted-foreground); + font-family: var(--font-sans); + font-size: var(--font-size-xs); +} + dd { margin: var(--space-1) 0 0; overflow: hidden; diff --git a/packages/paperclip-runner/devtools/browser/static-replay.spec.ts b/packages/paperclip-runner/devtools/browser/static-replay.spec.ts index 24bd47ee4c3..ed64d82c6d3 100644 --- a/packages/paperclip-runner/devtools/browser/static-replay.spec.ts +++ b/packages/paperclip-runner/devtools/browser/static-replay.spec.ts @@ -2,8 +2,9 @@ import { expect, test } from "@playwright/test"; test("validates and renders the shared Phase 1 fixture", async ({ page }, testInfo) => { await page.goto("/"); + await expect(page.getByRole("heading", { name: "Live runner diagnostics" })).toBeVisible(); await page.getByRole("button", { name: "Static replay" }).click(); - await expect(page.getByRole("heading", { name: "Durable recovery diagnostics" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Static protocol replay" })).toBeVisible(); await expect(page.getByTestId("terminal-badge")).toHaveText("Succeeded"); await expect(page.getByTestId("timeline").getByRole("listitem")).toHaveCount(9); await expect(page.getByTestId("timeline")).not.toContainText("workspace_preparing"); @@ -19,10 +20,41 @@ test("validates and renders the shared Phase 1 fixture", async ({ page }, testIn test("shows Phase 3 recovery diagnostics without secrets", async ({ page }, testInfo) => { await page.goto("/"); await page.getByRole("button", { name: "Recovery", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Durable recovery diagnostics" })).toBeVisible(); + const fault = page.getByLabel("Fault", { exact: true }); + await expect(fault).toHaveValue("lost-ack"); + expect(await fault.locator("option").allTextContents()).toEqual([ + "Normal connection", + "Socket drop", + "Lost ACK", + "Duplicate command", + "Runner restart", + "Harness restart", + "Malformed input", + "Lease expiry", + "Storage pressure", + "Drain", + "Revoke", + ]); + await fault.selectOption("none"); await page.getByRole("button", { name: "Run Phase 3 recovery" }).click(); + await expect(fault).toBeDisabled(); await expect(page.getByText("complete", { exact: true })).toBeVisible(); - await expect(page.getByTestId("phase3-diagnostics")).toContainText("Replayed"); + await expect(fault).toBeEnabled(); + await expect(page.getByTestId("phase3-connection-state")).toHaveText("Stopped"); + await expect(page.getByTestId("phase3-connections")).toHaveText("1"); + await expect(page.getByTestId("phase3-reconnects")).toHaveText("0"); + await expect(page.getByTestId("phase3-runner-restarts")).toHaveText("0"); + await expect(page.getByTestId("phase3-fresh-bootstraps")).toHaveText("1"); + await expect(page.getByTestId("phase3-diagnostics")).toContainText("At-least-once redeliveries"); await expect(page.getByTestId("phase3-diagnostics")).toContainText("Duplicate commands"); + await expect(page.getByTestId("phase3-storage")).toContainText("0 bytes of 65536 max"); + await expect(page.getByTestId("phase3-outcome")).toHaveText("Recovered"); + await expect(page.getByTestId("phase3-outcome")).toHaveAttribute("data-tone", "success"); + await expect(page.getByTestId("recovery-history").getByRole("listitem")).toHaveCount(9); + const firstRecoveryEvent = page.getByTestId("recovery-history").getByRole("listitem").first(); + await expect(firstRecoveryEvent).toContainText("workspace.ready"); + await expect(firstRecoveryEvent.getByText("delivered 2×")).toBeVisible(); await expect(page.getByText("Preserved", { exact: true })).toBeVisible(); await expect(page.getByText("Yes", { exact: true })).toBeVisible(); await expect(page.locator("body")).not.toContainText("leaseToken"); @@ -30,6 +62,45 @@ test("shows Phase 3 recovery diagnostics without secrets", async ({ page }, test await page.screenshot({ path: testInfo.outputPath("phase-03-recovery-diagnostics.png"), fullPage: true }); }); +test("tones failure outcomes and storage backpressure", async ({ page }) => { + await page.goto("/"); + await page.getByRole("button", { name: "Recovery", exact: true }).click(); + const fault = page.getByLabel("Fault", { exact: true }); + const run = page.getByRole("button", { name: "Run Phase 3 recovery" }); + + await fault.selectOption("revoke"); + await run.click(); + await expect(page.getByTestId("phase3-outcome")).toHaveText("Revoked"); + await expect(page.getByTestId("phase3-outcome")).toHaveAttribute("data-tone", "warning"); + await expect(page.getByText("Revoke completed", { exact: true })).toBeVisible(); + + await fault.selectOption("storage-pressure"); + await run.click(); + await expect(page.getByTestId("phase3-storage")).toContainText("0 bytes of 16384 max"); + await expect(page.getByTestId("phase3-storage").getByText("Backpressure")).toHaveAttribute( + "data-tone", + "warning", + ); +}); + +test("keeps recovery diagnostics within the narrow viewport", async ({ page }, testInfo) => { + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto("/"); + await page.getByRole("button", { name: "Recovery", exact: true }).click(); + await page.getByLabel("Fault", { exact: true }).selectOption("none"); + await page.getByRole("button", { name: "Run Phase 3 recovery" }).click(); + await expect(page.getByTestId("recovery-history")).toBeVisible(); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= document.documentElement.clientWidth, + ), + ).toBe(true); + await page.screenshot({ + path: testInfo.outputPath("phase-03-recovery-diagnostics-mobile.png"), + fullPage: true, + }); +}); + test("shows duplicate and unsupported-version replay states", async ({ page }) => { await page.goto("/"); await page.getByRole("button", { name: "Static replay" }).click(); diff --git a/packages/paperclip-runner/docs/tutorials/phase-03-break-recovery.md b/packages/paperclip-runner/docs/tutorials/phase-03-break-recovery.md index 6e062e33561..0f9914ae566 100644 --- a/packages/paperclip-runner/docs/tutorials/phase-03-break-recovery.md +++ b/packages/paperclip-runner/docs/tutorials/phase-03-break-recovery.md @@ -160,13 +160,34 @@ Run: pnpm --filter @paperclipai/paperclip-runner browser:dev --host 127.0.0.1 --port 4179 ``` -Open `http://127.0.0.1:4179/`. Select **Recovery**. Select -**Run Phase 3 recovery**. +Open `http://127.0.0.1:4179/` and select **Recovery**. Use the **Fault** selector +to run these cases with **Run Phase 3 recovery**: + +1. **Normal connection** +2. **Socket drop** +3. **Lost ACK** +4. **Runner restart** +5. **Revoke** + +Also run **Storage pressure** when checking the bounded-storage state. Confirm: -- recovery status becomes **complete**; -- connection, lease, outbox, ACK, replay, storage, and outcome fields appear; +- recovery status becomes **complete** after every run; +- **Outcome** is **Recovered** for the normal, socket-drop, lost-ACK, and + runner-restart cases, while revoke shows the warning-toned **Revoked** outcome + and its reason; +- **Connections**, **Reconnects**, **Runner restarts**, and **Fresh bootstraps** + distinguish the normal, reconnect, process-restart, and lease-bootstrap paths; +- **Recovery history** lists every committed source sequence and event type, with + a `delivered N×` marker on events delivered more than once; +- **At-least-once redeliveries** can be non-zero for a normal connection because + the runner may resend a still-unacknowledged in-flight outbox batch. Use the + reconnect counters and per-event delivery markers to identify injected + recovery; a lost ACK adds an extra delivery to the affected event; +- **Storage** reports current bytes and the maximum, and storage pressure shows a + warning-toned **Backpressure** badge; +- the connection lifecycle is human-readable, such as **Stopped**; - **Same runner and session** says **Preserved**; - **Secrets redacted** says **Yes**; - no bootstrap ticket or connection lease token appears.