feat(runner): add durable Phase 3 recovery transport

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/devtools/browser/src/App.tsx

@@ -7,6 +7,7 @@ import type {
   Phase2RunTrace,
   Phase2Scenario,
 } from "../../../src/contracts/phase2";
+import type { Phase3RunTrace } from "../../../src/contracts/phase3";
 import {
   applyPhase2LiveEvent,
   createPhase2LiveSnapshot,
@@ -54,7 +55,7 @@ interface BrowserFixture {
   source: string;
 }
 
-type BrowserMode = "live" | "replay";
+type BrowserMode = "recovery" | "live" | "replay";
 type LiveStatus = "idle" | "starting" | "running" | "terminal" | "error";
 
 type Phase2StreamRecord =
@@ -89,6 +90,94 @@ function humanizeProtocolLabel(value: string): string {
   return words.length === 0 ? words : `${words[0]?.toUpperCase()}${words.slice(1)}`;
 }
 
+function RecoveryDiagnostics() {
+  const [trace, setTrace] = useState<Phase3RunTrace | null>(null);
+  const [status, setStatus] = useState<"idle" | "running" | "complete" | "error">("idle");
+  const [error, setError] = useState<string | null>(null);
+
+  async function runRecovery() {
+    setStatus("running");
+    setError(null);
+    try {
+      const response = await fetch("/api/phase3/recovery", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ fault: "lost-ack" }),
+      });
+      const result = (await response.json()) as Phase3RunTrace & { error?: string };
+      if (!response.ok) throw new Error(result.error ?? "The recovery trace failed.");
+      setTrace(result);
+      setStatus("complete");
+    } catch (cause) {
+      setStatus("error");
+      setError(cause instanceof Error ? cause.message : String(cause));
+    }
+  }
+
+  return (
+    <div className="workspace-grid recovery-workspace">
+      <Card aria-labelledby="recovery-title">
+        <CardHeader>
+          <CardTitle id="recovery-title">Break recovery on purpose</CardTitle>
+          <CardDescription>
+            Lose one ACK, drop the socket, restart the runner, and finish the same session.
+          </CardDescription>
+        </CardHeader>
+        <CardContent className="editor-content">
+          <Button type="button" disabled={status === "running"} onClick={() => void runRecovery()}>
+            {status === "running" ? "Running recovery…" : "Run Phase 3 recovery"}
+          </Button>
+          <div className="live-status" aria-live="polite">
+            <span>Recovery status</span>
+            <Badge tone={status === "error" ? "danger" : status === "complete" ? "success" : "neutral"}>
+              {status}
+            </Badge>
+          </div>
+          {error ? <div className="validation-errors" role="alert">{error}</div> : null}
+          <p className="diagnostic-note">
+            Diagnostics show IDs and counters only. Bootstrap tickets and lease tokens are hidden.
+          </p>
+        </CardContent>
+      </Card>
+
+      <Card aria-labelledby="recovery-state-title">
+        <CardHeader>
+          <CardTitle id="recovery-state-title">Recovery state</CardTitle>
+          <CardDescription>Inspect connection, lease, outbox, ACK, replay, storage, drain, and revoke state.</CardDescription>
+        </CardHeader>
+        <CardContent>
+          {trace === null ? (
+            <div className="empty-live-state"><p>Run the trace to collect recovery diagnostics.</p></div>
+          ) : (
+            <>
+              <dl className="recovery-grid" data-testid="phase3-diagnostics">
+                <div><dt>Connection</dt><dd>{trace.diagnostics.connection.state}</dd></div>
+                <div><dt>Lease</dt><dd>{trace.diagnostics.connection.leaseId ?? "closed"}</dd></div>
+                <div><dt>Outbox</dt><dd>{trace.diagnostics.outbox.events}</dd></div>
+                <div><dt>Last ACK</dt><dd>{trace.diagnostics.cursors.runnerAckedSourceSeq}</dd></div>
+                <div><dt>Replayed</dt><dd>{trace.diagnostics.recovery.replayDeliveries}</dd></div>
+                <div><dt>Duplicate commands</dt><dd>{trace.diagnostics.commands.duplicateDeliveries}</dd></div>
+                <div><dt>Storage</dt><dd>{trace.diagnostics.outbox.bytes} bytes</dd></div>
+                <div><dt>Peak storage</dt><dd>{trace.diagnostics.outbox.peakBytes} bytes</dd></div>
+                <div><dt>Drain or revoke</dt><dd>{trace.diagnostics.recovery.outcome}</dd></div>
+                <div><dt>Secrets</dt><dd>{trace.diagnostics.security.secretLeakCount} leaks</dd></div>
+              </dl>
+              <div className="parity-line">
+                <span>Same runner and session</span>
+                <Badge tone={trace.assertions.stableIdentity ? "success" : "danger"}>{trace.assertions.stableIdentity ? "Preserved" : "Changed"}</Badge>
+              </div>
+              <div className="parity-line">
+                <span>Secrets redacted</span>
+                <Badge tone={trace.assertions.secretsRedacted ? "success" : "danger"}>{trace.assertions.secretsRedacted ? "Yes" : "No"}</Badge>
+              </div>
+            </>
+          )}
+        </CardContent>
+      </Card>
+    </div>
+  );
+}
+
 function liveStatusTone(status: LiveStatus, snapshot: SessionSnapshot | null) {
   if (status === "error") {
     return "danger" as const;
@@ -571,16 +660,23 @@ export function App() {
     <main className="app-shell">
       <header className="page-header">
         <div>
-          <p className="eyebrow">Paperclip Runner Protocol · Phase 2</p>
-          <h1>Live local runner</h1>
+          <p className="eyebrow">Paperclip Runner Protocol · Phase 3</p>
+          <h1>Durable recovery diagnostics</h1>
           <p>
-            Run the Rust supervisor and fake harness, then compare the live state with replay.
+            Break the outbound transport, recover the same session, and inspect durable state.
           </p>
         </div>
         <Badge tone="neutral">Standalone · no Paperclip core</Badge>
       </header>
 
       <nav className="mode-switch" aria-label="Runner devtool mode">
+        <Button
+          type="button"
+          aria-pressed={mode === "recovery"}
+          onClick={() => setMode("recovery")}
+        >
+          Recovery
+        </Button>
         <Button
           type="button"
           aria-pressed={mode === "live"}
@@ -597,7 +693,7 @@ export function App() {
         </Button>
       </nav>
 
-      {mode === "live" ? <LiveRunner /> : <StaticReplay />}
+      {mode === "recovery" ? <RecoveryDiagnostics /> : mode === "live" ? <LiveRunner /> : <StaticReplay />}
     </main>
   );
 }

packages/paperclip-runner/devtools/browser/src/styles.css

@@ -342,6 +342,20 @@ select {
   border-block: 0.0625rem solid var(--border);
 }
 
+.recovery-grid {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: var(--space-4);
+  margin: 0;
+  padding-block: var(--space-5);
+  border-block: 0.0625rem solid var(--border);
+}
+
+.diagnostic-note {
+  color: var(--muted-foreground);
+  font-size: var(--font-size-sm);
+}
+
 .snapshot-grid div {
   min-width: 0;
 }

packages/paperclip-runner/devtools/browser/static-replay.spec.ts

@@ -3,7 +3,7 @@ import { expect, test } from "@playwright/test";
 test("validates and renders the shared Phase 1 fixture", async ({ page }, testInfo) => {
   await page.goto("/");
   await page.getByRole("button", { name: "Static replay" }).click();
-  await expect(page.getByRole("heading", { name: "Live local runner" })).toBeVisible();
+  await expect(page.getByRole("heading", { name: "Durable recovery diagnostics" })).toBeVisible();
   await expect(page.getByTestId("terminal-badge")).toHaveText("Succeeded");
   await expect(page.getByTestId("timeline").getByRole("listitem")).toHaveCount(9);
   await expect(page.getByTestId("timeline")).not.toContainText("workspace_preparing");
@@ -16,6 +16,20 @@ test("validates and renders the shared Phase 1 fixture", async ({ page }, testIn
   });
 });
 
+test("shows Phase 3 recovery diagnostics without secrets", async ({ page }, testInfo) => {
+  await page.goto("/");
+  await page.getByRole("button", { name: "Recovery", exact: true }).click();
+  await page.getByRole("button", { name: "Run Phase 3 recovery" }).click();
+  await expect(page.getByText("complete", { exact: true })).toBeVisible();
+  await expect(page.getByTestId("phase3-diagnostics")).toContainText("Replayed");
+  await expect(page.getByTestId("phase3-diagnostics")).toContainText("Duplicate commands");
+  await expect(page.getByText("Preserved", { exact: true })).toBeVisible();
+  await expect(page.getByText("Yes", { exact: true })).toBeVisible();
+  await expect(page.locator("body")).not.toContainText("leaseToken");
+  await expect(page.locator("body")).not.toContainText("bootstrapTicket");
+  await page.screenshot({ path: testInfo.outputPath("phase-03-recovery-diagnostics.png"), fullPage: true });
+});
+
 test("shows duplicate and unsupported-version replay states", async ({ page }) => {
   await page.goto("/");
   await page.getByRole("button", { name: "Static replay" }).click();

packages/paperclip-runner/package.json

@@ -44,9 +44,13 @@
     "replay:phase1": "pnpm run build:typescript && node dist/cli/phase1-replay.js",
     "trace:phase2": "pnpm run build:typescript && pnpm run build:phase2-binaries && node dist/cli/phase2-mock-core.js",
     "record:phase2": "pnpm run trace:phase2 -- --quiet --output knowledge/evidence/phase-02-happy-path.json",
+    "trace:phase3": "pnpm run build:typescript && pnpm run build:phase2-binaries && node dist/cli/phase3-recovery.js",
+    "diagnose:phase3": "pnpm run trace:phase3",
+    "record:phase3": "pnpm run build:typescript && pnpm run build:phase2-binaries && node scripts/record-phase3-evidence.mjs",
+    "record:phase3:lost-ack": "pnpm run trace:phase3 -- --fault lost-ack --output knowledge/evidence/phase-03-lost-ack.json",
     "browser:dev": "pnpm run build:typescript && pnpm run build:phase2-binaries && vite --config vite.config.ts",
     "browser:preview": "vite preview --config vite.config.ts",
-    "verify": "pnpm run build && pnpm run typecheck && pnpm run check:phase1-goldens && pnpm run test && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run check:forbidden-imports && pnpm run docs:validate && pnpm run check:phase0-parity && pnpm run check:phase1-parity && pnpm run trace:phase0 && pnpm run replay:phase1 && pnpm run trace:phase2 -- --quiet",
+    "verify": "pnpm run build && pnpm run typecheck && pnpm run check:phase1-goldens && pnpm run test && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run check:forbidden-imports && pnpm run docs:validate && pnpm run check:phase0-parity && pnpm run check:phase1-parity && pnpm run trace:phase0 && pnpm run replay:phase1 && pnpm run trace:phase2 -- --quiet && pnpm run trace:phase3",
     "verify:rootless": "bash scripts/verify-rootless-linux.sh"
   },
   "devDependencies": {

packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs

@@ -3,6 +3,9 @@ use std::process::ExitCode;
 use std::time::Duration;
 
 use paperclip_runner_core::phase2::{run_local_runner, Phase2Error, RunnerConfig};
+use paperclip_runner_core::phase3::{
+    capture_bootstrap_ticket, run_durable_runner, BootstrapTicket, DurableRunnerConfig,
+};
 
 fn value(args: &[String], name: &str) -> Result<String, Phase2Error> {
     let index = args
@@ -34,8 +37,54 @@ fn usize_value(args: &[String], name: &str, default: usize) -> Result<usize, Pha
     })
 }
 
-fn run() -> Result<(), Phase2Error> {
+fn duration_value(args: &[String], name: &str, default: u64) -> Result<Duration, Phase2Error> {
+    Ok(Duration::from_millis(
+        optional_u64(args, name)?.unwrap_or(default),
+    ))
+}
+
+fn run_phase3(
+    args: &[String],
+    bootstrap_ticket: Option<BootstrapTicket>,
+) -> Result<(), Phase2Error> {
+    let config = DurableRunnerConfig {
+        connect_url: value(args, "--connect-url")?,
+        state_dir: PathBuf::from(value(args, "--state-dir")?),
+        runner_instance_id: value(args, "--runner-id")?,
+        environment_lease_id: value(args, "--environment-lease-id")?,
+        run_id: value(args, "--run-id")?,
+        normalized_session_id: value(args, "--session-id")?,
+        turn_id: value(args, "--turn-id")?,
+        item_id: value(args, "--item-id")?,
+        runner_version: value(args, "--runner-version")?,
+        runner_digest: value(args, "--runner-digest")?,
+        fake_harness_path: args
+            .iter()
+            .any(|argument| argument == "--fake-harness")
+            .then(|| value(args, "--fake-harness").map(PathBuf::from))
+            .transpose()?,
+        fake_harness_script_path: args
+            .iter()
+            .any(|argument| argument == "--fake-harness-script")
+            .then(|| value(args, "--fake-harness-script").map(PathBuf::from))
+            .transpose()?,
+        max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 64 * 1024)?,
+        p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 32 * 1024)?,
+        max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?,
+        reconnect_delay: duration_value(args, "--reconnect-delay-ms", 25)?,
+        max_runtime: duration_value(args, "--max-runtime-ms", 15_000)?,
+    };
+    let bootstrap_ticket = bootstrap_ticket
+        .ok_or_else(|| Phase2Error::invalid("runner bootstrap ticket is not available"))?;
+    run_durable_runner(config, bootstrap_ticket)
+        .map_err(|error| Phase2Error::invalid(error.to_string()))
+}
+
+fn run(bootstrap_ticket: Option<BootstrapTicket>) -> Result<(), Phase2Error> {
     let args = std::env::args().skip(1).collect::<Vec<_>>();
+    if args.iter().any(|argument| argument == "--connect-url") {
+        return run_phase3(&args, bootstrap_ticket);
+    }
     run_local_runner(RunnerConfig {
         run_id: value(&args, "--run-id")?,
         normalized_session_id: value(&args, "--session-id")?,
@@ -53,7 +102,15 @@ fn run() -> Result<(), Phase2Error> {
 }
 
 fn main() -> ExitCode {
-    match run() {
+    // Capture and remove the bootstrap capability before argument parsing or child work.
+    let bootstrap_ticket = match capture_bootstrap_ticket() {
+        Ok(ticket) => ticket,
+        Err(error) => {
+            eprintln!("paperclip-runnerd: {error}");
+            return ExitCode::FAILURE;
+        }
+    };
+    match run(bootstrap_ticket) {
         Ok(()) => ExitCode::SUCCESS,
         Err(error) => {
             eprintln!("paperclip-runnerd: {error}");

packages/paperclip-runner/runner/crates/runner-core/src/lib.rs

@@ -2,6 +2,7 @@
 
 pub mod phase1;
 pub mod phase2;
+pub mod phase3;
 
 use std::error::Error;
 use std::fmt::{self, Display, Formatter};

packages/paperclip-runner/scripts/phase2-browser-server.mjs

@@ -1,4 +1,7 @@
 import { randomUUID } from "node:crypto";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 
 export const PHASE2_BROWSER_LIMITS = Object.freeze({
   maxActiveRuns: 4,
@@ -230,9 +233,18 @@ async function loadPhase2Runner() {
   return import(runnerModuleUrl);
 }
 
+async function loadPhase3Runner() {
+  const runnerModuleUrl = new URL(
+    "../dist/mock-core/phase3-recovery.js",
+    import.meta.url,
+  ).href;
+  return import(runnerModuleUrl);
+}
+
 export function createPhase2BrowserMiddleware(options = {}) {
   const limits = normalizeLimits(options.limits);
   const loadRunner = options.loadRunner ?? loadPhase2Runner;
+  const loadRecoveryRunner = options.loadRecoveryRunner ?? loadPhase3Runner;
   const runs = new Map();
   let startingRuns = 0;
 
@@ -252,11 +264,39 @@ export function createPhase2BrowserMiddleware(options = {}) {
 
   return async function middleware(request, response, next) {
     const url = new URL(request.url ?? "/", "http://phase2.local");
-    if (!url.pathname.startsWith("/api/phase2/")) {
+    if (!url.pathname.startsWith("/api/phase2/") && url.pathname !== "/api/phase3/recovery") {
       next();
       return;
     }
 
+    if (request.method === "POST" && url.pathname === "/api/phase3/recovery") {
+      let storageDirectory = null;
+      try {
+        assertTrustedTransport(request);
+        rejectDeclaredOversize(request, limits.maxRequestBodyBytes);
+        const body = await readJson(request, limits.maxRequestBodyBytes);
+        const recovery = await loadRecoveryRunner();
+        const scratchRoot =
+          process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
+          process.env.PAPERCLIP_SCRATCH_DIR ??
+          tmpdir();
+        storageDirectory = await mkdtemp(
+          join(scratchRoot, "paperclip-runner-browser-phase3-"),
+        );
+        json(response, 200, await recovery.runPhase3Recovery({
+          fault: body.fault ?? "lost-ack",
+          stateDirectory: storageDirectory,
+        }));
+      } catch (error) {
+        sendError(response, error);
+      } finally {
+        if (storageDirectory !== null) {
+          await rm(storageDirectory, { recursive: true, force: true });
+        }
+      }
+      return;
+    }
+
     try {
       assertTrustedTransport(request);
       rejectDeclaredOversize(request, limits.maxRequestBodyBytes);

packages/paperclip-runner/scripts/phase2-browser-server.test.mjs

@@ -128,6 +128,31 @@ afterEach(async () => {
 });
 
 describe("Phase 2 browser transport hardening", () => {
+  it("applies the same loopback and origin guards to Phase 3 recovery", async () => {
+    let recoveryCalls = 0;
+    const loadRecoveryRunner = async () => ({
+      async runPhase3Recovery() {
+        recoveryCalls += 1;
+        return { schema: "paperclip.runner.phase3.trace.v1" };
+      },
+    });
+    const { port } = await startServer({ loadRecoveryRunner });
+
+    const denied = await sendRequest(port, {
+      path: "/api/phase3/recovery",
+      origin: "https://attacker.example",
+      body: JSON.stringify({ fault: "lost-ack" }),
+    });
+    const accepted = await sendRequest(port, {
+      path: "/api/phase3/recovery",
+      body: JSON.stringify({ fault: "lost-ack" }),
+    });
+
+    expect(denied.status).toBe(403);
+    expect(accepted.status).toBe(200);
+    expect(recoveryCalls).toBe(1);
+  });
+
   it("denies cross-origin starts and applies the guard before route lookup", async () => {
     const runner = fakeRunner();
     const { port } = await startServer({ loadRunner: () => runner.load() });

packages/paperclip-runner/scripts/record-phase3-evidence.mjs

@@ -0,0 +1,49 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { PHASE3_FAULTS, runPhase3Recovery } from "../dist/index.js";
+
+const packageRoot = fileURLToPath(new URL("..", import.meta.url));
+const evidenceRoot = resolve(packageRoot, "knowledge/evidence");
+const faultRoot = resolve(evidenceRoot, "phase-03-faults");
+mkdirSync(faultRoot, { recursive: true });
+
+const summaries = [];
+for (const fault of PHASE3_FAULTS) {
+  const trace = await runPhase3Recovery({ fault });
+  writeFileSync(
+    resolve(faultRoot, `${fault}.json`),
+    `${JSON.stringify(trace, null, 2)}\n`,
+  );
+  if (fault === "lost-ack") {
+    writeFileSync(
+      resolve(evidenceRoot, "phase-03-recovery-trace.json"),
+      `${JSON.stringify(trace, null, 2)}\n`,
+    );
+  }
+  summaries.push({
+    fault,
+    outcome: trace.diagnostics.recovery.outcome,
+    connectionCount: trace.diagnostics.connection.connectionCount,
+    replayDeliveries: trace.diagnostics.recovery.replayDeliveries,
+    runnerRestarts: trace.diagnostics.recovery.runnerRestarts,
+    harnessRestarts: trace.diagnostics.recovery.harnessRestarts,
+    commands: trace.diagnostics.commands,
+    outbox: trace.diagnostics.outbox,
+    security: trace.diagnostics.security,
+    assertions: trace.assertions,
+  });
+}
+
+writeFileSync(
+  resolve(evidenceRoot, "phase-03-fault-matrix.json"),
+  `${JSON.stringify({ schema: "paperclip.runner.phase3.evidence.v1", traces: summaries }, null, 2)}\n`,
+);
+
+const passed = summaries.every((summary) =>
+  Object.values(summary.assertions).every(Boolean),
+);
+process.stdout.write(
+  `Recorded ${summaries.length} Phase 3 fault traces; every trace passed: ${passed}.\n`,
+);

packages/paperclip-runner/src/cli/phase3-recovery.ts

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+
+import {
+  PHASE3_FAULTS,
+  type Phase3Fault,
+  type Phase3RunTrace,
+} from "../contracts/phase3.js";
+import { runPhase3Recovery } from "../mock-core/phase3-recovery.js";
+
+interface CliOptions {
+  fault: Phase3Fault;
+  json: boolean;
+  output: string | null;
+  stateDirectory: string | null;
+  keepState: boolean;
+}
+
+function optionValue(args: readonly string[], name: string): string | null {
+  const index = args.indexOf(name);
+  if (index < 0) return null;
+  const value = args[index + 1];
+  if (value === undefined || value.startsWith("--")) {
+    throw new Error(`${name} requires a value`);
+  }
+  return value;
+}
+
+function parseOptions(args: readonly string[]): CliOptions {
+  const fault = optionValue(args, "--fault") ?? "lost-ack";
+  if (!PHASE3_FAULTS.includes(fault as Phase3Fault)) {
+    throw new Error(
+      `--fault must be one of: ${PHASE3_FAULTS.join(", ")}`,
+    );
+  }
+  return {
+    fault: fault as Phase3Fault,
+    json: args.includes("--json"),
+    output: optionValue(args, "--output"),
+    stateDirectory: optionValue(args, "--state-dir"),
+    keepState: args.includes("--keep-state"),
+  };
+}
+
+function assertionLine(name: string, passed: boolean): string {
+  return `${passed ? "PASS" : "FAIL"} ${name}`;
+}
+
+export function formatPhase3Diagnostics(trace: Phase3RunTrace): string {
+  const { diagnostics, assertions } = trace;
+  const lines = [
+    `Phase 3 recovery: ${diagnostics.fault}`,
+    `Outcome: ${diagnostics.recovery.outcome} (${diagnostics.recovery.reason})`,
+    `Connection: ${diagnostics.connection.state}; ${diagnostics.connection.connectionCount} connection(s), ${diagnostics.connection.reconnectCount} reconnect(s)`,
+    `Lease: ${diagnostics.connection.leaseId ?? "none"}; expires ${diagnostics.connection.leaseExpiresAt ?? "not available"}`,
+    `Identity: runner=${diagnostics.identity.runnerInstanceId} session=${diagnostics.identity.normalizedSessionId} turn=${diagnostics.identity.turnId} item=${diagnostics.identity.itemId}`,
+    `Cursors: runner ACK ${diagnostics.cursors.runnerAckedSourceSeq}; core ACK ${diagnostics.cursors.coreAckedSourceSeq}; next source ${diagnostics.cursors.runnerNextSourceSeq}`,
+    `Outbox: ${diagnostics.outbox.events} pending / ${diagnostics.outbox.bytes} bytes; peak ${diagnostics.outbox.peakBytes} / ${diagnostics.outbox.maxBytes}; P0 lost ${diagnostics.outbox.p0Lost}`,
+    `Commands: ${diagnostics.commands.completed} completed, ${diagnostics.commands.rejected} rejected, ${diagnostics.commands.duplicateDeliveries} duplicate delivery(s), ${diagnostics.commands.logicalEffects} logical effect(s)`,
+    `Recovery: ${diagnostics.recovery.replayDeliveries} replay delivery(s), ${diagnostics.recovery.runnerRestarts} runner restart(s), ${diagnostics.recovery.harnessRestarts} harness restart(s)`,
+    `Security: ${diagnostics.security.secretLeakCount} persisted secret leak(s)`,
+    "Assertions:",
+    assertionLine("stable identity", assertions.stableIdentity),
+    assertionLine("source cursor continuity", assertions.sourceCursorContinuous),
+    assertionLine(
+      "one logical effect per accepted command",
+      assertions.oneLogicalEffectPerAcceptedCommand,
+    ),
+    assertionLine("no duplicate logical events", assertions.noDuplicateLogicalEvents),
+    assertionLine("P0 events preserved", assertions.p0Preserved),
+    assertionLine("bounded storage", assertions.boundedStorage),
+    assertionLine("secrets redacted", assertions.secretsRedacted),
+  ];
+  return `${lines.join("\n")}\n`;
+}
+
+async function main(): Promise<void> {
+  const options = parseOptions(process.argv.slice(2));
+  const trace = await runPhase3Recovery({
+    fault: options.fault,
+    stateDirectory:
+      options.stateDirectory === null ? undefined : resolve(options.stateDirectory),
+    keepState: options.keepState,
+  });
+  if (options.output !== null) {
+    const output = resolve(options.output);
+    await mkdir(dirname(output), { recursive: true });
+    await writeFile(output, `${JSON.stringify(trace, null, 2)}\n`);
+  }
+  process.stdout.write(
+    options.json ? `${JSON.stringify(trace, null, 2)}\n` : formatPhase3Diagnostics(trace),
+  );
+  if (Object.values(trace.assertions).some((passed) => !passed)) {
+    process.exitCode = 1;
+  }
+}
+
+main().catch((error) => {
+  process.stderr.write(
+    `phase3-recovery: ${error instanceof Error ? error.message : String(error)}\n`,
+  );
+  process.exitCode = 1;
+});

packages/paperclip-runner/src/contracts/phase3.ts

@@ -0,0 +1,158 @@
+export const PHASE3_FAULTS = [
+  "none",
+  "socket-drop",
+  "lost-ack",
+  "duplicate-command",
+  "runner-restart",
+  "harness-restart",
+  "malformed-input",
+  "lease-expiry",
+  "storage-pressure",
+  "drain",
+  "revoke",
+] as const;
+
+export type Phase3Fault = (typeof PHASE3_FAULTS)[number];
+
+export interface Phase3Identity {
+  runnerInstanceId: string;
+  environmentLeaseId: string;
+  runId: string;
+  normalizedSessionId: string;
+  turnId: string;
+  itemId: string;
+}
+
+export interface Phase3StoredEvent {
+  sourceSeq: number;
+  sourceEventId: string;
+  priority: 0 | 1 | 2;
+  eventType: string;
+  itemId?: string;
+  envelope: Record<string, unknown>;
+  byteSize: number;
+}
+
+export interface Phase3ProcessedCommand {
+  commandId: string;
+  controllerSeq: number;
+  commandDigest: string;
+  status: "completed" | "failed" | "rejected";
+  logicalEffectCount: number;
+  result: Record<string, unknown>;
+}
+
+export interface Phase3RunnerState extends Phase3Identity {
+  schema: "paperclip.runner.phase3.state.v1";
+  lifecycle:
+    | "connecting"
+    | "ready"
+    | "terminal"
+    | "backpressure"
+    | "draining"
+    | "revoked"
+    | "stopped"
+    | "recoverable_failure"
+    | "unrecoverable";
+  nextSourceSeq: number;
+  ackedSourceSeq: number;
+  lastControllerCommandSeq: number;
+  reconnectCount: number;
+  maxOutboxBytes: number;
+  peakOutboxBytes: number;
+  outbox: Phase3StoredEvent[];
+  processedCommands: Record<string, Phase3ProcessedCommand>;
+  diagnostics: string[];
+  backpressure: boolean;
+  recoverableFailure: string | null;
+  unrecoverableOutcome: string | null;
+  harnessGeneration: number;
+  stopAfterFlush: boolean;
+}
+
+export interface Phase3CoreCommand {
+  schema: "paperclip.prp.command.v1";
+  commandId: string;
+  controllerSeq: number;
+  type: string;
+  issuedAt: string;
+  payload: Record<string, unknown>;
+  status: "pending" | "completed" | "failed" | "rejected";
+  result: Record<string, unknown> | null;
+}
+
+export interface Phase3CommittedEvent {
+  sourceSeq: number;
+  sourceEventId: string;
+  eventType: string;
+  priority: 0 | 1 | 2;
+  envelope: Record<string, unknown>;
+  deliveryCount: number;
+  logicalEffectCount: number;
+}
+
+export interface Phase3Diagnostics {
+  schema: "paperclip.runner.phase3.diagnostics.v1";
+  fault: Phase3Fault;
+  connection: {
+    state: string;
+    connectionCount: number;
+    reconnectCount: number;
+    leaseId: string | null;
+    leaseExpiresAt: string | null;
+  };
+  identity: Phase3Identity;
+  cursors: {
+    runnerAckedSourceSeq: number;
+    runnerNextSourceSeq: number;
+    coreAckedSourceSeq: number;
+    highestCommittedSourceSeq: number;
+  };
+  outbox: {
+    events: number;
+    bytes: number;
+    peakBytes: number;
+    maxBytes: number;
+    backpressure: boolean;
+    p0Committed: number;
+    p0Lost: number;
+  };
+  commands: {
+    issued: number;
+    completed: number;
+    rejected: number;
+    logicalEffects: number;
+    duplicateDeliveries: number;
+  };
+  recovery: {
+    replayDeliveries: number;
+    runnerRestarts: number;
+    harnessRestarts: number;
+    malformedFrames: number;
+    freshBootstraps: number;
+    outcome: "recovered" | "drained" | "revoked" | "unrecoverable";
+    reason: string;
+  };
+  security: {
+    bootstrapTicketPersisted: boolean;
+    connectionLeaseTokenPersisted: boolean;
+    secretLeakCount: number;
+  };
+  committedEvents: Phase3CommittedEvent[];
+}
+
+export interface Phase3RunTrace {
+  schema: "paperclip.runner.phase3.trace.v1";
+  diagnostics: Phase3Diagnostics;
+  runnerState: Phase3RunnerState;
+  commands: Phase3CoreCommand[];
+  assertions: {
+    stableIdentity: boolean;
+    sourceCursorContinuous: boolean;
+    oneLogicalEffectPerAcceptedCommand: boolean;
+    noDuplicateLogicalEvents: boolean;
+    p0Preserved: boolean;
+    boundedStorage: boolean;
+    secretsRedacted: boolean;
+  };
+}

packages/paperclip-runner/src/index.ts

@@ -2,9 +2,11 @@ export * from "./contracts/control-plane-port.js";
 export * from "./contracts/harness-driver.js";
 export * from "./contracts/native-session-backend.js";
 export * from "./contracts/phase2.js";
+export * from "./contracts/phase3.js";
 export * from "./contracts/types.js";
 export * from "./mock-core/mock-control-plane-adapter.js";
 export * from "./mock-core/phase2-local-runner.js";
+export * from "./mock-core/phase3-recovery.js";
 export * from "./protocol/phase0-fixture.js";
 export * from "./protocol/phase1-contract.js";
 export * from "./protocol/phase1-loader.js";

packages/paperclip-runner/src/mock-core/phase3-recovery.test.ts

@@ -0,0 +1,379 @@
+import {
+  chmodSync,
+  lstatSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { connect, type Socket } from "node:net";
+import { tmpdir } from "node:os";
+import { resolve } from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import {
+  Phase3MockCore,
+  phase3Internals,
+  runPhase3Recovery,
+} from "./phase3-recovery.js";
+
+async function upgradeStatus(url: string, ticket: string): Promise<number> {
+  const parsed = new URL(url);
+  return new Promise((resolveStatus, rejectStatus) => {
+    const socket = connect(Number(parsed.port), parsed.hostname);
+    let response = "";
+    socket.once("error", rejectStatus);
+    socket.once("connect", () => {
+      socket.write(
+        [
+          `GET ${parsed.pathname} HTTP/1.1`,
+          `Host: ${parsed.host}`,
+          "Upgrade: websocket",
+          "Connection: Upgrade",
+          "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==",
+          "Sec-WebSocket-Version: 13",
+          `Authorization: Bearer ${ticket}`,
+          "\r\n",
+        ].join("\r\n"),
+      );
+    });
+    socket.on("data", (chunk) => {
+      response += chunk.toString("utf8");
+      if (response.includes("\r\n\r\n")) {
+        const status = Number(response.match(/^HTTP\/1\.1 (\d{3})/)?.[1]);
+        socket.destroy();
+        resolveStatus(status);
+      }
+    });
+  });
+}
+
+async function upgradeSocket(url: string, ticket: string): Promise<Socket> {
+  const parsed = new URL(url);
+  return new Promise((resolveSocket, rejectSocket) => {
+    const socket = connect(Number(parsed.port), parsed.hostname);
+    let response = Buffer.alloc(0);
+    const onError = (error: Error): void => rejectSocket(error);
+    const onData = (chunk: Buffer): void => {
+      response = Buffer.concat([response, chunk]);
+      const boundary = response.indexOf("\r\n\r\n");
+      if (boundary === -1) return;
+      socket.off("error", onError);
+      socket.off("data", onData);
+      const status = Number(response.toString("utf8").match(/^HTTP\/1\.1 (\d{3})/)?.[1]);
+      if (status !== 101) {
+        socket.destroy();
+        rejectSocket(new Error(`WebSocket upgrade returned ${String(status)}`));
+        return;
+      }
+      resolveSocket(socket);
+    };
+    socket.once("error", onError);
+    socket.once("connect", () => {
+      socket.write(
+        [
+          `GET ${parsed.pathname} HTTP/1.1`,
+          `Host: ${parsed.host}`,
+          "Upgrade: websocket",
+          "Connection: Upgrade",
+          "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==",
+          "Sec-WebSocket-Version: 13",
+          `Authorization: Bearer ${ticket}`,
+          "\r\n",
+        ].join("\r\n"),
+      );
+    });
+    socket.on("data", onData);
+  });
+}
+
+function sendMaskedJson(socket: Socket, value: unknown): void {
+  const payload = Buffer.from(JSON.stringify(value));
+  const mask = Buffer.from([0x11, 0x22, 0x33, 0x44]);
+  const header: number[] = [0x81];
+  if (payload.length <= 125) {
+    header.push(0x80 | payload.length);
+  } else {
+    header.push(0x80 | 126, (payload.length >>> 8) & 0xff, payload.length & 0xff);
+  }
+  const masked = Buffer.from(payload);
+  for (let index = 0; index < masked.length; index += 1) {
+    masked[index] = masked[index]! ^ mask[index % mask.length]!;
+  }
+  socket.write(Buffer.concat([Buffer.from(header), mask, masked]));
+}
+
+async function expectSocketClosed(socket: Socket): Promise<void> {
+  await new Promise<void>((resolveClose, rejectClose) => {
+    const timer = setTimeout(() => rejectClose(new Error("WebSocket was not rejected.")), 2_000);
+    socket.once("close", () => {
+      clearTimeout(timer);
+      resolveClose();
+    });
+  });
+}
+
+function hello(identity: ReturnType<typeof phase3Internals.phase3Identity>): Record<string, unknown> {
+  return {
+    protocol: "paperclip.runner",
+    version: 1,
+    envelopeId: "hello_auth_regression",
+    kind: "hello",
+    runnerInstanceId: identity.runnerInstanceId,
+    payload: {
+      protocolMin: 1,
+      protocolMax: 1,
+      runnerVersion: "0.3.0",
+      runnerDigest: "sha256:phase3-approved",
+      environmentLeaseId: identity.environmentLeaseId,
+    },
+  };
+}
+
+describe.sequential("Phase 3 durable transport and recovery", () => {
+  it("accepts a short-lived bootstrap ticket once and rejects its reuse", async () => {
+    const scratch =
+      process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
+      process.env.PAPERCLIP_SCRATCH_DIR ??
+      tmpdir();
+    const root = mkdtempSync(resolve(scratch, "phase3-ticket-test-"));
+    const core = new Phase3MockCore({
+      stateDirectory: root,
+      identity: phase3Internals.phase3Identity(),
+      fault: "none",
+    });
+    try {
+      await core.start();
+      const ticket = core.issueBootstrapTicket();
+
+      await expect(upgradeStatus(core.connectUrl, ticket)).resolves.toBe(101);
+      await expect(upgradeStatus(core.connectUrl, ticket)).resolves.toBe(401);
+      const expiredTicket = core.issueBootstrapTicket(-1);
+      await expect(upgradeStatus(core.connectUrl, expiredTicket)).resolves.toBe(401);
+    } finally {
+      await core.stop();
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  it("rejects bootstrap and resumed lease credentials bound to a different runner identity", async () => {
+    const scratch =
+      process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
+      process.env.PAPERCLIP_SCRATCH_DIR ??
+      tmpdir();
+    const root = mkdtempSync(resolve(scratch, "phase3-identity-auth-test-"));
+    const identity = phase3Internals.phase3Identity();
+    const core = new Phase3MockCore({
+      stateDirectory: root,
+      identity,
+      fault: "none",
+    });
+    try {
+      await core.start();
+
+      const bootstrap = core.issueBootstrapTicket();
+      const bootstrapSocket = await upgradeSocket(core.connectUrl, bootstrap);
+      sendMaskedJson(bootstrapSocket, {
+        ...hello(identity),
+        runnerInstanceId: "runner_wrong_identity",
+      });
+      await expectSocketClosed(bootstrapSocket);
+
+      const leaseToken = "lease_wrong_identity_regression";
+      const digest = phase3Internals.tokenDigest(leaseToken);
+      core.store.state.leases[digest] = {
+        digest,
+        leaseId: "lease_wrong_identity",
+        runnerInstanceId: "runner_other_identity",
+        expiresAt: new Date(Date.now() + 30_000).toISOString(),
+        revokedAt: null,
+      };
+      core.store.save();
+      const leaseSocket = await upgradeSocket(core.connectUrl, leaseToken);
+      sendMaskedJson(leaseSocket, hello(identity));
+      await expectSocketClosed(leaseSocket);
+    } finally {
+      await core.stop();
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  it.runIf(process.platform !== "win32")(
+    "atomically stores mock-core state without following a preplanted predictable symlink",
+    () => {
+      const scratch =
+        process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
+        process.env.PAPERCLIP_SCRATCH_DIR ??
+        tmpdir();
+      const root = mkdtempSync(resolve(scratch, "phase3-core-state-symlink-"));
+      const stateDirectory = resolve(root, "mock-core");
+      const victim = resolve(root, "victim.txt");
+      mkdirSync(stateDirectory, { mode: 0o700 });
+      chmodSync(stateDirectory, 0o700);
+      writeFileSync(victim, "unchanged", { mode: 0o600 });
+      const planted = resolve(stateDirectory, "mock-core-state.json.next");
+      symlinkSync(victim, planted);
+      try {
+        const core = new Phase3MockCore({
+          stateDirectory,
+          identity: phase3Internals.phase3Identity(),
+          fault: "none",
+        });
+        core.issueBootstrapTicket();
+        const statePath = resolve(stateDirectory, "mock-core-state.json");
+
+        expect(readFileSync(victim, "utf8")).toBe("unchanged");
+        expect(lstatSync(planted).isSymbolicLink()).toBe(true);
+        expect(lstatSync(stateDirectory).mode & 0o777).toBe(0o700);
+        expect(lstatSync(statePath).mode & 0o777).toBe(0o600);
+      } finally {
+        rmSync(root, { recursive: true, force: true });
+      }
+    },
+  );
+
+  it.each(["socket-drop", "lost-ack", "malformed-input"] as const)(
+    "recovers the %s reconnect path with stable identities and cursors",
+    async (fault) => {
+      const trace = await runPhase3Recovery({ fault });
+
+      expect(trace.diagnostics.recovery.outcome).toBe("recovered");
+      expect(trace.diagnostics.connection.connectionCount).toBeGreaterThan(1);
+      expect(trace.assertions.stableIdentity).toBe(true);
+      expect(trace.assertions.sourceCursorContinuous).toBe(true);
+      expect(trace.assertions.noDuplicateLogicalEvents).toBe(true);
+      expect(trace.assertions.p0Preserved).toBe(true);
+      expect(trace.assertions.secretsRedacted).toBe(true);
+      expect(trace.diagnostics.security).toMatchObject({
+        bootstrapTicketPersisted: false,
+        connectionLeaseTokenPersisted: false,
+        secretLeakCount: 0,
+      });
+      if (fault === "lost-ack") {
+        expect(trace.diagnostics.recovery.replayDeliveries).toBeGreaterThan(0);
+        expect(
+          trace.diagnostics.committedEvents.some((event) => event.deliveryCount > 1),
+        ).toBe(true);
+      }
+      if (fault === "malformed-input") {
+        expect(trace.diagnostics.recovery.malformedFrames).toBe(1);
+        expect(trace.runnerState.diagnostics.join(" ")).toContain(
+          "malformed WebSocket JSON",
+        );
+      }
+    },
+  );
+
+  it("replays a processed command without repeating its logical effect", async () => {
+    const trace = await runPhase3Recovery({ fault: "duplicate-command" });
+
+    expect(trace.diagnostics.commands.duplicateDeliveries).toBeGreaterThan(0);
+    expect(trace.assertions.oneLogicalEffectPerAcceptedCommand).toBe(true);
+    expect(trace.runnerState.processedCommands.command_phase3_001?.logicalEffectCount).toBe(1);
+    expect(
+      trace.diagnostics.committedEvents.filter(
+        (event) => event.eventType === "workspace.ready",
+      ),
+    ).toHaveLength(1);
+  });
+
+  it("restores the same runner session after a real runner process restart", async () => {
+    const trace = await runPhase3Recovery({ fault: "runner-restart" });
+
+    expect(trace.diagnostics.recovery.runnerRestarts).toBe(1);
+    expect(trace.diagnostics.recovery.freshBootstraps).toBe(2);
+    expect(trace.diagnostics.recovery.replayDeliveries).toBeGreaterThan(0);
+    expect(
+      trace.diagnostics.committedEvents.some(
+        (event) => event.eventType === "runner.reconciled",
+      ),
+    ).toBe(true);
+    expect(trace.assertions.stableIdentity).toBe(true);
+    expect(trace.assertions.sourceCursorContinuous).toBe(true);
+  });
+
+  it("reconciles a restarted harness without replacing session, turn, or item IDs", async () => {
+    const trace = await runPhase3Recovery({ fault: "harness-restart" });
+    const reconciled = trace.diagnostics.committedEvents.find(
+      (event) => event.eventType === "session.reconciled",
+    );
+
+    expect(trace.diagnostics.recovery.harnessRestarts).toBe(1);
+    expect(reconciled).toBeDefined();
+    expect(reconciled?.envelope).toMatchObject({
+      runId: "run_phase3_stable",
+      normalizedSessionId: "session_phase3_stable",
+      turnId: "turn_phase3_stable",
+    });
+    expect(trace.assertions.stableIdentity).toBe(true);
+  });
+
+  it("uses a fresh bootstrap after lease expiry and resumes the durable cursor", async () => {
+    const trace = await runPhase3Recovery({ fault: "lease-expiry" });
+
+    expect(trace.diagnostics.recovery.runnerRestarts).toBe(1);
+    expect(trace.diagnostics.recovery.freshBootstraps).toBe(2);
+    expect(trace.assertions.sourceCursorContinuous).toBe(true);
+    expect(trace.diagnostics.recovery.outcome).toBe("recovered");
+  });
+
+  it("bounds storage, coalesces P2, preserves P0, and rejects new turns", async () => {
+    const trace = await runPhase3Recovery({ fault: "storage-pressure" });
+    const deltaEvents = trace.diagnostics.committedEvents.filter(
+      (event) => event.eventType === "item.delta",
+    );
+    const turn = trace.commands.find((command) => command.type === "turn.start");
+
+    expect(trace.diagnostics.outbox.backpressure).toBe(true);
+    expect(trace.diagnostics.outbox.peakBytes).toBeLessThanOrEqual(
+      trace.diagnostics.outbox.maxBytes,
+    );
+    expect(deltaEvents).toHaveLength(1);
+    expect(trace.diagnostics.committedEvents.some((event) => event.eventType === "runner.backpressure"))
+      .toBe(true);
+    expect(turn?.status).toBe("rejected");
+    expect(trace.assertions.boundedStorage).toBe(true);
+    expect(trace.assertions.p0Preserved).toBe(true);
+  });
+
+  it.each(["drain", "revoke"] as const)(
+    "%s stops new work only after preserving the durable event cursor",
+    async (fault) => {
+      const trace = await runPhase3Recovery({ fault });
+
+      expect(trace.diagnostics.recovery.outcome).toBe(fault === "drain" ? "drained" : "revoked");
+      expect(trace.diagnostics.outbox.events).toBe(0);
+      expect(trace.assertions.sourceCursorContinuous).toBe(true);
+      expect(trace.assertions.p0Preserved).toBe(true);
+      if (fault === "drain") {
+        expect(trace.commands.find((command) => command.type === "turn.start")?.status)
+          .toBe("rejected");
+      } else {
+        const afterRevoke = trace.commands.find(
+          (command) => command.commandId === "command_after_revoke",
+        );
+        expect(trace.runnerState.lifecycle).toBe("revoked");
+        expect(trace.diagnostics.recovery.replayDeliveries).toBeGreaterThan(0);
+        expect(afterRevoke?.status).toBe("rejected");
+        expect(afterRevoke?.result?.logicalEffectCount).toBe(0);
+        expect(
+          trace.diagnostics.committedEvents.filter(
+            (event) => event.eventType === "turn.accepted",
+          ),
+        ).toHaveLength(1);
+      }
+    },
+  );
+
+  it("passes only the bootstrap capability and platform basics to the runner", () => {
+    const environment = phase3Internals.runnerEnvironment("opaque-ticket");
+
+    expect(environment.PAPERCLIP_RUNNER_BOOTSTRAP_TICKET).toBe("opaque-ticket");
+    expect(environment.PAPERCLIP_API_KEY).toBeUndefined();
+    expect(environment.OPENAI_API_KEY).toBeUndefined();
+    expect(environment.EMAIL_AGENTMAIL_GENERAL_API_KEY).toBeUndefined();
+  });
+});