test(e2e): normalize plan observations

tests/runner-e2e/run-observations.ts

@@ -0,0 +1,23 @@
+export interface ObservableRunState {
+  status?: string | null;
+  errorCode?: string | null;
+}
+
+export function isNonExecutingReviewFenceRun(run: ObservableRunState) {
+  return (
+    run.status === "cancelled" &&
+    run.errorCode === "issue_continuation_waiting_on_review"
+  );
+}
+
+export function numberedPlanStepCount(body: string | null | undefined) {
+  return (body ?? "").split(/\r?\n/).filter((line) => {
+    const normalized = line
+      .replaceAll("**", "")
+      .replaceAll("__", "")
+      .replaceAll("`", "");
+    return /^\s*(?:#{1,6}\s*)?(?:[-*+]\s*)?(?:step\s+)?\d+(?:[.)]|\s*[-—:])(?:\s|$)/i.test(
+      normalized,
+    );
+  }).length;
+}

tests/runner-e2e/runner.spec.ts

@@ -7,6 +7,10 @@ import { runnerExecutionById } from "./catalog.js";
 import { classifyFailure } from "./failure-classifier.js";
 import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";
 import { evaluateMatchers, type MatcherResult } from "./matchers.js";
+import {
+  isNonExecutingReviewFenceRun,
+  numberedPlanStepCount,
+} from "./run-observations.js";
 import {
   assertSecretFree,
   findSecretLeakInJsonValues,
@@ -228,6 +232,7 @@ function matchingRuns(runs: RunRecord[], issue: IssueRecord) {
     [issue.executionRunId, issue.checkoutRunId].filter(Boolean),
   );
   return runs.filter((run) => {
+    if (isNonExecutingReviewFenceRun(run)) return false;
     const context = record(run.contextSnapshot);
     return (
       context.issueId === issue.id ||
@@ -262,10 +267,6 @@ function renderedMarkerPattern(marker: string) {
   );
 }
 
-function numberedPlanStepCount(body: string | null | undefined) {
-  return [...(body ?? "").matchAll(/^\s*\d+[.)]\s+/gm)].length;
-}
-
 for (const execution of executions) {
   const privateDir = path.join(privateRoot, "cases", execution.task.id);
   const resultPath = path.join(privateDir, "result.json");

tests/runner-e2e/support.test.ts

@@ -21,6 +21,10 @@ import {
   sanitizeJson,
 } from "./redaction.js";
 import { parseDarwinSharedMemory } from "./shared-memory.js";
+import {
+  isNonExecutingReviewFenceRun,
+  numberedPlanStepCount,
+} from "./run-observations.js";
 import { runnerE2EWebServerCommand } from "./web-server-command.js";
 
 const cleanupDirectories: string[] = [];
@@ -210,6 +214,39 @@ describe("runner E2E matchers", () => {
   });
 });
 
+describe("runner E2E run observations", () => {
+  it("counts provider-equivalent numbered Plan step formats", () => {
+    expect(numberedPlanStepCount("1. First\n2) Second")).toBe(2);
+    expect(
+      numberedPlanStepCount(
+        "# Plan\n\nStep 1 — First\n\nStep 2 — Second\n\nStep 3: Verify",
+      ),
+    ).toBe(3);
+    expect(numberedPlanStepCount("## **Step 1** — First\n- **2.** Second")).toBe(2);
+  });
+
+  it("excludes only queued continuations fenced while awaiting review", () => {
+    expect(
+      isNonExecutingReviewFenceRun({
+        status: "cancelled",
+        errorCode: "issue_continuation_waiting_on_review",
+      }),
+    ).toBe(true);
+    expect(
+      isNonExecutingReviewFenceRun({
+        status: "failed",
+        errorCode: "issue_continuation_waiting_on_review",
+      }),
+    ).toBe(false);
+    expect(
+      isNonExecutingReviewFenceRun({
+        status: "cancelled",
+        errorCode: "provider_failure",
+      }),
+    ).toBe(false);
+  });
+});
+
 describe("runner E2E failure policy", () => {
   it("retries only transient infrastructure failures", () => {
     expect(