diff --git a/tests/runner-e2e/run-observations.ts b/tests/runner-e2e/run-observations.ts new file mode 100644 index 00000000000..4a99b121469 --- /dev/null +++ b/tests/runner-e2e/run-observations.ts @@ -0,0 +1,23 @@ +export interface ObservableRunState { + status?: string | null; + errorCode?: string | null; +} + +export function isNonExecutingReviewFenceRun(run: ObservableRunState) { + return ( + run.status === "cancelled" && + run.errorCode === "issue_continuation_waiting_on_review" + ); +} + +export function numberedPlanStepCount(body: string | null | undefined) { + return (body ?? "").split(/\r?\n/).filter((line) => { + const normalized = line + .replaceAll("**", "") + .replaceAll("__", "") + .replaceAll("`", ""); + return /^\s*(?:#{1,6}\s*)?(?:[-*+]\s*)?(?:step\s+)?\d+(?:[.)]|\s*[-—:])(?:\s|$)/i.test( + normalized, + ); + }).length; +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index cc18180178e..04fa5d9f015 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -7,6 +7,10 @@ import { runnerExecutionById } from "./catalog.js"; import { classifyFailure } from "./failure-classifier.js"; import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js"; import { evaluateMatchers, type MatcherResult } from "./matchers.js"; +import { + isNonExecutingReviewFenceRun, + numberedPlanStepCount, +} from "./run-observations.js"; import { assertSecretFree, findSecretLeakInJsonValues, @@ -228,6 +232,7 @@ function matchingRuns(runs: RunRecord[], issue: IssueRecord) { [issue.executionRunId, issue.checkoutRunId].filter(Boolean), ); return runs.filter((run) => { + if (isNonExecutingReviewFenceRun(run)) return false; const context = record(run.contextSnapshot); return ( context.issueId === issue.id || @@ -262,10 +267,6 @@ function renderedMarkerPattern(marker: string) { ); } -function numberedPlanStepCount(body: string | null | undefined) { - return [...(body ?? "").matchAll(/^\s*\d+[.)]\s+/gm)].length; -} - for (const execution of executions) { const privateDir = path.join(privateRoot, "cases", execution.task.id); const resultPath = path.join(privateDir, "result.json"); diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index f53b9d536d8..f36a6b5867c 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -21,6 +21,10 @@ import { sanitizeJson, } from "./redaction.js"; import { parseDarwinSharedMemory } from "./shared-memory.js"; +import { + isNonExecutingReviewFenceRun, + numberedPlanStepCount, +} from "./run-observations.js"; import { runnerE2EWebServerCommand } from "./web-server-command.js"; const cleanupDirectories: string[] = []; @@ -210,6 +214,39 @@ describe("runner E2E matchers", () => { }); }); +describe("runner E2E run observations", () => { + it("counts provider-equivalent numbered Plan step formats", () => { + expect(numberedPlanStepCount("1. First\n2) Second")).toBe(2); + expect( + numberedPlanStepCount( + "# Plan\n\nStep 1 — First\n\nStep 2 — Second\n\nStep 3: Verify", + ), + ).toBe(3); + expect(numberedPlanStepCount("## **Step 1** — First\n- **2.** Second")).toBe(2); + }); + + it("excludes only queued continuations fenced while awaiting review", () => { + expect( + isNonExecutingReviewFenceRun({ + status: "cancelled", + errorCode: "issue_continuation_waiting_on_review", + }), + ).toBe(true); + expect( + isNonExecutingReviewFenceRun({ + status: "failed", + errorCode: "issue_continuation_waiting_on_review", + }), + ).toBe(false); + expect( + isNonExecutingReviewFenceRun({ + status: "cancelled", + errorCode: "provider_failure", + }), + ).toBe(false); + }); +}); + describe("runner E2E failure policy", () => { it("retries only transient infrastructure failures", () => { expect(