test(evals): require a useful visible native blocker explanation

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/plans/2026-10-02-native-completion-live-comparison.md

@@ -0,0 +1,67 @@
+# Native completion guidance qualification — 2026-10-02
+
+**Partial report, updated 19:34 UTC.** No completed native qualification result
+is available yet. The selected scope is three native profiles, each with one
+completion and one concrete blocker case: six cells per variant. Missing cells
+are pending, not passing. [Draft PR #14961](https://github.com/paperclipai/paperclip/pull/14961)
+remains stacked on [draft PR #14948](https://github.com/paperclipai/paperclip/pull/14948).
+
+## Results available now
+
+The earlier [manual/shared-prompt comparison](2026-10-02-stock-harness-live-comparison.md)
+has all 48 results, including failures. Both variants pass 15 of 24 cases, with
+different outcomes. Classic Claude/OpenCode document delivery fails with the
+reduced instructions. That draft remains unqualified for merge. Those legacy
+results do not measure this native completion-tool change.
+
+| Native profile | Completion baseline | Completion candidate | Blocker baseline | Blocker candidate |
+| --- | --- | --- | --- | --- |
+| Codex | Pending | Running at `55ce97b675e1ed68cc171fe44b729147209a7e24` | Pending | Pending |
+| ACPX Claude | Pending | Pending | Pending | Pending |
+| OpenCode | Pending | Pending | Pending | Pending |
+
+## Attempts and provenance
+
+- The protected [candidate completion pilot](https://github.com/paperclipai/paperclip/actions/runs/37053897522)
+  runs from the trusted default-branch workflow against immutable candidate
+  `55ce97b675e1ed68cc171fe44b729147209a7e24`. It selects only native Codex's skill
+  completion case. Full exact-head prerequisites precede provider admission.
+- The first [historical setup](https://github.com/paperclipai/paperclip/actions/runs/37054642871)
+  selected six cells at `9060f7ee4` on `codex/native-completion-previous-guidance`.
+  It was cancelled before provider cells started after review identified a gap
+  in the blocker grader. This partial setup remains recorded. Both variants
+  will receive the same stronger visible-explanation checks before that case
+  runs. The completion fixture is unchanged.
+- The comparison restores only five native production sources containing tool
+  descriptions and their session fingerprint. The tiny hire manual, reduced
+  shared prompts, and merged Codex base fix #14920 remain constant. Twelve
+  behavioral/fixture sources were byte-identical before the shared grader fix;
+  all six profile/model/auth/effort fixture configurations match. Recheck those
+  hashes after the fix before dispatch.
+- Historical structural assertions expect the prior descriptions in actual
+  authenticated/wire/provider catalogs. The historical v13 branch tests its
+  prior compatibility cases; the candidate separately requires v13-to-v14
+  rotation. Structural differences are not behavioral failures.
+
+## Verification and review
+
+At the pilot source, full repository typecheck and build pass locally. All 909
+E2E support tests pass; typecheck and 27-cell discovery pass. Exact-head
+credential-free prerequisites pass all 585 executed checks (584 TypeScript and
+one Rust), with retained source/binary hashes and
+zero provider calls. The receipt is retained under
+`tests/runner-e2e/results/stock-harness-preflight-2026-10-02T19-29-43.267Z/`.
+
+Greptile found that the blocker oracle could accept a visible marker while only
+checking owner/action in backend state. The strengthened oracle now also checks
+the visible owner, exact requested action, and a blocking explanation; its 12
+positive/plausible-negative calibrations pass. All 913 current support tests and
+E2E typecheck pass after that fix. The first added calibration caught
+the marker itself matching the word "blocked"; word boundaries corrected that
+grader defect. The initial failed calibration remains retained. Candidate
+production instructions and independent completion graders were not changed.
+
+Fresh CI/review for the corrected fixture and paid outcomes remain pending.
+Cost is unknown until retained results arrive; no speed, spending, coding-quality,
+or broad reliability improvement is claimed. Native finish/block documentation
+must remain separate from legacy Paperclip skill/API completion guidance.

tests/runner-e2e/native-completion-case.test.ts

@@ -4,7 +4,7 @@ import { nativeBlockedReportTask } from "./native-completion-case.js";
 import type { MatrixExecution } from "./types.js";
 
 function observation(): MatcherObservation {
-  return { message: "BLOCKED_probe. Release Owner must Grant deployment access.",
+  return { message: "BLOCKED_probe. Deployment is blocked. Release Owner must Grant deployment access.",
     issueStatus: "blocked", runStatus: "succeeded", runtimeMode: "native",
     json: { run: { resultJson: { nativeResult: { reportedWorkDisposition: "blocked",
       blocker: { owner: { name: "Release Owner" }, unblockAction: "Grant deployment access", scope: "task_wide" } } } } } };
@@ -16,7 +16,8 @@ describe("native completion blocker oracle", () => {
     expect((await evaluateMatchers(matchers, observation())).every(row => row.passed)).toBe(true);
     expect(nativeBlockedReportTask.buildPrompt("probe")).not.toMatch(/paperclip_(?:finish|block)|reportedWorkDisposition|json/i);
   });
-  it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed"])(
+  it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed",
+    "visible-marker-only", "visible-owner-missing", "visible-action-missing", "visible-reason-missing"])(
     "rejects %s without trusting the visible completion claim", async variant => {
       const current = observation();
       const result = (current.json as { run: { resultJson: { nativeResult?: { reportedWorkDisposition: string;
@@ -28,6 +29,10 @@ describe("native completion blocker oracle", () => {
       if (variant === "done-report") result.nativeResult!.reportedWorkDisposition = "done";
       if (variant === "legacy-runtime") current.runtimeMode = "legacy";
       if (variant === "run-failed") current.runStatus = "failed";
+      if (variant === "visible-marker-only") current.message = "BLOCKED_probe";
+      if (variant === "visible-owner-missing") current.message = current.message!.replace("Release Owner", "Someone");
+      if (variant === "visible-action-missing") current.message = current.message!.replace("Grant deployment access", "Take action");
+      if (variant === "visible-reason-missing") current.message = "BLOCKED_probe. Release Owner: Grant deployment access.";
       expect((await evaluateMatchers(matchers, current)).some(row => !row.passed)).toBe(true);
     });
 });

tests/runner-e2e/native-completion-case.ts

@@ -23,6 +23,9 @@ export const nativeBlockedReportTask: RunnerTaskFixture = {
     { kind: "run_status", expected: "succeeded" },
     { kind: "runtime_mode", expected: "native" },
     { kind: "message_contains", expected: `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}` },
+    { kind: "message_contains", expected: "Release Owner" },
+    { kind: "message_contains", expected: "Grant deployment access" },
+    { kind: "message_regex", pattern: "\\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\\b", flags: "i" },
     { kind: "json_path", path: "run.resultJson.nativeResult.reportedWorkDisposition", expected: "blocked" },
     { kind: "json_path", path: "run.resultJson.nativeResult.blocker.owner.name", expected: "Release Owner" },
     { kind: "json_path", path: "run.resultJson.nativeResult.blocker.unblockAction", expected: "Grant deployment access" },