diff --git a/doc/plans/2026-10-02-native-completion-live-comparison.md b/doc/plans/2026-10-02-native-completion-live-comparison.md new file mode 100644 index 00000000000..1ddd892e6d7 --- /dev/null +++ b/doc/plans/2026-10-02-native-completion-live-comparison.md @@ -0,0 +1,67 @@ +# Native completion guidance qualification — 2026-10-02 + +**Partial report, updated 19:34 UTC.** No completed native qualification result +is available yet. The selected scope is three native profiles, each with one +completion and one concrete blocker case: six cells per variant. Missing cells +are pending, not passing. [Draft PR #14961](https://github.com/paperclipai/paperclip/pull/14961) +remains stacked on [draft PR #14948](https://github.com/paperclipai/paperclip/pull/14948). + +## Results available now + +The earlier [manual/shared-prompt comparison](2026-10-02-stock-harness-live-comparison.md) +has all 48 results, including failures. Both variants pass 15 of 24 cases, with +different outcomes. Classic Claude/OpenCode document delivery fails with the +reduced instructions. That draft remains unqualified for merge. Those legacy +results do not measure this native completion-tool change. + +| Native profile | Completion baseline | Completion candidate | Blocker baseline | Blocker candidate | +| --- | --- | --- | --- | --- | +| Codex | Pending | Running at `55ce97b675e1ed68cc171fe44b729147209a7e24` | Pending | Pending | +| ACPX Claude | Pending | Pending | Pending | Pending | +| OpenCode | Pending | Pending | Pending | Pending | + +## Attempts and provenance + +- The protected [candidate completion pilot](https://github.com/paperclipai/paperclip/actions/runs/37053897522) + runs from the trusted default-branch workflow against immutable candidate + `55ce97b675e1ed68cc171fe44b729147209a7e24`. It selects only native Codex's skill + completion case. Full exact-head prerequisites precede provider admission. +- The first [historical setup](https://github.com/paperclipai/paperclip/actions/runs/37054642871) + selected six cells at `9060f7ee4` on `codex/native-completion-previous-guidance`. + It was cancelled before provider cells started after review identified a gap + in the blocker grader. This partial setup remains recorded. Both variants + will receive the same stronger visible-explanation checks before that case + runs. The completion fixture is unchanged. +- The comparison restores only five native production sources containing tool + descriptions and their session fingerprint. The tiny hire manual, reduced + shared prompts, and merged Codex base fix #14920 remain constant. Twelve + behavioral/fixture sources were byte-identical before the shared grader fix; + all six profile/model/auth/effort fixture configurations match. Recheck those + hashes after the fix before dispatch. +- Historical structural assertions expect the prior descriptions in actual + authenticated/wire/provider catalogs. The historical v13 branch tests its + prior compatibility cases; the candidate separately requires v13-to-v14 + rotation. Structural differences are not behavioral failures. + +## Verification and review + +At the pilot source, full repository typecheck and build pass locally. All 909 +E2E support tests pass; typecheck and 27-cell discovery pass. Exact-head +credential-free prerequisites pass all 585 executed checks (584 TypeScript and +one Rust), with retained source/binary hashes and +zero provider calls. The receipt is retained under +`tests/runner-e2e/results/stock-harness-preflight-2026-10-02T19-29-43.267Z/`. + +Greptile found that the blocker oracle could accept a visible marker while only +checking owner/action in backend state. The strengthened oracle now also checks +the visible owner, exact requested action, and a blocking explanation; its 12 +positive/plausible-negative calibrations pass. All 913 current support tests and +E2E typecheck pass after that fix. The first added calibration caught +the marker itself matching the word "blocked"; word boundaries corrected that +grader defect. The initial failed calibration remains retained. Candidate +production instructions and independent completion graders were not changed. + +Fresh CI/review for the corrected fixture and paid outcomes remain pending. +Cost is unknown until retained results arrive; no speed, spending, coding-quality, +or broad reliability improvement is claimed. Native finish/block documentation +must remain separate from legacy Paperclip skill/API completion guidance. diff --git a/tests/runner-e2e/native-completion-case.test.ts b/tests/runner-e2e/native-completion-case.test.ts index 963296d37b0..49eaccc0a27 100644 --- a/tests/runner-e2e/native-completion-case.test.ts +++ b/tests/runner-e2e/native-completion-case.test.ts @@ -4,7 +4,7 @@ import { nativeBlockedReportTask } from "./native-completion-case.js"; import type { MatrixExecution } from "./types.js"; function observation(): MatcherObservation { - return { message: "BLOCKED_probe. Release Owner must Grant deployment access.", + return { message: "BLOCKED_probe. Deployment is blocked. Release Owner must Grant deployment access.", issueStatus: "blocked", runStatus: "succeeded", runtimeMode: "native", json: { run: { resultJson: { nativeResult: { reportedWorkDisposition: "blocked", blocker: { owner: { name: "Release Owner" }, unblockAction: "Grant deployment access", scope: "task_wide" } } } } } }; @@ -16,7 +16,8 @@ describe("native completion blocker oracle", () => { expect((await evaluateMatchers(matchers, observation())).every(row => row.passed)).toBe(true); expect(nativeBlockedReportTask.buildPrompt("probe")).not.toMatch(/paperclip_(?:finish|block)|reportedWorkDisposition|json/i); }); - it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed"])( + it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed", + "visible-marker-only", "visible-owner-missing", "visible-action-missing", "visible-reason-missing"])( "rejects %s without trusting the visible completion claim", async variant => { const current = observation(); const result = (current.json as { run: { resultJson: { nativeResult?: { reportedWorkDisposition: string; @@ -28,6 +29,10 @@ describe("native completion blocker oracle", () => { if (variant === "done-report") result.nativeResult!.reportedWorkDisposition = "done"; if (variant === "legacy-runtime") current.runtimeMode = "legacy"; if (variant === "run-failed") current.runStatus = "failed"; + if (variant === "visible-marker-only") current.message = "BLOCKED_probe"; + if (variant === "visible-owner-missing") current.message = current.message!.replace("Release Owner", "Someone"); + if (variant === "visible-action-missing") current.message = current.message!.replace("Grant deployment access", "Take action"); + if (variant === "visible-reason-missing") current.message = "BLOCKED_probe. Release Owner: Grant deployment access."; expect((await evaluateMatchers(matchers, current)).some(row => !row.passed)).toBe(true); }); }); diff --git a/tests/runner-e2e/native-completion-case.ts b/tests/runner-e2e/native-completion-case.ts index ed633be07a6..864371407c7 100644 --- a/tests/runner-e2e/native-completion-case.ts +++ b/tests/runner-e2e/native-completion-case.ts @@ -23,6 +23,9 @@ export const nativeBlockedReportTask: RunnerTaskFixture = { { kind: "run_status", expected: "succeeded" }, { kind: "runtime_mode", expected: "native" }, { kind: "message_contains", expected: `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}` }, + { kind: "message_contains", expected: "Release Owner" }, + { kind: "message_contains", expected: "Grant deployment access" }, + { kind: "message_regex", pattern: "\\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\\b", flags: "i" }, { kind: "json_path", path: "run.resultJson.nativeResult.reportedWorkDisposition", expected: "blocked" }, { kind: "json_path", path: "run.resultJson.nativeResult.blocker.owner.name", expected: "Release Owner" }, { kind: "json_path", path: "run.resultJson.nativeResult.blocker.unblockAction", expected: "Grant deployment access" },