From 8792aed8ac8e9d004406afcb4b9a4ffddccd72c8 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 14:36:57 -0500 Subject: [PATCH] test(evals): align historical blocker explanation oracle Co-Authored-By: Paperclip --- tests/runner-e2e/native-completion-case.test.ts | 9 +++++++-- tests/runner-e2e/native-completion-case.ts | 3 +++ tests/runner-e2e/native-completion-comparison.json | 9 +++++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/native-completion-case.test.ts b/tests/runner-e2e/native-completion-case.test.ts index 963296d37b..49eaccc0a2 100644 --- a/tests/runner-e2e/native-completion-case.test.ts +++ b/tests/runner-e2e/native-completion-case.test.ts @@ -4,7 +4,7 @@ import { nativeBlockedReportTask } from "./native-completion-case.js"; import type { MatrixExecution } from "./types.js"; function observation(): MatcherObservation { - return { message: "BLOCKED_probe. Release Owner must Grant deployment access.", + return { message: "BLOCKED_probe. Deployment is blocked. Release Owner must Grant deployment access.", issueStatus: "blocked", runStatus: "succeeded", runtimeMode: "native", json: { run: { resultJson: { nativeResult: { reportedWorkDisposition: "blocked", blocker: { owner: { name: "Release Owner" }, unblockAction: "Grant deployment access", scope: "task_wide" } } } } } }; @@ -16,7 +16,8 @@ describe("native completion blocker oracle", () => { expect((await evaluateMatchers(matchers, observation())).every(row => row.passed)).toBe(true); expect(nativeBlockedReportTask.buildPrompt("probe")).not.toMatch(/paperclip_(?:finish|block)|reportedWorkDisposition|json/i); }); - it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed"])( + it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed", + "visible-marker-only", "visible-owner-missing", "visible-action-missing", "visible-reason-missing"])( "rejects %s without trusting the visible completion claim", async variant => { const current = observation(); const result = (current.json as { run: { resultJson: { nativeResult?: { reportedWorkDisposition: string; @@ -28,6 +29,10 @@ describe("native completion blocker oracle", () => { if (variant === "done-report") result.nativeResult!.reportedWorkDisposition = "done"; if (variant === "legacy-runtime") current.runtimeMode = "legacy"; if (variant === "run-failed") current.runStatus = "failed"; + if (variant === "visible-marker-only") current.message = "BLOCKED_probe"; + if (variant === "visible-owner-missing") current.message = current.message!.replace("Release Owner", "Someone"); + if (variant === "visible-action-missing") current.message = current.message!.replace("Grant deployment access", "Take action"); + if (variant === "visible-reason-missing") current.message = "BLOCKED_probe. Release Owner: Grant deployment access."; expect((await evaluateMatchers(matchers, current)).some(row => !row.passed)).toBe(true); }); }); diff --git a/tests/runner-e2e/native-completion-case.ts b/tests/runner-e2e/native-completion-case.ts index ed633be07a..864371407c 100644 --- a/tests/runner-e2e/native-completion-case.ts +++ b/tests/runner-e2e/native-completion-case.ts @@ -23,6 +23,9 @@ export const nativeBlockedReportTask: RunnerTaskFixture = { { kind: "run_status", expected: "succeeded" }, { kind: "runtime_mode", expected: "native" }, { kind: "message_contains", expected: `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}` }, + { kind: "message_contains", expected: "Release Owner" }, + { kind: "message_contains", expected: "Grant deployment access" }, + { kind: "message_regex", pattern: "\\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\\b", flags: "i" }, { kind: "json_path", path: "run.resultJson.nativeResult.reportedWorkDisposition", expected: "blocked" }, { kind: "json_path", path: "run.resultJson.nativeResult.blocker.owner.name", expected: "Release Owner" }, { kind: "json_path", path: "run.resultJson.nativeResult.blocker.unblockAction", expected: "Grant deployment access" }, diff --git a/tests/runner-e2e/native-completion-comparison.json b/tests/runner-e2e/native-completion-comparison.json index 8dd508d1a8..ad421c73cb 100644 --- a/tests/runner-e2e/native-completion-comparison.json +++ b/tests/runner-e2e/native-completion-comparison.json @@ -1,6 +1,6 @@ { "schema": "paperclip.native-completion.matched-baseline/v1", - "candidateSha": "55ce97b675e1ed68cc171fe44b729147209a7e24", + "candidateSha": "ae34843731ac338bd806a4106cb447134bff402c", "historicalInstructionSourceSha": "5837aa44425ea134fc33a96c502c22d6a32b6866", "heldConstant": [ "merged native Codex base fix #14920", @@ -17,8 +17,8 @@ ], "historicalStructuralOracles": "Same authenticated/wire/catalog gates expect the explicit historical descriptions. The v13 baseline does not run the candidate-only v13-to-v14 rotation assertion; its prior compatibility cases remain mandatory. Candidate graders are unchanged.", "behavioralSourceHashes": { - "tests/runner-e2e/native-completion-case.ts": "6ecd17a78aad20eea36d5ea0f039f22255ea2e61115e16a5e2b1543c72e9a622", - "tests/runner-e2e/native-completion-case.test.ts": "253910919bef2dce5509e4005a635b16c518d4940a870c7bbc100898ce8afe83", + "tests/runner-e2e/native-completion-case.ts": "ebe9c5caeef5da5e0cdd82cbab8ed841b79e7d8bf54e891fbae78ad01cc7c8a7", + "tests/runner-e2e/native-completion-case.test.ts": "1889cecd7efef36020ce6d8c407f8375f3d79af7fd9351e1eb6b46f08bdbaa4e", "tests/runner-e2e/context-integrity-cases.ts": "201d05f8257c1121a94dd9d8a22f08b9db80555e388013ffeabb65e301499fd2", "tests/runner-e2e/context-integrity-flow.ts": "63bcb7bdba4c80b15ff5872f16383844456eab0c5a10627da5d3487a26821c6a", "tests/runner-e2e/context-integrity-scoring.ts": "50d370044e45386ba87fb23ca63c228b96bb8373d67bb7e82e8b2a89eb23f252", @@ -42,5 +42,6 @@ ], "expectedCells": 6, "expectedProviderTurns": 6 - } + }, + "sharedGraderCorrection": "Requires visible owner, exact unblock action and a blocking explanation; same corrected grader in both variants." }