mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-07 07:23:08 +02:00
test(evals): align historical blocker explanation oracle
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
9060f7ee4f
commit
8792aed8ac
3 files changed
+15
-6
No files matched your search
@@ -4,7 +4,7 @@ import { nativeBlockedReportTask } from "./native-completion-case.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
|
||||
function observation(): MatcherObservation {
|
||||
return { message: "BLOCKED_probe. Release Owner must Grant deployment access.",
|
||||
return { message: "BLOCKED_probe. Deployment is blocked. Release Owner must Grant deployment access.",
|
||||
issueStatus: "blocked", runStatus: "succeeded", runtimeMode: "native",
|
||||
json: { run: { resultJson: { nativeResult: { reportedWorkDisposition: "blocked",
|
||||
blocker: { owner: { name: "Release Owner" }, unblockAction: "Grant deployment access", scope: "task_wide" } } } } } };
|
||||
@@ -16,7 +16,8 @@ describe("native completion blocker oracle", () => {
|
||||
expect((await evaluateMatchers(matchers, observation())).every(row => row.passed)).toBe(true);
|
||||
expect(nativeBlockedReportTask.buildPrompt("probe")).not.toMatch(/paperclip_(?:finish|block)|reportedWorkDisposition|json/i);
|
||||
});
|
||||
it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed"])(
|
||||
it.each(["visible-claim-only", "wrong-owner", "wrong-action", "wrong-scope", "done-report", "legacy-runtime", "run-failed",
|
||||
"visible-marker-only", "visible-owner-missing", "visible-action-missing", "visible-reason-missing"])(
|
||||
"rejects %s without trusting the visible completion claim", async variant => {
|
||||
const current = observation();
|
||||
const result = (current.json as { run: { resultJson: { nativeResult?: { reportedWorkDisposition: string;
|
||||
@@ -28,6 +29,10 @@ describe("native completion blocker oracle", () => {
|
||||
if (variant === "done-report") result.nativeResult!.reportedWorkDisposition = "done";
|
||||
if (variant === "legacy-runtime") current.runtimeMode = "legacy";
|
||||
if (variant === "run-failed") current.runStatus = "failed";
|
||||
if (variant === "visible-marker-only") current.message = "BLOCKED_probe";
|
||||
if (variant === "visible-owner-missing") current.message = current.message!.replace("Release Owner", "Someone");
|
||||
if (variant === "visible-action-missing") current.message = current.message!.replace("Grant deployment access", "Take action");
|
||||
if (variant === "visible-reason-missing") current.message = "BLOCKED_probe. Release Owner: Grant deployment access.";
|
||||
expect((await evaluateMatchers(matchers, current)).some(row => !row.passed)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -23,6 +23,9 @@ export const nativeBlockedReportTask: RunnerTaskFixture = {
|
||||
{ kind: "run_status", expected: "succeeded" },
|
||||
{ kind: "runtime_mode", expected: "native" },
|
||||
{ kind: "message_contains", expected: `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}` },
|
||||
{ kind: "message_contains", expected: "Release Owner" },
|
||||
{ kind: "message_contains", expected: "Grant deployment access" },
|
||||
{ kind: "message_regex", pattern: "\\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\\b", flags: "i" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.reportedWorkDisposition", expected: "blocked" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.blocker.owner.name", expected: "Release Owner" },
|
||||
{ kind: "json_path", path: "run.resultJson.nativeResult.blocker.unblockAction", expected: "Grant deployment access" },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schema": "paperclip.native-completion.matched-baseline/v1",
|
||||
"candidateSha": "55ce97b675e1ed68cc171fe44b729147209a7e24",
|
||||
"candidateSha": "ae34843731ac338bd806a4106cb447134bff402c",
|
||||
"historicalInstructionSourceSha": "5837aa44425ea134fc33a96c502c22d6a32b6866",
|
||||
"heldConstant": [
|
||||
"merged native Codex base fix #14920",
|
||||
@@ -17,8 +17,8 @@
|
||||
],
|
||||
"historicalStructuralOracles": "Same authenticated/wire/catalog gates expect the explicit historical descriptions. The v13 baseline does not run the candidate-only v13-to-v14 rotation assertion; its prior compatibility cases remain mandatory. Candidate graders are unchanged.",
|
||||
"behavioralSourceHashes": {
|
||||
"tests/runner-e2e/native-completion-case.ts": "6ecd17a78aad20eea36d5ea0f039f22255ea2e61115e16a5e2b1543c72e9a622",
|
||||
"tests/runner-e2e/native-completion-case.test.ts": "253910919bef2dce5509e4005a635b16c518d4940a870c7bbc100898ce8afe83",
|
||||
"tests/runner-e2e/native-completion-case.ts": "ebe9c5caeef5da5e0cdd82cbab8ed841b79e7d8bf54e891fbae78ad01cc7c8a7",
|
||||
"tests/runner-e2e/native-completion-case.test.ts": "1889cecd7efef36020ce6d8c407f8375f3d79af7fd9351e1eb6b46f08bdbaa4e",
|
||||
"tests/runner-e2e/context-integrity-cases.ts": "201d05f8257c1121a94dd9d8a22f08b9db80555e388013ffeabb65e301499fd2",
|
||||
"tests/runner-e2e/context-integrity-flow.ts": "63bcb7bdba4c80b15ff5872f16383844456eab0c5a10627da5d3487a26821c6a",
|
||||
"tests/runner-e2e/context-integrity-scoring.ts": "50d370044e45386ba87fb23ca63c228b96bb8373d67bb7e82e8b2a89eb23f252",
|
||||
@@ -42,5 +42,6 @@
|
||||
],
|
||||
"expectedCells": 6,
|
||||
"expectedProviderTurns": 6
|
||||
}
|
||||
},
|
||||
"sharedGraderCorrection": "Requires visible owner, exact unblock action and a blocking explanation; same corrected grader in both variants."
|
||||
}
|
||||
Reference in new issue
Block a user