test(evals): freeze prior native guidance for matched comparison

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-28 12:04:41 +01:00 · 2026-10-02 14:29:30 -05:00
1 parent 55ce97b675
commit 4cf05747be
@@ -1,12 +1,13 @@
+// Explicit historical oracle for the matched native-guidance baseline.
+const PRP_COMPLETION_TOOL_DESCRIPTION = "Return the one semantic completion result for this task, including an explicit response_wake yield when waiting for the next response.";
+const PRP_BLOCK_TOOL_DESCRIPTION = "Return the one semantic result when the task cannot continue.";
 import Ajv2020 from "ajv/dist/2020.js";
 import { describe, expect, it } from "vitest";
 import {
  PRP_BLOCK_RESULT_OUTPUT_SCHEMA,
  PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_BLOCK_TOOL_DESCRIPTION,
  PRP_COMPLETION_RESULT_OUTPUT_SCHEMA,
  PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_COMPLETION_TOOL_DESCRIPTION,
 } from "./completion-result.js";
 import { codexSemanticToolSpecs } from "../drivers/codex/codex-driver-values.js";

@@ -1,11 +1,6 @@
 /** Provider-neutral semantic completion tools and their strict model-facing schemas. */
 export const PRP_COMPLETION_TOOL_NAME = "paperclip_finish" as const;
 export const PRP_BLOCK_TOOL_NAME = "paperclip_block" as const;
-/** Native Runner guidance; legacy adapters use their own skill/API completion paths. */
-export const PRP_COMPLETION_TOOL_DESCRIPTION =
-  "Report completed work (done), work requiring review (needs_review), or an explicit wait for the next response (yielded with response_wake). Use the current completion contract and supporting evidence. If rejected, correct the report and retry. After acceptance, read the returned outcome; do not claim completion while gated. Explain any required approval with its supplied link and action, then write the final response and end the turn without further tool calls.";
-export const PRP_BLOCK_TOOL_DESCRIPTION =
-  "Report work that cannot continue because of a concrete blocker. Identify the blocker, its owner, and the action needed to unblock it; use the current completion contract and supporting evidence. If rejected, correct the report and retry. After acceptance, read the returned outcome, explain the blocker and any required action in the final response, and end the turn without further tool calls.";
 export const PRP_SEMANTIC_TOOL_NAMES = [
  PRP_COMPLETION_TOOL_NAME,
  PRP_BLOCK_TOOL_NAME,
@@ -1,8 +1,9 @@
+// Explicit historical oracle for the matched native-guidance baseline.
+const PRP_COMPLETION_TOOL_DESCRIPTION = "Return the one semantic completion result for this task, including an explicit response_wake yield when waiting for the next response.";
+const PRP_BLOCK_TOOL_DESCRIPTION = "Return the one semantic result when the task cannot continue.";
 import {
  PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_BLOCK_TOOL_DESCRIPTION,
  PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_COMPLETION_TOOL_DESCRIPTION,
 } from "../../contracts/completion-result.js";
 import {
  CODEX_BLOCK_RESULT_OUTPUT_SCHEMA,
@@ -1,9 +1,5 @@
 import type { PersistedHarnessProviderIdentity } from "../../contracts/harness-driver.js";
 import type { NativeUserMessage } from "../../contracts/types.js";
-import {
-  PRP_BLOCK_TOOL_DESCRIPTION,
-  PRP_COMPLETION_TOOL_DESCRIPTION,
-} from "../../contracts/completion-result.js";
 import {
  CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
  CODEX_BLOCK_TOOL_NAME,
@@ -236,7 +232,8 @@ export function differingJsonPaths(
 function finishToolSpec(): Record<string, unknown> {
  return {
    name: CODEX_COMPLETION_TOOL_NAME,
-    description: PRP_COMPLETION_TOOL_DESCRIPTION,
+    description:
+      "Return the one semantic completion result for this task, including an explicit response_wake yield when waiting for the next response.",
    inputSchema: CODEX_RESULT_PROVIDER_INPUT_SCHEMA,
  };
 }
@@ -244,7 +241,8 @@ function finishToolSpec(): Record<string, unknown> {
 function blockToolSpec(): Record<string, unknown> {
  return {
    name: CODEX_BLOCK_TOOL_NAME,
-    description: PRP_BLOCK_TOOL_DESCRIPTION,
+    description:
+      "Return the one semantic result when the task cannot continue.",
    inputSchema: CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
  };
 }
@@ -1,9 +1,10 @@
+// Explicit historical oracle for the matched native-guidance baseline.
+const PRP_COMPLETION_TOOL_DESCRIPTION = "Return the semantic completion result.";
+const PRP_BLOCK_TOOL_DESCRIPTION = "Return the semantic blocked result.";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import {
  PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_BLOCK_TOOL_DESCRIPTION,
  PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_COMPLETION_TOOL_DESCRIPTION,
 } from "../../contracts/completion-result.js";

 import { canonicalOpenCodeMcpToolName, startOpenCodeMcpBridge, type OpenCodeMcpBridge } from "./mcp-bridge.js";
@@ -4,10 +4,8 @@ import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js";

 import {
  PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_BLOCK_TOOL_DESCRIPTION,
  PRP_BLOCK_TOOL_NAME,
  PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_COMPLETION_TOOL_DESCRIPTION,
  PRP_COMPLETION_TOOL_NAME,
 } from "../../contracts/completion-result.js";

@@ -119,8 +117,8 @@ function normalizeTools(
    seen.add(name);
  }
  if (includeTerminalTools) for (const terminal of [
-    { name: PRP_COMPLETION_TOOL_NAME, description: PRP_COMPLETION_TOOL_DESCRIPTION, inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA },
-    { name: PRP_BLOCK_TOOL_NAME, description: PRP_BLOCK_TOOL_DESCRIPTION, inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA },
+    { name: PRP_COMPLETION_TOOL_NAME, description: "Return the semantic completion result.", inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA },
+    { name: PRP_BLOCK_TOOL_NAME, description: "Return the semantic blocked result.", inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA },
  ]) {
    if (!seen.has(terminal.name)) tools.push(terminal);
  }
@@ -1,11 +1,12 @@
+// Explicit historical oracle for the matched native-guidance baseline.
+const PRP_COMPLETION_TOOL_DESCRIPTION = "Return the semantic completion result.";
+const PRP_BLOCK_TOOL_DESCRIPTION = "Return the semantic blocked result.";
 import { connect } from "node:net";

 import { afterEach, describe, expect, it, vi } from "vitest";
 import {
  PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_BLOCK_TOOL_DESCRIPTION,
  PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_COMPLETION_TOOL_DESCRIPTION,
 } from "../contracts/completion-result.js";

 import {
@@ -10,10 +10,8 @@ import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js";

 import {
  PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_BLOCK_TOOL_DESCRIPTION,
  PRP_BLOCK_TOOL_NAME,
  PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
-  PRP_COMPLETION_TOOL_DESCRIPTION,
  PRP_COMPLETION_TOOL_NAME,
 } from "../contracts/completion-result.js";

@@ -72,12 +70,12 @@ const MAX_RESULT_CHUNK_BYTES = 64 * 1024;
 const RESERVED_TOOLS: readonly RunnerToolDefinition[] = [
  {
    name: PRP_COMPLETION_TOOL_NAME,
-    description: PRP_COMPLETION_TOOL_DESCRIPTION,
+    description: "Return the semantic completion result.",
    inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
  },
  {
    name: PRP_BLOCK_TOOL_NAME,
-    description: PRP_BLOCK_TOOL_DESCRIPTION,
+    description: "Return the semantic blocked result.",
    inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
  },
 ];
@@ -1,3 +1,6 @@
+// Explicit historical oracle for the matched native-guidance baseline.
+const PRP_COMPLETION_TOOL_DESCRIPTION = "Return the one semantic completion result for this task, including an explicit response_wake yield when waiting for the next response.";
+const PRP_BLOCK_TOOL_DESCRIPTION = "Return the one semantic result when the task cannot continue.";
 import {
  chmod,
  cp,
@@ -55,8 +58,6 @@ import {
  createCodexTaskEnvelope,
 } from "../contracts/codex.js";
 import {
-  PRP_BLOCK_TOOL_DESCRIPTION,
-  PRP_COMPLETION_TOOL_DESCRIPTION,
 } from "../contracts/completion-result.js";
 import {
  CodexAppServerDriver,
@@ -1840,12 +1840,6 @@ describe("rebindNativeSessionCheckpoint", () => {
      retainedFingerprint:
        "sha256:5b7b302db36f7ed6686f9ea1ba70bbf79ebd7fabf86953b548d966a2bc38b648",
    },
-    {
-      contract: "native completion tool guidance",
-      // Deployed v13 local catalog before canonical finish/block descriptions.
-      retainedFingerprint:
-        "sha256:68a51d34e091c55ee5d0d2b563153454dd727d72db16e6a27c358d342ae489c9",
-    },
    {
      contract: "task-bound human-input description",
      // Deployed v9 still advertises the generic mock-task question description.
@@ -23,7 +23,7 @@ export function nativeToolContractFingerprintForTarget(
  return `sha256:${createHash("sha256")
    .update(
      JSON.stringify({
-        schema: "paperclip.native-tool-contract.v14",
+        schema: "paperclip.native-tool-contract.v13",
        executionTargetKind,
        advertisementPolicy: {
          // Direct provider threads retain declarations from thread/start.
@@ -0,0 +1,46 @@
+{
+  "schema": "paperclip.native-completion.matched-baseline/v1",
+  "candidateSha": "55ce97b675e1ed68cc171fe44b729147209a7e24",
+  "historicalInstructionSourceSha": "5837aa44425ea134fc33a96c502c22d6a32b6866",
+  "heldConstant": [
+    "merged native Codex base fix #14920",
+    "tiny default hire manual",
+    "reduced shared startup/resume prompts",
+    "all fixtures, models, effort, permissions, auth and behavioral graders"
+  ],
+  "productionSourcesRestored": [
+    "packages/paperclip-runner/src/contracts/completion-result.ts",
+    "packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts",
+    "packages/paperclip-runner/src/drivers/runner-tool-bridge.ts",
+    "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts",
+    "server/src/services/native-runtime/native-session-resume.ts"
+  ],
+  "historicalStructuralOracles": "Same authenticated/wire/catalog gates expect the explicit historical descriptions. The v13 baseline does not run the candidate-only v13-to-v14 rotation assertion; its prior compatibility cases remain mandatory. Candidate graders are unchanged.",
+  "behavioralSourceHashes": {
+    "tests/runner-e2e/native-completion-case.ts": "6ecd17a78aad20eea36d5ea0f039f22255ea2e61115e16a5e2b1543c72e9a622",
+    "tests/runner-e2e/native-completion-case.test.ts": "253910919bef2dce5509e4005a635b16c518d4940a870c7bbc100898ce8afe83",
+    "tests/runner-e2e/context-integrity-cases.ts": "201d05f8257c1121a94dd9d8a22f08b9db80555e388013ffeabb65e301499fd2",
+    "tests/runner-e2e/context-integrity-flow.ts": "63bcb7bdba4c80b15ff5872f16383844456eab0c5a10627da5d3487a26821c6a",
+    "tests/runner-e2e/context-integrity-scoring.ts": "50d370044e45386ba87fb23ca63c228b96bb8373d67bb7e82e8b2a89eb23f252",
+    "tests/runner-e2e/matchers.ts": "852dc82ca9b4d5d1b2e902407c1df5ff0343313e71175805b6a444c55ff69957",
+    "tests/runner-e2e/runner.spec.ts": "14d57e0f9131b3f1981ac593464738d01362008eefe85ea836075c794681fa85",
+    "tests/runner-e2e/live-fixtures.ts": "1ef80209cc53cc514a8e58af4c2225e38ec15b20acf17ca7564506ca6617ac3e",
+    "tests/runner-e2e/catalog.ts": "7d6fd220dcac80296ba0c73a0febf9975a9b6f28085359d52ca605b598c40ec7",
+    "tests/runner-e2e/stock-harness.ts": "9ce135aabe82f532eca9f16bfd00689c3279bbacf9f68e247a4332ba8a0d4c3c",
+    "tests/runner-e2e/stock-harness-admission.ts": "4cad5e7e703029651d79b5ace4b22368da3b326fed46fe38366c6f95d24b4451",
+    "tests/runner-e2e/launch.ts": "b020d6fd8a4ae00e008215a49b937bbc65208e66341113a4e548c3d4b492ff4a"
+  },
+  "selection": {
+    "profiles": [
+      "runner-codex",
+      "runner-acpx-claude",
+      "runner-opencode"
+    ],
+    "cases": [
+      "assigned-skill-explicit-invocation",
+      "native-blocked-report"
+    ],
+    "expectedCells": 6,
+    "expectedProviderTurns": 6
+  }
+}
@@ -48,10 +48,10 @@ export const stockHarnessGates = [
    "preserves stock Codex instructions on prepared recovery", "preserves stock Codex instructions on direct recovery",
    ...["codex", "opencode", "claude_managed", "aws_agentcore", "acpx"].map(provider =>
      `preserves answer and internal wait descriptions in the serialized native ${provider} tool catalog`)] },
-  { id: "SH-4-resume", name: "Refresh retained native completion guidance", cwd: ".", files: [
+  { id: "SH-4-resume", name: "Historical native fingerprint and retained-session compatibility", cwd: ".", files: [
    "server/src/services/native-runtime/native-session-resume.test.ts",
  ], testPattern: "refreshes retained",
-  required: ["native completion tool guidance"] },
+  required: ["refreshes retained completion descriptions without changing the Paperclip task or prior history"] },
  { id: "SH-eval", name: "Independent oracle and qualification admission", cwd: ".",
    config: "tests/runner-e2e/vitest.config.ts",
    files: ["tests/runner-e2e/stock-harness.test.ts", "tests/runner-e2e/stock-harness-checks.test.mjs",