PaperClipAI/packages/paperclip-runner/src/eval/eval-slice.test.ts

import { describe, expect, it } from "vitest";

import type { EvalBundle } from "./eval-bundle.js";
import type { EvalObservation } from "./eval-scoring.js";
import { buildEvalSliceReport, renderEvalSliceMarkdown } from "./eval-slice.js";

function bundle(): EvalBundle {
  return {
    schema: "paperclip.runner.eval-bundle.v1",
    provider: { runtime: "runnerd", transport: "codex-app-server", protocolVersion: "prp.v1" },
    model: { id: "gpt-5-codex" },
    launchContext: { workingDirectoryClass: "ephemeral-fixture", scenarioId: "slice", turnTimeoutMs: 60_000 },
    promptPolicy: { id: "exact-single-call", callTemplate: "Call {op}.", restraintTemplate: "No tools." },
    grants: ["discovery:tasks:read"],
    runner: { package: "@paperclipai/paperclip-runner", binary: "paperclip-runnerd", version: "0.0.0" },
    controlPlaneAdapter: { kind: "mock", contract: "paperclip.capability.control-plane.v1" },
    faultInjection: [],
  };
}

const trace = { runId: "r", sessionId: "s", turnId: "t", itemId: "i", receiptIds: ["rc"], terminalPresent: true };

function green(caseId: string): EvalObservation {
  return {
    caseId,
    controlPlaneOwned: false,
    expectedCalls: ["finish_task"],
    observedCalls: ["finish_task"],
    forbiddenCalls: ["checkout_task"],
    finalState: { expected: "mutated", observed: "mutated" },
    authorization: { expected: "allowed", observed: "allowed" },
    trace,
  };
}

function gateFail(caseId: string): EvalObservation {
  return { ...green(caseId), observedCalls: ["finish_task", "checkout_task"] };
}

describe("buildEvalSliceReport", () => {
  it("aggregates scorecards and content-addresses the bundle", () => {
    const report = buildEvalSliceReport(bundle(), [green("a"), gateFail("b")]);
    expect(report.bundle.id).toMatch(/^evb-[0-9a-f]{16}$/);
    expect(report.aggregate.caseCount).toBe(2);
    expect(report.aggregate.passed).toBe(1);
    expect(report.aggregate.gateFailures).toBe(1);
    expect(report.aggregate.dimensionMeans.semantic_outcome).toBe(1);
    // Every scorecard carries the report bundle id.
    for (const entry of report.cases) expect(entry.scorecard.bundleId).toBe(report.bundle.id);
  });

  it("is deterministic", () => {
    const a = buildEvalSliceReport(bundle(), [green("a")]);
    const b = buildEvalSliceReport(bundle(), [green("a")]);
    expect(JSON.stringify(a)).toBe(JSON.stringify(b));
  });

  it("refuses to build a report from a secret-carrying bundle", () => {
    const leaky = { ...bundle(), grants: ["not-a-grant"] };
    expect(() => buildEvalSliceReport(leaky, [green("a")])).toThrow();
  });

  it("does not persist free-form bundle declarations", () => {
    const candidate = bundle();
    candidate.promptPolicy.callTemplate = "Unique instructions that stay in memory.";
    const report = buildEvalSliceReport(candidate, [green("a")]);
    const serialized = JSON.stringify(report);

    expect(report.bundle.declaration.promptPolicy.callTemplateSha256).toMatch(
      /^sha256:[0-9a-f]{64}$/,
    );
    expect(serialized).not.toContain(candidate.promptPolicy.callTemplate);
  });

  it("scans the final serialized report before returning it", () => {
    const observation = green("Bearer abcdef0123456789abcdef");
    expect(() => buildEvalSliceReport(bundle(), [observation])).toThrow(/bearer-token/);
  });
});

describe("renderEvalSliceMarkdown", () => {
  it("renders an inspectable table with no secrets", () => {
    const md = renderEvalSliceMarkdown(buildEvalSliceReport(bundle(), [green("a"), gateFail("b")]));
    expect(md).toContain("Runner eval slice");
    expect(md).toContain("| a |");
    expect(md).toContain("| b |");
    expect(md).toContain("FAIL");
    expect(md).not.toContain("sk-");
  });
});