PaperClipAI/packages/paperclip-runner/src/cli/eval-session-contract.test.ts

import { createHash } from "node:crypto";
import { chmod, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { describe, expect, it, vi } from "vitest";
import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js";
import { sumAttemptCosts } from "../../scripts/runner-protocol-eval-metrics.mjs";

import { parseNativeRuntimeContext } from "../contracts/runtime-context.js";
import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";
import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js";
import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js";
import {
  evalSessionUsage,
  parseEvalSessionRequest,
} from "./eval-session-contract.js";
import {
  boundedEvalSessionUsage,
  runEvalSessionCli,
  evalRuntimeSystemInstructions,
  evalSessionProviderVersion,
  prepareEvalRuntimeContext,
  parseEvalSessionCliArgs,
} from "./eval-session.js";

function request(overrides: Record<string, unknown> = {}): unknown {
  return {
    schema: "paperclip-runner/eval-session-request/v1",
    attemptId: "attempt-1",
    prompt: "Inspect the governed task.",
    model: "gpt-5.6-sol",
    provider: "codex",
    runnerd: { path: "/tmp/paperclip-runnerd", sha256: "a".repeat(64) },
    limits: {
      turnTimeoutMs: 120_000,
      maxAgentTurns: 1,
      maxEstimatedCostNanodollars: 100_000_000,
    },
    session: {},
    ...(overrides.acpxAgent === "pi" ? { piThinkingLevel: "low", session: { piThinkingLevel: "low" } } : {}),
    ...overrides,
  };
}

it("rejects a stale qualified Claude SDK before provider creation", async () => {
  const workspace = await mkdtemp(join(tmpdir(), "eval-qualified-profile-"));
  try {
    const path = join(workspace, "request.json");
    const expected = resolveQualifiedAcpxProfile("claude", "claude-sonnet-5");
    await writeFile(path, JSON.stringify(request({
      provider: "acpx", driver: "acpx_runtime", acpxAgent: "claude", model: "claude-sonnet-5",
      runnerd: { path: join(workspace, "missing-runnerd"), sha256: "a".repeat(64) },
      session: { workingDirectory: workspace },
    })));
    const serviceFactory = vi.fn();
    const args = ["--request", path, "--output", join(workspace, "artifact.json"), "--expected-acpx-profile"];
    await expect(runEvalSessionCli([...args, JSON.stringify({ ...expected, agentRuntimeVersion: "0.3.263" })], { serviceFactory }))
      .rejects.toThrow("does not match the built runner profile");
    await expect(runEvalSessionCli([...args, JSON.stringify(expected)], { serviceFactory })).rejects.toThrow("ENOENT");
    expect(serviceFactory).not.toHaveBeenCalled();
    expect(await readdir(workspace)).toEqual(["request.json"]);
  } finally { await rm(workspace, { recursive: true, force: true }); }
});

function agentCoreProfile(overrides: Record<string, unknown> = {}) {
  return {
    profileId: "agentcore-qualified",
    region: "us-east-1",
    accountId: "123456789012",
    harnessArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:harness/test",
    harnessVersion: "1",
    endpointArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:harness-endpoint/test",
    endpointQualifier: "paperclip",
    agentRuntimeArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/test",
    memoryArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/test",
    memoryId: "memory-test",
    invocationRoleArn: "arn:aws:iam::123456789012:role/paperclip-agentcore",
    contextBucket: "paperclip-agentcore-context",
    contextPrefix: "paperclip/agentcore/test",
    contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test",
    qualificationRevision: "aws-agentcore-harness-context-v2",
    eventExpiryDays: 90,
    maxEstimatedSessionCostUsd: 1,
    maxIterations: 8,
    maxOutputTokens: 4_096,
    timeoutSeconds: 300,
    ...overrides,
  };
}

describe("eval-session request contract", () => {
  it.each(["copilot"] as const)("admits %s only with the matching CLI diagnostic opt-in", (agent) => {
    const value = request({ provider: "acpx", acpxAgent: agent, model: "explicit-provider-model" });
    expect(() => parseEvalSessionRequest(value)).toThrow("--candidate-profile");
    expect(parseEvalSessionRequest(value, { candidateProfile: agent })).toMatchObject({ acpxAgent: agent, model: "explicit-provider-model" });
    expect(() => parseEvalSessionRequest(value, { candidateProfile: agent === "pi" ? "cursor" : "pi" })).toThrow("must match");
    expect(() => parseEvalSessionRequest(request({ provider: "acpx", acpxAgent: agent, model: "" }), { candidateProfile: agent })).toThrow("request.model");
    expect(() => parseEvalSessionRequest(request({ provider: "acpx", acpxAgent: "codex", session: { acpxAgent: agent } }))).toThrow("session.acpxAgent must match");
    expect(() => parseEvalSessionRequest(request({ provider: "acpx", acpxAgent: agent, candidateProfile: agent }))).toThrow("--candidate-profile");
  });

  it("admits Pi without a diagnostic flag and retains matching historical opt-in", () => {
    const value = request({ provider: "acpx", acpxAgent: "pi", model: "openrouter/deepseek/deepseek-v4-flash-0731" });
    expect(parseEvalSessionRequest(value)).toMatchObject({ acpxAgent: "pi" });
    expect(parseEvalSessionRequest(value, { candidateProfile: "pi" })).toMatchObject({ acpxAgent: "pi" });
    expect(() => parseEvalSessionRequest(value, { candidateProfile: "cursor" })).toThrow("must match");
  });

  it("requires exact matching Pi mode in request and session and rejects aliases", () => {
    const base = { provider: "acpx", acpxAgent: "pi", model: "openrouter/deepseek/deepseek-v4-flash-0731" };
    for (const value of [undefined, "medium", "minimal", "xhigh", ""]) expect(() => parseEvalSessionRequest(request({ ...base, piThinkingLevel: value }))).toThrow();
    expect(() => parseEvalSessionRequest(request({ ...base, piThinkingLevel: "low", session: { piThinkingLevel: "high" } }))).toThrow(/must match/);
    expect(() => parseEvalSessionRequest(request({ piThinkingLevel: "low" }))).toThrow(/only supported/);
  });

  it("accepts only known diagnostic flags and rejects ambiguous repeated arguments", () => {
    const args = ["--request", "/tmp/request.json", "--output", "/tmp/result.json"];
    const expected = resolveQualifiedAcpxProfile("pi", "openrouter/deepseek/deepseek-v4-flash-0731");
    expect(parseEvalSessionCliArgs([...args, "--candidate-profile", "pi", "--expected-acpx-profile", JSON.stringify(expected)]))
      .toMatchObject({ candidateProfile: "pi", expectedAcpxProfile: expected });
    expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile", "pi"])).toThrow("require --expected-acpx-profile");
    expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile", "codex"])).toThrow("must be pi, cursor, or copilot");
    expect(() => parseEvalSessionCliArgs([...args, "--candidate-profile"])).toThrow("missing");
    expect(() => parseEvalSessionCliArgs([...args, "--request", "/tmp/other.json"])).toThrow("duplicate");
    expect(() => parseEvalSessionRequest(request(), { candidateProfile: "pi" })).toThrow("must match");
    for (const invalid of ["null", "[]", "true", "{", JSON.stringify("text")]) {
      expect(() => parseEvalSessionCliArgs([...args, "--expected-acpx-profile", invalid])).toThrow("JSON object");
    }
    expect(() => parseEvalSessionCliArgs([...args, "--expected-acpx-profile", " ".repeat(4097)])).toThrow("4096-byte bound");
  });

  it.each(["pi", "cursor", "copilot"] as const)("rejects stale or incomplete %s identities before runtime construction", async (agent) => {
    const workspace = await mkdtemp(join(tmpdir(), "paperclip-eval-profile-"));
    const model = agent === "pi" ? "openrouter/deepseek/deepseek-v4-flash-0731" : "explicit-provider-model";
    const expected = resolveQualifiedAcpxProfile(agent, model);
    const requestPath = join(workspace, "request.json");
    const args = ["--request", requestPath, "--output", join(workspace, "output.json"), "--candidate-profile", agent];
    const serviceFactory = vi.fn(() => { throw new Error("provider must not start"); });
    try {
      await writeFile(requestPath, JSON.stringify(request({ provider: "acpx", acpxAgent: agent, model,
        runnerd: { path: join(workspace, "missing-runnerd"), sha256: "a".repeat(64) },
        session: { workingDirectory: workspace, ...(agent === "pi" ? { piThinkingLevel: "low" } : {}) },
      })));
      const { commandDigest: _removed, ...incomplete } = expected;
      for (const stale of [
        {}, incomplete, { ...expected, unexpected: true },
        { ...expected, agentProfileVersion: expected.agentProfileVersion - 1 },
        { ...expected, commandDigest: `sha256:${"0".repeat(64)}` },
        { ...expected, reportedModelId: "another-model" },
        { ...expected, agentServerVersion: "previous-binary" },
      ]) {
        await expect(runEvalSessionCli([...args, "--expected-acpx-profile", JSON.stringify(stale)], { serviceFactory }))
          .rejects.toThrow("does not match the built runner profile");
      }
      await expect(runEvalSessionCli(args, { serviceFactory })).rejects.toThrow("require --expected-acpx-profile");
      // A matching identity reaches the independent runner-binary digest gate.
      await expect(runEvalSessionCli([...args, "--expected-acpx-profile", JSON.stringify(expected)], { serviceFactory }))
        .rejects.toThrow("ENOENT");
      expect(serviceFactory).not.toHaveBeenCalled();
      expect(await readdir(workspace)).toEqual(["request.json"]);
    } finally { await rm(workspace, { recursive: true, force: true }); }
  });

  it("requires bounded exact Copilot profile admission before loading provider state", async () => {
    const args = ["--request", "/tmp/request.json", "--output", "/tmp/result.json", "--candidate-profile", "copilot"];
    expect(() => parseEvalSessionCliArgs(args)).toThrow("require --expected-acpx-profile");
    for (const invalid of ["[]", "null", "malformed", JSON.stringify({ overflow: "x".repeat(4096) })]) {
      expect(() => parseEvalSessionCliArgs([...args, "--expected-acpx-profile", invalid])).toThrow();
    }
    const profile = resolveQualifiedAcpxProfile("copilot", "gpt-5.6-luna");
    expect(parseEvalSessionCliArgs([...args, "--expected-acpx-profile", JSON.stringify(profile)]).expectedAcpxProfile).toEqual(profile);
    const workspace = await mkdtemp(join(tmpdir(), "eval-profile-admission-"));
    try {
      const path = join(workspace, "request.json");
      await writeFile(path, JSON.stringify(request({ provider: "acpx", acpxAgent: "copilot", model: "gpt-5.6-luna" })));
      const factory = vi.fn();
      await expect(runEvalSessionCli(["--request", path, "--output", join(workspace, "output.json"), "--candidate-profile", "copilot", "--expected-acpx-profile", JSON.stringify({ ...profile, commandDigest: `sha256:${"0".repeat(64)}` })], { serviceFactory: factory })).rejects.toThrow("does not match the built runner profile");
      expect(factory).not.toHaveBeenCalled();
      expect(await readdir(workspace)).toEqual(["request.json"]);
    } finally { await rm(workspace, { recursive: true, force: true }); }
  });

  it("materializes a production-v3 runtime context for direct live providers", async () => {
    const workspace = await mkdtemp(join(tmpdir(), "paperclip-eval-context-"));
    let instructionRoot: string | null = null;
    try {
      const context = await prepareEvalRuntimeContext(workspace);
      instructionRoot = context.instructions.bundle.rootPath;
      expect(parseNativeRuntimeContext(context)).toEqual(context);
      expect(context.skills).toEqual([]);
      expect(context.mcp.assignmentSetId).toBe("paperclip-runner-direct-eval-v1");
      expect(context.instructions.entryPath).toBe("AGENTS.md");
      expect(await readFile(
        join(context.instructions.bundle.rootPath, "AGENTS.md"),
        "utf8",
      )).toContain("Paperclip direct live evaluation");
      const systemInstructions = evalRuntimeSystemInstructions(context);
      expect(systemInstructions).toContain("Paperclip direct live evaluation");
      expect(systemInstructions).toContain("Task-state changes in this mock control plane use finish_task and block_task");
      expect(systemInstructions).toContain("The current user request defines the work for this turn");
      expect(systemInstructions).toContain("read only the context needed for that request, perform the requested action, and end the turn");
      expect(systemInstructions).toContain("Do not finish or block the mock task unless the current request asks for that state change");
      expect(systemInstructions).toContain(
        `Read-only instruction sibling root: ${context.instructions.bundle.rootPath}`,
      );
      expect((await stat(context.instructions.bundle.rootPath)).mode & 0o777)
        .toBe(0o555);
    } finally {
      if (instructionRoot !== null) {
        await chmod(instructionRoot, 0o700).catch(() => undefined);
      }
      await rm(workspace, { recursive: true, force: true });
    }
  });

  it("normalizes the current local live-session provider contract", () => {
    expect(parseEvalSessionRequest(request())).toMatchObject({
      provider: "codex",
      driver: "codex_app_server",
      model: "gpt-5.6-sol",
    });
    expect(parseEvalSessionRequest(request({
      provider: "acpx",
      driver: "acpx_runtime",
      acpxAgent: "claude",
      model: "claude-sonnet-5",
    }))).toMatchObject({ provider: "acpx", acpxAgent: "claude" });
  });

  it("accepts null optional fields from the original Evalbook v1 producer", () => {
    const parsed = parseEvalSessionRequest(request({
      acpxAgent: null,
      agentCoreProfile: null,
      opencodeVersion: null,
    }));
    expect(parsed).toMatchObject({
      provider: "codex",
      driver: "codex_app_server",
    });
    expect(parsed).not.toHaveProperty("acpxAgent");
    expect(parsed).not.toHaveProperty("agentCoreProfile");
    expect(parsed).not.toHaveProperty("opencodeVersion");
  });

  it("attributes managed providers to their immutable deployed revisions", () => {
    expect(evalSessionProviderVersion(parseEvalSessionRequest(request({
      provider: "aws_agentcore",
      driver: "aws_agentcore_harness_api",
      model: "global.anthropic.claude-sonnet-4-6",
      agentCoreProfile: agentCoreProfile(),
    })))).toBe("aws-agentcore-harness-context-v2");
    expect(evalSessionProviderVersion(parseEvalSessionRequest(request({
      provider: "claude_managed",
      driver: "claude_managed_agents_api",
      model: "claude-sonnet-5",
      managedProfile: {
        profileId: "managed-qualified",
        anthropicAgentId: "agent-test",
        agentVersion: "17",
        environmentId: "environment-test",
        betaVersion: "managed-agents-2026-04-01",
        maxSessionListCostUsd: 1,
      },
    })))).toBe("17");
  });

  it("admits Pi and both qualified remote provider profiles", () => {
    expect(parseEvalSessionRequest(request({
      provider: "acpx",
      acpxAgent: "pi",
      piThinkingLevel: "low",
    }))).toMatchObject({ acpxAgent: "pi", piThinkingLevel: "low" });
    expect(parseEvalSessionRequest(request({
      provider: "aws_agentcore",
      driver: "aws_agentcore_harness_api",
      model: "global.anthropic.claude-sonnet-4-6",
      agentCoreProfile: agentCoreProfile(),
    }))).toMatchObject({
      provider: "aws_agentcore",
      agentCoreProfile: {
        contextBucket: "paperclip-agentcore-context",
        contextPrefix: "paperclip/agentcore/test",
        contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test",
      },
    });
    expect(parseEvalSessionRequest(request({
      provider: "claude_managed",
      driver: "claude_managed_agents_api",
      model: "claude-sonnet-5",
      managedProfile: {
        profileId: "managed-qualified",
        anthropicAgentId: "agent-test",
        agentVersion: "1",
        environmentId: "environment-test",
        betaVersion: "managed-agents-2026-04-01",
        maxSessionListCostUsd: 1,
      },
    }))).toMatchObject({ provider: "claude_managed" });
  });

  it("requires complete remote profiles and AgentCore S3/KMS qualification", () => {
    expect(() => parseEvalSessionRequest(request({
      provider: "aws_agentcore",
      model: "global.anthropic.claude-sonnet-4-6",
    }))).toThrow("request.agentCoreProfile must be an object");
    expect(() => parseEvalSessionRequest(request({
      provider: "aws_agentcore",
      model: "global.anthropic.claude-sonnet-4-6",
      agentCoreProfile: agentCoreProfile({ contextKmsKeyArn: "" }),
    }))).toThrow("contextKmsKeyArn");
  });

  it.each(["latest", "0", "01", "2147483648"])(
    "rejects noncanonical Managed agentVersion %s",
    (agentVersion) => {
      expect(() => parseEvalSessionRequest(request({
        provider: "claude_managed",
        model: "claude-sonnet-5",
        managedProfile: {
          profileId: "managed-qualified",
          anthropicAgentId: "agent-test",
          agentVersion,
          environmentId: "environment-test",
          betaVersion: "managed-agents-2026-04-01",
          maxSessionListCostUsd: 1,
        },
      }))).toThrow("canonical positive int32 string");
    },
  );

  it("rejects contradictory session and execution inputs", () => {
    expect(() => parseEvalSessionRequest(request({
      provider: "opencode",
      driver: "codex_app_server",
    }))).toThrow("provider/driver mismatch");
    expect(() => parseEvalSessionRequest(request({
      session: { requestedModel: "different-model" },
    }))).toThrow("requestedModel must match");
    expect(() => parseEvalSessionRequest(request({
      includeCollaborationModeInstructions: false,
    }))).toThrow("requires collaboration-mode instructions");
  });
});

describe("eval-session usage", () => {
  it("preserves Grok's missing usage as unknown instead of manufacturing zero tokens or cost", () => {
    const parsed = { ...parseEvalSessionRequest(request()), provider: "acpx" as const, acpxAgent: "grok" as const };
    const turn = { turnId: "turn-1", status: "completed" as const, assistantText: "done", snapshot: { usageLedger: [] } as unknown as CapabilityLiveSessionSnapshot };
    expect(() => boundedEvalSessionUsage(parsed, turn)).toThrow("budget cost coverage is unavailable");
    expect(turn.status).toBe("completed");
    expect(turn.snapshot.usageLedger).toEqual([]);
    expect(() => boundedEvalSessionUsage({ ...parsed, provider: "codex", acpxAgent: undefined }, turn)).toThrow();
  });

  it("retains durable failed turns even when their reported usage exceeds completed-turn limits", () => {
    const parsed = parseEvalSessionRequest(request());
    const snapshot = {
      usageLedger: [{
        receiptId: "receipt-failed",
        attemptId: "attempt-1",
        providerResponseId: "response-failed",
        turnId: "turn-failed",
        observedAt: "2026-09-05T00:00:00.000Z",
        providerCalls: 2,
        providerRequests: 2,
        inputTokens: 1_000,
        outputTokens: 100,
        cachedInputTokens: 0,
        reasoningTokens: 0,
        costNanodollars: 200_000_000,
      }],
    } as unknown as CapabilityLiveSessionSnapshot;
    const failedTurn = {
      turnId: "turn-failed",
      status: "failed" as const,
      assistantText: "",
      snapshot,
    };

    expect(boundedEvalSessionUsage(parsed, failedTurn)).toMatchObject({
      agentTurns: 2,
      providerReportedCostNanodollars: 200_000_000,
    });
    expect(() => boundedEvalSessionUsage(parsed, {
      ...failedTurn,
      status: "completed",
    })).toThrow("agent turn limit exceeded");
  });

  it.each(["pi", "cursor", "copilot"] as const)("keeps %s oracle results when the current turn explicitly has unavailable usage", (agent) => {
    const parsed = parseEvalSessionRequest(request({
      provider: "acpx", acpxAgent: agent, model: "exact-provider-model",
    }), { candidateProfile: agent });
    const unavailable = {
      turnId: "turn-candidate", attemptId: "attempt-1", agent,
      reason: "provider_did_not_report_usage" as const,
      tokenUsage: null, costNanodollars: null, observedAt: "2026-09-28T19:18:10.000Z",
    };
    const snapshot = {
      config: { provider: "acpx", acpxAgent: agent },
      usageLedger: [], usageUnavailable: [unavailable],
    } as unknown as CapabilityLiveSessionSnapshot;
    const turn = { turnId: unavailable.turnId, status: "completed" as const, assistantText: "Upgrade your plan to continue", snapshot };
    expect(() => boundedEvalSessionUsage(parsed, turn)).toThrow("budget cost coverage is unavailable");
    expect(turn.assistantText).toBe("Upgrade your plan to continue");
    expect(snapshot.usageLedger).toEqual([]);
    expect(() => boundedEvalSessionUsage(parsed, { ...turn, turnId: "other-turn" }))
      .toThrow("omitted usage accounting");
    expect(() => boundedEvalSessionUsage(parsed, {
      ...turn, snapshot: { ...snapshot, usageUnavailable: [{ ...unavailable, agent: agent === "pi" ? "cursor" : "pi" }] },
    })).toThrow("omitted usage accounting");
    expect(() => boundedEvalSessionUsage(parsed, {
      ...turn, snapshot: { ...snapshot, usageUnavailable: [unavailable, { ...unavailable, turnId: "second-turn" }] },
    })).toThrow("agent turn limit exceeded");
  });

  it("counts prior unavailable candidate turns alongside a later measured turn without claiming a complete total", () => {
    const parsed = parseEvalSessionRequest(request({
      provider: "acpx", acpxAgent: "cursor", model: "gpt-5.6-sol",
    }), { candidateProfile: "cursor" });
    const unavailable = {
      turnId: "prior-turn", attemptId: "attempt-1", agent: "cursor" as const,
      reason: "provider_did_not_report_usage" as const,
      tokenUsage: null, costNanodollars: null, observedAt: "2026-09-28T19:18:10.000Z",
    };
    const snapshot = {
      config: { provider: "acpx", acpxAgent: "cursor" },
      usageUnavailable: [unavailable, { ...unavailable }],
      usageLedger: [{
        receiptId: "measured-receipt", turnId: "measured-turn", providerCalls: 1,
        providerRequests: 1, inputTokens: 100, outputTokens: 10,
        cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0,
      }],
    } as unknown as CapabilityLiveSessionSnapshot;
    const turn = { turnId: "measured-turn", status: "completed" as const, assistantText: "Done", snapshot };
    expect(() => boundedEvalSessionUsage(parsed, turn)).toThrow("agent turn limit exceeded");
    expect(() => boundedEvalSessionUsage({ ...parsed, limits: { ...parsed.limits, maxAgentTurns: 2 } }, turn)).toThrow("budget cost coverage is unavailable");
    expect(boundedEvalSessionUsage(parsed, {
      ...turn,
      snapshot: { ...snapshot, usageUnavailable: [{ ...unavailable, turnId: "measured-turn" }] },
    })).toMatchObject({ agentTurns: 1, inputTokens: 100, outputTokens: 10 });
  });

  it("deduplicates receipts and applies the versioned model price", () => {
    const receipt = {
      receiptId: "receipt-1",
      attemptId: "attempt-1",
      providerResponseId: "response-1",
      turnId: "turn-1",
      observedAt: "2026-09-01T00:00:00.000Z",
      providerCalls: 1,
      providerRequests: 2,
      inputTokens: 1_000,
      outputTokens: 100,
      cachedInputTokens: 400,
      reasoningTokens: 50,
      costNanodollars: 6_000_000,
    };
    const snapshot = {
      usageLedger: [receipt, { ...receipt }],
    } as unknown as CapabilityLiveSessionSnapshot;
    expect(evalSessionUsage("gpt-5.6-sol", snapshot)).toMatchObject({
      agentTurns: 1,
      providerRequests: 2,
      inputTokens: 1_000,
      cachedInputTokens: 400,
      outputTokens: 100,
      reasoningTokens: 50,
      providerReportedCostNanodollars: 6_000_000,
      estimatedCostNanodollars: 6_200_000,
    });
  });

  it.each([0, 6_000_000])("does not label Codex legacy ledger cost %d as a provider USD receipt", (legacyCost) => {
    const snapshot = {
      config: { provider: "codex" },
      usageLedger: [{
        receiptId: "codex-receipt", providerCalls: 1, providerRequests: 1,
        inputTokens: 1_000, outputTokens: 100, cachedInputTokens: 400,
        reasoningTokens: 50, costNanodollars: legacyCost,
      }],
    } as unknown as CapabilityLiveSessionSnapshot;
    const usage = boundedEvalSessionUsage(parseEvalSessionRequest(request()), {
      turnId: "codex-turn", status: "completed", assistantText: "Completed", snapshot,
    });
    expect(usage).toMatchObject({
      agentTurns: 1, providerRequests: 1, inputTokens: 1_000, outputTokens: 100,
      providerReportedCostNanodollars: null,
      providerReportedCostProvenance: "unavailable",
      estimatedCostNanodollars: 6_200_000,
      legacyLedgerCostNanodollars: legacyCost,
      costCoverage: "estimated",
    });
    expect(sumAttemptCosts([usage])).toMatchObject({
      providerReported: { nanodollars: null, recordedAttempts: 0 },
      estimated: { nanodollars: 6_200_000, recordedAttempts: 1 },
    });
    expect(snapshot.usageLedger[0]?.costNanodollars).toBe(legacyCost);
    const limitedRequest = parseEvalSessionRequest(request());
    limitedRequest.limits.maxEstimatedCostNanodollars = 10_000_000;
    const costlySnapshot = { ...snapshot, usageLedger: [{ ...snapshot.usageLedger[0]!, costNanodollars: 10_000_001 }] };
    expect(() => boundedEvalSessionUsage(limitedRequest, {
      turnId: "codex-turn", status: "completed", assistantText: "Completed", snapshot: costlySnapshot,
    })).toThrow("legacy ledger cost limit exceeded");
  });

  it("fails closed when a completed turn has no usage receipt", () => {
    expect(() => evalSessionUsage(
      "gpt-5.6-sol",
      { usageLedger: [] } as unknown as CapabilityLiveSessionSnapshot,
    )).toThrow("omitted usage accounting");
  });

  it.each(["pi", "cursor", "copilot"])("retains unpriced %s usage without inventing an invoice or substituting a model", (agent) => {
    const snapshot = {
      config: { provider: "acpx", acpxAgent: agent },
      usageLedger: [{
        receiptId: "candidate-turn-receipt", providerCalls: 1, providerRequests: 2,
        inputTokens: 1_000, outputTokens: 100, cachedInputTokens: 400,
        reasoningTokens: 50, costNanodollars: 0,
      }],
    } as unknown as CapabilityLiveSessionSnapshot;
    expect(evalSessionUsage("exact-provider-model[context=272k]", snapshot)).toMatchObject({
      agentTurns: 1, providerRequests: 2, inputTokens: 1_000, outputTokens: 100,
      providerReportedCostNanodollars: null,
      providerReportedCostProvenance: "unavailable",
      estimatedCostNanodollars: null, pricingVersion: null, ratesUsdPerMillionTokens: null,
      costCoverage: "unpriced",
    });
    const parsed = parseEvalSessionRequest(request({ provider: "acpx", acpxAgent: agent, model: "exact-provider-model[context=272k]" }), { candidateProfile: agent as "pi" | "cursor" | "copilot" });
    expect(() => boundedEvalSessionUsage(parsed, {
      turnId: "candidate-turn", status: "completed", assistantText: "Completed provider response", snapshot,
    })).toThrow("budget cost coverage is unavailable");
    const priced = evalSessionUsage("openrouter/deepseek/deepseek-v4-flash-0731", snapshot);
    expect(priced.providerReportedCostNanodollars).toBeNull();
    expect(priced.costCoverage).toBe("estimated");
    expect(priced.estimatedCostNanodollars).toBeGreaterThan(0);
    expect(priced.pricingVersion).toBe("provider-list-prices-2026-09-07-openrouter");
    expect(priced.ratesUsdPerMillionTokens).toEqual({ input: 0.14, cachedInput: 0.028, output: 0.28 });
    expect(() => evalSessionUsage("unpriced-qualified-model", {
      ...snapshot, config: { provider: "codex" },
    } as unknown as CapabilityLiveSessionSnapshot)).toThrow("model pricing unavailable");
  });
});

describe("eval-session budget settlement", () => {
  it.each(["no_receipt", "grok_no_receipt", "unpriced", "mixed", "over_limit", "suspension"] as const)("retains evidence while failing %s settlement", async (kind) => {
    const workspace = await mkdtemp(join(tmpdir(), "eval-budget-settlement-"));
    try {
      const binary = join(workspace, "runnerd");
      await writeFile(binary, "unused fake runner");
      const agent = kind === "grok_no_receipt" ? "grok" : "copilot";
      const noReceipt = kind === "no_receipt" || kind === "grok_no_receipt";
      const model = kind === "grok_no_receipt" ? "grok-4.7" : kind === "unpriced" || kind === "no_receipt" ? "exact-unpriced-candidate" : "gpt-5.6-sol";
      const unavailable = { turnId: kind === "mixed" ? "prior-turn" : "turn-1", attemptId: "attempt-1", agent: "copilot", reason: "provider_did_not_report_usage", tokenUsage: null, costNanodollars: null, observedAt: "2026-09-28T19:00:00.000Z" };
      const state = JSON.stringify(createCapabilityFixtureState({}));
      const snapshot = {
        sessionId: "session-1", revision: 1, providerThreadId: "provider-1", providerSessionId: "provider-1",
        providerModel: { id: model, provider: agent === "grok" ? "xai" : "github" }, status: "idle", activeTurnId: null,
        createdAt: "2026-09-28T19:00:00.000Z", updatedAt: "2026-09-28T19:00:01.000Z",
        authority: { companyId: "company-1", actorId: "actor-1", taskId: "task-1", runId: "run-1", sessionId: "session-1", scenarioId: "budget-test" },
        config: { provider: "acpx", acpxAgent: agent, driver: "acpx_runtime", seedState: state, workingDirectory: workspace, scenario: { id: "budget-test" }, capabilities: [], explicitClaims: [], turnTimeoutMs: 1000 },
        mockState: state, process: null, networkEvidence: { realPaperclipRequests: 0, childPaperclipEnvironmentKeys: [] },
        transcript: [{ id: "assistant-1", role: "assistant", text: "Retained actual provider response", turnId: "turn-1", at: "2026-09-28T19:00:01.000Z" }],
        evidence: [], authorizationRecords: [], attempts: [], terminalTurns: [{ turnId: "turn-1", status: "completed" }],
        usageLedger: noReceipt ? [] : [{ receiptId: "receipt-1", turnId: "turn-1", providerCalls: 1, providerRequests: 1, inputTokens: 100, outputTokens: kind === "over_limit" ? 1_000_000 : 10, cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0 }],
        usageUnavailable: kind === "no_receipt" || kind === "mixed" ? [unavailable] : [],
      } as unknown as CapabilityLiveSessionSnapshot;
      const sendMessage = vi.fn(async () => ({ turnId: "turn-1", status: "completed", assistantText: "Retained actual provider response", snapshot }));
      const completeAttempt = vi.fn(async () => undefined);
      const shutdown = vi.fn(async () => {
        snapshot.status = "closed";
        if (kind === "suspension") throw new NativeSessionCloseUnrecoverableError({
          runnerSuspended: false, providerDrained: true,
          suspensionState: { commandStatus: "pending", runnerLifecycle: "ready", runnerIdentityMatches: true },
          semanticTools: { privateToolId: "private-value-must-not-be-copied" },
          unrelated: "private-value-must-not-be-copied",
        });
      });
      const input = request({ provider: "acpx", acpxAgent: agent, model,
        runnerd: { path: binary, sha256: createHash("sha256").update("unused fake runner").digest("hex") },
        session: { workingDirectory: workspace, ...(agent === "pi" ? { piThinkingLevel: "low" } : {}) }, limits: { turnTimeoutMs: 1000, maxAgentTurns: 2, maxEstimatedCostNanodollars: 100_000_000 },
      });
      const requestPath = join(workspace, "request.json");
      const outputPath = join(workspace, "output.json");
      await writeFile(requestPath, JSON.stringify(input));
      const exitCode = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, ...(agent === "grok" ? [] : ["--candidate-profile", "copilot", "--expected-acpx-profile", JSON.stringify(resolveQualifiedAcpxProfile("copilot", model))])], {
        serviceFactory: () => ({ create: async () => ({ sendMessage, completeAttempt, shutdown, snapshot: () => snapshot }) }) as never,
      });
      const artifact = JSON.parse(await readFile(outputPath, "utf8"));
      expect(exitCode).toBe(2);
      if (kind === "suspension") {
        expect(artifact.infrastructureFailure).toEqual({
          class: "runner_infrastructure_failure", category: "runner_infrastructure", retryable: false,
          diagnostics: { runnerSuspended: false, providerDrained: true,
            suspensionCommandStatus: "pending", runnerLifecycle: "ready", runnerIdentityMatches: true },
        });
        expect(JSON.stringify(artifact)).not.toContain("private-value-must-not-be-copied");
        expect(artifact.snapshot.terminalTurns).toEqual([{ turnId: "turn-1", status: "completed" }]);
        expect(sendMessage).toHaveBeenCalledTimes(1);
        expect(shutdown).toHaveBeenCalledTimes(1);
        expect(completeAttempt).toHaveBeenCalledWith("succeeded", null);
        return;
      }
      expect(artifact).not.toHaveProperty("infrastructureError");
      expect(artifact.accountingFailure).toMatchObject({ class: kind === "over_limit" ? "provider_budget_reached" : "provider_budget_coverage_unknown", retryable: false });
      expect(artifact.turn).toMatchObject({ status: "completed", assistantText: "Retained actual provider response" });
      expect(artifact.snapshot.status).toBe("closed");
      expect(artifact.acpxProfile.agent).toBe(agent);
      expect(artifact.devtools).toBeDefined();
      expect(artifact.issueThread).toBeDefined();
      if (noReceipt) expect(artifact).not.toHaveProperty("usage");
      else expect(artifact.usage.providerReportedCostNanodollars).toBeNull();
      expect(sendMessage).toHaveBeenCalledTimes(1);
      expect(shutdown).toHaveBeenCalledTimes(1);
      expect(completeAttempt).toHaveBeenCalledWith("failed", artifact.accountingFailure.class);
    } finally {
      for (const entry of await readdir(workspace)) {
        if (entry.startsWith(".paperclip-eval-runtime-context-")) await chmod(join(workspace, entry, "instructions"), 0o700).catch(() => undefined);
      }
      await rm(workspace, { recursive: true, force: true });
    }
  });
});