PaperClipAI/tests/runner-e2e/native-instruction-consolidation.test.ts

import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { pathToFileURL } from "node:url";
import { gunzipSync } from "node:zlib";
import { describe, expect, it } from "vitest";
import { runnerSuites, buildRunnerMatrix } from "./catalog.js";
import { gradeNativeDefault, NATIVE_MASTER_DEFAULT_SHA256 } from "./native-completion-defaults.js";
import { nativeCompletionTasks } from "./native-completion-cases.js";
import { assertNativeInstructionLineage, assertNativeInstructionSelection, nativeInstructionVariant, validateNativeInstructionMeasurement, NATIVE_INSTRUCTION_BASE_SHA, NATIVE_INSTRUCTION_DEFAULT_SHA256, NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_VARIANTS } from "./native-instruction-consolidation.js";

describe("native instruction comparison admission", () => {
  it("proves real hosted ancestry when the source is more than eight commits from the base", () => {
    const temporary = mkdtempSync(join(tmpdir(), "native-instruction-lineage-"));
    const source = join(temporary, "source");
    const checkout = join(temporary, "checkout");
    const environment = { ...process.env, GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.invalid",
      GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.invalid" };
    const fixtureGit = (cwd: string, args: string[]) => execFileSync("git", args, {
      cwd, env: environment, encoding: "utf8", stdio: ["pipe", "pipe", "pipe"],
    }).trim();
    try {
      fixtureGit(temporary, ["init", "--quiet", source]);
      writeFileSync(join(source, "source.txt"), "Immutable source fixture\n");
      fixtureGit(source, ["add", "source.txt"]);
      const tree = fixtureGit(source, ["write-tree"]);
      const base = fixtureGit(source, ["commit-tree", tree, "-m", "base"]);
      let head = base;
      for (let index = 0; index < 12; index++)
        head = fixtureGit(source, ["commit-tree", tree, "-p", head, "-m", `source ${index}`]);
      fixtureGit(source, ["update-ref", "refs/heads/fixture", head]);
      fixtureGit(temporary, ["clone", "--quiet", "--depth=1", "--branch=fixture", pathToFileURL(source).href, checkout]);
      const run = (...args: string[]) => fixtureGit(checkout, ["--no-replace-objects", ...args.map(value =>
        value === NATIVE_INSTRUCTION_BASE_SHA ? base
          : value === "https://github.com/paperclipai/paperclip.git" ? pathToFileURL(source).href : value)]);
      expect(() => run("merge-base", "--is-ancestor", NATIVE_INSTRUCTION_BASE_SHA, head)).toThrow();
      expect(() => assertNativeInstructionLineage(head, run, true)).not.toThrow();
      expect(run("rev-parse", "HEAD")).toBe(head);
      expect(() => run("merge-base", "--is-ancestor", NATIVE_INSTRUCTION_BASE_SHA, head)).not.toThrow();
    } finally {
      rmSync(temporary, { recursive: true, force: true });
    }
  });
  it("hydrates bounded hosted history and still requires exact HEAD and real base ancestry", () => {
    const head = "a".repeat(40);
    for (const scenario of ["valid", "changed-head", "no-base", "local", "not-shallow"]) {
      let hydrated = false;
      const calls: string[][] = [];
      const run = (...args: string[]) => {
        calls.push(args);
        if (args[0] === "merge-base") {
          if (!hydrated || scenario === "no-base") throw new Error("no declared ancestor");
          return "";
        }
        if (args[1] === "--is-shallow-repository") return scenario === "not-shallow" ? "false" : "true";
        if (args.includes("fetch")) { hydrated = true; return ""; }
        return scenario === "changed-head" ? "b".repeat(40) : head;
      };
      if (scenario === "valid") expect(() => assertNativeInstructionLineage(head, run, true)).not.toThrow();
      else expect(() => assertNativeInstructionLineage(head, run, scenario !== "local")).toThrow();
      const fetches = calls.filter(args => args.includes("fetch"));
      const depths = ["local", "not-shallow"].includes(scenario) ? [] : scenario === "no-base" ? [8, 32, 128] : [8];
      expect(fetches).toEqual(depths.map(depth => ["-c", "credential.helper=", "-c", "core.hooksPath=/dev/null",
        "fetch", "--no-tags", `--depth=${depth}`, "https://github.com/paperclipai/paperclip.git", head]));
    }
  });
  it("accepts each complete source variant and rejects a mixed variant", () => {
    const files = Object.keys(NATIVE_INSTRUCTION_VARIANTS.baseline);
    const sources = JSON.parse(gunzipSync(readFileSync(new URL("./fixtures/native-instruction-sources.json.gz", import.meta.url))).toString()) as
      Record<string, { sourceSha: string; files: Record<string, string> }>;
    expect(sources.baseline!.sourceSha).toBe(NATIVE_INSTRUCTION_BASE_SHA);
    expect(sources.opencodeFeedbackResponses!.sourceSha).toBe("a386a599983519eb1d399f8b770bfccdb2a74762");
    for (const source of Object.values(sources)) expect(Object.keys(source.files).sort()).toEqual([...files].sort());
    const baseline = new Map(files.map(file => [file, Buffer.from(sources.baseline!.files[file]!)]));
    expect(nativeInstructionVariant(file => baseline.get(file)!)).toBe("baseline");
    const candidate = new Map(files.map(file => [file, Buffer.from(sources.opencodeFeedbackResponses!.files[file]!)]));
    const currentVariant = nativeInstructionVariant(file => candidate.get(file)!);
    expect(currentVariant).toBe("opencodeFeedbackResponses");
    candidate.set(files[0]!, baseline.get(files[0]!)!);
    expect(() => nativeInstructionVariant(file => candidate.get(file)!)).toThrow("Mixed or unknown");
    baseline.set(files[0]!, Buffer.from("unknown source"));
    expect(() => nativeInstructionVariant(file => baseline.get(file)!)).toThrow("Mixed or unknown");
  });

  it("declares six explicit single-attempt cells with the original task prompts and oracle", () => {
    const suite = runnerSuites.find(value => value.id === NATIVE_INSTRUCTION_SUITE)!;
    expect(suite.tasks).toEqual(nativeCompletionTasks);
    expect(suite.manualOnly).toBe(true);
    expect(suite.environments.map(value => value.id)).toEqual(["local"]);
    const executions = buildRunnerMatrix([suite]);
    expect(executions).toHaveLength(6);
    expect(() => assertNativeInstructionSelection(executions)).not.toThrow();
    for (const execution of executions) {
      expect(execution.task.expectedRunCount).toBe(1);
      expect(execution.task.automaticRetryPolicy).toBe("single_attempt");
      expect(() => assertNativeInstructionSelection([{ ...execution, environment: { ...execution.environment, id: "daytona" } }])).toThrow();
      expect(() => assertNativeInstructionSelection([{ ...execution, task: { ...execution.task, automaticRetryPolicy: undefined } }])).toThrow();
    }
  });

  it("uses the new common production default without qualifying old default evidence", () => {
    const receipt = { schema: "paperclip.native-completion-default.v1" as const, companyId: "company", agentId: "agent", entryFile: "AGENTS.md",
      files: [{ path: "AGENTS.md", sha256: NATIVE_INSTRUCTION_DEFAULT_SHA256, bytes: 40 }], budgets: { company: 1000, agent: 1000 } };
    expect(gradeNativeDefault(receipt, NATIVE_INSTRUCTION_DEFAULT_SHA256).passed).toBe(true);
    expect(gradeNativeDefault(receipt).passed).toBe(false);
    receipt.files[0]!.sha256 = NATIVE_MASTER_DEFAULT_SHA256;
    expect(gradeNativeDefault(receipt, NATIVE_INSTRUCTION_DEFAULT_SHA256).passed).toBe(false);
    receipt.files[0]!.sha256 = NATIVE_INSTRUCTION_DEFAULT_SHA256;
    receipt.budgets.company = 0;
    expect(gradeNativeDefault(receipt, NATIVE_INSTRUCTION_DEFAULT_SHA256).passed).toBe(false);
  });

  it("rejects duplicate or incomplete capture, dirty or paid source, and stale fixture evidence", () => {
    const measurement = {
      schema: "paperclip.native-instruction-measurement.v2", sourceSha: "frozen", sourceDirty: false, providerCalls: 0,
      fixtureSha256: createHash("sha256").update(readFileSync(new URL("../../packages/paperclip-runner/src/backends/native-instruction-measurement.test.ts", import.meta.url))).digest("hex"),
      sourceHashes: Object.fromEntries(Object.entries(NATIVE_INSTRUCTION_VARIANTS.baseline).map(([file, digest]) => [file.split('/').at(-1)!, digest])),
      directOpenCodeReceipts: ["v4", "v5"].flatMap(schema => ["start", "resume", "continuation"].map(phase => ({ provider: "opencode", schema, phase }))),
      receipts: ['codex', 'acpx', 'opencode'].flatMap(provider => ['v4', 'v5'].flatMap(schema => ['start', 'resume', 'continuation'].map(phase => ({ provider, schema, phase })))),
    };
    const source = { sourceSha: "frozen", variant: "baseline" };
    expect(() => validateNativeInstructionMeasurement(measurement, source)).not.toThrow();
    for (const invalid of [
      { ...measurement, receipts: measurement.receipts.slice(1) },
      { ...measurement, directOpenCodeReceipts: undefined },
      { ...measurement, directOpenCodeReceipts: measurement.directOpenCodeReceipts.slice(1) },
      { ...measurement, directOpenCodeReceipts: measurement.directOpenCodeReceipts.map(() => measurement.directOpenCodeReceipts[0]!) },
      { ...measurement, receipts: measurement.receipts.map(() => measurement.receipts[0]!) },
      { ...measurement, sourceDirty: true }, { ...measurement, providerCalls: 1 },
      { ...measurement, sourceSha: "other" }, { ...measurement, fixtureSha256: "stale" },
      { ...measurement, sourceHashes: { ...measurement.sourceHashes, "runtime-context.ts": "wrong" } },
      { ...measurement, sourceHashes: { ...measurement.sourceHashes, "opencode-app-server-proxy.ts": "wrong" } },
    ]) expect(() => validateNativeInstructionMeasurement(invalid, source)).toThrow();
  });
});