PaperClipAI/tests/runner-e2e/hermes-source.test.ts

import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { devNull, tmpdir } from "node:os";
import { afterEach, describe, expect, it } from "vitest";
import { prepareHermesQualificationSource } from "./hermes-source.js";
import { resolveRunnerE2ESource } from "./source.js";
import type { MatrixExecution } from "./types.js";

const selected = [{ profile: { qualificationCandidate: "hermes" } }] as MatrixExecution[];
const roots: string[] = [];
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
function checkout() {
  const root = realpathSync(mkdtempSync(join(tmpdir(), "hermes-source-test-"))); roots.push(root);
  const git = (...args: string[]) => execFileSync("git", args, { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"],
    env: { PATH: process.env.PATH, GIT_CONFIG_GLOBAL: devNull, GIT_CONFIG_NOSYSTEM: "1" } }).trim();
  git("init", "-b", "qualification");
  writeFileSync(join(root, "source.txt"), "original\n");
  writeFileSync(join(root, "pnpm-lock.yaml"), "original lock\n");
  git("add", "source.txt", "pnpm-lock.yaml");
  git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "-c", "commit.gpgsign=false", "-c", `core.hooksPath=${devNull}`, "commit", "-m", "fixture");
  return { root, git, sha: git("rev-parse", "HEAD") };
}

describe("Hermes qualification controller source admission", () => {
  it("records the real checkout automatically in child/result provenance instead of a workflow SHA", () => {
    const f = checkout(); const env = { PATH: process.env.PATH, GITHUB_SHA: "b".repeat(40), GITHUB_REF: "refs/heads/master" };
    expect(prepareHermesQualificationSource(selected, f.root, env)).toMatchObject({ sha: f.sha, ref: "refs/heads/qualification", workingTreeClean: true });
    expect(resolveRunnerE2ESource(null, env)).toMatchObject({ sha: f.sha, ref: "refs/heads/qualification" });
  });
  it("preserves a requested target ref only after independently matching its SHA", () => {
    const f = checkout(); const env = { PATH: process.env.PATH, PAPERCLIP_RUNNER_E2E_SOURCE_SHA: f.sha, PAPERCLIP_RUNNER_E2E_SOURCE_REF: "refs/pull/123/merge" };
    expect(prepareHermesQualificationSource(selected, f.root, env)).toMatchObject({ sha: f.sha, ref: "refs/pull/123/merge" });
    expect(() => prepareHermesQualificationSource(selected, f.root, { ...env, PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "a".repeat(40) })).toThrow("differs");
  });
  it("records detached HEAD instead of borrowing another workflow ref", () => {
    const f = checkout(); f.git("checkout", "--detach", f.sha);
    const env = { PATH: process.env.PATH, GITHUB_REF: "refs/heads/master" };
    expect(prepareHermesQualificationSource(selected, f.root, env)).toMatchObject({ sha: f.sha, ref: "HEAD" });
  });
  it.each(["tracked", "untracked"])("rejects %s source changes without returning a source receipt", kind => {
    const f = checkout(); writeFileSync(join(f.root, kind === "tracked" ? "source.txt" : "extra.txt"), "changed\n");
    const env: NodeJS.ProcessEnv = { PATH: process.env.PATH };
    expect(() => prepareHermesQualificationSource(selected, f.root, env)).toThrow("clean checkout before credentials");
    expect(env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA).toBeUndefined();
  });
  it("does not run Git or alter provenance for other providers or missing selections", () => {
    const env = { PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "existing" };
    expect(prepareHermesQualificationSource([], "/does-not-exist", env)).toBeNull();
    expect(prepareHermesQualificationSource([{ profile: { qualificationCandidate: "cursor" } }] as MatrixExecution[], "/does-not-exist", env)).toBeNull();
    expect(env).toEqual({ PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "existing" });
  });
  it("ignores ambient Git redirection and keeps credentials out of the source receipt", () => {
    const f = checkout();
    const env = { PATH: process.env.PATH, GIT_DIR: "/does-not-exist", GIT_WORK_TREE: "/does-not-exist",
      OPENROUTER_API_KEY: "fixture-private-token", FUTURE_PROVIDER_API_KEY: "fixture-private-token" };
    const receipt = prepareHermesQualificationSource(selected, f.root, env);
    expect(receipt).toMatchObject({ sha: f.sha });
    expect(JSON.stringify(receipt)).not.toContain("fixture-private-token");
  });
  it("rejects unreadable source without accepting an operator-supplied SHA", () => {
    const env = { PATH: process.env.PATH, PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "a".repeat(40) };
    expect(() => prepareHermesQualificationSource(selected, "/does-not-exist", env)).toThrow("readable Git source before credentials");
  });
});

describe("approved qualification dependency lock", () => {
  const resolved = "reviewed resolved lock\n";
  const digest = createHash("sha256").update(resolved).digest("hex");
  it("admits only the independently verified lock replacement and records its digest and dirty state", () => {
    const f = checkout(); writeFileSync(join(f.root, "pnpm-lock.yaml"), resolved);
    const env = { PATH: process.env.PATH, PAPERCLIP_RUNNER_E2E_LOCK_SHA256: digest };
    expect(prepareHermesQualificationSource(selected, f.root, env)).toMatchObject({
      sha: f.sha, workingTreeClean: false, approvedLockSha256: digest,
    });
    expect(resolveRunnerE2ESource(null, env).sha).toBe(f.sha);
  });
  it("still records a verified lock when it already matches the commit", () => {
    const f = checkout();
    const originalDigest = createHash("sha256").update("original lock\n").digest("hex");
    expect(prepareHermesQualificationSource(selected, f.root, {
      PATH: process.env.PATH, PAPERCLIP_RUNNER_E2E_LOCK_SHA256: originalDigest,
    })).toMatchObject({ workingTreeClean: true, approvedLockSha256: originalDigest });
  });
  it.each(["missing", "incorrect", "malformed"])("rejects a replacement with %s approval before recording source", approval => {
    const f = checkout(); writeFileSync(join(f.root, "pnpm-lock.yaml"), resolved);
    const env: NodeJS.ProcessEnv = { PATH: process.env.PATH,
      ...(approval === "missing" ? {} : { PAPERCLIP_RUNNER_E2E_LOCK_SHA256: approval === "incorrect" ? "a".repeat(64) : "not-a-digest" }) };
    expect(() => prepareHermesQualificationSource(selected, f.root, env)).toThrow();
    expect(env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA).toBeUndefined();
  });
  it.each(["source", "untracked", "staged", "deleted", "symlink"])("rejects %s changes even alongside an approved digest", kind => {
    const f = checkout(); const lock = join(f.root, "pnpm-lock.yaml"); writeFileSync(lock, resolved);
    if (kind === "source") writeFileSync(join(f.root, "source.txt"), "changed\n");
    if (kind === "untracked") writeFileSync(join(f.root, "extra.txt"), "changed\n");
    if (kind === "staged") f.git("add", "pnpm-lock.yaml");
    if (kind === "deleted" || kind === "symlink") rmSync(lock);
    if (kind === "symlink") { writeFileSync(join(f.root, "resolved.txt"), resolved); symlinkSync("resolved.txt", lock); }
    expect(() => prepareHermesQualificationSource(selected, f.root, {
      PATH: process.env.PATH, PAPERCLIP_RUNNER_E2E_LOCK_SHA256: digest,
    })).toThrow();
  });
});

describe("generated runtime source admission", () => {
  const assetRoot = "packages/paperclip-runner/provider-assets/hermes/linux-x64";
  function generated(root: string, relative: string) {
    const directory = join(root, relative); mkdirSync(directory, { recursive: true });
    writeFileSync(join(directory, "manifest.json"), "fixture runtime bytes\n");
  }
  it("admits untracked Mac/Linux runtime assets and a verified pack without hiding source changes", () => {
    const f = checkout();
    for (const root of [assetRoot, "packages/paperclip-runner/provider-assets/hermes/darwin-arm64", "packages/paperclip-runner/provider-pack"]) generated(f.root, root);
    expect(prepareHermesQualificationSource(selected, f.root, { PATH: process.env.PATH })).toMatchObject({ sha: f.sha, workingTreeClean: true });
    writeFileSync(join(f.root, "source.txt"), "changed source\n");
    expect(() => prepareHermesQualificationSource(selected, f.root, { PATH: process.env.PATH })).toThrow("clean checkout before credentials");
  });
  it.each(["runner-e2e-build", "runner-e2e-provider-pack", "packages/paperclip-runner/provider-assets/hermes/win32-x64", "packages/paperclip-runner/provider-assets/unmanaged"])("rejects untracked files in %s", root => {
    const f = checkout(); generated(f.root, assetRoot); generated(f.root, root);
    expect(() => prepareHermesQualificationSource(selected, f.root, { PATH: process.env.PATH })).toThrow("clean checkout before credentials");
  });
  it.each(["modified", "staged", "deleted"])("rejects %s tracked files inside a generated asset root", kind => {
    const f = checkout(); generated(f.root, assetRoot); f.git("add", assetRoot);
    f.git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "-c", "commit.gpgsign=false", "-c", `core.hooksPath=${devNull}`, "commit", "-m", "tracked asset fixture");
    const file = join(f.root, assetRoot, "manifest.json");
    if (kind === "deleted") rmSync(file); else writeFileSync(file, "changed tracked asset\n");
    if (kind === "staged") f.git("add", assetRoot);
    expect(() => prepareHermesQualificationSource(selected, f.root, { PATH: process.env.PATH })).toThrow("clean checkout before credentials");
  });
  it("rejects a runtime-root symlink instead of treating it as generated content", () => {
    const f = checkout(); const parent = join(f.root, "packages/paperclip-runner/provider-assets/hermes"); mkdirSync(parent, { recursive: true });
    symlinkSync(f.root, join(parent, "linux-x64"));
    expect(() => prepareHermesQualificationSource(selected, f.root, { PATH: process.env.PATH })).toThrow("clean checkout before credentials");
  });
});