test(evals): synchronize SDK public source pins offline

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/source-sdk-public-inputs.mjs

@@ -0,0 +1,73 @@
+// Offline source-pin maintenance only. Runtime/compiler/parser/zod pins are never regenerated here.
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { lstat, readFile, readdir, realpath, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const registryPath = "tests/runner-e2e/source-sdk-public-inputs.json";
+const sourceRoots = ["packages/shared/src", "packages/plugins/sdk/src"];
+const buildInputs = ["package.json", "packages/plugins/sdk/package.json", "packages/plugins/sdk/tsconfig.json",
+  "packages/shared/package.json", "packages/shared/tsconfig.json", "pnpm-lock.yaml",
+  "scripts/ensure-plugin-build-deps.mjs", "tsconfig.base.json", "tsconfig.json"];
+const sourcePath = file => sourceRoots.some(root => file.startsWith(`${root}/`));
+const blob = bytes => createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex");
+const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");
+const fail = () => { throw new Error("SDK public input synchronization refused"); };
+const git = (root, args) => execFileSync("git", args, { cwd: root, encoding: "utf8", maxBuffer: 4 * 1024 * 1024, stdio: ["ignore", "pipe", "pipe"] });
+
+export async function synchronizeSdkPublicInputs(root, revision, { write = false } = {}) {
+  if (!/^[a-f0-9]{40}$/.test(revision) || await realpath(root) !== root || git(root, ["rev-parse", "HEAD"]).trim() !== revision) fail();
+  // The reviewed committed registry supplies every non-source identity. Never trust a locally edited pin.
+  const baselineBytes = git(root, ["show", `${revision}:${registryPath}`]);
+  const baseline = JSON.parse(baselineBytes);
+  if (!baseline.inputs || Object.keys(baseline.inputs).filter(file => !sourcePath(file)).sort().join("\0") !== [...buildInputs].sort().join("\0")) fail();
+  const entries = git(root, ["ls-tree", "-rz", revision, "--", ...sourceRoots, ...buildInputs]).split("\0").filter(Boolean);
+  const inputs = {};
+  for (const entry of entries) {
+    const match = /^(100644|100755) blob ([a-f0-9]{40})\t([A-Za-z0-9._/-]+)$/.exec(entry);
+    if (!match || match[3].split("/").includes("..") || Object.hasOwn(inputs, match[3])) fail();
+    inputs[match[3]] = match[2];
+  }
+  if (buildInputs.some(file => !inputs[file]) || sourceRoots.some(directory => !Object.keys(inputs).some(file => file.startsWith(`${directory}/`)))) fail();
+  const actualSources = [];
+  const collect = async directory => {
+    if (await realpath(path.join(root, directory)) !== path.join(root, directory)) fail();
+    for (const entry of await readdir(path.join(root, directory), { withFileTypes: true })) {
+      const file = `${directory}/${entry.name}`;
+      if (entry.isDirectory()) await collect(file);
+      else if (entry.isFile()) actualSources.push(file);
+      else fail();
+    }
+  };
+  for (const directory of sourceRoots) await collect(directory);
+  if (actualSources.sort().join("\0") !== Object.keys(inputs).filter(sourcePath).sort().join("\0")) fail();
+  for (const [file, expected] of Object.entries(inputs)) {
+    const filename = path.join(root, file), stat = await lstat(filename);
+    if (!stat.isFile() || stat.isSymbolicLink() || await realpath(filename) !== filename || stat.size > 8 * 1024 * 1024) fail();
+    const bytes = await readFile(filename);
+    // Preserve the existing, separately authenticated restored-lock exception.
+    if (blob(bytes) !== expected && !(file === "pnpm-lock.yaml" && sha256(bytes) === baseline.lockSha256)) fail();
+  }
+  const updated = { ...baseline, inputs: Object.fromEntries(Object.entries(inputs).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) };
+  const output = `${JSON.stringify(updated, null, 2)}\n`;
+  const filename = path.join(root, registryPath), stat = await lstat(filename);
+  if (!stat.isFile() || stat.isSymbolicLink() || await realpath(filename) !== filename) fail();
+  const current = await readFile(filename, "utf8");
+  // A second invocation may check the exact generated bytes, but may not replace unrelated edits.
+  if (current !== baselineBytes && current !== output) fail();
+  if (git(root, ["rev-parse", "HEAD"]).trim() !== revision) fail();
+  const result = { revision, inputs: Object.keys(inputs).length, changed: current !== output,
+    registrySha256: sha256(output), runtimePinsChanged: false };
+  if (write && result.changed) await writeFile(filename, output);
+  return result;
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const [revision, mode] = process.argv.slice(2);
+  if (process.argv.length !== 4 || !["--check", "--write"].includes(mode)) throw new Error("Usage: node tests/runner-e2e/source-sdk-public-inputs.mjs <HEAD-40hex> --check|--write");
+  const root = await realpath(path.resolve(fileURLToPath(new URL("../..", import.meta.url))));
+  const result = await synchronizeSdkPublicInputs(root, revision, { write: mode === "--write" });
+  console.log(JSON.stringify(result));
+  if (mode === "--check" && result.changed) process.exitCode = 1;
+}

tests/runner-e2e/source-sdk-public-inputs.test.mjs

@@ -0,0 +1,72 @@
+import { strict as assert } from "node:assert";
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, test } from "node:test";
+import { synchronizeSdkPublicInputs } from "./source-sdk-public-inputs.mjs";
+
+const registry = "tests/runner-e2e/source-sdk-public-inputs.json";
+const roots = [];
+const digest = bytes => createHash("sha256").update(bytes).digest("hex");
+const blob = bytes => createHash("sha1").update(`blob ${Buffer.byteLength(bytes)}\0`).update(bytes).digest("hex");
+const git = (root, ...args) => execFileSync("git", ["-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", ...args], { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
+async function put(root, file, bytes) { await mkdir(path.dirname(path.join(root, file)), { recursive: true }); await writeFile(path.join(root, file), bytes); }
+async function fixture() {
+  const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "sdk-input-sync-"))); roots.push(root);
+  const files = ["package.json", "packages/plugins/sdk/package.json", "packages/plugins/sdk/tsconfig.json",
+    "packages/shared/package.json", "packages/shared/tsconfig.json", "pnpm-lock.yaml", "scripts/ensure-plugin-build-deps.mjs",
+    "tsconfig.base.json", "tsconfig.json", "packages/shared/src/schema.ts", "packages/plugins/sdk/src/types.ts"];
+  for (const file of files) await put(root, file, `synthetic original ${file}\n`);
+  const baseline = { inputs: Object.fromEntries(files.map(file => [file, blob(`synthetic stale ${file}\n`)])),
+    lockSha256: digest("synthetic authenticated restored lock"), compilerPackageSha256: "a".repeat(64),
+    parserPackageSha256: "b".repeat(64), parserEntrySha256: "c".repeat(64),
+    zodPackageSha256: "d".repeat(64), zodEntrySha256: "e".repeat(64), zodTypeFiles: { "index.d.cts": "f".repeat(64) } };
+  await put(root, registry, `${JSON.stringify(baseline, null, 2)}\n`);
+  git(root, "init", "-q"); git(root, "add", ".");
+  git(root, "-c", "user.name=Synthetic unit", "-c", "user.email=synthetic@example.invalid", "commit", "-qm", "synthetic source");
+  return { root, baseline, revision: git(root, "rev-parse", "HEAD").trim() };
+}
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+
+test("synchronizes the committed source graph, preserves every runtime pin and is idempotent", async () => {
+  const { root, baseline } = await fixture();
+  await put(root, "packages/shared/src/added.ts", "synthetic added source\n");
+  git(root, "add", "."); git(root, "-c", "user.name=Synthetic unit", "-c", "user.email=synthetic@example.invalid", "commit", "-qm", "additional source");
+  const revision = git(root, "rev-parse", "HEAD").trim();
+  assert.equal((await synchronizeSdkPublicInputs(root, revision)).changed, true);
+  assert.deepEqual(JSON.parse(await readFile(path.join(root, registry), "utf8")), baseline, "check does not write");
+  const changed = await synchronizeSdkPublicInputs(root, revision, { write: true });
+  const updated = JSON.parse(await readFile(path.join(root, registry), "utf8"));
+  assert.equal(updated.inputs["packages/shared/src/added.ts"], blob("synthetic added source\n"));
+  for (const [file, pin] of Object.entries(updated.inputs)) assert.equal(pin, git(root, "rev-parse", `${revision}:${file}`).trim());
+  const { inputs: oldInputs, ...runtime } = baseline, { inputs: newInputs, ...afterRuntime } = updated;
+  assert.deepEqual(afterRuntime, runtime); assert.equal(changed.runtimePinsChanged, false);
+  assert.equal(changed.registrySha256, digest(await readFile(path.join(root, registry))));
+  assert.equal((await synchronizeSdkPublicInputs(root, revision)).changed, false);
+});
+for (const mutation of ["dirty", "missing", "extra", "symlink", "registry", "revision"]) {
+  test(`refuses ${mutation} source custody without writing the registry`, async () => {
+    const { root, revision } = await fixture();
+    const target = path.join(root, "packages/shared/src/schema.ts");
+    if (mutation === "dirty") await writeFile(target, "uncommitted synthetic source");
+    if (mutation === "missing") await rm(target);
+    if (mutation === "extra") await put(root, "packages/shared/src/uncommitted.ts", "uncommitted");
+    if (mutation === "symlink") { await rm(target); await symlink("../package.json", target); }
+    if (mutation === "registry") { const current = JSON.parse(await readFile(path.join(root, registry), "utf8")); current.parserEntrySha256 = "0".repeat(64); await writeFile(path.join(root, registry), JSON.stringify(current)); }
+    const before = await readFile(path.join(root, registry));
+    await assert.rejects(synchronizeSdkPublicInputs(root, mutation === "revision" ? "0".repeat(40) : revision, { write: true }));
+    assert.deepEqual(await readFile(path.join(root, registry)), before);
+  });
+}
+test("admits only the separately pinned restored lock overlay without changing its identities", async () => {
+  const { root, baseline, revision } = await fixture();
+  await writeFile(path.join(root, "pnpm-lock.yaml"), "synthetic authenticated restored lock");
+  await synchronizeSdkPublicInputs(root, revision, { write: true });
+  const updated = JSON.parse(await readFile(path.join(root, registry), "utf8"));
+  assert.equal(updated.lockSha256, baseline.lockSha256);
+  assert.equal(updated.inputs["pnpm-lock.yaml"], git(root, "rev-parse", `${revision}:pnpm-lock.yaml`).trim());
+  await writeFile(path.join(root, "pnpm-lock.yaml"), "unknown lock overlay");
+  await assert.rejects(synchronizeSdkPublicInputs(root, revision));
+});

tests/runner-e2e/source-sdk-public-type.test.ts

@@ -73,6 +73,24 @@ it("rejects changed source and unknown source initializers", async () => {
   const scanner = createSdkArtifactScanner(root, [], expectations); expect(await scanner.scan(file, Buffer.from(declaration))).toBe(false);
   const other = createSdkArtifactScanner(root, [], await fixture(source.replace("z.string().min(1)", "privateFactory()"))); expect(await other.scan(file, Buffer.from(declaration))).toBe(false);
 });
+it("keeps stale, missing and extra source graph inputs ineligible after pin synchronization", async () => {
+  const expectations = await fixture();
+  const missing = { ...expectations.inputs }; delete missing[inputFile];
+  for (const inputs of [{ ...expectations.inputs, [inputFile]: "0".repeat(40) }, missing]) {
+    const scanner = createSdkArtifactScanner(root, [], { ...expectations, inputs });
+    expect(await scanner.scan(file, Buffer.from(declaration))).toBe(false);
+    expect((await scanner.audit()).publicSource).toBeNull();
+  }
+  const extra = path.join(root, "packages/shared/src/unpinned-synthetic.ts");
+  try {
+    await writeFile(extra, "export const synthetic = true;");
+    const scanner = createSdkArtifactScanner(root, [], expectations);
+    expect(await scanner.scan(file, Buffer.from(declaration))).toBe(false);
+    expect((await scanner.audit()).publicSource).toBeNull();
+  } finally { await rm(extra, { force: true }); }
+  const scanner = createSdkArtifactScanner(root, [], expectations);
+  expect(await scanner.scan(file, Buffer.from(declaration))).toBe(true);
+});
 it("keeps shape overflow blocked with bounded retained observations", async () => {
   const scanner = createSdkArtifactScanner(root, [], await fixture()); expect(await scanner.scan(file, Buffer.from((declaration+'\n').repeat(40)))).toBe(false);
   const audit = await scanner.audit(); expect(audit.bounded).toBe(false); expect(audit.observations).toHaveLength(32);