PaperClipAI/packages/paperclip-runner/scripts/materialize-kimi-distribution.mjs

import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { chmod, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, unlink, writeFile } from "node:fs/promises";
import { dirname, isAbsolute, join, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { KIMI_DISTRIBUTION_CLOSURE_SHA256, KIMI_DISTRIBUTION_LOCK_SHA256, KIMI_NODE_DISTRIBUTIONS, KIMI_NODE_VERSION, KIMI_RUNTIME_VERSION } from "../src/drivers/acpx/kimi-closure-pins.ts";
import { inventoryKimiRuntimeFiles, kimiRuntimeClosureDigest } from "../src/drivers/acpx/kimi-runtime-files.ts";
import { patchKimiRuntime } from "./patch-kimi-runtime.mjs";

const run = promisify(execFile);
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const repositoryRoot = resolve(packageRoot, "../..");
const hash = bytes => createHash("sha256").update(bytes).digest("hex");
export const KIMI_DISTRIBUTION_ENTRYPOINT = "node_modules/@moonshot-ai/kimi-code/dist/main.mjs";

export async function runKimiBuildCommand(command, args, options, checkCancelled = () => {}, signal) {
  checkCancelled(); signal?.throwIfAborted();
  const pending = run(command, args, { ...options, signal, killSignal: "SIGKILL" });
  const closed = new Promise(resolve => pending.child.once("close", resolve));
  try {
    const result = await pending;
    checkCancelled(); signal?.throwIfAborted();
    return result;
  } finally { await closed; }
}

export function kimiDistributionInstallCommand() {
  return ["ci", "--ignore-scripts", "--omit=optional", "--omit=dev", "--no-audit", "--no-fund", "--registry=https://registry.npmjs.org", "--userconfig=.npmrc", "--globalconfig=.npmrc-global"];
}

export function kimiDistributionEnvironment(source = process.env) {
  return Object.fromEntries(["PATH", "LANG", "LC_ALL", "HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY", "https_proxy", "http_proxy", "no_proxy", "SSL_CERT_FILE", "SSL_CERT_DIR", "NODE_EXTRA_CA_CERTS"].flatMap(key => typeof source[key] === "string" ? [[key, source[key]]] : []));
}

export async function verifyLockedKimiPackageGraph(root, lock) {
  if (lock.lockfileVersion !== 3 || !lock.packages || lock.packages[""]?.dependencies?.["@moonshot-ai/kimi-code"] !== KIMI_RUNTIME_VERSION) throw new Error("Kimi distribution lock is invalid");
  let packages = 0;
  for (const [path, entry] of Object.entries(lock.packages)) {
    if (!path) continue;
    if (!path.startsWith("node_modules/") || /[\\\u0000-\u001f\u007f]/.test(path) || path.split("/").some(part => !part || part === "." || part === "..") || entry.link || typeof entry.version !== "string" || !/^https:\/\/registry\.npmjs\.org\//.test(entry.resolved ?? "") || !/^sha512-[A-Za-z0-9+/]+=*$/.test(entry.integrity ?? "")) throw new Error("Kimi distribution lock contains an unpinned package");
    if (entry.optional === true) {
      try { await lstat(join(root, path)); } catch (error) { if (error.code === "ENOENT") continue; throw error; }
      throw new Error("Kimi headless distribution unexpectedly installed an optional TUI dependency");
    }
    const file = join(root, path, "package.json");
    if (await realpath(file) !== file || !(await lstat(file)).isFile()) throw new Error("Kimi distribution package metadata is not a regular file");
    const actual = JSON.parse(await readFile(file, "utf8"));
    if (actual.version !== entry.version) throw new Error(`Kimi distribution package differs from its lock: ${path}`);
    packages++;
  }
  return packages;
}

export function assertKimiNodeSystemDependencies(listing, platform) {
  for (const line of listing.split("\n").map(value => value.trim()).filter(Boolean)) {
    if (platform === "darwin") {
      if (line.endsWith(":")) continue;
      if (!line.startsWith("/usr/lib/") && !line.startsWith("/System/Library/")) throw new Error("Kimi Node depends on an unbundled library");
    } else if (!line.startsWith("linux-vdso.so.") && !/^(?:[^\s]+ => )?\/(?:usr\/)?lib(?:64)?\//.test(line)) throw new Error("Kimi Node depends on an unbundled library");
  }
}

async function nodeDistribution(staging, target, archivePath, environment, runOwned, signal) {
  const pin = KIMI_NODE_DISTRIBUTIONS[target];
  if (!pin) throw new Error(`Kimi distribution target is unsupported: ${target}`);
  let bytes;
  if (archivePath) bytes = await readFile(archivePath);
  else {
    const timeout = AbortSignal.timeout(60_000);
    const response = await fetch(`https://nodejs.org/dist/v${KIMI_NODE_VERSION}/node-v${KIMI_NODE_VERSION}-${target}.tar.gz`, { redirect: "error", signal: signal ? AbortSignal.any([signal, timeout]) : timeout });
    if (!response.ok || !response.body) throw new Error("Pinned Kimi Node download failed");
    const chunks = []; let length = 0;
    for await (const chunk of response.body) {
      length += chunk.length;
      if (length > 128 * 1024 * 1024) throw new Error("Kimi Node archive exceeds its bound");
      chunks.push(Buffer.from(chunk));
    }
    bytes = Buffer.concat(chunks);
  }
  if (bytes.length > 128 * 1024 * 1024 || hash(bytes) !== pin.archiveSha256) throw new Error("Kimi Node archive does not match its release pin");
  const archive = join(staging, `${target}.tar.gz`); await writeFile(archive, bytes);
  const output = join(staging, `node-${target}`); await mkdir(output);
  await runOwned("tar", ["-xzf", archive, "-C", output, "--strip-components=1", `node-v${KIMI_NODE_VERSION}-${target}/bin/node`, `node-v${KIMI_NODE_VERSION}-${target}/lib/node_modules/npm`], { env: environment, timeout: 30_000 });
  const executable = join(output, "bin/node");
  if (hash(await readFile(executable)) !== pin.executableSha256 || (await lstat(executable)).size !== pin.executableSize) throw new Error("Kimi Node executable does not match its target pin");
  const npm = join(output, "lib/node_modules/npm/bin/npm-cli.js");
  if (JSON.parse(await readFile(join(output, "lib/node_modules/npm/package.json"), "utf8")).version !== "11.19.0") throw new Error("Kimi Node archive has an unexpected npm version");
  return { executable, npm };
}

async function removePackageManagerLinks(root) {
  const visit = async directory => {
    for (const name of await readdir(directory)) {
      const path = join(directory, name), stat = await lstat(path);
      if (stat.isSymbolicLink()) {
        if (dirname(path).endsWith("/node_modules/.bin")) await unlink(path);
        else throw new Error("Kimi package payload contains an unsupported symbolic link");
      } else if (stat.isDirectory()) await visit(path);
      else if (!stat.isFile()) throw new Error("Kimi package payload contains a non-file resource");
    }
  };
  await visit(root);
  await rm(join(root, "node_modules/.package-lock.json"), { force: true });
}

export async function buildKimiDistribution({ outputRoot, target = `${process.platform}-${process.arch}`, lockDirectory = join(packageRoot, "scripts/kimi-distribution"), nodeArchives = {}, checkCancelled = () => {}, signal }) {
  checkCancelled(); signal?.throwIfAborted();
  if (typeof outputRoot !== "string" || !isAbsolute(outputRoot)) throw new Error("Kimi distribution output must be absolute");
  const output = resolve(outputRoot);
  if (["/", packageRoot, repositoryRoot].includes(output)) throw new Error("Refusing unsafe Kimi distribution output");
  if (!Object.hasOwn(KIMI_NODE_DISTRIBUTIONS, target)) throw new Error("Kimi distribution target is unsupported");
  const host = `${process.platform}-${process.arch}`;
  if (!Object.hasOwn(KIMI_NODE_DISTRIBUTIONS, host)) throw new Error("Kimi distribution build host is unsupported");
  try { await lstat(output); throw new Error("Kimi distribution output already exists"); } catch (error) { if (error.code !== "ENOENT") throw error; }
  const lockBytes = await readFile(join(lockDirectory, "package-lock.json"));
  if (hash(lockBytes) !== KIMI_DISTRIBUTION_LOCK_SHA256) throw new Error("Kimi distribution lock differs from its source pin");
  await mkdir(dirname(output), { recursive: true });
  const staging = await mkdtemp(join(await realpath(dirname(output)), ".paperclip-kimi-distribution-"));
  const environment = kimiDistributionEnvironment();
  const runOwned = (command, args, options) => runKimiBuildCommand(command, args, options, checkCancelled, signal);
  const runtime = join(staging, "runtime");
  try {
    const buildHome = join(staging, "build-home"); await mkdir(buildHome); environment.HOME = buildHome;
    const installedNode = await nodeDistribution(staging, target, nodeArchives[target], environment, runOwned, signal);
    const buildNode = host === target ? installedNode : await nodeDistribution(staging, host, nodeArchives[host], environment, runOwned, signal);
    if ((await runOwned(buildNode.executable, ["--version"], { env: {}, timeout: 10_000 })).stdout.trim() !== `v${KIMI_NODE_VERSION}`) throw new Error("Kimi distribution build Node cannot execute");
    await mkdir(runtime);
    for (const name of ["package.json", "package-lock.json"]) await copyFile(join(lockDirectory, name), join(runtime, name));
    await writeFile(join(runtime, ".npmrc"), "registry=https://registry.npmjs.org/\nignore-scripts=true\naudit=false\nfund=false\n");
    await writeFile(join(runtime, ".npmrc-global"), "");
    await runOwned(buildNode.executable, [buildNode.npm, ...kimiDistributionInstallCommand()], { cwd: runtime, env: environment, timeout: 300_000, maxBuffer: 4 * 1024 * 1024 });
    if (!(await readFile(join(runtime, "package-lock.json"))).equals(lockBytes)) throw new Error("Kimi installation changed its committed lock");
    const packageCount = await verifyLockedKimiPackageGraph(runtime, JSON.parse(lockBytes.toString("utf8")));
    const patch = await patchKimiRuntime(join(runtime, KIMI_DISTRIBUTION_ENTRYPOINT));
    await mkdir(join(runtime, "node/bin"), { recursive: true });
    await copyFile(installedNode.executable, join(runtime, "node/bin/node")); await chmod(join(runtime, "node/bin/node"), 0o755);
    if (host === target) {
      const listing = process.platform === "darwin"
        ? await runOwned("/usr/bin/otool", ["-L", join(runtime, "node/bin/node")], { env: {}, timeout: 10_000 })
        : await runOwned("ldd", [join(runtime, "node/bin/node")], { env: { PATH: "/usr/bin:/bin" }, timeout: 10_000 });
      assertKimiNodeSystemDependencies(listing.stdout, process.platform);
      if ((await runOwned(join(runtime, "node/bin/node"), ["--version"], { env: {}, timeout: 10_000 })).stdout.trim() !== `v${KIMI_NODE_VERSION}`) throw new Error("Kimi Node cannot execute after relocation");
    }
    await removePackageManagerLinks(runtime);
    const entries = await inventoryKimiRuntimeFiles(runtime);
    const closure = kimiRuntimeClosureDigest(entries);
    const metadata = { schema: "paperclip.kimi-distribution.v1", version: KIMI_RUNTIME_VERSION, target, nodeVersion: KIMI_NODE_VERSION, runtimeRoot: "runtime", entrypoint: KIMI_DISTRIBUTION_ENTRYPOINT, lockSha256: KIMI_DISTRIBUTION_LOCK_SHA256, patch, packageCount, nativeClosureSha256: closure };
    await writeFile(join(staging, "native-closure.json"), `${JSON.stringify({ entries })}\n`);
    await writeFile(join(staging, "kimi-distribution.json"), `${JSON.stringify(metadata, null, 2)}\n`);
    for (const name of await readdir(staging)) if (!["runtime", "native-closure.json", "kimi-distribution.json"].includes(name)) await rm(join(staging, name), { recursive: true, force: true });
    checkCancelled();
    await rename(staging, output);
    return { outputRoot: output, runtimeRoot: join(output, "runtime"), version: KIMI_RUNTIME_VERSION, closureDigest: `sha256:${closure}`, metadata };
  } catch (error) { await rm(staging, { recursive: true, force: true }); throw error; }
}

export async function materializeKimiDistribution(options) {
  if (typeof options.outputRoot !== "string" || !isAbsolute(options.outputRoot)) throw new Error("Kimi distribution output must be absolute");
  const output = resolve(options.outputRoot);
  if (["/", packageRoot, repositoryRoot].includes(output)) throw new Error("Refusing unsafe Kimi distribution output");
  const target = options.target ?? `${process.platform}-${process.arch}`;
  if (!Object.hasOwn(KIMI_DISTRIBUTION_CLOSURE_SHA256, target)) throw new Error("Kimi distribution target is unsupported");
  try { await lstat(output); throw new Error("Kimi distribution output already exists"); } catch (error) { if (error.code !== "ENOENT") throw error; }
  await mkdir(dirname(output), { recursive: true });
  const parent = await realpath(dirname(output));
  const temporary = await mkdtemp(join(parent, ".paperclip-kimi-admission-"));
  const candidate = join(temporary, "candidate");
  try {
    const result = await buildKimiDistribution({ ...options, outputRoot: candidate });
    if (result.metadata.nativeClosureSha256 !== KIMI_DISTRIBUTION_CLOSURE_SHA256[target]) throw new Error("Kimi native closure differs from its trusted target pin");
    try { await lstat(options.outputRoot); throw new Error("Kimi distribution output already exists"); } catch (error) { if (error.code !== "ENOENT") throw error; }
    await rename(candidate, options.outputRoot);
    return { ...result, outputRoot: options.outputRoot, runtimeRoot: join(options.outputRoot, "runtime") };
  } finally { await rm(temporary, { recursive: true, force: true }); }
}

if (import.meta.url.endsWith("/materialize-kimi-distribution.mjs") && process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
  const args = process.argv.slice(2);
  if (args.length !== 1) throw new Error("Usage: node scripts/materialize-kimi-distribution.mjs /absolute/output-directory");
  materializeKimiDistribution({ outputRoot: args[0] }).then(result => console.log(JSON.stringify(result))).catch(error => { console.error(error.message); process.exitCode = 1; });
}