PaperClipAI/scripts/create-provider-pack-archive.mjs

import { createHash } from 'node:crypto';
import { createReadStream, createWriteStream } from 'node:fs';
import { lstat, mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises';
import { spawn } from 'node:child_process';
import { pipeline } from 'node:stream/promises';
import { constants, createZstdCompress } from 'node:zlib';
import { resolve } from 'node:path';
import { pathToFileURL } from 'node:url';

export async function createProviderPackArchive(root) {
  const manifest = JSON.parse(await readFile(resolve(root, 'provider-pack.json'), 'utf8'));
  if (manifest.schema !== 'paperclip-runner/remote-provider-pack/v1'
    || !/^sha256:[a-f0-9]{64}$/.test(manifest.digest ?? '')) throw Error('Invalid provider pack identity');
  const archive = `${root}.tar.zst`;
  const tar = spawn('tar', ['--no-xattrs', '-cf', '-', '-C', root, '.'], {
    env: { ...process.env, COPYFILE_DISABLE: '1' }, stdio: ['ignore', 'pipe', 'inherit'],
  });
  const completed = new Promise((resolve, reject) => {
    tar.on('error', reject);
    tar.on('close', code => code === 0 ? resolve() : reject(Error('Provider pack archive creation failed')));
  });
  try {
    await Promise.all([completed, pipeline(tar.stdout,
      createZstdCompress({ params: { [constants.ZSTD_c_compressionLevel]: 3 } }), createWriteStream(archive))]);
  } catch (error) {
    tar.kill();
    throw error;
  }
  const hash = createHash('sha256');
  for await (const chunk of createReadStream(archive)) hash.update(chunk);
  const receipt = {
    schema: 'paperclip-runner/provider-pack-archive/v1',
    encoding: 'tar+zstd',
    providerPackDigest: manifest.digest,
    sha256: hash.digest('hex'),
    bytes: (await stat(archive)).size,
  };
  await writeFile(`${root}.archive.json`, JSON.stringify(receipt) + '\n');
  return receipt;
}

// Keep large immutable files separate so a fresh sandbox can reuse identical
// image bytes without trusting that image's older provider-pack identity.
export async function createProviderPackLayers(root) {
  const manifest = JSON.parse(await readFile(resolve(root, 'provider-pack.json'), 'utf8'));
  if (manifest.schema !== 'paperclip-runner/remote-provider-pack/v1'
    || !/^sha256:[a-f0-9]{64}$/.test(manifest.digest ?? '')) throw Error('Invalid provider pack identity');
  const directory = `${root}.archives`;
  await mkdir(directory, { recursive: true });
  const small = [], files = [];
  const visit = async (relative = '') => {
    for (const name of (await readdir(resolve(root, relative))).sort()) {
      const path = relative ? `${relative}/${name}` : name;
      const metadata = await lstat(resolve(root, path));
      if (metadata.isDirectory()) { small.push(path); await visit(path); }
      else if (metadata.isFile() && metadata.size >= 1024 * 1024) {
        const hash = createHash('sha256');
        for await (const chunk of createReadStream(resolve(root, path))) hash.update(chunk);
        files.push({ path, bytes: metadata.size, mode: metadata.mode & 0o777, sha256: hash.digest('hex') });
      } else if (metadata.isFile() || metadata.isSymbolicLink()) small.push(path);
      else throw Error('Unsupported provider pack entry');
    }
  };
  await visit();
  const archive = async (name, paths) => {
    const target = resolve(directory, name);
    // Explicit NUL-separated names and no recursion avoid options, duplicate
    // traversal, and hardlinks from an archive to files in a different layer.
    const tar = spawn('tar', ['--no-xattrs', '--no-recursion', '-cf', '-', '-C', root,
      '--null', '-T', '-'], { env: { ...process.env, COPYFILE_DISABLE: '1' }, stdio: ['pipe', 'pipe', 'inherit'] });
    const completed = new Promise((resolve, reject) => {
      tar.on('error', reject);
      tar.on('close', code => code === 0 ? resolve() : reject(Error('Provider pack layer creation failed')));
    });
    tar.stdin.on('error', () => {}); // The child/pipeline supplies the causal failure.
    tar.stdin.end(paths.map(path => `./${path}\0`).join(''));
    try {
      await Promise.all([completed, pipeline(tar.stdout,
        createZstdCompress({ params: { [constants.ZSTD_c_compressionLevel]: 3 } }), createWriteStream(target))]);
    } catch (error) { tar.kill(); throw error; }
    const hash = createHash('sha256');
    for await (const chunk of createReadStream(target)) hash.update(chunk);
    return { sha256: hash.digest('hex'), bytes: (await stat(target)).size };
  };
  const overlay = await archive('overlay.tar.zst', small);
  for (const [index, file] of files.entries()) file.archive = await archive(`${index}.tar.zst`, [file.path]);
  const receipt = { schema: 'paperclip-runner/provider-pack-archive/v2', encoding: 'tar+zstd',
    providerPackDigest: manifest.digest, overlay, files };
  await writeFile(`${root}.archive.json`, JSON.stringify(receipt) + '\n');
  return receipt;
}

if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
  if (!process.argv[2] || process.argv[3]) throw Error('Usage: create-provider-pack-archive.mjs PACK_ROOT');
  await createProviderPackLayers(resolve(process.argv[2]));
}