PaperClipAI/scripts/release-runner-artifacts.mjs

// Data transfer for the existing release assembler. Producer jobs have no
// publishing credentials; consumers never execute binaries from the archive.
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { existsSync, lstatSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, isAbsolute, join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { gzipSync, gunzipSync } from 'node:zlib';
import { runnerBinaryTarget } from '../packages/paperclip-runner/src/live/runner-binary.ts';
import { verifyReleaseProviderPack } from '../packages/paperclip-runner/scripts/release-provider-pack.mjs';

const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..');
export const RELEASE_RUNNER_TARGETS = ['darwin-arm64', 'darwin-x64', 'linux-x64'];
const portableLinuxBuildTarget = 'x86_64-unknown-linux-musl';
const dataPaths = [...RELEASE_RUNNER_TARGETS.map(target => `bin/${target}/paperclip-runnerd`),
  'bin/release-manifest.json', 'remote-provider-packs/linux-x64/provider-pack.json'];
const archivePaths = [...dataPaths, 'assembly-receipt.json'];
const maxBinary = 128 * 1024 * 1024;
const maxArchive = 3 * maxBinary + 2 * 1024 * 1024;
const digest = value => `sha256:${createHash('sha256').update(value).digest('hex')}`;
const revision = value => assert.match(value ?? '', /^[a-f0-9]{40}$/, 'Runner artifacts require a full source SHA');
function file(path, limit = maxBinary) {
  const stat = lstatSync(path);
  assert.ok(stat.isFile() && !stat.isSymbolicLink() && stat.size <= limit, `Unsafe or oversized artifact ${path}`);
  return readFileSync(path);
}
function inside(root, path, limit = maxBinary) {
  const components = path.split('/');
  for (let length = 0; length < components.length; length++) {
    const directory = length === 0 ? root : join(root, ...components.slice(0, length));
    const stat = lstatSync(directory);
    assert.ok(stat.isDirectory() && !stat.isSymbolicLink(), 'Artifact directories must not be symlinks');
  }
  return file(join(root, path), limit);
}
function destination(root, path) {
  assert.ok(archivePaths.includes(path), 'Unknown artifact destination');
  const components = path.split('/');
  for (let length = 0; length < components.length; length++) {
    const directory = length === 0 ? root : join(root, ...components.slice(0, length));
    if (!existsSync(directory)) mkdirSync(directory);
    const stat = lstatSync(directory);
    assert.ok(stat.isDirectory() && !stat.isSymbolicLink(), 'Artifact output directories must not be symlinks');
  }
  const output = join(root, path);
  let existing;
  try { existing = lstatSync(output); }
  catch (error) { if (error.code !== 'ENOENT') throw error; }
  assert.ok(!existing || existing.isFile() && !existing.isSymbolicLink() && existing.nlink === 1,
    'Refuse to overwrite an artifact link or non-file');
  return output;
}
function newDirectory(path) {
  assert.ok(isAbsolute(path) && resolve(path) === path && path !== '/' && !existsSync(path), 'Artifact output must be a new absolute owned directory');
  mkdirSync(path, { recursive: true, mode: 0o700 });
}
function writeJson(path, value) { writeFileSync(path, JSON.stringify(value, null, 2) + '\n'); }
function imageProvenance(path, sourceRevision, requestedImage = '') {
  const [image] = JSON.parse(file(path, 512 * 1024));
  assert.equal(image.Os, 'linux'); assert.equal(image.Architecture, 'amd64');
  assert.equal(image.Config?.Labels?.['org.opencontainers.image.revision'], sourceRevision);
  assert.match(image.Id, /^sha256:[a-f0-9]{64}$/);
  if (requestedImage) {
    assert.match(requestedImage, /^ghcr\.io\/paperclipai\/paperclip@sha256:[a-f0-9]{64}$/);
    assert.ok(image.RepoDigests?.includes(requestedImage));
  }
  return { sourceRevision, platform: 'linux/amd64', imageId: image.Id, requestedImage, repoDigests: image.RepoDigests ?? [] };
}

function sourcePins(sourceRoot = repo) {
  return { profiles: file(join(sourceRoot, 'packages/paperclip-runner/acpx-profiles.json'), 512 * 1024),
    distributions: file(join(sourceRoot, 'packages/paperclip-runner/cursor-distributions.json'), 512 * 1024) };
}
// The unbuilt producer cannot import the compiled eval join. Check its same
// daemon contracts here, including the durable/remote capabilities needed by
// production. Metadata is a compatibility claim, not embedded source proof.
export function verifyReleaseDaemonMetadata(metadata) {
  assert.equal(metadata?.schema, 'paperclip-runner/runnerd-build-metadata/v1');
  assert.equal(metadata.binaryName, 'paperclip-runnerd', 'Unexpected release daemon name');
  assert.equal(metadata.packageName, '@paperclipai/paperclip-runner', 'Unexpected release daemon package');
  assert.ok(typeof metadata.packageVersion === 'string' && metadata.packageVersion.trim(), 'Missing daemon package version');
  for (const [field, expected] of [['binaryContractVersion', 2], ['nativeExecutionVersion', 1], ['harnessDriverVersion', 1]]) {
    assert.equal(metadata[field], expected, `Incompatible daemon ${field}`);
  }
  assert.equal(metadata.prp?.name, 'paperclip.runner', 'Incompatible daemon protocol family');
  assert.ok(Number.isSafeInteger(metadata.prp.minimumVersion) && metadata.prp.minimumVersion === 1 &&
    Number.isSafeInteger(metadata.prp.maximumVersion) && metadata.prp.maximumVersion >= 2, 'Incompatible daemon protocol versions');
  for (const [field, required] of [
    ['durableSessionCapabilities', ['unlimited_runtime', 'connection_lease_renewal']],
    ['prpTransportModes', ['dial_ws_loopback', 'dial_wss', 'listen_ws']],
  ]) {
    assert.ok(Array.isArray(metadata[field]) && required.every(capability => metadata[field].includes(capability)),
      `Missing release daemon ${field}`);
  }
  return metadata;
}
// npm hosts need not use the Docker image's libc. Validate the executable's
// actual load contract, rather than trusting a producer's static-link claim.
export function verifyPortableLinuxDaemon(bytes) {
  assert.ok(bytes.length >= 64 && bytes.length <= maxBinary && runnerBinaryTarget(bytes) === 'linux-x64'
    && bytes[4] === 2 && bytes[5] === 1 && bytes[6] === 1 && [2, 3].includes(bytes.readUInt16LE(16))
    && bytes.readUInt16LE(52) === 64, 'Portable Linux daemon requires a complete x64 ELF header');
  const offset = Number(bytes.readBigUInt64LE(32)), size = bytes.readUInt16LE(54), count = bytes.readUInt16LE(56);
  assert.ok(Number.isSafeInteger(offset) && offset >= 64 && size === 56 && count > 0 && count <= 256
    && offset + size * count <= bytes.length, 'Portable Linux daemon has an invalid program table');
  let executableLoad = false;
  for (let index = 0; index < count; index++) {
    const header = offset + index * size, type = bytes.readUInt32LE(header);
    assert.notEqual(type, 3, 'Portable Linux daemon cannot require a libc interpreter');
    if (type === 1 && bytes.readUInt32LE(header + 4) & 1) executableLoad = true;
    if (type !== 2) continue;
    const start = Number(bytes.readBigUInt64LE(header + 8)), length = Number(bytes.readBigUInt64LE(header + 32));
    assert.ok(Number.isSafeInteger(start) && Number.isSafeInteger(length) && length > 0 && length <= 1024 * 1024
      && length % 16 === 0 && start >= 64 && start + length <= bytes.length, 'Portable Linux daemon has an invalid dynamic table');
    let terminated = false;
    for (let entry = start; entry < start + length; entry += 16) {
      const tag = bytes.readBigUInt64LE(entry);
      assert.notEqual(tag, 1n, 'Portable Linux daemon cannot require a shared library');
      if (tag === 0n) { terminated = true; break; }
    }
    assert.ok(terminated, 'Portable Linux daemon has an unterminated dynamic table');
  }
  assert.ok(executableLoad, 'Portable Linux daemon requires an executable load segment');
  return { buildTarget: portableLinuxBuildTarget, linkage: 'static' };
}
export function verifyRunnerReleaseData(files, sourceRevision, pins = sourcePins()) {
  revision(sourceRevision);
  assert.deepEqual([...files.keys()].sort(), [...archivePaths].sort(), 'Release archive must contain exactly the known data files');
  const receipt = JSON.parse(files.get('assembly-receipt.json'));
  assert.equal(receipt.schema, 'paperclip.runner.release-assembly.v1');
  assert.equal(receipt.sourceRevision, sourceRevision, 'Assembly source mismatch');
  assert.deepEqual(Object.keys(receipt.files ?? {}).sort(), [...dataPaths].sort());
  for (const path of dataPaths) assert.equal(digest(files.get(path)), receipt.files[path], `Assembly artifact digest mismatch: ${path}`);
  const manifest = JSON.parse(files.get('bin/release-manifest.json'));
  assert.equal(manifest.schema, 'paperclip.runner.release-binaries.v1');
  assert.equal(manifest.sourceRevision, sourceRevision, 'Daemon source mismatch');
  assert.deepEqual(Object.keys(manifest.platforms ?? {}).sort(), [...RELEASE_RUNNER_TARGETS].sort(), 'Missing or duplicate release target');
  assert.deepEqual(Object.keys(receipt.daemonMetadata ?? {}).sort(), [...RELEASE_RUNNER_TARGETS].sort(), 'Missing daemon compatibility receipt');
  for (const target of RELEASE_RUNNER_TARGETS) {
    const path = `bin/${target}/paperclip-runnerd`, artifact = manifest.platforms[target];
    assert.equal(artifact.path, `${target}/paperclip-runnerd`, 'Unsafe daemon path');
    assert.equal(runnerBinaryTarget(files.get(path)), target, 'Daemon architecture mismatch');
    assert.equal(digest(files.get(path)), artifact.sha256, 'Daemon digest mismatch');
    verifyReleaseDaemonMetadata(receipt.daemonMetadata[target]);
  }
  assert.equal(receipt.sourcePins?.profiles, digest(pins.profiles), 'Checked-out source profile snapshot mismatch');
  assert.equal(receipt.sourcePins?.distributions, digest(pins.distributions), 'Checked-out source distribution snapshot mismatch');
  const pack = verifyReleaseProviderPack(JSON.parse(files.get('remote-provider-packs/linux-x64/provider-pack.json')), sourceRevision,
    JSON.parse(pins.profiles), JSON.parse(pins.distributions));
  assert.equal(manifest.remoteProviderPack?.target, 'linux-x64');
  assert.equal(manifest.remoteProviderPack?.digest, pack.digest, 'Provider pack assembly identity mismatch');
  assert.equal(manifest.remoteProviderPack?.sourceRevision, sourceRevision);
  assert.equal(receipt.providerImage?.sourceRevision, sourceRevision, 'Provider image source mismatch');
  assert.equal(receipt.providerImage?.platform, 'linux/amd64');
  assert.match(receipt.providerImage?.imageId ?? '', /^sha256:[a-f0-9]{64}$/);
  if (receipt.providerImage.requestedImage) {
    assert.match(receipt.providerImage.requestedImage, /^ghcr\.io\/paperclipai\/paperclip@sha256:[a-f0-9]{64}$/);
    assert.ok(receipt.providerImage.repoDigests?.includes(receipt.providerImage.requestedImage), 'Provider image digest mismatch');
  }
  assert.deepEqual(receipt.linuxDaemon, { sourceRevision, ...verifyPortableLinuxDaemon(files.get('bin/linux-x64/paperclip-runnerd')) },
    'Linux npm daemon must bind the portable exact-source build');
  return receipt;
}

// A deliberately small ustar data envelope: no directories, links, extensions,
// executable hooks or arbitrary paths. Validate the entire envelope before any
// destination file exists, including compressed/uncompressed bounds and hashes.
export function readRunnerReleaseArchive(bytes, sourceRevision, checksum, pins) {
  assert.ok(bytes.length <= maxArchive, 'Release archive is oversized');
  assert.equal(digest(bytes), checksum, 'Release archive checksum mismatch');
  const tar = gunzipSync(bytes, { maxOutputLength: maxArchive });
  const files = new Map();
  const octal = bytes => {
    const value = bytes.toString('ascii').replace(/\0.*$/s, '').trim();
    assert.match(value, /^[0-7]+$/, 'Invalid archive numeric field');
    return parseInt(value, 8);
  };
  let offset = 0;
  while (offset + 512 <= tar.length) {
    const header = tar.subarray(offset, offset + 512);
    if (header.every(byte => byte === 0)) {
      assert.ok(tar.subarray(offset).every(byte => byte === 0), 'Unexpected archive trailer');
      break;
    }
    const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, '');
    assert.ok(archivePaths.includes(name) && !files.has(name), 'Unsafe, unexpected or duplicate archive path');
    assert.ok(header.subarray(345, 500).every(byte => byte === 0), 'Archive prefixes are unsupported');
    assert.ok(header[156] === 0 || header[156] === 48, 'Archive links and extensions are forbidden');
    assert.ok(header.subarray(157, 257).every(byte => byte === 0), 'Archive link target is forbidden');
    const sum = [...header].reduce((sum, byte, index) => sum + (index >= 148 && index < 156 ? 32 : byte), 0);
    assert.equal(octal(header.subarray(148, 156)), sum, 'Archive header checksum mismatch');
    const size = octal(header.subarray(124, 136));
    assert.ok(size <= (name.endsWith('paperclip-runnerd') ? maxBinary : 512 * 1024) && offset + 512 + size <= tar.length, 'Oversized or truncated archive member');
    const mode = octal(header.subarray(100, 108));
    assert.equal(mode, name.endsWith('paperclip-runnerd') ? 0o755 : 0o644, 'Unexpected archive file permissions');
    files.set(name, tar.subarray(offset + 512, offset + 512 + size));
    offset += 512 + Math.ceil(size / 512) * 512;
  }
  assert.ok(offset + 1024 <= tar.length, 'Archive must have a complete end marker');
  verifyRunnerReleaseData(files, sourceRevision, pins);
  return files;
}

export function makeRunnerReleaseArchive(files, sourceRevision) {
  verifyRunnerReleaseData(files, sourceRevision);
  const chunks = [];
  for (const path of archivePaths) {
    const bytes = files.get(path), header = Buffer.alloc(512);
    header.write(path);
    const field = (offset, width, value) => header.write(value.toString(8).padStart(width - 1, '0') + '\0', offset, width, 'ascii');
    field(100, 8, path.endsWith('paperclip-runnerd') ? 0o755 : 0o644);
    field(108, 8, 0); field(116, 8, 0); field(124, 12, bytes.length); field(136, 12, 0);
    header.fill(32, 148, 156); header[156] = 48;
    header.write('ustar\0', 257, 6, 'ascii'); header.write('00', 263, 2, 'ascii');
    const sum = [...header].reduce((sum, byte) => sum + byte, 0);
    header.write(sum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii');
    chunks.push(header, bytes, Buffer.alloc((512 - bytes.length % 512) % 512));
  }
  chunks.push(Buffer.alloc(1024));
  return gzipSync(Buffer.concat(chunks));
}

async function main() {
  const [mode, sourceRevision, ...args] = process.argv.slice(2);
  revision(sourceRevision);
  if (mode === 'record-binary') {
    const [target, binary, output, ...extra] = args;
    assert.ok(RELEASE_RUNNER_TARGETS.includes(target) && !extra.length);
    const bytes = file(binary); assert.equal(runnerBinaryTarget(bytes), target);
    assert.equal(`${process.platform}-${process.arch}`, target, 'Execute metadata only on the daemon native target');
    const portable = target === 'linux-x64' ? verifyPortableLinuxDaemon(bytes) : undefined;
    const metadata = JSON.parse(execFileSync(binary, ['--build-metadata'], { timeout: 15_000, maxBuffer: 128 * 1024 }));
    verifyReleaseDaemonMetadata(metadata);
    newDirectory(output); writeFileSync(join(output, 'paperclip-runnerd'), bytes, { mode: 0o755 });
    writeJson(join(output, 'receipt.json'), { sourceRevision, target, sha256: digest(bytes), metadata,
      ...(portable ? { linuxDaemon: { sourceRevision, ...portable } } : {}) });
  } else if (mode === 'record-image-binary') {
    const [binary, metadataPath, imageMetadata, output, requestedImage = '', ...extra] = args;
    assert.equal(extra.length, 0);
    const bytes = file(binary); assert.equal(runnerBinaryTarget(bytes), 'linux-x64');
    // Metadata was executed in this same image, whose libc matches the daemon.
    // The host validates data only and must never execute the extracted ELF.
    const metadata = verifyReleaseDaemonMetadata(JSON.parse(file(metadataPath, 128 * 1024)));
    const providerImage = imageProvenance(imageMetadata, sourceRevision, requestedImage);
    newDirectory(output); writeFileSync(join(output, 'paperclip-runnerd'), bytes, { mode: 0o755 });
    writeJson(join(output, 'receipt.json'), { sourceRevision, target: 'linux-x64', sha256: digest(bytes), metadata, providerImage });
  } else if (mode === 'record-pack') {
    const [packPath, imageMetadata, output, requestedImage = '', ...extra] = args; assert.equal(extra.length, 0);
    const bytes = file(packPath, 512 * 1024); verifyReleaseProviderPack(JSON.parse(bytes), sourceRevision);
    const providerImage = imageProvenance(imageMetadata, sourceRevision, requestedImage);
    newDirectory(output); writeFileSync(join(output, 'provider-pack.json'), bytes);
    writeJson(join(output, 'receipt.json'), { sourceRevision, sha256: digest(bytes), providerImage });
  } else if (mode === 'assemble') {
    const [artifacts, output, ...extra] = args; assert.equal(extra.length, 0);
    const platforms = {}, daemonMetadata = {};
    let linuxDaemon;
    assert.deepEqual((await import('node:fs')).readdirSync(artifacts).sort(), [...RELEASE_RUNNER_TARGETS, 'provider-pack'].sort());
    for (const target of RELEASE_RUNNER_TARGETS) {
      const path = join(artifacts, target, 'paperclip-runnerd'), bytes = inside(artifacts, `${target}/paperclip-runnerd`);
      const receipt = JSON.parse(inside(artifacts, `${target}/receipt.json`, 128 * 1024));
      assert.equal(receipt.sourceRevision, sourceRevision); assert.equal(receipt.target, target);
      daemonMetadata[target] = verifyReleaseDaemonMetadata(receipt.metadata);
      if (target === 'linux-x64') {
        linuxDaemon = { sourceRevision, ...verifyPortableLinuxDaemon(bytes) };
        assert.deepEqual(receipt.linuxDaemon, linuxDaemon, 'Linux producer must bind the portable exact-source build');
      }
      assert.equal(receipt.sha256, digest(bytes)); assert.equal(runnerBinaryTarget(bytes), target);
      platforms[target] = { path: resolve(path), sha256: receipt.sha256 };
    }
    const packPath = join(artifacts, 'provider-pack/provider-pack.json'), bytes = inside(artifacts, 'provider-pack/provider-pack.json', 512 * 1024);
    const packReceipt = JSON.parse(inside(artifacts, 'provider-pack/receipt.json', 128 * 1024));
    assert.equal(packReceipt.sourceRevision, sourceRevision); assert.equal(packReceipt.sha256, digest(bytes));
    verifyReleaseProviderPack(JSON.parse(bytes), sourceRevision);
    newDirectory(output);
    const manifestPath = join(output, 'assembler-input.json');
    writeJson(manifestPath, { sourceRevision, platforms, remoteProviderPack: { path: resolve(packPath), sha256: packReceipt.sha256 } });
    // The existing assembler owns source/profile-aware staging. Never replace
    // it with a second daemon assembly implementation in this transfer helper.
    execFileSync(process.execPath, [join(repo, 'packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs'), manifestPath], { stdio: 'inherit' });
    const files = new Map(dataPaths.map(path => [path, file(join(repo, 'packages/paperclip-runner/dist', path), path.endsWith('paperclip-runnerd') ? maxBinary : 512 * 1024)]));
    const pins = sourcePins();
    const receipt = { schema: 'paperclip.runner.release-assembly.v1', sourceRevision,
      sourcePins: { profiles: digest(pins.profiles), distributions: digest(pins.distributions) }, daemonMetadata,
      providerImage: packReceipt.providerImage, linuxDaemon, files: Object.fromEntries([...files].map(([path, bytes]) => [path, digest(bytes)])) };
    files.set('assembly-receipt.json', Buffer.from(JSON.stringify(receipt, null, 2) + '\n'));
    const archive = makeRunnerReleaseArchive(files, sourceRevision);
    writeFileSync(join(output, 'runner-release-assets.tar.gz'), archive);
    writeFileSync(join(output, 'runner-release-assets.tar.gz.sha256'), digest(archive) + '\n');
  } else if (mode === 'unpack') {
    const [archive, checksum, output, sourceRoot = repo, ...extra] = args; assert.equal(extra.length, 0);
    const files = readRunnerReleaseArchive(file(archive, maxArchive), sourceRevision, file(checksum, 256).toString().trim(), sourcePins(sourceRoot));
    newDirectory(output);
    for (const [path, bytes] of files) {
      writeFileSync(destination(output, path), bytes, { mode: path.endsWith('paperclip-runnerd') ? 0o755 : 0o644 });
    }
    console.log(JSON.stringify({ sourceRevision, targets: RELEASE_RUNNER_TARGETS, archiveSha256: digest(file(archive, maxArchive)), providerCalls: 0 }));
  } else throw new Error('Unknown release artifact operation');
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) await main();