PaperClipAI/server/src/services/skill-package-runtime.ts

import { measureRunPhase } from "./execution-preparation-timing.js";
import { promises as fs } from "node:fs";
import path from "node:path";
import type { Db } from "@paperclipai/db";
import type { CompanySkillVersionFileInventoryEntry } from "@paperclipai/shared";
import { skillPackageStore, type StoredSkillVersion } from "./skill-package-store.js";
import { assertSkillPackageFileSize, packageSha256, verifySkillPackageDirectory, relocateSkillPackageVerification, type SkillPackageFile, type SkillPackageManifest } from "./skill-package-files.js";
import { REPOSITORY_CONTENT_DIR, repositorySkillProjection } from "./skill-repository-package.js";
import { assertSkillSnapshotPath, skillFileBytes } from "./skill-snapshot.js";
import { ensureSkillCacheDirectory } from "./skill-version-cache.js";

type ProjectedFile = SkillPackageFile & { sourcePath?: string; generated?: Buffer };
// Only the wrapper/policy bytes and file metadata are retained, never a decoded repository.
const projections = new Map<string, { files: ProjectedFile[]; bytes: number }>();
let projectionBytes = 0;
const MAX_PROJECTION_BYTES = 16 * 1024 * 1024;
const MAX_PROJECTIONS = 256;
const repositoryFiles = new WeakMap<SkillPackageManifest, ProjectedFile[]>();
function originalFiles(version: StoredSkillVersion, manifest: SkillPackageManifest): ProjectedFile[] {
  if (version.repositorySnapshotId) {
    let shared = repositoryFiles.get(manifest);
    if (!shared) {
      shared = manifest.files.map(file => ({ ...file, sourcePath: file.path, path: `${REPOSITORY_CONTENT_DIR}/${file.path}` }));
      repositoryFiles.set(manifest, shared);
    }
    return [...shared];
  }
  if (!version.packageFileMap) return manifest.files.map(file => ({ ...file, sourcePath: file.path }));
  const byPath = new Map(manifest.files.map(file => [file.path, file]));
  return version.packageFileMap.map(mapping => {
    assertSkillSnapshotPath(mapping.path);
    assertSkillSnapshotPath(mapping.sourcePath);
    const file = byPath.get(mapping.sourcePath);
    if (!file) throw new Error("Skill version references a missing package file");
    return { ...file, ...mapping, encoding: mapping.encoding ?? file.encoding };
  });
}
export function skillPackageRuntime(db: Db) {
  const store = skillPackageStore(db);
  function readOriginalFile(version: StoredSkillVersion, sourcePath: string, maxBytes?: number, projected?: ProjectedFile) {
    return store.readVersionFile({ ...version, packageSnapshotId: version.repositorySnapshotId ?? version.packageSnapshotId,
      packageFileMap: projected ? [{ path: sourcePath, sourcePath, kind: projected.kind, encoding: projected.encoding }] : null }, sourcePath, true, maxBytes);
  }
  async function inventory(version: StoredSkillVersion) {
    const id = version.repositorySnapshotId ?? version.packageSnapshotId;
    if (!id) throw new Error("Skill version has no package");
    const manifest = await store.manifest(version.companyId, id);
    const files = originalFiles(version, manifest).map(({ path, kind }) => ({ path, kind }));
    if (version.repositorySnapshotId) {
      files.unshift({ path: "SKILL.md", kind: "skill" });
      const policy = path.posix.join(path.posix.dirname(version.repositorySkillPath!), "agents/openai.yaml");
      const file = manifest.files.find(file => file.path === policy);
      if (file) files.push({ path: "agents/openai.yaml", kind: file.kind });
    }
    return files;
  }
  async function projection(version: StoredSkillVersion, maxBytes?: number): Promise<ProjectedFile[]> {
    const id = version.repositorySnapshotId ?? version.packageSnapshotId;
    if (!id) throw new Error("Skill version has no package");
    const manifest = await store.manifest(version.companyId, id);
    const key = `${version.companyId}:${version.id}:${manifest.contentHash}`;
    const files = originalFiles(version, manifest);
    const cached = projections.get(key);
    if (cached) return [...files, ...cached.files];
    const generatedFiles: ProjectedFile[] = [];
    if (version.repositorySnapshotId) {
      if (!version.repositorySkillPath) throw new Error("Repository skill entrypoint is missing");
      const wanted = new Set([version.repositorySkillPath, path.posix.join(path.posix.dirname(version.repositorySkillPath), "agents/openai.yaml")]);
      for (const file of manifest.files.filter(file => wanted.has(file.path))) assertSkillPackageFileSize(file.size, maxBytes);
      const selected: CompanySkillVersionFileInventoryEntry[] = [];
      for (const file of manifest.files.filter(file => wanted.has(file.path))) {
        const original = await readOriginalFile(version, file.path, maxBytes);
        if (!original) throw new Error("Repository skill file is missing");
        selected.push(original);
      }
      const generated = repositorySkillProjection(selected, version.repositorySkillPath!).filter(file => !file.path.startsWith(`${REPOSITORY_CONTENT_DIR}/`));
      for (const file of generated) {
        const bytes = skillFileBytes(file);
        generatedFiles.push({ path: file.path, kind: file.kind, encoding: file.encoding ?? "utf8", executable: Boolean(file.executable),
          size: bytes.length, sha256: packageSha256(bytes), generated: bytes });
      }
    }
    // Retain only tiny generated wrappers/policies per version. Shared package
    // file metadata follows the bounded manifest cache through the WeakMap.
    const bytes = generatedFiles.reduce((sum, file) => sum + (file.generated?.length ?? 0) + 512, 0);
    while (projections.size && (projections.size >= MAX_PROJECTIONS || projectionBytes + bytes > MAX_PROJECTION_BYTES)) {
      const first = projections.keys().next().value!;
      projectionBytes -= projections.get(first)!.bytes;
      projections.delete(first);
    }
    if (generatedFiles.length && bytes <= MAX_PROJECTION_BYTES) { projections.set(key, { files: generatedFiles, bytes }); projectionBytes += bytes; }
    return [...files, ...generatedFiles];
  }
  async function materialize(version: StoredSkillVersion, directory: string) {
    const files = await measureRunPhase("skills.repository.read", () => projection(version));
    const parent = path.dirname(directory);
    await ensureSkillCacheDirectory(parent, true);
    if (await measureRunPhase("skills.version.verify", () => verifySkillPackageDirectory(directory, files), { fileCount: files.length })) return directory;
    return store.withPackage(version.companyId, (version.repositorySnapshotId ?? version.packageSnapshotId)!, async root => {
      const staging = await fs.mkdtemp(path.join(parent, `.${version.id}-`));
      try {
        for (const file of files) {
          const target = path.join(staging, file.path);
          await fs.mkdir(path.dirname(target), { recursive: true });
          if (file.generated) await fs.writeFile(target, file.generated);
          else await fs.copyFile(path.join(root, file.sourcePath!), target);
          await fs.chmod(target, file.executable ? 0o555 : 0o444);
        }
        if (!await verifySkillPackageDirectory(staging, files)) throw new Error("Prepared skill version failed integrity verification");
        await fs.rm(directory, { recursive: true, force: true });
        await fs.rename(staging, directory);
        relocateSkillPackageVerification(staging, directory, files);
        return directory;
      } finally { await fs.rm(staging, { recursive: true, force: true }); }
    });
  }
  async function readFile(version: StoredSkillVersion, relative: string, maxBytes?: number) {
    const id = (version.repositorySnapshotId ?? version.packageSnapshotId)!;
    const manifest = await store.manifest(version.companyId, id);
    const file = originalFiles(version, manifest).find(file => file.path === relative)
      ?? (await projection(version, maxBytes)).find(file => file.path === relative);
    if (!file) return null;
    assertSkillPackageFileSize(file.size, maxBytes);
    const content = file.generated ? file.generated.toString(file.encoding)
      : (await readOriginalFile(version, file.sourcePath!, maxBytes, file))?.content;
    if (content === undefined) return null;
    return { path: file.path, kind: file.kind, content, encoding: file.encoding, executable: file.executable };
  }
  return { inventory, materialize, readFile };
}