PaperClipAI/server/src/services/native-runtime/runtime-context.ts

import { measureRunPhase } from "../execution-preparation-timing.js";
import { spekoToolsForSession } from "../voice/speko-agent-tools.js";
import { githubBotConnectionIdsForRun } from "../chat-github-tools.js";
import { boundRunIssueIdSql } from "../run-task-binding.js";
import { isBrowserUseConnection } from "../browser-use-client.js";
import { restConnectorFor } from "../connectors/rest-registry.js";
import { createHash, randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, type Db } from "@paperclipai/db";
import type { PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils";
import { isToolConnectionAttentionHealth } from "@paperclipai/shared";
import {
  PAPERCLIP_OPERATIONAL_SKILL_KEY,
  resolvePaperclipDesiredSkillNames,
} from "@paperclipai/adapter-utils/server-utils";
import {
  NATIVE_RUNTIME_ASSET_SCHEMA,
  PAPERCLIP_EXECUTION_PROMPT,
  PAPERCLIP_EXECUTION_PROMPT_REVISION,
  canonicalNativeRuntimeContextDigest,
  nativeRuntimePromptDigest,
  parseNativeRuntimeContext,
  type NativeRuntimeAssetReference,
  type NativeRuntimeContextSnapshot,
} from "../../vendor/paperclip-runner/index.js";
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
import { agentInstructionsService, agentInstructionsBundleMode } from "../agent-instructions.js";
import { agentInstructionRevisionService } from "../agent-instruction-revisions.js";
import { toolAccessService } from "../tool-access.js";
import { filterResolvedGitHubConnectionsForRun } from "../git-credentials.js";

const MAX_ASSET_FILES = 10_000;
const MAX_ASSET_BYTES = 64 * 1024 * 1024;
type RuntimeAgent = { id: string; companyId: string; name: string; adapterType?: string | null; adapterConfig: unknown };
type AssetFile = { path: string; content: Buffer; mode: number };

const sha256 = (value: string | Buffer) => createHash("sha256").update(value).digest("hex");
const safeRelativePath = (value: string, label: string) => {
  const normalized = value.replaceAll("\\", "/");
  if (!normalized || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => !part || part === "." || part === "..")) {
    throw new Error(`${label} must stay within its bundle root`);
  }
  return normalized;
};

async function collectDirectoryFiles(sourceRoot: string): Promise<AssetFile[]> {
  const root = path.resolve(sourceRoot);
  const rootStat = await fs.lstat(root).catch(() => null);
  if (!rootStat?.isDirectory() || rootStat.isSymbolicLink()) throw new Error(`runtime context source is not a safe directory: ${root}`);
  const files: AssetFile[] = [];
  let totalBytes = 0;
  async function visit(relativeDirectory: string): Promise<void> {
    const entries = await fs.readdir(relativeDirectory ? path.join(root, relativeDirectory) : root, { withFileTypes: true });
    entries.sort((a, b) => a.name.localeCompare(b.name));
    for (const entry of entries) {
      const relativePath = safeRelativePath(relativeDirectory ? `${relativeDirectory}/${entry.name}` : entry.name, "runtime context path");
      const absolutePath = path.join(root, relativePath);
      const stat = await fs.lstat(absolutePath);
      if (stat.isSymbolicLink()) throw new Error(`runtime context bundles do not permit symlinks: ${relativePath}`);
      if (stat.isDirectory()) { await visit(relativePath); continue; }
      if (!stat.isFile()) throw new Error(`runtime context bundle contains an unsupported file: ${relativePath}`);
      const content = await fs.readFile(absolutePath);
      totalBytes += content.byteLength;
      if (files.length + 1 > MAX_ASSET_FILES || totalBytes > MAX_ASSET_BYTES) throw new Error("runtime context bundle exceeds its bound");
      files.push({ path: relativePath, content, mode: stat.mode & 0o555 });
    }
  }
  await visit("");
  if (!files.length) throw new Error(`runtime context source is empty: ${root}`);
  return files;
}

async function makeDirectoriesReadOnly(directory: string): Promise<void> {
  for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
    if (entry.isDirectory()) await makeDirectoriesReadOnly(path.join(directory, entry.name));
  }
  await fs.chmod(directory, 0o555);
}

async function verifyMaterializedAsset(
  rootPath: string,
  manifestFiles: Array<{ path: string; sha256: string; mode: number; size: number }>,
): Promise<void> {
  const actual = await measureRunPhase("runtime_context.asset_read", () => collectDirectoryFiles(rootPath));
  const actualByPath = new Map(actual.map((file) => [file.path, file]));
  if (actualByPath.size !== manifestFiles.length) throw new Error("runtime context asset file count mismatch");
  for (const expected of manifestFiles) {
    const file = actualByPath.get(expected.path);
    if (!file || file.content.byteLength !== expected.size || sha256(file.content) !== expected.sha256 || (file.mode & 0o555) !== expected.mode) {
      throw new Error(`runtime context asset digest mismatch: ${expected.path}`);
    }
  }
}

export async function materializeAsset(files: AssetFile[]): Promise<NativeRuntimeAssetReference> {
  const { sorted, manifestFiles, totalBytes, assetDigest, manifestText, manifestDigest } = await measureRunPhase("runtime_context.asset_manifest", () => {
    const sorted = [...files].sort((a, b) => a.path.localeCompare(b.path));
    const manifestFiles = sorted.map((file) => ({ path: safeRelativePath(file.path, "runtime context path"), sha256: sha256(file.content), mode: file.mode & 0o555, size: file.content.byteLength }));
    const totalBytes = manifestFiles.reduce((sum, file) => sum + file.size, 0);
    if (!manifestFiles.length || manifestFiles.length > MAX_ASSET_FILES || totalBytes > MAX_ASSET_BYTES) throw new Error("runtime context bundle is empty or exceeds its bound");
    const assetDigest = sha256(JSON.stringify(manifestFiles));
    const manifestText = `${JSON.stringify({ schema: "paperclip.runtime-asset-manifest.v1", digest: assetDigest, fileCount: manifestFiles.length, totalBytes, files: manifestFiles })}\n`;
    const manifestDigest = sha256(manifestText);
    return { sorted, manifestFiles, totalBytes, assetDigest, manifestText, manifestDigest };
  });
  const assetsRoot = path.join(resolvePaperclipInstanceRoot(), "runtime-context-assets");
  const rootPath = path.join(assetsRoot, "bundles", assetDigest);
  const manifestPath = path.join(assetsRoot, "manifests", `${assetDigest}.json`);
  const existing = await measureRunPhase("runtime_context.asset_cache", () => fs.readFile(manifestPath, "utf8").catch(() => null));
  if (existing !== null) {
    if (sha256(existing) !== manifestDigest || !(await fs.stat(rootPath).catch(() => null))?.isDirectory()) throw new Error(`runtime context asset verification failed: ${assetDigest}`);
    await measureRunPhase("runtime_context.asset_verify", () => verifyMaterializedAsset(rootPath, manifestFiles), { fileCount: manifestFiles.length, bytes: totalBytes });
    return { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest: assetDigest, manifestDigest, rootPath, fileCount: manifestFiles.length, totalBytes };
  }
  await measureRunPhase("runtime_context.asset_materialize", async () => {
    const stagingRoot = path.join(assetsRoot, ".staging", randomUUID());
    await fs.mkdir(stagingRoot, { recursive: true, mode: 0o700 });
    try {
      for (const file of sorted) {
        const target = path.join(stagingRoot, file.path);
        await fs.mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
        await fs.writeFile(target, file.content, { mode: file.mode & 0o555 });
        await fs.chmod(target, file.mode & 0o555);
      }
      await fs.mkdir(path.dirname(rootPath), { recursive: true, mode: 0o700 });
      let publishedRoot = false;
      await fs.rename(stagingRoot, rootPath).then(() => {
        publishedRoot = true;
      }).catch(async (error: NodeJS.ErrnoException) => {
        // Concurrent publishers can surface EACCES on macOS when the winning
        // destination has already been made read-only. Treat that as a race only
        // when the content-addressed destination now exists; a genuine permission
        // failure with no published destination must still fail loudly.
        const publishedByPeer = (await fs.stat(rootPath).catch(() => null))?.isDirectory() === true;
        if (error.code !== "EEXIST" && error.code !== "ENOTEMPTY" && !publishedByPeer) throw error;
        await fs.rm(stagingRoot, { recursive: true, force: true });
      });
      // Publish the writable staging directory first. Renaming a read-only
      // directory can fail with EACCES on macOS even when both parents are owned
      // and writable. Files are already immutable; directories become immutable
      // immediately after this process wins publication.
      if (publishedRoot) await makeDirectoriesReadOnly(rootPath);
      await fs.mkdir(path.dirname(manifestPath), { recursive: true, mode: 0o700 });
      await fs.writeFile(manifestPath, manifestText, { flag: "wx", mode: 0o400 }).catch(async (error: NodeJS.ErrnoException) => {
        if (error.code !== "EEXIST") throw error;
        if (sha256(await measureRunPhase("runtime_context.asset_cache", () => fs.readFile(manifestPath, "utf8"))) !== manifestDigest) throw new Error(`runtime context manifest race mismatch: ${assetDigest}`);
      });
      await measureRunPhase("runtime_context.asset_verify", () => verifyMaterializedAsset(rootPath, manifestFiles), { fileCount: manifestFiles.length, bytes: totalBytes });
    } catch (error) {
      await fs.rm(stagingRoot, { recursive: true, force: true }).catch(() => undefined);
      throw error;
    }
  }, { fileCount: manifestFiles.length, bytes: totalBytes });
  return { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest: assetDigest, manifestDigest, rootPath, fileCount: manifestFiles.length, totalBytes };
}

async function materializeInstructionBundle(db: Db, agent: RuntimeAgent, agentFiles = false) {
  if (agentFiles) {
    const current = await agentInstructionRevisionService(db).readCommittedForRuntime({ companyId: agent.companyId, agentId: agent.id });
    if (!current) throw new Error("Configured instruction entry is missing");
    const entryPath = safeRelativePath(current.revision.entryFile, "instruction entry path");
    return { entryPath, bundle: await materializeAsset([{ path: entryPath, content: Buffer.from(current.content, "utf8"), mode: 0o444 }]) };
  }
  const exported = await agentInstructionsService().exportFiles(agent, { rejectSymlinks: true });
  if (agentInstructionsBundleMode(agent) === "managed") {
    // The database head remains authoritative when a saved revision could not
    // update its disk projection. Overlay the exact committed snapshot directly;
    // a repair followed by a separate export can race another pending commit.
    const current = await agentInstructionRevisionService(db).readCommittedForRuntime({ companyId: agent.companyId, agentId: agent.id });
    if (current) {
      exported.entryFile = current.revision.entryFile;
      exported.files[current.revision.entryFile] = current.content;
    }
  }
  const entryPath = safeRelativePath(exported.entryFile, "instruction entry path");
  if (!(entryPath in exported.files)) throw new Error(`configured instruction entry is missing: ${entryPath}`);
  const files = Object.entries(exported.files).map(([relativePath, content]) => ({ path: safeRelativePath(relativePath, "instruction path"), content: Buffer.from(content, "utf8"), mode: 0o444 }));
  return { entryPath, bundle: await materializeAsset(files) };
}

async function materializeSelectedSkills(runtimeConfig: Record<string, unknown>, entries: PaperclipSkillEntry[], omitLegacy: boolean) {
  const desiredKeys = resolvePaperclipDesiredSkillNames(runtimeConfig, entries).filter(
    (key) => !omitLegacy || key !== PAPERCLIP_OPERATIONAL_SKILL_KEY,
  );
  const byKey = new Map(entries.map((entry) => [entry.key, entry]));
  return Promise.all(desiredKeys.sort().map(async (key) => {
    const entry = byKey.get(key);
    if (!entry) throw new Error(`assigned runtime skill is missing from the company library: ${key}`);
    if (entry.sourceStatus === "missing") throw new Error(entry.missingDetail ?? `assigned runtime skill source is missing: ${key}`);
    return { key: entry.key, runtimeName: safeRelativePath(entry.runtimeName, `runtime name for ${key}`), versionId: entry.versionId ?? entry.currentVersionId ?? null, bundle: await materializeAsset(await measureRunPhase("runtime_context.asset_read", () => collectDirectoryFiles(entry.source))) };
  }));
}

export async function resolveNativeRuntimeMcpSnapshot(input: { db: Db; agent: Pick<RuntimeAgent, "id" | "companyId">; runId: string }) {
  const effective = await toolAccessService(input.db).getEffectiveToolAssignmentsForAgent(input.agent.companyId, input.agent.id);
  const permitted = new Set([...effective.entries.filter((entry) => entry.effect === "include" && entry.connectionId).map((entry) => entry.connectionId!), ...effective.allowedTools.map((tool) => tool.connectionId)]);
  const hasGitHubConnection = effective.installedConnections.some((connection) => {
    const config = connection.config && typeof connection.config === "object"
      ? connection.config as Record<string, unknown>
      : {};
    const transportConfig = connection.transportConfig && typeof connection.transportConfig === "object"
      ? connection.transportConfig as Record<string, unknown>
      : {};
    return config.sourceTemplateKey === "github" || transportConfig.sourceTemplateKey === "github";
  });
  const hasVoiceConnection = effective.installedConnections.some(connection => connection.transport === "voice");
  const [runIdentity] = hasGitHubConnection || hasVoiceConnection
    ? await input.db.select({ issueId: boundRunIssueIdSql(), responsibleUserId: heartbeatRuns.responsibleUserId, activeIdentityContextId: heartbeatRuns.activeIdentityContextId })
      .from(heartbeatRuns)
      .where(and(
        eq(heartbeatRuns.id, input.runId),
        eq(heartbeatRuns.companyId, input.agent.companyId),
        eq(heartbeatRuns.agentId, input.agent.id),
      ))
      .limit(1)
    : [];
  // Broker runs pin the permitted tool catalog, not one person’s credential.
  // Selection happens at each operation and can change through steering.
  const resolvedInstalledConnections = runIdentity?.activeIdentityContextId
    ? effective.installedConnections : await filterResolvedGitHubConnectionsForRun({
    db: input.db,
    companyId: input.agent.companyId,
    agentId: input.agent.id,
    responsibleUserId: runIdentity?.responsibleUserId ?? null,
    connections: effective.installedConnections,
  });
  // App access is optional runtime context. Keep usable assignments pinned, but
  // do not stop unrelated work because an assigned app needs attention.
  const githubBotConnectionIds = await githubBotConnectionIdsForRun(input.db, input.agent.companyId, input.agent.id, input.runId);
  const spekoConnectionIds = hasVoiceConnection ? new Set((await spekoToolsForSession(input.db, { companyId: input.agent.companyId, agentId: input.agent.id, runId: input.runId, issueId: runIdentity?.issueId ?? null, identityContextId: runIdentity?.activeIdentityContextId })).map(tool => tool.connectionId)) : new Set<string>();
  const availableConnectionIds = new Set(resolvedInstalledConnections.filter((connection) =>
    permitted.has(connection.id)
    && connection.status === "active"
    && connection.enabled
    && (Boolean(runIdentity?.activeIdentityContextId) && (connection.config?.sourceTemplateKey === "github" || connection.transportConfig?.sourceTemplateKey === "github")
      || connection.credentialPolicy === "per_user"
      || !isToolConnectionAttentionHealth(connection.healthStatus))
    && (["mcp_remote", "local_stdio"].includes(connection.transport) || isBrowserUseConnection(connection) || restConnectorFor(connection) || githubBotConnectionIds.has(connection.id) || spekoConnectionIds.has(connection.id))
  ).map((connection) => connection.id));
  const assignment = {
    version: 1,
    agentId: input.agent.id,
    connections: [...availableConnectionIds].sort(),
    tools: effective.allowedTools
      .filter((tool) => availableConnectionIds.has(tool.connectionId))
      .map((tool) => tool.id)
      .sort(),
  };
  const assignmentDigest = sha256(JSON.stringify(assignment));
  return { assignmentSetId: `sha256:${assignmentDigest}`, digest: assignmentDigest, bindingId: assignment.connections.length ? `native-mcp:${input.runId}` : null };
}

export async function buildNativeRuntimeContext(input: { db: Db; agent: RuntimeAgent; runId: string; runtimeConfig: Record<string, unknown>; runtimeSkillEntries: PaperclipSkillEntry[]; instructionWorkingCopy?: { rootPath: string; entryPath: string; kind?: "agent_files" } }): Promise<NativeRuntimeContextSnapshot> {
  const preconfigured = input.runtimeConfig.homeMode === "preconfigured";
  const emptyMcpDigest = sha256("preconfigured");
  const [instructions, skills, mcp] = await Promise.all([
    measureRunPhase("runtime_context.instructions", () => materializeInstructionBundle(input.db, input.agent, input.instructionWorkingCopy?.kind === "agent_files")),
    measureRunPhase("runtime_context.skills", () => preconfigured ? [] : materializeSelectedSkills(input.runtimeConfig, input.runtimeSkillEntries, input.agent.adapterType === "paperclip_runner")),
    measureRunPhase("runtime_context.mcp", () => preconfigured ? { assignmentSetId: `sha256:${emptyMcpDigest}`, digest: emptyMcpDigest, bindingId: null } : resolveNativeRuntimeMcpSnapshot({ db: input.db, agent: input.agent, runId: input.runId })),
  ]);
  const snapshot = {
    prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() },
    instructions: { ...instructions, ...(input.instructionWorkingCopy ? { workingCopy: input.instructionWorkingCopy } : {}) },
    skills,
    mcp,
    ...(input.runtimeConfig.paperclipConnectionInstructions ? {
      connectionInstructions: input.runtimeConfig.paperclipConnectionInstructions as { text: string; digest: string },
    } : {}),
  };
  return parseNativeRuntimeContext({ ...snapshot, aggregateDigest: canonicalNativeRuntimeContextDigest(snapshot) });
}