PaperClipAI/tests/runner-acceptance/catalog.ts

import { createHash } from "node:crypto";

import { DEFAULT_CODEX_LOCAL_MODEL } from "../../packages/adapters/codex-local/src/index.js";
import { ACPX_QUALIFICATION_MODELS } from "../acpx-qualification-models.js";
import { QUALIFIED_OPENCODE_MODEL } from "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js";
import { BUILTIN_ADAPTER_TYPES } from "../../server/src/adapters/builtin-adapter-types.js";

import { findSensitiveValue } from "./redaction.js";
import type {
  JsonValue,
  RunnerAcceptanceCase,
  RunnerAcceptanceCell,
  RunnerAcceptanceProfile,
} from "./types.js";

const excludedBuiltInAdapterTypes = new Set(["paperclip_runner", "pi_local"]);

const directBuiltInAdapterTypes = [
  "acpx_local",
  "claude_local",
  "codex_local",
  "cursor_cloud",
  "cursor",
  "gemini_local",
  "grok_local",
  "hermes_gateway",
  "hermes_local",
  "kimi_local",
  "openclaw_gateway",
  "opencode_local",
  "process",
  "http",
] as const;

function directProfile(adapterType: string): RunnerAcceptanceProfile {
  return {
    id: `direct-${adapterType.replaceAll("_", "-")}`,
    label: `Direct ${adapterType}`,
    generation: "direct",
    adapterScope: "built_in",
    adapterType,
    expectedRuntimeMode: "legacy",
    provider: null,
    model: null,
    adapterConfig: {},
    invariants: [
      "runtime.mode=legacy",
      "runnerd.start_count=0",
      "native_record.count=0",
      "finalization.path=direct_adapter",
      "task_controls.mode=classic",
    ],
  };
}

function nativeProfile(input: {
  id: string;
  label: string;
  provider: "codex" | "opencode" | "acpx";
  model: string;
  adapterConfig: Record<string, JsonValue>;
}): RunnerAcceptanceProfile {
  return {
    ...input,
    generation: "native",
    adapterScope: "built_in",
    adapterType: "paperclip_runner",
    expectedRuntimeMode: "native",
    invariants: [
      "runtime.mode=native",
      "runnerd.start_count=1",
      "native_record.count>=1",
      "finalization.path=native",
      "provider.persisted=true",
    ],
  };
}

export const directAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [
  ...directBuiltInAdapterTypes.map(directProfile),
  {
    id: "direct-external-plugin-contract",
    label: "Direct external plugin contract",
    generation: "direct",
    adapterScope: "external_plugin_contract",
    adapterType: "external_plugin:*",
    expectedRuntimeMode: "legacy",
    provider: null,
    model: null,
    adapterConfig: {},
    invariants: [
      "runtime.mode=legacy",
      "runnerd.start_count=0",
      "native_record.count=0",
      "finalization.path=plugin_adapter",
      "task_controls.mode=classic",
    ],
  },
] as const;

export const nativeAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [
  nativeProfile({
    id: "runner-codex",
    label: "Paperclip Runner Codex",
    provider: "codex",
    model: DEFAULT_CODEX_LOCAL_MODEL,
    adapterConfig: {
      provider: "codex",
      codexPermissionMode: "untrusted",
      lifecycleMode: "per_turn",
    },
  }),
  nativeProfile({
    id: "runner-opencode",
    label: "Paperclip Runner OpenCode",
    provider: "opencode",
    model: QUALIFIED_OPENCODE_MODEL,
    adapterConfig: {
      provider: "opencode",
      model: QUALIFIED_OPENCODE_MODEL,
      opencodePermissionMode: "ask",
      lifecycleMode: "per_turn",
    },
  }),
  nativeProfile({
    id: "runner-acpx-claude",
    label: "Paperclip Runner ACPX Claude",
    provider: "acpx",
    model: ACPX_QUALIFICATION_MODELS.claude,
    adapterConfig: {
      provider: "acpx",
      acpxAgent: "claude",
      model: ACPX_QUALIFICATION_MODELS.claude,
      acpxPermissionMode: "approve-reads",
      lifecycleMode: "per_turn",
    },
  }),
  nativeProfile({
    id: "runner-acpx-codex",
    label: "Paperclip Runner ACPX Codex",
    provider: "acpx",
    model: ACPX_QUALIFICATION_MODELS.codex,
    adapterConfig: {
      provider: "acpx",
      acpxAgent: "codex",
      model: ACPX_QUALIFICATION_MODELS.codex,
      acpxPermissionMode: "approve-reads",
      lifecycleMode: "per_turn",
    },
  }),
] as const;

export const runnerAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [
  ...directAcceptanceProfiles,
  ...nativeAcceptanceProfiles,
];

export const runnerAcceptanceCases: readonly RunnerAcceptanceCase[] = [
  {
    id: "runtime-selection",
    label: "Runtime selection remains isolated",
    revision: 1,
    appliesTo: "all",
    assertions: [
      "runtime_selection.is_stable",
      "status_authority.matches_runtime",
      "execution_occurs_once",
    ],
  },
  {
    id: "task-thread-baseline",
    label: "Task thread preserves baseline behavior",
    revision: 1,
    appliesTo: "all",
    assertions: [
      "task_thread.active_state",
      "task_thread.settled_state",
      "task_thread.empty_transcript_state",
      "task_thread.classic_interface_state",
    ],
  },
  {
    id: "question-round-trip",
    label: "Structured question round trip",
    revision: 1,
    appliesTo: "all",
    assertions: [
      "question.rendered_once",
      "question.response_validated",
      "question.resume_occurs_once",
      "finalization.byte_stable",
    ],
  },
  {
    id: "flagged-native-recovery",
    label: "Persisted native recovery after flag change",
    revision: 1,
    appliesTo: "native",
    assertions: [
      "fresh_start.flag_required",
      "persisted_run.readable_when_disabled",
      "persisted_run.recoverable_when_disabled",
      "provider.identity_does_not_drift",
    ],
  },
] as const;

function suiteDefinitionHash() {
  return createHash("sha256")
    .update(JSON.stringify({
      profiles: runnerAcceptanceProfiles.map((profile) => ({
        id: profile.id,
        adapterType: profile.adapterType,
        generation: profile.generation,
        provider: profile.provider,
        model: profile.model,
        adapterConfig: profile.adapterConfig,
        invariants: profile.invariants,
      })),
      cases: runnerAcceptanceCases.map((acceptanceCase) => ({
        id: acceptanceCase.id,
        revision: acceptanceCase.revision,
        appliesTo: acceptanceCase.appliesTo,
        assertions: acceptanceCase.assertions,
      })),
    }))
    .digest("hex");
}

export const runnerAcceptanceSuiteDefinitionHash = suiteDefinitionHash();

export function buildRunnerAcceptanceMatrix(): RunnerAcceptanceCell[] {
  return runnerAcceptanceProfiles.flatMap((profile) =>
    runnerAcceptanceCases
      .filter((acceptanceCase) =>
        acceptanceCase.appliesTo === "all"
        || acceptanceCase.appliesTo === profile.generation)
      .map((acceptanceCase) => ({
        id: `${profile.id}.${acceptanceCase.id}`,
        suiteId: "runner-compatibility" as const,
        suiteDefinitionHash: runnerAcceptanceSuiteDefinitionHash,
        profile,
        acceptanceCase,
        assertions: [...new Set([
          ...profile.invariants,
          ...acceptanceCase.assertions,
        ])],
      })),
  );
}

function duplicates(values: readonly string[]) {
  const seen = new Set<string>();
  return values.filter((value) => {
    if (seen.has(value)) return true;
    seen.add(value);
    return false;
  });
}

function assertSafeConfig(value: JsonValue, path = "adapterConfig") {
  if (typeof value === "string") {
    const leak = findSensitiveValue(value);
    if (leak) throw new Error(`Acceptance catalog contains ${leak} at ${path}`);
    return;
  }
  if (Array.isArray(value)) {
    value.forEach((entry, index) => assertSafeConfig(entry, `${path}[${index}]`));
    return;
  }
  if (!value || typeof value !== "object") return;
  for (const [key, entry] of Object.entries(value)) {
    if (/(?:authorization|api.?key|token|password|secret|credential|env)/i.test(key)) {
      throw new Error(`Acceptance catalog contains prohibited field ${path}.${key}`);
    }
    assertSafeConfig(entry, `${path}.${key}`);
  }
}

export function validateRunnerAcceptanceCatalog() {
  const duplicateProfileIds = duplicates(runnerAcceptanceProfiles.map(({ id }) => id));
  if (duplicateProfileIds.length > 0) {
    throw new Error(`Duplicate runner acceptance profile ids: ${duplicateProfileIds.join(", ")}`);
  }
  const duplicateCaseIds = duplicates(runnerAcceptanceCases.map(({ id }) => id));
  if (duplicateCaseIds.length > 0) {
    throw new Error(`Duplicate runner acceptance case ids: ${duplicateCaseIds.join(", ")}`);
  }

  const expectedDirectBuiltIns = new Set(
    [...BUILTIN_ADAPTER_TYPES].filter((type) => !excludedBuiltInAdapterTypes.has(type)),
  );
  const catalogedDirectBuiltIns = new Set(
    directAcceptanceProfiles
      .filter(({ adapterScope }) => adapterScope === "built_in")
      .map(({ adapterType }) => adapterType),
  );
  const missing = [...expectedDirectBuiltIns].filter((type) => !catalogedDirectBuiltIns.has(type));
  const unexpected = [...catalogedDirectBuiltIns].filter((type) => !expectedDirectBuiltIns.has(type));
  if (missing.length > 0 || unexpected.length > 0) {
    throw new Error(
      `Direct adapter catalog drift; missing=${missing.join(",") || "none"}; unexpected=${unexpected.join(",") || "none"}`,
    );
  }

  for (const profile of runnerAcceptanceProfiles) {
    assertSafeConfig(profile.adapterConfig);
    if (profile.adapterType === "pi_local") {
      throw new Error("Pi is outside the acceptance foundation boundary");
    }
    if (
      profile.provider === "acpx"
      && profile.adapterConfig.acpxAgent !== "claude"
      && profile.adapterConfig.acpxAgent !== "codex"
    ) {
      throw new Error(`Unqualified ACPX profile ${profile.id}`);
    }
  }

  const matrix = buildRunnerAcceptanceMatrix();
  const duplicateCellIds = duplicates(matrix.map(({ id }) => id));
  if (duplicateCellIds.length > 0) {
    throw new Error(`Duplicate runner acceptance cell ids: ${duplicateCellIds.join(", ")}`);
  }
  return matrix;
}

export const runnerAcceptanceMatrix = validateRunnerAcceptanceCatalog();