PaperClipAI/packages/adapters/grok-local/src/server/test.ts

import type {
  AdapterEnvironmentCheck,
  AdapterEnvironmentTestContext,
  AdapterEnvironmentTestResult,
} from "@paperclipai/adapter-utils";
import {
  asNumber,
  asString,
  asStringArray,
  ensurePathInEnv,
  parseObject,
} from "@paperclipai/adapter-utils/server-utils";
import {
  describeAdapterExecutionTarget,
  prepareAdapterExecutionTargetRuntime,
  ensureAdapterExecutionTargetCommandResolvable,
  ensureAdapterExecutionTargetDirectory,
  resolveAdapterExecutionTargetCwd,
  runAdapterExecutionTargetProcess,
} from "@paperclipai/adapter-utils/execution-target";
import { mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { stageGrokHomeForSync } from "./grok-home.js";
import { copyBackGrokAuth } from "./grok-auth-copyback.js";
import { DEFAULT_GROK_LOCAL_MODEL } from "../index.js";
import { parseGrokJsonl } from "./parse.js";
import { ADAPTER_AUTH_MISSING_CHECK_CODE } from "@paperclipai/shared";

export interface GrokModelsProbe {
  authenticated: boolean;
  defaultModel: string | null;
  models: string[];
}

function summarizeStatus(checks: AdapterEnvironmentCheck[]): AdapterEnvironmentTestResult["status"] {
  if (checks.some((check) => check.level === "error")) return "fail";
  if (checks.some((check) => check.level === "warn")) return "warn";
  return "pass";
}

function firstNonEmptyLine(text: string): string {
  return (
    text
      .split(/\r?\n/)
      .map((line) => line.trim())
      .find(Boolean) ?? ""
  );
}

function summarizeProbeDetail(stdout: string, stderr: string, parsedError: string | null): string | null {
  const raw = parsedError?.trim() || firstNonEmptyLine(stderr) || firstNonEmptyLine(stdout);
  if (!raw) return null;
  const clean = raw.replace(/\s+/g, " ").trim();
  const max = 240;
  return clean.length > max ? `${clean.slice(0, max - 3)}...` : clean;
}

function normalizeEnv(input: unknown): Record<string, string> {
  if (typeof input !== "object" || input === null || Array.isArray(input)) return {};
  const env: Record<string, string> = {};
  for (const [key, value] of Object.entries(input as Record<string, unknown>)) {
    if (typeof value === "string") env[key] = value;
  }
  return env;
}

const GROK_AUTH_REQUIRED_RE =
  /(?:not\s+logged\s+in|login\s+required|run\s+`?grok\s+login`?|authentication\s+required|unauthorized|invalid\s+credentials)/i;

export function parseGrokModelsOutput(stdout: string): GrokModelsProbe {
  const trimmedLines = stdout
    .split(/\r?\n/)
    .map((line) => line.trim())
    .filter(Boolean);
  const models: string[] = [];
  let defaultModel: string | null = null;
  let authenticated = false;
  let inModelsBlock = false;

  for (const line of trimmedLines) {
    if (/logged in/i.test(line)) authenticated = true;
    const defaultMatch = /^Default model:\s*(.+)$/i.exec(line);
    if (defaultMatch?.[1]) {
      defaultModel = defaultMatch[1].trim();
      continue;
    }
    if (/^Available models:/i.test(line)) {
      inModelsBlock = true;
      continue;
    }
    if (!inModelsBlock) continue;
    const bulletMatch = /^[*-]\s*(.+?)(?:\s+\(default\))?$/.exec(line);
    if (bulletMatch?.[1]) {
      models.push(bulletMatch[1].trim());
      continue;
    }
    if (line.length > 0) {
      models.push(line.replace(/\s+\(default\)$/, "").trim());
    }
  }

  return {
    authenticated,
    defaultModel,
    models: Array.from(new Set(models.filter(Boolean))),
  };
}

export async function testEnvironment(
  ctx: AdapterEnvironmentTestContext,
): Promise<AdapterEnvironmentTestResult> {
  const checks: AdapterEnvironmentCheck[] = [];
  const config = parseObject(ctx.config);
  const command = asString(config.command, "grok");
  const target = ctx.executionTarget ?? null;
  const targetIsRemote = target?.kind === "remote";
  const targetIsSandbox = target?.kind === "remote" && target.transport === "sandbox";
  const cwd = resolveAdapterExecutionTargetCwd(target, asString(config.cwd, ""), process.cwd());
  const targetLabel = targetIsRemote
    ? ctx.environmentName ?? describeAdapterExecutionTarget(target)
    : null;
  const runId = `grok-envtest-${Date.now()}-${Math.random().toString(16).slice(2)}`;

  if (targetLabel) {
    checks.push({
      code: "grok_environment_target",
      level: "info",
      message: `Probing inside environment: ${targetLabel}`,
    });
  }

  try {
    await ensureAdapterExecutionTargetDirectory(runId, target, cwd, {
      cwd,
      env: {},
      createIfMissing: true,
    });
    checks.push({
      code: "grok_cwd_valid",
      level: "info",
      message: `Working directory is valid: ${cwd}`,
    });
  } catch (err) {
    checks.push({
      code: "grok_cwd_invalid",
      level: "error",
      message: err instanceof Error ? err.message : "Invalid working directory",
      detail: cwd,
    });
  }

  const env = normalizeEnv(config.env);
  let stagedHome: string | undefined;
  let runtimeWorkspaceLocalDir: string | undefined;
  let restore: (() => Promise<void>) | undefined;
  try {
    if (config.managedAiConnection && targetIsRemote) {
      try {
        const hostHome = env.GROK_HOME;
        stagedHome = await stageGrokHomeForSync(hostHome, { runId });
        // `cwd` is the remote target path here, never a host directory, so the
        // runtime gets an empty host workspace to stage from and the remote
        // path separately — the same split codex-local and opencode-local
        // draw. Handing it the remote path as `workspaceLocalDir` made the
        // host-side ignore scan fail on a directory that does not exist.
        runtimeWorkspaceLocalDir = await mkdtemp(
          path.join(os.tmpdir(), `paperclip-grok-envtest-${runId}-`),
        );
        const prepared = await prepareAdapterExecutionTargetRuntime({
          runId, target, adapterKey: "grok",
          workspaceLocalDir: runtimeWorkspaceLocalDir,
          workspaceRemoteDir: cwd,
          assets: [{ key: "home", localDir: stagedHome, followSymlinks: true,
            restore: async ({ assetDir, readFile }) => { await copyBackGrokAuth({
              readSandboxAuth: () => readFile(path.posix.join(assetDir, "auth.json")),
              hostHomeDir: hostHome, log: () => {},
            }); },
          }],
        });
        env.GROK_HOME = prepared.assetDirs.home;
        restore = () => prepared.restoreWorkspace(() => {});
      } catch (err) {
        // The environment test reports what is wrong with the environment; a
        // credential-staging failure is a finding, not a crash.
        checks.push({
          code: "grok_environment_unprepared",
          level: "error",
          message: err instanceof Error ? err.message : "Could not stage the managed account into the environment",
        });
      }
    }
  const runtimeEnv = ensurePathInEnv({ ...process.env, ...env });

  try {
    await ensureAdapterExecutionTargetCommandResolvable(command, target, cwd, runtimeEnv);
    checks.push({
      code: "grok_command_resolvable",
      level: "info",
      message: `Command is executable: ${command}`,
    });
  } catch (err) {
    checks.push({
      code: "grok_command_unresolvable",
      level: "error",
      message: err instanceof Error ? err.message : "Command is not executable",
      detail: command,
    });
  }

  const canRunProbe =
    checks.every((check) =>
      check.code !== "grok_cwd_invalid" &&
      check.code !== "grok_command_unresolvable" &&
      check.code !== "grok_environment_unprepared");

  const configuredModel = asString(config.model, DEFAULT_GROK_LOCAL_MODEL).trim();

  if (canRunProbe) {
    const modelsProbe = await runAdapterExecutionTargetProcess(
      runId,
      target,
      command,
      ["models"],
      {
        cwd,
        env,
        timeoutSec: Math.max(1, asNumber(config.helloProbeTimeoutSec, 45)),
        graceSec: 5,
        onLog: async () => {},
      },
    );

    const probeOutput = `${modelsProbe.stdout}\n${modelsProbe.stderr}`;
    const parsedModels = parseGrokModelsOutput(modelsProbe.stdout);
    const authRequired = GROK_AUTH_REQUIRED_RE.test(probeOutput);

    if (modelsProbe.timedOut) {
      checks.push({
        code: "grok_models_probe_timed_out",
        level: "warn",
        message: "`grok models` timed out.",
        hint: "Retry the probe. If this persists, run `grok models` manually from the target environment.",
      });
    } else if ((modelsProbe.exitCode ?? 1) !== 0) {
      checks.push({
        code: authRequired ? "grok_auth_required" : "grok_models_probe_failed",
        level: authRequired ? "warn" : "error",
        message: authRequired
          ? "Grok CLI is not authenticated."
          : "`grok models` failed.",
        detail: summarizeProbeDetail(modelsProbe.stdout, modelsProbe.stderr, null),
        hint: authRequired ? "Run `grok login` on the target host, then retry." : undefined,
      });
      if (authRequired && targetIsSandbox) {
        // Emit the neutral canonical check so the user interface can decide
        // login eligibility from a stable code. The user interface does not
        // read the message text or the top-level status.
        checks.push({
          code: ADAPTER_AUTH_MISSING_CHECK_CODE,
          level: "warn",
          message: "This environment has no ready authentication for this adapter.",
          detail: summarizeProbeDetail(modelsProbe.stdout, modelsProbe.stderr, null),
          hint: "Provide credentials for this adapter, or start login in the environment.",
        });
      }
    } else {
      checks.push({
        code: "grok_models_probe_passed",
        level: "info",
        message: parsedModels.authenticated
          ? "Grok CLI authentication is configured."
          : "`grok models` completed.",
        detail: parsedModels.defaultModel ? `Default model: ${parsedModels.defaultModel}` : undefined,
      });
      if (parsedModels.models.length > 0) {
        checks.push({
          code: "grok_models_discovered",
          level: "info",
          message: `Discovered ${parsedModels.models.length} Grok model(s).`,
        });
      } else {
        checks.push({
          code: "grok_models_empty",
          level: "warn",
          message: "Grok returned no available models.",
          hint: "Run `grok models` manually and verify the account has access to a model.",
        });
      }
      if (configuredModel) {
        // The default model id is a sentinel meaning "let the Grok CLI pick its
        // own default" — execute.ts only passes `--model` when the value differs
        // from it, so the sentinel is never sent to grok and must not be checked
        // against the discovered list (real grok never lists "grok-build", which
        // otherwise produced a spurious "not found" warning on every probe).
        const usesCliDefault = configuredModel === DEFAULT_GROK_LOCAL_MODEL;
        const available = usesCliDefault || parsedModels.models.includes(configuredModel);
        checks.push({
          code: available ? "grok_model_configured" : "grok_model_not_found",
          level: available ? "info" : "warn",
          message: usesCliDefault
            ? `Using the Grok CLI's default model${parsedModels.defaultModel ? ` (${parsedModels.defaultModel})` : ""}.`
            : available
              ? `Configured model: ${configuredModel}`
              : `Configured model "${configuredModel}" not found in available models.`,
          hint: available ? undefined : "Run `grok models` and choose an available model id.",
        });
      }
    }
  }

  if (canRunProbe) {
    const probeArgs = [
      "--output-format",
      "streaming-json",
      "--always-approve",
      "--permission-mode",
      "dontAsk",
      "--disable-web-search",
    ];
    if (configuredModel && configuredModel !== DEFAULT_GROK_LOCAL_MODEL) {
      probeArgs.push("--model", configuredModel);
    }
    probeArgs.push("--single", "Respond with exactly hello.");

    const helloProbe = await runAdapterExecutionTargetProcess(
      runId,
      target,
      command,
      probeArgs,
      {
        cwd,
        env,
        timeoutSec: Math.max(1, asNumber(config.helloProbeTimeoutSec, 45)),
        graceSec: 5,
        onLog: async () => {},
      },
    );
    const parsed = parseGrokJsonl(helloProbe.stdout);
    const detail = summarizeProbeDetail(helloProbe.stdout, helloProbe.stderr, parsed.errorMessage);
    const authRequired = GROK_AUTH_REQUIRED_RE.test(`${helloProbe.stdout}\n${helloProbe.stderr}`);

    if (helloProbe.timedOut) {
      checks.push({
        code: "grok_hello_probe_timed_out",
        level: "warn",
        message: "Grok hello probe timed out.",
        hint: "Retry the probe. If this persists, verify Grok can run a simple `--single` prompt manually.",
      });
    } else if ((helloProbe.exitCode ?? 1) !== 0) {
      checks.push({
        code: authRequired ? "grok_hello_probe_auth_required" : "grok_hello_probe_failed",
        level: authRequired ? "warn" : "error",
        message: authRequired
          ? "Grok CLI could not answer the hello probe because authentication is missing."
          : "Grok hello probe failed.",
        ...(detail ? { detail } : {}),
        hint: authRequired ? "Run `grok login` on the target host, then retry." : undefined,
      });
      if (authRequired && targetIsSandbox) {
        // Emit the neutral canonical check so the user interface can decide
        // login eligibility from a stable code. The user interface does not
        // read the message text or the top-level status.
        checks.push({
          code: ADAPTER_AUTH_MISSING_CHECK_CODE,
          level: "warn",
          message: "This environment has no ready authentication for this adapter.",
          ...(detail ? { detail } : {}),
          hint: "Provide credentials for this adapter, or start login in the environment.",
        });
      }
    } else if (/\bhello\b/i.test(parsed.summary)) {
      checks.push({
        code: "grok_hello_probe_passed",
        level: "info",
        message: "Grok hello probe succeeded.",
      });
    } else {
      checks.push({
        code: "grok_hello_probe_unexpected_output",
        level: "warn",
        message: "Grok hello probe succeeded but returned unexpected output.",
        ...(detail ? { detail } : {}),
      });
    }
  }

  return {
    adapterType: "grok_local",
    status: summarizeStatus(checks),
    checks,
    testedAt: new Date().toISOString(),
  };
  } finally {
    try { await restore?.(); } finally {
      // Both temporary directories are removed even when one removal fails,
      // and neither failure turns an answered environment test into a thrown
      // error: these are best-effort temp directories, and the caller asked
      // for checks.
      await Promise.allSettled([
        stagedHome ? rm(stagedHome, { recursive: true, force: true }) : undefined,
        runtimeWorkspaceLocalDir ? rm(runtimeWorkspaceLocalDir, { recursive: true, force: true }) : undefined,
      ]);
    }
  }
}