PaperClipAI/tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts

import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js";
import { expect } from "@playwright/test";
import { createHash } from "node:crypto";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { checks, observeInternal } from "./internal-support.js";
import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
import { privilegedDefinition, privilegedObservation } from "./privileged-support.js";
import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli } from "./privileged-cli-support.js";

export function inspectRunCommands(reply: string, runId: string) {
  const matches = [...reply.matchAll(/```(?:bash|sh|shell|zsh|console)?\s*\n([\s\S]*?)```/g)]
    .flatMap(match => match[1]!.replace(/\\\r?\n/g, " ").split(/\r?\n/))
    .map(line => line.trim().replace(/^\$\s+/, "")).filter(line => /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+run\s+(?:get|cancel)\b/.test(line));
  const commands: string[][] = [], bindings: Array<{ placeholder: string; fixtureValue: string }> = [];
  for (const line of matches) {
    const parsed = line.match(/^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+run\s+(get|cancel)\s+(\S+)(?:\s+(--json))?$/);
    if (!parsed) throw new Error("Unsupported run command flags or shell syntax");
    let id = parsed[2]!;
    if (/^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(id)) { bindings.push({ placeholder: id, fixtureValue: runId }); id = runId; }
    if (id !== runId) throw new Error("Run command names a different native run");
    commands.push(["run", parsed[1]!, id, ...(parsed[3] ? [parsed[3]] : [])]);
  }
  if (commands.length !== 2 || commands[0]?.[1] !== "get" || commands[1]?.[1] !== "cancel") throw new Error("Read the exact run before the separate authorized cancellation command");
  return { commands, bindings };
}
export function gradeInspectRunCli(value: unknown) {
  const c = checks(), o = privilegedObservation(value);
  c.add("run-companion-observation", !!o, "Actual read-before-cancel state and the owned native worker run are required."); if (!o) return c.result();
  c.add("read-exact-last-morgan-run", o.state.initialRunReallyRunning === true && o.state.readRunId === o.state.workerRunId
    && o.state.readAgentId === (o.state.worker as { id?: string } | undefined)?.id && o.state.readCompanyId === o.companyId
    && o.state.readStatus === "running", "The actual CLI read selects Morgan's exact current run, rather than a stale or other-company run.");
  // Who may cancel is set by the product (the agent bearer probe); how the reply
  // words that is not graded by keywords and is read by a person instead.
  c.add("board-only-control", o.state.agentCancelStatus === 403,
    "A real agent bearer is denied cancellation, so only the person's own CLI can perform the later stop.");
  c.add("actual-cancelled-result", o.state.cancelRunId === o.state.workerRunId && o.state.finalWorkerStatus === "cancelled"
    && o.state.cancelOnlyAfterRead === true && o.runs.length === 2
    && o.runs.filter(run => run.status === "cancelled" && run.id === o.state.workerRunId).length === 1
    && o.runs.filter(run => run.status === "succeeded" && run.agentId === o.leadId).length === 1,
    "The owned human CLI actually cancels the same running worker, with one successful canonical guidance run and no duplicate admission.");
  return c.result();
}
const probeKeys = new WeakMap<ChatCatalogContext, { id: string; token: string }>();
const inspect = privilegedDefinition("cli-operations.inspect-run-cli", "privileged-run-cli", {
  async setup(ctx) {
    const operator = await prepareOwnedCli(ctx), busy = await busyRuntimeWorker(ctx);
    if (!ctx.input.privateSecrets) throw new Error("Private actor probe redaction is unavailable");
    const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `catalog-run-control-${ctx.input.nonce}` });
    ctx.input.privateSecrets.push(key.token); probeKeys.set(ctx, key);
    const response = await fetch(new URL(`/api/heartbeat-runs/${busy.workerRunId}/cancel`, ctx.input.api.baseURL), {
      method: "POST", redirect: "error", signal: AbortSignal.timeout(15_000),
      headers: { Authorization: `Bearer ${key.token}`, "Content-Type": "application/json" }, body: "{}" });
    const agentCancelStatus = response.status; await response.body?.cancel();
    return { ...operator, ...busy, agentCancelStatus, initialRunReallyRunning: busy.workerWasReallyWaiting };
  },
  async act(ctx, state) {
    const initial = await pollUntil({ label: "Canonical run-control guidance", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150_000),
      load: () => observeInternal(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && (!!o.reply && ctx.issue()?.conversationState === "waiting"
        && o.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded").length === 1) });
    const returned = inspectRunCommands(initial.reply, String(state.workerRunId));
    await runOwnedCli(ctx, returned.commands[0]!);
    const read = await ctx.input.api.get<{ id: string; agentId: string; companyId: string; status: string }>(`/api/heartbeat-runs/${state.workerRunId}`);
    state.readRunId = read.id; state.readAgentId = read.agentId; state.readCompanyId = read.companyId; state.readStatus = read.status;
    if (read.id !== state.workerRunId || read.companyId !== ctx.input.fixtures.company.id || read.agentId !== (state.worker as { id: string }).id) throw new Error("CLI read did not verify the exact owned Morgan run");
    if (read.status === "running") {
      await runOwnedCli(ctx, returned.commands[1]!);
      state.cancelRunId = read.id; state.cancelOnlyAfterRead = true;
    }
    const final = await pollUntil({ label: "Actual owned run cancellation", deadlineAt: Date.now() + 60_000,
      load: () => ctx.input.api.get<{ id: string; status: string }>(`/api/heartbeat-runs/${state.workerRunId}`), accept: row => row.status === "cancelled" });
    state.finalWorkerStatus = final.status;
    state.commandHashes = returned.commands.map(argv => createHash("sha256").update(JSON.stringify(argv)).digest("hex"));
    state.unresolvedRunBindings = returned.bindings;
    await ctx.input.evidence("chat-catalog-cli-run-control.json", { workerRunId: state.workerRunId, readRunId: state.readRunId,
      readStatus: state.readStatus, finalStatus: final.status, actor: "authenticated-human", commandHashes: state.commandHashes });
  }, grade: gradeInspectRunCli,
  async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
});
inspect.oracleVersion = "paperclip.chat-catalog.cli-run.v2";
inspect.expectedRunCount = 2; inspect.expectedHumanMessageCount = 1; inspect.attemptTimeoutMs = 600_000;
inspect.runPolicy = { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 1 } };
inspect.sourceFiles = [...inspect.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "chat-catalog-domains/runtime-busy-worker.ts", "../../cli/src/index.ts"];
inspect.cleanup = async (ctx, state) => {
  await releaseRuntimeWorker(state);
  try { const key = probeKeys.get(ctx); if (key) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`); probeKeys.delete(ctx); } }
  finally { await cleanupOwnedCli(ctx); }
};
export const privilegedRunCliCases = [inspect];