test(evals): exercise human CLI authorization and revocation

tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.test.ts

@@ -0,0 +1,18 @@
+import { describe, expect, it } from "vitest";
+import { gradeCliAuthorization } from "./identity-cli-authorization.js";
+import type { IdentityObservation } from "./identity-support.js";
+function valid(): IdentityObservation {
+  return { scenarioId: "access.cli-authorization", origin: "http://fixture.test", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [],
+    interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], activity: [], links: [], runs: [], humanQueries: [],
+    reply: "Use paperclipai auth login and approve its request in your browser.", state: { humanId: "alex", agentApproveStatus: 401, reviewVisible: true, reviewCompanyVisible: true,
+      requestedAccess: "board", requestedCompanyId: "acorn", approvedByUserId: "alex", cliUserId: "alex", cliSource: "board_key", cliHasCompany: true, cliExitCode: 0,
+      logoutRevoked: true, localCredentialRemoved: true, revokedReadStatus: 401, agentBeforeId: "ceo", agentAfterId: "ceo", agentAfterStatus: 200 } };
+}
+describe("human CLI authorization oracle", () => {
+  it("accepts complete real CLI/UI/independent revocation evidence", () => { expect(gradeCliAuthorization(valid()).passed).toBe(true); });
+  it.each([{ cliSource: "agent" }, { approvedByUserId: "riley" }, { requestedCompanyId: "birch" }, { agentApproveStatus: 200 },
+    { revokedReadStatus: 200 }, { localCredentialRemoved: false }, { agentAfterStatus: 401 }, { cliExitCode: 1 }])("rejects wrong identity, scope or revocation %j", change => {
+    const observation = valid(); Object.assign(observation.state, change); expect(gradeCliAuthorization(observation).passed).toBe(false);
+  });
+  it("rejects missing evidence and prose-only success", () => { expect(gradeCliAuthorization(undefined).passed).toBe(false); expect(gradeCliAuthorization({ reply: "Connected as you." }).passed).toBe(false); });
+});

tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.ts

@@ -0,0 +1,130 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { appendFile, mkdir, readFile, rm } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { expect } from "@playwright/test";
+import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
+import { pollUntil } from "../api.js";
+import { checks } from "./internal-support.js";
+import { identityDefinition, identityObservation, probeAgentPermission, readAgentResource, waitForReply } from "./identity-support.js";
+
+interface CliCredential { token: string; userId: string; apiBase: string }
+interface Owned { directory: string; store: string; processes: Set<ChildProcess>; token?: string; keyId?: string; challenge?: { id: string; token: string }; revoked: boolean }
+const owned = new WeakMap<ChatCatalogContext, Owned>();
+const root = fileURLToPath(new URL("../../../", import.meta.url));
+export function gradeCliAuthorization(value: unknown) {
+  const c = checks(); const o = identityObservation(value);
+  c.add("complete-human-cli-proof", !!o && typeof o.state.humanId === "string" && o.state.humanId !== "local-board", "An actual authenticated person authorizes an actual source CLI process."); if (!o) return c.result();
+  c.add("agent-guidance-keeps-identities-separate", /(?:auth\s+login|cli|terminal)/i.test(o.reply) && /approv|authoriz|sign.?in|browser/i.test(o.reply)
+    && !/\bI(?:'ve| have)?\s+(?:connected|authenticated|logged you in)\b/i.test(o.reply) && o.state.agentApproveStatus === 401,
+    "The answer guides the browser authorization and the actual agent bearer cannot grant a board credential.");
+  c.add("real-human-company-scope", o.state.reviewVisible === true && o.state.reviewCompanyVisible === true && o.state.requestedAccess === "board"
+    && o.state.requestedCompanyId === o.companyId && o.state.approvedByUserId === o.state.humanId && o.state.cliUserId === o.state.humanId
+    && o.state.cliSource === "board_key" && o.state.cliHasCompany === true && o.state.cliExitCode === 0,
+    "The CLI challenge's displayed command, Board access and audit company are explicitly approved by the person; real CLI whoami resolves that same user and accessible company.");
+  c.add("revocation-proven", o.state.logoutRevoked === true && o.state.localCredentialRemoved === true && o.state.revokedReadStatus === 401,
+    "Actual CLI logout revokes its saved credential, removes the private local copy and a fresh bearer request is rejected.");
+  c.add("agent-still-independent", o.state.agentBeforeId === o.leadId && o.state.agentAfterId === o.leadId && o.state.agentAfterStatus === 200,
+    "Revoking the human CLI key leaves the separately issued agent identity usable with its original company authority.");
+  return c.result();
+}
+export const cliAuthorizationIdentityCases: readonly ChatCatalogCase[] = [identityDefinition("access.cli-authorization", async ctx => {
+  const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session");
+  if (!session.user?.id || session.user.id === "local-board" || !ctx.input.privateSecrets) throw new Error("CLI authorization requires a real human session and private credential redaction");
+  const privateRoot = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR;
+  if (!privateRoot || !path.isAbsolute(privateRoot)) throw new Error("CLI credentials require the attempt-private result root outside the model workspace");
+  const directory = path.join(privateRoot, "cases", ctx.scenario.id, `cli-auth-${ctx.input.nonce}`);
+  if (path.relative(ctx.input.workspacePath, directory).startsWith("..") === false) throw new Error("CLI credential home cannot be placed inside the model workspace");
+  await mkdir(directory, { recursive: true, mode: 0o700 });
+  owned.set(ctx, { directory, store: path.join(directory, "auth.json"), processes: new Set(), revoked: false });
+  const before = await readAgentResource(ctx, `/api/agents/${ctx.input.fixtures.agent.id}`);
+  return { humanId: session.user.id, agentBeforeId: (before.data as { id?: string })?.id };
+}, gradeCliAuthorization, {
+  authentication: "human-session", attemptTimeoutMs: 420_000,
+  sourceFiles: ["chat-catalog-domains/identity-cli-authorization.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/access.yaml",
+    "../../cli/src/index.ts", "../../cli/src/client/board-auth.ts", "../../cli/src/commands/client/auth.ts", "../../cli/src/commands/client/common.ts", "../../pnpm-lock.yaml"],
+  async act(ctx, state) {
+    await waitForReply(ctx); const resources = owned.get(ctx); if (!resources) throw new Error("Owned CLI home missing");
+    const login = runCli(ctx, ["auth", "login", "--company-id", ctx.input.fixtures.company.id, "--no-browser"], "login");
+    void login.catch(() => {}); // Later awaited; browser failures still trigger owned cleanup.
+    // Observe the CLI's own public challenge, never synthesize its success or
+    // copy an agent key into its private store. Raw output is kept private.
+    const approval = await pollUntil({ label: "source CLI approval URL", deadlineAt: Date.now() + 45_000, intervalMs: 300,
+      load: async () => { const text = await readFile(path.join(resources.directory, "login.stderr"), "utf8").catch(() => "");
+        const match = text.match(/https?:\/\/[^\s]+\/cli-auth\/[^\s]+/); return match?.[0]; }, accept: value => !!value });
+    const url = new URL(approval!); if (url.origin !== ctx.input.api.baseURL) throw new Error("CLI approval escaped owned instance");
+    const token = url.searchParams.get("token"); const id = url.pathname.split("/").at(-1); if (!token || !id) throw new Error("CLI returned incomplete challenge");
+    resources.challenge = { id, token };
+    ctx.input.privateSecrets!.push(approval!, token);
+    const challenge = await privateChallenge(ctx, id, token);
+    state.requestedAccess = challenge.requestedAccess; state.requestedCompanyId = challenge.requestedCompanyId;
+    state.agentApproveStatus = await probeAgentPermission(ctx, `/api/cli-auth/challenges/${id}/approve`, "POST", { token });
+    const source = ctx.input.page.url(); await ctx.input.page.goto(approval!, { waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByRole("heading", { name: "Approve Paperclip CLI access", exact: true })).toBeVisible();
+    await expect(ctx.input.page.getByText("paperclipai auth login", { exact: true })).toBeVisible(); state.reviewVisible = true;
+    await expect(ctx.input.page.getByText(ctx.input.fixtures.company.name, { exact: true }).last()).toBeVisible(); state.reviewCompanyVisible = true;
+    const approved = ctx.input.page.waitForResponse(response => response.request().method() === "POST" && new URL(response.url()).pathname === `/api/cli-auth/challenges/${id}/approve`);
+    await ctx.input.page.getByRole("button", { name: "Approve CLI access", exact: true }).click();
+    const approvalResult = await approved; if (!approvalResult.ok()) throw new Error("Human CLI approval failed");
+    resources.keyId = (await approvalResult.json() as { keyId: string }).keyId;
+    await expect(ctx.input.page.getByRole("heading", { name: "CLI access approved", exact: true })).toBeVisible();
+    const loggedIn = await login; state.cliExitCode = loggedIn.code;
+    if (loggedIn.code !== 0) throw new Error("Real CLI login failed; private process logs retained");
+    const store = JSON.parse(await readFile(resources.store, "utf8")) as { credentials: Record<string, CliCredential> };
+    const credential = store.credentials[ctx.input.api.baseURL]; if (!credential?.token) throw new Error("CLI did not persist its own credential");
+    resources.token = credential.token; ctx.input.privateSecrets!.push(credential.token);
+    const decision = await privateChallenge(ctx, id, token); state.approvedByUserId = decision.approvedByUser?.id;
+    const identity = await runCli(ctx, ["auth", "whoami"], "whoami"); if (identity.code !== 0) throw new Error("Real authenticated CLI whoami failed");
+    const me = JSON.parse(identity.stdout) as { userId: string; source: string; companyIds: string[] };
+    state.cliUserId = me.userId; state.cliSource = me.source; state.cliHasCompany = me.companyIds.includes(ctx.input.fixtures.company.id);
+    const logout = await runCli(ctx, ["auth", "logout"], "logout"); if (logout.code !== 0) throw new Error("Real CLI logout failed");
+    const result = JSON.parse(logout.stdout) as { revoked: boolean; removedLocalCredential: boolean };
+    resources.revoked = result.revoked; state.logoutRevoked = result.revoked; state.localCredentialRemoved = result.removedLocalCredential
+      && !(JSON.parse(await readFile(resources.store, "utf8")) as { credentials: Record<string, unknown> }).credentials[ctx.input.api.baseURL];
+    const revoked = await fetch(`${ctx.input.api.baseURL}/api/cli-auth/me`, { headers: { Authorization: `Bearer ${credential.token}` }, redirect: "error", signal: AbortSignal.timeout(15_000) });
+    state.revokedReadStatus = revoked.status; await revoked.body?.cancel();
+    const after = await readAgentResource(ctx, `/api/agents/${ctx.input.fixtures.agent.id}`); state.agentAfterStatus = after.status; state.agentAfterId = (after.data as { id?: string })?.id;
+    await ctx.input.evidence("identity-cli-human-authorization.json", { userId: state.cliUserId, requestedCompanyId: state.requestedCompanyId, access: state.requestedAccess,
+      source: state.cliSource, approvedByUserId: state.approvedByUserId, logoutRevoked: state.logoutRevoked, revokedReadStatus: state.revokedReadStatus, agentAfterId: state.agentAfterId });
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+  async cleanup(ctx) {
+    const resources = owned.get(ctx); if (!resources) return;
+    try {
+      await Promise.all([...resources.processes].map(child => new Promise<void>(resolve => {
+        if (child.exitCode !== null) return resolve();
+        const timer = setTimeout(() => child.kill("SIGKILL"), 5_000);
+        child.once("close", () => { clearTimeout(timer); resolve(); }); child.kill("SIGTERM");
+      })));
+      const credential = resources.token ?? await readFile(resources.store, "utf8").then(text => (JSON.parse(text) as { credentials: Record<string, CliCredential> }).credentials[ctx.input.api.baseURL]?.token).catch(() => undefined);
+      if (credential && !resources.revoked) {
+        ctx.input.privateSecrets!.push(credential);
+        const response = await fetch(`${ctx.input.api.baseURL}/api/cli-auth/revoke-current`, { method: "POST", headers: { Authorization: `Bearer ${credential}`, "content-type": "application/json" }, body: "{}", redirect: "error", signal: AbortSignal.timeout(15_000) });
+        const ok = response.ok || response.status === 401; await response.body?.cancel(); if (!ok) throw new Error("Owned CLI credential cleanup failed");
+      }
+      if (resources.keyId && !resources.revoked) await ctx.input.api.delete(`/api/board-api-keys/${resources.keyId}`, { allowNotFound: true });
+      else if (!resources.keyId && resources.challenge) await ctx.input.api.postSensitive(`/api/cli-auth/challenges/${resources.challenge.id}/cancel`, { token: resources.challenge.token });
+    } finally { await rm(resources.directory, { recursive: true, force: true }); owned.delete(ctx); }
+  },
+})];
+async function privateChallenge(ctx: ChatCatalogContext, id: string, token: string) {
+  const response = await fetch(`${ctx.input.api.baseURL}/api/cli-auth/challenges/${id}?token=${encodeURIComponent(token)}`, { redirect: "error", signal: AbortSignal.timeout(15_000) });
+  if (!response.ok) { await response.body?.cancel(); throw new Error(`Public CLI challenge read HTTP ${response.status}; response withheld`); }
+  return response.json() as Promise<{ requestedAccess: string; requestedCompanyId: string; approvedByUser?: { id: string } }>;
+}
+async function runCli(ctx: ChatCatalogContext, args: string[], name: string): Promise<{ code: number | null; stdout: string }> {
+  const resources = owned.get(ctx); if (!resources) throw new Error("Owned CLI process state missing");
+  const environment: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: resources.directory, USER: process.env.USER, LANG: process.env.LANG, TMPDIR: process.env.TMPDIR,
+    PAPERCLIP_HOME: resources.directory, PAPERCLIP_AUTH_STORE: resources.store, PAPERCLIP_NO_BROWSER: "1", DO_NOT_TRACK: "1" };
+  const child = spawn(process.execPath, ["--import", path.join(root, "cli/node_modules/tsx/dist/loader.mjs"), path.join(root, "cli/src/index.ts"), ...args,
+    "--api-base", ctx.input.api.baseURL, "--json"], { cwd: root, env: environment, stdio: ["ignore", "pipe", "pipe"] });
+  resources.processes.add(child); let stdout = ""; let writes = Promise.resolve();
+  child.stdout.on("data", chunk => { stdout += String(chunk); writes = writes.then(() => appendFile(path.join(resources.directory, `${name}.stdout`), chunk, { mode: 0o600 })); });
+  child.stderr.on("data", chunk => { writes = writes.then(() => appendFile(path.join(resources.directory, `${name}.stderr`), chunk, { mode: 0o600 })); });
+  let killTimer: ReturnType<typeof setTimeout> | undefined;
+  const timer = setTimeout(() => { child.kill("SIGTERM"); killTimer = setTimeout(() => child.kill("SIGKILL"), 5_000); }, 120_000);
+  try {
+    const code = await new Promise<number | null>((resolve, reject) => { child.once("error", () => reject(new Error("Owned source CLI process failed; details withheld"))); child.once("close", resolve); });
+    await writes; return { code, stdout };
+  } finally { clearTimeout(timer); clearTimeout(killTimer); resources.processes.delete(child); }
+}