diff --git a/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.test.ts new file mode 100644 index 00000000000..92416fe1750 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { gradeCliAuthorization } from "./identity-cli-authorization.js"; +import type { IdentityObservation } from "./identity-support.js"; +function valid(): IdentityObservation { + return { scenarioId: "access.cli-authorization", origin: "http://fixture.test", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], + interactions: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], activity: [], links: [], runs: [], humanQueries: [], + reply: "Use paperclipai auth login and approve its request in your browser.", state: { humanId: "alex", agentApproveStatus: 401, reviewVisible: true, reviewCompanyVisible: true, + requestedAccess: "board", requestedCompanyId: "acorn", approvedByUserId: "alex", cliUserId: "alex", cliSource: "board_key", cliHasCompany: true, cliExitCode: 0, + logoutRevoked: true, localCredentialRemoved: true, revokedReadStatus: 401, agentBeforeId: "ceo", agentAfterId: "ceo", agentAfterStatus: 200 } }; +} +describe("human CLI authorization oracle", () => { + it("accepts complete real CLI/UI/independent revocation evidence", () => { expect(gradeCliAuthorization(valid()).passed).toBe(true); }); + it.each([{ cliSource: "agent" }, { approvedByUserId: "riley" }, { requestedCompanyId: "birch" }, { agentApproveStatus: 200 }, + { revokedReadStatus: 200 }, { localCredentialRemoved: false }, { agentAfterStatus: 401 }, { cliExitCode: 1 }])("rejects wrong identity, scope or revocation %j", change => { + const observation = valid(); Object.assign(observation.state, change); expect(gradeCliAuthorization(observation).passed).toBe(false); + }); + it("rejects missing evidence and prose-only success", () => { expect(gradeCliAuthorization(undefined).passed).toBe(false); expect(gradeCliAuthorization({ reply: "Connected as you." }).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.ts b/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.ts new file mode 100644 index 00000000000..d665482b54a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-cli-authorization.ts @@ -0,0 +1,130 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { appendFile, mkdir, readFile, rm } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect } from "@playwright/test"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { checks } from "./internal-support.js"; +import { identityDefinition, identityObservation, probeAgentPermission, readAgentResource, waitForReply } from "./identity-support.js"; + +interface CliCredential { token: string; userId: string; apiBase: string } +interface Owned { directory: string; store: string; processes: Set; token?: string; keyId?: string; challenge?: { id: string; token: string }; revoked: boolean } +const owned = new WeakMap(); +const root = fileURLToPath(new URL("../../../", import.meta.url)); +export function gradeCliAuthorization(value: unknown) { + const c = checks(); const o = identityObservation(value); + c.add("complete-human-cli-proof", !!o && typeof o.state.humanId === "string" && o.state.humanId !== "local-board", "An actual authenticated person authorizes an actual source CLI process."); if (!o) return c.result(); + c.add("agent-guidance-keeps-identities-separate", /(?:auth\s+login|cli|terminal)/i.test(o.reply) && /approv|authoriz|sign.?in|browser/i.test(o.reply) + && !/\bI(?:'ve| have)?\s+(?:connected|authenticated|logged you in)\b/i.test(o.reply) && o.state.agentApproveStatus === 401, + "The answer guides the browser authorization and the actual agent bearer cannot grant a board credential."); + c.add("real-human-company-scope", o.state.reviewVisible === true && o.state.reviewCompanyVisible === true && o.state.requestedAccess === "board" + && o.state.requestedCompanyId === o.companyId && o.state.approvedByUserId === o.state.humanId && o.state.cliUserId === o.state.humanId + && o.state.cliSource === "board_key" && o.state.cliHasCompany === true && o.state.cliExitCode === 0, + "The CLI challenge's displayed command, Board access and audit company are explicitly approved by the person; real CLI whoami resolves that same user and accessible company."); + c.add("revocation-proven", o.state.logoutRevoked === true && o.state.localCredentialRemoved === true && o.state.revokedReadStatus === 401, + "Actual CLI logout revokes its saved credential, removes the private local copy and a fresh bearer request is rejected."); + c.add("agent-still-independent", o.state.agentBeforeId === o.leadId && o.state.agentAfterId === o.leadId && o.state.agentAfterStatus === 200, + "Revoking the human CLI key leaves the separately issued agent identity usable with its original company authority."); + return c.result(); +} +export const cliAuthorizationIdentityCases: readonly ChatCatalogCase[] = [identityDefinition("access.cli-authorization", async ctx => { + const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + if (!session.user?.id || session.user.id === "local-board" || !ctx.input.privateSecrets) throw new Error("CLI authorization requires a real human session and private credential redaction"); + const privateRoot = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR; + if (!privateRoot || !path.isAbsolute(privateRoot)) throw new Error("CLI credentials require the attempt-private result root outside the model workspace"); + const directory = path.join(privateRoot, "cases", ctx.scenario.id, `cli-auth-${ctx.input.nonce}`); + if (path.relative(ctx.input.workspacePath, directory).startsWith("..") === false) throw new Error("CLI credential home cannot be placed inside the model workspace"); + await mkdir(directory, { recursive: true, mode: 0o700 }); + owned.set(ctx, { directory, store: path.join(directory, "auth.json"), processes: new Set(), revoked: false }); + const before = await readAgentResource(ctx, `/api/agents/${ctx.input.fixtures.agent.id}`); + return { humanId: session.user.id, agentBeforeId: (before.data as { id?: string })?.id }; +}, gradeCliAuthorization, { + authentication: "human-session", attemptTimeoutMs: 420_000, + sourceFiles: ["chat-catalog-domains/identity-cli-authorization.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/access.yaml", + "../../cli/src/index.ts", "../../cli/src/client/board-auth.ts", "../../cli/src/commands/client/auth.ts", "../../cli/src/commands/client/common.ts", "../../pnpm-lock.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const resources = owned.get(ctx); if (!resources) throw new Error("Owned CLI home missing"); + const login = runCli(ctx, ["auth", "login", "--company-id", ctx.input.fixtures.company.id, "--no-browser"], "login"); + void login.catch(() => {}); // Later awaited; browser failures still trigger owned cleanup. + // Observe the CLI's own public challenge, never synthesize its success or + // copy an agent key into its private store. Raw output is kept private. + const approval = await pollUntil({ label: "source CLI approval URL", deadlineAt: Date.now() + 45_000, intervalMs: 300, + load: async () => { const text = await readFile(path.join(resources.directory, "login.stderr"), "utf8").catch(() => ""); + const match = text.match(/https?:\/\/[^\s]+\/cli-auth\/[^\s]+/); return match?.[0]; }, accept: value => !!value }); + const url = new URL(approval!); if (url.origin !== ctx.input.api.baseURL) throw new Error("CLI approval escaped owned instance"); + const token = url.searchParams.get("token"); const id = url.pathname.split("/").at(-1); if (!token || !id) throw new Error("CLI returned incomplete challenge"); + resources.challenge = { id, token }; + ctx.input.privateSecrets!.push(approval!, token); + const challenge = await privateChallenge(ctx, id, token); + state.requestedAccess = challenge.requestedAccess; state.requestedCompanyId = challenge.requestedCompanyId; + state.agentApproveStatus = await probeAgentPermission(ctx, `/api/cli-auth/challenges/${id}/approve`, "POST", { token }); + const source = ctx.input.page.url(); await ctx.input.page.goto(approval!, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("heading", { name: "Approve Paperclip CLI access", exact: true })).toBeVisible(); + await expect(ctx.input.page.getByText("paperclipai auth login", { exact: true })).toBeVisible(); state.reviewVisible = true; + await expect(ctx.input.page.getByText(ctx.input.fixtures.company.name, { exact: true }).last()).toBeVisible(); state.reviewCompanyVisible = true; + const approved = ctx.input.page.waitForResponse(response => response.request().method() === "POST" && new URL(response.url()).pathname === `/api/cli-auth/challenges/${id}/approve`); + await ctx.input.page.getByRole("button", { name: "Approve CLI access", exact: true }).click(); + const approvalResult = await approved; if (!approvalResult.ok()) throw new Error("Human CLI approval failed"); + resources.keyId = (await approvalResult.json() as { keyId: string }).keyId; + await expect(ctx.input.page.getByRole("heading", { name: "CLI access approved", exact: true })).toBeVisible(); + const loggedIn = await login; state.cliExitCode = loggedIn.code; + if (loggedIn.code !== 0) throw new Error("Real CLI login failed; private process logs retained"); + const store = JSON.parse(await readFile(resources.store, "utf8")) as { credentials: Record }; + const credential = store.credentials[ctx.input.api.baseURL]; if (!credential?.token) throw new Error("CLI did not persist its own credential"); + resources.token = credential.token; ctx.input.privateSecrets!.push(credential.token); + const decision = await privateChallenge(ctx, id, token); state.approvedByUserId = decision.approvedByUser?.id; + const identity = await runCli(ctx, ["auth", "whoami"], "whoami"); if (identity.code !== 0) throw new Error("Real authenticated CLI whoami failed"); + const me = JSON.parse(identity.stdout) as { userId: string; source: string; companyIds: string[] }; + state.cliUserId = me.userId; state.cliSource = me.source; state.cliHasCompany = me.companyIds.includes(ctx.input.fixtures.company.id); + const logout = await runCli(ctx, ["auth", "logout"], "logout"); if (logout.code !== 0) throw new Error("Real CLI logout failed"); + const result = JSON.parse(logout.stdout) as { revoked: boolean; removedLocalCredential: boolean }; + resources.revoked = result.revoked; state.logoutRevoked = result.revoked; state.localCredentialRemoved = result.removedLocalCredential + && !(JSON.parse(await readFile(resources.store, "utf8")) as { credentials: Record }).credentials[ctx.input.api.baseURL]; + const revoked = await fetch(`${ctx.input.api.baseURL}/api/cli-auth/me`, { headers: { Authorization: `Bearer ${credential.token}` }, redirect: "error", signal: AbortSignal.timeout(15_000) }); + state.revokedReadStatus = revoked.status; await revoked.body?.cancel(); + const after = await readAgentResource(ctx, `/api/agents/${ctx.input.fixtures.agent.id}`); state.agentAfterStatus = after.status; state.agentAfterId = (after.data as { id?: string })?.id; + await ctx.input.evidence("identity-cli-human-authorization.json", { userId: state.cliUserId, requestedCompanyId: state.requestedCompanyId, access: state.requestedAccess, + source: state.cliSource, approvedByUserId: state.approvedByUserId, logoutRevoked: state.logoutRevoked, revokedReadStatus: state.revokedReadStatus, agentAfterId: state.agentAfterId }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async cleanup(ctx) { + const resources = owned.get(ctx); if (!resources) return; + try { + await Promise.all([...resources.processes].map(child => new Promise(resolve => { + if (child.exitCode !== null) return resolve(); + const timer = setTimeout(() => child.kill("SIGKILL"), 5_000); + child.once("close", () => { clearTimeout(timer); resolve(); }); child.kill("SIGTERM"); + }))); + const credential = resources.token ?? await readFile(resources.store, "utf8").then(text => (JSON.parse(text) as { credentials: Record }).credentials[ctx.input.api.baseURL]?.token).catch(() => undefined); + if (credential && !resources.revoked) { + ctx.input.privateSecrets!.push(credential); + const response = await fetch(`${ctx.input.api.baseURL}/api/cli-auth/revoke-current`, { method: "POST", headers: { Authorization: `Bearer ${credential}`, "content-type": "application/json" }, body: "{}", redirect: "error", signal: AbortSignal.timeout(15_000) }); + const ok = response.ok || response.status === 401; await response.body?.cancel(); if (!ok) throw new Error("Owned CLI credential cleanup failed"); + } + if (resources.keyId && !resources.revoked) await ctx.input.api.delete(`/api/board-api-keys/${resources.keyId}`, { allowNotFound: true }); + else if (!resources.keyId && resources.challenge) await ctx.input.api.postSensitive(`/api/cli-auth/challenges/${resources.challenge.id}/cancel`, { token: resources.challenge.token }); + } finally { await rm(resources.directory, { recursive: true, force: true }); owned.delete(ctx); } + }, +})]; +async function privateChallenge(ctx: ChatCatalogContext, id: string, token: string) { + const response = await fetch(`${ctx.input.api.baseURL}/api/cli-auth/challenges/${id}?token=${encodeURIComponent(token)}`, { redirect: "error", signal: AbortSignal.timeout(15_000) }); + if (!response.ok) { await response.body?.cancel(); throw new Error(`Public CLI challenge read HTTP ${response.status}; response withheld`); } + return response.json() as Promise<{ requestedAccess: string; requestedCompanyId: string; approvedByUser?: { id: string } }>; +} +async function runCli(ctx: ChatCatalogContext, args: string[], name: string): Promise<{ code: number | null; stdout: string }> { + const resources = owned.get(ctx); if (!resources) throw new Error("Owned CLI process state missing"); + const environment: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: resources.directory, USER: process.env.USER, LANG: process.env.LANG, TMPDIR: process.env.TMPDIR, + PAPERCLIP_HOME: resources.directory, PAPERCLIP_AUTH_STORE: resources.store, PAPERCLIP_NO_BROWSER: "1", DO_NOT_TRACK: "1" }; + const child = spawn(process.execPath, ["--import", path.join(root, "cli/node_modules/tsx/dist/loader.mjs"), path.join(root, "cli/src/index.ts"), ...args, + "--api-base", ctx.input.api.baseURL, "--json"], { cwd: root, env: environment, stdio: ["ignore", "pipe", "pipe"] }); + resources.processes.add(child); let stdout = ""; let writes = Promise.resolve(); + child.stdout.on("data", chunk => { stdout += String(chunk); writes = writes.then(() => appendFile(path.join(resources.directory, `${name}.stdout`), chunk, { mode: 0o600 })); }); + child.stderr.on("data", chunk => { writes = writes.then(() => appendFile(path.join(resources.directory, `${name}.stderr`), chunk, { mode: 0o600 })); }); + let killTimer: ReturnType | undefined; + const timer = setTimeout(() => { child.kill("SIGTERM"); killTimer = setTimeout(() => child.kill("SIGKILL"), 5_000); }, 120_000); + try { + const code = await new Promise((resolve, reject) => { child.once("error", () => reject(new Error("Owned source CLI process failed; details withheld"))); child.once("close", resolve); }); + await writes; return { code, stdout }; + } finally { clearTimeout(timer); clearTimeout(killTimer); resources.processes.delete(child); } +}