fix(evals): observe supported owned CLI profile guidance

tests/runner-e2e/chat-catalog-domains/privileged-cli-profile.test.ts

@@ -0,0 +1,73 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { contextGuidanceArgv, runOwnedCli } from "./privileged-cli-support.js";
+import { gradeChooseCliContext, privilegedCliCases } from "./privileged-cli.js";
+import { settlePrivilegedChat } from "./privileged-support.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { readContext, upsertProfile, setCurrentProfile } from "../../../cli/src/client/context.js";
+vi.mock("./internal-definition.js", () => ({ fixtureWorker: vi.fn() }));
+vi.mock("./privileged-cli-support.js", async original => ({ ...await original<typeof import("./privileged-cli-support.js")>(), runOwnedCli: vi.fn() }));
+vi.mock("./privileged-support.js", async original => ({ ...await original<typeof import("./privileged-support.js")>(), settlePrivilegedChat: vi.fn() }));
+const fixture = { company: "owned-company", apiBase: "http://fixture.test" };
+const fence = (text: string) => `\`\`\`bash\n${text}\n\`\`\``;
+const exact = "paperclipai context set --profile acorn --company-id owned-company --persona board --use\npaperclipai company current\npaperclipai auth whoami";
+const directories: string[] = [];
+afterEach(async () => { vi.clearAllMocks(); await Promise.all(directories.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });
+it("accepts actual supported board-persona and company/identity readback with inherited configured API, without rewriting argv", () => {
+  expect(contextGuidanceArgv(fence(exact), fixture)).toEqual({ commands: [
+    ["context", "set", "--profile", "acorn", "--company-id", "owned-company", "--persona", "board", "--use"],
+    ["company", "current"], ["auth", "whoami"]], bindings: [] });
+});
+it("accepts selecting the existing configured profile then reading its scoped company", () => {
+  expect(contextGuidanceArgv(fence("paperclipai context use acorn\npaperclipai company current --json"), fixture)?.commands).toEqual([["context", "use", "acorn"], ["company", "current", "--json"]]);
+});
+it.each([exact.replace("owned-company", "foreign"), exact.replace("board", "agent"), exact.replace("--use", "--api-base https://foreign.invalid --use"),
+  exact.replace("owned-company", "<acorn-company-id>"), exact.replace("owned-company", "<COMPANY_ID>"), exact.replace("owned-company", "$COMPANY_ID"),
+  exact.replace("--persona board", "--agent-id attacker"), exact.replace("--persona board", "--api-key secret"), exact.replace("--persona board", "--invented yes"),
+  exact.replace("--use", "--use --use"), exact.replace("acorn", "birch"), exact.replace("acorn", "Acorn"), exact.replace("company current", "company current --company-id foreign"),
+  exact.replace("company current", "company list"), exact.replace("auth whoami", "auth login"), exact.replace("company current", "issue create --title nope"), exact + " ; curl bad.invalid",
+  "paperclipai company current\npaperclipai context use acorn", "paperclipai context use acorn\npaperclipai auth whoami"])("rejects unsafe, unresolved, foreign or missing scoped readback: %s", text => {
+  expect(contextGuidanceArgv(fence(text), fixture)).toBeUndefined();
+});
+it("actual production partial profile update preserves configured API and Birch while explicit board persona is retained", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "owned-cli-profile-")); directories.push(dir); const file = path.join(dir, "context.json");
+  for (const [name, company] of [["acorn", fixture.company], ["birch", "other-company"]]) upsertProfile(name!, { apiBase: fixture.apiBase, companyId: company, persona: "board" }, file);
+  setCurrentProfile("birch", file); const before = readContext(file);
+  upsertProfile("acorn", { companyId: fixture.company, persona: "board" }, file); setCurrentProfile("acorn", file); const after = readContext(file);
+  expect(after.currentProfile).toBe("acorn"); expect(after.profiles.acorn).toEqual({ apiBase: fixture.apiBase, companyId: fixture.company, persona: "board" });
+  expect(after.profiles.birch).toEqual(before.profiles.birch);
+});
+it("requires independently executed context, actual scoped company and board persona rather than grammar alone", () => {
+  const o = { scenarioId: "cli", origin: fixture.apiBase, companyId: fixture.company, leadId: "lead", projects: [], tasks: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{id:"native",status:"succeeded"}], reply: fence(exact),
+    state: { initialProfile: "birch", finalProfile: "acorn", selectedCompanyId: fixture.company, selectedApiBase: fixture.apiBase, selectedPersona: "board", actualScopedCompanyId: fixture.company,
+      contextCommandsExecuted: true, otherProfilePreserved: true, onlyOwnedContextWritten: true, operatorUserId: "person", identityIncludesSelectedCompany: true } };
+  expect(gradeChooseCliContext(o).passed).toBe(true);
+  for (const delta of [{ selectedPersona: "agent" }, { selectedPersona: undefined }, { actualScopedCompanyId: "foreign" }, { actualScopedCompanyId: undefined },
+    { contextCommandsExecuted: false }, { identityIncludesSelectedCompany: false }, { otherProfilePreserved: false }])
+    expect(gradeChooseCliContext({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false);
+});
+
+it.each([fixture.company, "foreign"])("owning act executes unmodified returned commands then independently observes actual scoped company %s", async savedCompany => {
+  const commands = contextGuidanceArgv(fence(exact), fixture)!.commands;
+  const otherProfile = { companyId: "birch-company", apiBase: fixture.apiBase, persona: "board" };
+  const profile = { companyId: fixture.company, apiBase: fixture.apiBase, persona: "board" };
+  vi.mocked(settlePrivilegedChat).mockResolvedValue({ reply: fence(exact) } as never);
+  vi.mocked(runOwnedCli).mockImplementation(async (_ctx, argv) => {
+    // The first three calls are the returned sequence. Verification uses fresh
+    // JSON reads after it, never treats a suggested command as an outcome.
+    const index = vi.mocked(runOwnedCli).mock.calls.length;
+    if (index <= commands.length) return "{}";
+    if (argv[0] === "context") return JSON.stringify({ currentProfile: "acorn", contextPath: "/owned/context.json", profile, profiles: { acorn: profile, birch: otherProfile } });
+    if (argv[0] === "company") return JSON.stringify({ id: savedCompany });
+    return JSON.stringify({ userId: "person", companyIds: [fixture.company] });
+  });
+  const ctx = { observationDeadlineAt: Date.now() + 10_000, input: { api: { baseURL: fixture.apiBase }, fixtures: { company: { id: fixture.company } }, evidence: vi.fn() } } as unknown as ChatCatalogContext;
+  const state: Record<string, unknown> = { initialProfile: "birch", operatorDirectory: "/owned", otherProfile };
+  await privilegedCliCases.find(row => row.scenarioId === "cli-operations.choose-cli-context")!.act!(ctx, state);
+  expect(vi.mocked(runOwnedCli).mock.calls.map(call => call[1])).toEqual([...commands, ["context", "show", "--json"], ["company", "current", "--json"], ["auth", "whoami", "--json"]]);
+  expect(state.unresolvedContextBindings).toEqual([]); expect(state.actualScopedCompanyId).toBe(savedCompany);
+  const observation = { scenarioId: "cli", origin: fixture.apiBase, companyId: fixture.company, leadId: "lead", projects: [], tasks: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "native", status: "succeeded" }], reply: fence(exact), state };
+  expect(gradeChooseCliContext(observation).passed).toBe(savedCompany === fixture.company);
+});

tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts

@@ -77,21 +77,30 @@ export function websiteCreateArgv(reply: string, fixture: { company: string; pro
 export function contextGuidanceArgv(reply: string, fixture: { company: string; apiBase: string }) {
   const lines = [...reply.matchAll(/```(?:bash|sh|shell)?\s*\n([\s\S]*?)```/g)]
     .flatMap(match => match[1]!.replace(/\\\r?\n/g, " ").split(/\r?\n/))
-    .map(line => line.trim()).filter(line => /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+context\b/.test(line));
+    .map(line => line.trim()).filter(line => line && !line.startsWith("#"));
   let selected = false, readback = false;
   const commands: string[][] = [], bindings: Array<{ commandIndex: number; flag: string; placeholder: string; fixtureValue: string }> = [];
   for (const line of lines) {
     if (/[`;|&\n\r]|\$\(/.test(line)) return;
+    const prefix = /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+/;
+    if (!prefix.test(line)) return;
     let tokens: string[];
-    try { tokens = line.match(/"(?:[^"\\]|\\.)*"|'[^']*'|[^\s]+/g)!.map(token => token.startsWith('"') ? JSON.parse(token) as string : token.startsWith("'") ? token.slice(1, -1) : token); }
+    try { tokens = line.replace(prefix, "").match(/"(?:[^"\\]|\\.)*"|'[^']*'|[^\s]+/g)!.map(token => token.startsWith('"') ? JSON.parse(token) as string : token.startsWith("'") ? token.slice(1, -1) : token); }
     catch { return; }
-    const original = tokens.slice(tokens.indexOf("context"));
+    const original = tokens;
+    const family = original[0];
     const command = original[1], args = original.slice(2);
-    if (command === "use") { if (args.shift()?.toLowerCase() !== "acorn") return; selected = true; }
-    else if (command === "show") readback ||= selected;
-    else if (command !== "set" && command !== "list") return;
-    let profile: string | undefined, company: string | undefined, apiBase: string | undefined, use = false;
-    const allowedFlags = command === "set" ? ["--profile", "--company-id", "--api-base", "--use", "--json"]
+    if (family === "company" || family === "auth") {
+      if (!selected || family === "company" && command !== "current" || family === "auth" && command !== "whoami"
+        || args.length > 1 || args.length === 1 && args[0] !== "--json") return;
+      if (family === "company") readback = true;
+      commands.push(original); continue;
+    }
+    if (family !== "context") return;
+    if (command === "use") { if (args.shift() !== "acorn") return; selected = true; }
+    else if (command !== "show" && command !== "set" && command !== "list") return;
+    let profile: string | undefined, company: string | undefined, apiBase: string | undefined, persona: string | undefined, use = false;
+    const allowedFlags = command === "set" ? ["--profile", "--company-id", "--api-base", "--persona", "--use", "--json"]
       : command === "show" ? ["--profile", "--json"] : command === "list" ? ["--json"] : [];
     const flags = new Set<string>();
     while (args.length) {
@@ -101,22 +110,21 @@ export function contextGuidanceArgv(reply: string, fixture: { company: string; a
       if (flag === "--json") continue;
       if (flag === "--use") { use = true; continue; }
       if (!args.length) return;
-      let value = args.shift()!;
-      const expected = flag === "--company-id" ? fixture.company : flag === "--api-base" ? fixture.apiBase : undefined;
-      if (expected && /^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(value)) {
-        const index = original.indexOf(flag) + 1;
-        bindings.push({ commandIndex: commands.length, flag, placeholder: value, fixtureValue: expected });
-        original[index] = expected; value = expected;
-      }
+      const value = args.shift()!;
       if (flag === "--profile") profile = value;
       if (flag === "--company-id") company = value;
       if (flag === "--api-base") apiBase = value;
+      if (flag === "--persona") persona = value;
     }
-    if (profile && profile.toLowerCase() !== "acorn") return;
+    if (profile !== undefined && profile !== "acorn") return;
     if (command === "set") {
-      if (profile?.toLowerCase() !== "acorn" || company !== fixture.company || apiBase !== fixture.apiBase) return;
+      // Acorn is independently configured. A genuine partial upsert may keep
+      // those values, but supplied values must match: never fix placeholders.
+      if (profile !== "acorn" || company !== undefined && company !== fixture.company
+        || apiBase !== undefined && apiBase !== fixture.apiBase || persona !== undefined && persona !== "board") return;
       selected ||= use;
     }
+    if (command === "show") readback ||= selected;
     commands.push(original);
   }
   return selected && readback ? { commands, bindings } : undefined;

tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts

@@ -27,7 +27,7 @@ describe("returned CLI command admission", () => {
     expect(validContextGuidance(valid.replace("acorn", "birch"), target)).toBe(false);
     expect(validContextGuidance(command("paperclipai context use acorn"), target)).toBe(false);
     expect(validContextGuidance(command('export PAPERCLIP_COMPANY_ID="owned-company"\nnpx paperclipai board setup\necho "$PAPERCLIP_COMPANY_ID"'), target)).toBe(false);
-    expect(validContextGuidance(command("paperclipai context set --profile acorn --api-base <API_BASE> --company-id <COMPANY_ID> --use\npaperclipai context show"), target)).toBe(true);
+    expect(validContextGuidance(command("paperclipai context set --profile acorn --api-base <API_BASE> --company-id <COMPANY_ID> --use\npaperclipai context show"), target)).toBe(false);
   });
 });
 const base = { scenarioId: "cli", origin: target.apiBase, companyId: target.company, leadId: "lead", state: {}, projects: [], goals: [],
@@ -51,6 +51,7 @@ describe("CLI public-state oracles", () => {
   it("requires selected identity/company and preserves the other operator profile", () => {
     const o = { ...base, tasks: [], reply: command("paperclipai context use acorn\npaperclipai context show --json"),
       state: { initialProfile: "birch", finalProfile: "acorn", selectedCompanyId: target.company, selectedApiBase: target.apiBase,
+        selectedPersona: "board", actualScopedCompanyId: target.company, contextCommandsExecuted: true,
         otherProfilePreserved: true, onlyOwnedContextWritten: true, operatorUserId: "synthetic-person", identityIncludesSelectedCompany: true } };
     expect(gradeChooseCliContext(o).passed).toBe(true);
     for (const delta of [{ otherProfilePreserved: false }, { selectedCompanyId: "foreign" }, { identityIncludesSelectedCompany: false }, { onlyOwnedContextWritten: false }])

tests/runner-e2e/chat-catalog-domains/privileged-cli.ts

@@ -98,7 +98,8 @@ export function gradeChooseCliContext(value: unknown) {
   c.add("supported-context-guidance", validContextGuidance(o.reply, { company: o.companyId, apiBase: o.origin }),
     "The answer supplies supported context/profile commands for the intended API and company instead of invented context flags.");
   c.add("actual-profile-selected", o.state.initialProfile === "birch" && o.state.finalProfile === "acorn"
-    && o.state.selectedCompanyId === o.companyId && o.state.selectedApiBase === o.origin,
+    && o.state.selectedCompanyId === o.companyId && o.state.selectedApiBase === o.origin
+    && o.state.selectedPersona === "board" && o.state.actualScopedCompanyId === o.companyId && o.state.contextCommandsExecuted === true,
     "Supported CLI commands select the real Acorn fixture at its actual API base.");
   c.add("other-profile-preserved", o.state.otherProfilePreserved === true && o.state.onlyOwnedContextWritten === true,
     "The other configured profile is byte-equivalent semantically, and only the explicit owned context path is used.");
@@ -112,7 +113,7 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi
     const operator = await prepareOwnedCli(ctx);
     const birch = await ctx.input.api.post<Entity>("/api/companies", { name: `Birch CLI fixture ${ctx.input.nonce}` });
     for (const [name, companyId] of [["acorn", ctx.input.fixtures.company.id], ["birch", birch.id]])
-      await runOwnedCli(ctx, ["context", "set", "--profile", name!, "--api-base", ctx.input.api.baseURL, "--company-id", companyId!, "--use", "--json"]);
+      await runOwnedCli(ctx, ["context", "set", "--profile", name!, "--api-base", ctx.input.api.baseURL, "--company-id", companyId!, "--persona", "board", "--use", "--json"]);
     const before = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"]));
     return { ...operator, initialProfile: before.currentProfile, otherProfile: before.profiles.birch };
   },
@@ -125,10 +126,13 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi
     for (const argv of returned.commands) await runOwnedCli(ctx, argv);
     state.returnedCommandHashes = returned.commands.map(argv => createHash("sha256").update(JSON.stringify(argv)).digest("hex"));
     state.unresolvedContextBindings = returned.bindings;
+    state.contextCommandsExecuted = true;
     const selected = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"]));
+    const scopedCompany = JSON.parse(await runOwnedCli(ctx, ["company", "current", "--json"]));
     const identity = JSON.parse(await runOwnedCli(ctx, ["auth", "whoami", "--json"]));
     state.finalProfile = selected.currentProfile; state.selectedCompanyId = selected.profile.companyId;
     state.selectedApiBase = selected.profile.apiBase;
+    state.selectedPersona = selected.profile.persona; state.actualScopedCompanyId = scopedCompany.id;
     state.otherProfilePreserved = JSON.stringify(selected.profiles.birch) === JSON.stringify(state.otherProfile);
     state.onlyOwnedContextWritten = selected.contextPath === `${state.operatorDirectory}/context.json`;
     state.operatorUserId = identity.userId; state.identityIncludesSelectedCompany = identity.companyIds?.includes(ctx.input.fixtures.company.id) === true;
@@ -136,6 +140,8 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi
   }, grade: gradeChooseCliContext,
   async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
 });
-context.sourceFiles = [...context.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"];
+context.sourceFiles = [...context.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts",
+  "../../cli/src/commands/client/context.ts", "../../cli/src/commands/client/company.ts", "../../cli/src/commands/client/auth.ts", "../../cli/src/client/context.ts"];
+context.oracleVersion = "paperclip.chat-catalog.cli-context.v2";
 context.cleanup = cleanupOwnedCli;
 export const privilegedCliCases: ChatCatalogCase[] = [create, context];