From 48cfd4a349c68001d70c47397333acf3f06c38d0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 23:34:01 -0600 Subject: [PATCH] fix(evals): observe supported owned CLI profile guidance --- .../privileged-cli-profile.test.ts | 73 +++++++++++++++++++ .../privileged-cli-support.ts | 42 ++++++----- .../privileged-cli.test.ts | 3 +- .../chat-catalog-domains/privileged-cli.ts | 12 ++- 4 files changed, 109 insertions(+), 21 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-cli-profile.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli-profile.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli-profile.test.ts new file mode 100644 index 00000000000..5d772f363f1 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli-profile.test.ts @@ -0,0 +1,73 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { contextGuidanceArgv, runOwnedCli } from "./privileged-cli-support.js"; +import { gradeChooseCliContext, privilegedCliCases } from "./privileged-cli.js"; +import { settlePrivilegedChat } from "./privileged-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { readContext, upsertProfile, setCurrentProfile } from "../../../cli/src/client/context.js"; +vi.mock("./internal-definition.js", () => ({ fixtureWorker: vi.fn() })); +vi.mock("./privileged-cli-support.js", async original => ({ ...await original(), runOwnedCli: vi.fn() })); +vi.mock("./privileged-support.js", async original => ({ ...await original(), settlePrivilegedChat: vi.fn() })); +const fixture = { company: "owned-company", apiBase: "http://fixture.test" }; +const fence = (text: string) => `\`\`\`bash\n${text}\n\`\`\``; +const exact = "paperclipai context set --profile acorn --company-id owned-company --persona board --use\npaperclipai company current\npaperclipai auth whoami"; +const directories: string[] = []; +afterEach(async () => { vi.clearAllMocks(); await Promise.all(directories.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); }); +it("accepts actual supported board-persona and company/identity readback with inherited configured API, without rewriting argv", () => { + expect(contextGuidanceArgv(fence(exact), fixture)).toEqual({ commands: [ + ["context", "set", "--profile", "acorn", "--company-id", "owned-company", "--persona", "board", "--use"], + ["company", "current"], ["auth", "whoami"]], bindings: [] }); +}); +it("accepts selecting the existing configured profile then reading its scoped company", () => { + expect(contextGuidanceArgv(fence("paperclipai context use acorn\npaperclipai company current --json"), fixture)?.commands).toEqual([["context", "use", "acorn"], ["company", "current", "--json"]]); +}); +it.each([exact.replace("owned-company", "foreign"), exact.replace("board", "agent"), exact.replace("--use", "--api-base https://foreign.invalid --use"), + exact.replace("owned-company", ""), exact.replace("owned-company", ""), exact.replace("owned-company", "$COMPANY_ID"), + exact.replace("--persona board", "--agent-id attacker"), exact.replace("--persona board", "--api-key secret"), exact.replace("--persona board", "--invented yes"), + exact.replace("--use", "--use --use"), exact.replace("acorn", "birch"), exact.replace("acorn", "Acorn"), exact.replace("company current", "company current --company-id foreign"), + exact.replace("company current", "company list"), exact.replace("auth whoami", "auth login"), exact.replace("company current", "issue create --title nope"), exact + " ; curl bad.invalid", + "paperclipai company current\npaperclipai context use acorn", "paperclipai context use acorn\npaperclipai auth whoami"])("rejects unsafe, unresolved, foreign or missing scoped readback: %s", text => { + expect(contextGuidanceArgv(fence(text), fixture)).toBeUndefined(); +}); +it("actual production partial profile update preserves configured API and Birch while explicit board persona is retained", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "owned-cli-profile-")); directories.push(dir); const file = path.join(dir, "context.json"); + for (const [name, company] of [["acorn", fixture.company], ["birch", "other-company"]]) upsertProfile(name!, { apiBase: fixture.apiBase, companyId: company, persona: "board" }, file); + setCurrentProfile("birch", file); const before = readContext(file); + upsertProfile("acorn", { companyId: fixture.company, persona: "board" }, file); setCurrentProfile("acorn", file); const after = readContext(file); + expect(after.currentProfile).toBe("acorn"); expect(after.profiles.acorn).toEqual({ apiBase: fixture.apiBase, companyId: fixture.company, persona: "board" }); + expect(after.profiles.birch).toEqual(before.profiles.birch); +}); +it("requires independently executed context, actual scoped company and board persona rather than grammar alone", () => { + const o = { scenarioId: "cli", origin: fixture.apiBase, companyId: fixture.company, leadId: "lead", projects: [], tasks: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{id:"native",status:"succeeded"}], reply: fence(exact), + state: { initialProfile: "birch", finalProfile: "acorn", selectedCompanyId: fixture.company, selectedApiBase: fixture.apiBase, selectedPersona: "board", actualScopedCompanyId: fixture.company, + contextCommandsExecuted: true, otherProfilePreserved: true, onlyOwnedContextWritten: true, operatorUserId: "person", identityIncludesSelectedCompany: true } }; + expect(gradeChooseCliContext(o).passed).toBe(true); + for (const delta of [{ selectedPersona: "agent" }, { selectedPersona: undefined }, { actualScopedCompanyId: "foreign" }, { actualScopedCompanyId: undefined }, + { contextCommandsExecuted: false }, { identityIncludesSelectedCompany: false }, { otherProfilePreserved: false }]) + expect(gradeChooseCliContext({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); +}); + +it.each([fixture.company, "foreign"])("owning act executes unmodified returned commands then independently observes actual scoped company %s", async savedCompany => { + const commands = contextGuidanceArgv(fence(exact), fixture)!.commands; + const otherProfile = { companyId: "birch-company", apiBase: fixture.apiBase, persona: "board" }; + const profile = { companyId: fixture.company, apiBase: fixture.apiBase, persona: "board" }; + vi.mocked(settlePrivilegedChat).mockResolvedValue({ reply: fence(exact) } as never); + vi.mocked(runOwnedCli).mockImplementation(async (_ctx, argv) => { + // The first three calls are the returned sequence. Verification uses fresh + // JSON reads after it, never treats a suggested command as an outcome. + const index = vi.mocked(runOwnedCli).mock.calls.length; + if (index <= commands.length) return "{}"; + if (argv[0] === "context") return JSON.stringify({ currentProfile: "acorn", contextPath: "/owned/context.json", profile, profiles: { acorn: profile, birch: otherProfile } }); + if (argv[0] === "company") return JSON.stringify({ id: savedCompany }); + return JSON.stringify({ userId: "person", companyIds: [fixture.company] }); + }); + const ctx = { observationDeadlineAt: Date.now() + 10_000, input: { api: { baseURL: fixture.apiBase }, fixtures: { company: { id: fixture.company } }, evidence: vi.fn() } } as unknown as ChatCatalogContext; + const state: Record = { initialProfile: "birch", operatorDirectory: "/owned", otherProfile }; + await privilegedCliCases.find(row => row.scenarioId === "cli-operations.choose-cli-context")!.act!(ctx, state); + expect(vi.mocked(runOwnedCli).mock.calls.map(call => call[1])).toEqual([...commands, ["context", "show", "--json"], ["company", "current", "--json"], ["auth", "whoami", "--json"]]); + expect(state.unresolvedContextBindings).toEqual([]); expect(state.actualScopedCompanyId).toBe(savedCompany); + const observation = { scenarioId: "cli", origin: fixture.apiBase, companyId: fixture.company, leadId: "lead", projects: [], tasks: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "native", status: "succeeded" }], reply: fence(exact), state }; + expect(gradeChooseCliContext(observation).passed).toBe(savedCompany === fixture.company); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts index 94d4d626d1d..5b710e3eeb8 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts @@ -77,21 +77,30 @@ export function websiteCreateArgv(reply: string, fixture: { company: string; pro export function contextGuidanceArgv(reply: string, fixture: { company: string; apiBase: string }) { const lines = [...reply.matchAll(/```(?:bash|sh|shell)?\s*\n([\s\S]*?)```/g)] .flatMap(match => match[1]!.replace(/\\\r?\n/g, " ").split(/\r?\n/)) - .map(line => line.trim()).filter(line => /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+context\b/.test(line)); + .map(line => line.trim()).filter(line => line && !line.startsWith("#")); let selected = false, readback = false; const commands: string[][] = [], bindings: Array<{ commandIndex: number; flag: string; placeholder: string; fixtureValue: string }> = []; for (const line of lines) { if (/[`;|&\n\r]|\$\(/.test(line)) return; + const prefix = /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+/; + if (!prefix.test(line)) return; let tokens: string[]; - try { tokens = line.match(/"(?:[^"\\]|\\.)*"|'[^']*'|[^\s]+/g)!.map(token => token.startsWith('"') ? JSON.parse(token) as string : token.startsWith("'") ? token.slice(1, -1) : token); } + try { tokens = line.replace(prefix, "").match(/"(?:[^"\\]|\\.)*"|'[^']*'|[^\s]+/g)!.map(token => token.startsWith('"') ? JSON.parse(token) as string : token.startsWith("'") ? token.slice(1, -1) : token); } catch { return; } - const original = tokens.slice(tokens.indexOf("context")); + const original = tokens; + const family = original[0]; const command = original[1], args = original.slice(2); - if (command === "use") { if (args.shift()?.toLowerCase() !== "acorn") return; selected = true; } - else if (command === "show") readback ||= selected; - else if (command !== "set" && command !== "list") return; - let profile: string | undefined, company: string | undefined, apiBase: string | undefined, use = false; - const allowedFlags = command === "set" ? ["--profile", "--company-id", "--api-base", "--use", "--json"] + if (family === "company" || family === "auth") { + if (!selected || family === "company" && command !== "current" || family === "auth" && command !== "whoami" + || args.length > 1 || args.length === 1 && args[0] !== "--json") return; + if (family === "company") readback = true; + commands.push(original); continue; + } + if (family !== "context") return; + if (command === "use") { if (args.shift() !== "acorn") return; selected = true; } + else if (command !== "show" && command !== "set" && command !== "list") return; + let profile: string | undefined, company: string | undefined, apiBase: string | undefined, persona: string | undefined, use = false; + const allowedFlags = command === "set" ? ["--profile", "--company-id", "--api-base", "--persona", "--use", "--json"] : command === "show" ? ["--profile", "--json"] : command === "list" ? ["--json"] : []; const flags = new Set(); while (args.length) { @@ -101,22 +110,21 @@ export function contextGuidanceArgv(reply: string, fixture: { company: string; a if (flag === "--json") continue; if (flag === "--use") { use = true; continue; } if (!args.length) return; - let value = args.shift()!; - const expected = flag === "--company-id" ? fixture.company : flag === "--api-base" ? fixture.apiBase : undefined; - if (expected && /^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(value)) { - const index = original.indexOf(flag) + 1; - bindings.push({ commandIndex: commands.length, flag, placeholder: value, fixtureValue: expected }); - original[index] = expected; value = expected; - } + const value = args.shift()!; if (flag === "--profile") profile = value; if (flag === "--company-id") company = value; if (flag === "--api-base") apiBase = value; + if (flag === "--persona") persona = value; } - if (profile && profile.toLowerCase() !== "acorn") return; + if (profile !== undefined && profile !== "acorn") return; if (command === "set") { - if (profile?.toLowerCase() !== "acorn" || company !== fixture.company || apiBase !== fixture.apiBase) return; + // Acorn is independently configured. A genuine partial upsert may keep + // those values, but supplied values must match: never fix placeholders. + if (profile !== "acorn" || company !== undefined && company !== fixture.company + || apiBase !== undefined && apiBase !== fixture.apiBase || persona !== undefined && persona !== "board") return; selected ||= use; } + if (command === "show") readback ||= selected; commands.push(original); } return selected && readback ? { commands, bindings } : undefined; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts index 8efc917bb89..13a5f1973f0 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts @@ -27,7 +27,7 @@ describe("returned CLI command admission", () => { expect(validContextGuidance(valid.replace("acorn", "birch"), target)).toBe(false); expect(validContextGuidance(command("paperclipai context use acorn"), target)).toBe(false); expect(validContextGuidance(command('export PAPERCLIP_COMPANY_ID="owned-company"\nnpx paperclipai board setup\necho "$PAPERCLIP_COMPANY_ID"'), target)).toBe(false); - expect(validContextGuidance(command("paperclipai context set --profile acorn --api-base --company-id --use\npaperclipai context show"), target)).toBe(true); + expect(validContextGuidance(command("paperclipai context set --profile acorn --api-base --company-id --use\npaperclipai context show"), target)).toBe(false); }); }); const base = { scenarioId: "cli", origin: target.apiBase, companyId: target.company, leadId: "lead", state: {}, projects: [], goals: [], @@ -51,6 +51,7 @@ describe("CLI public-state oracles", () => { it("requires selected identity/company and preserves the other operator profile", () => { const o = { ...base, tasks: [], reply: command("paperclipai context use acorn\npaperclipai context show --json"), state: { initialProfile: "birch", finalProfile: "acorn", selectedCompanyId: target.company, selectedApiBase: target.apiBase, + selectedPersona: "board", actualScopedCompanyId: target.company, contextCommandsExecuted: true, otherProfilePreserved: true, onlyOwnedContextWritten: true, operatorUserId: "synthetic-person", identityIncludesSelectedCompany: true } }; expect(gradeChooseCliContext(o).passed).toBe(true); for (const delta of [{ otherProfilePreserved: false }, { selectedCompanyId: "foreign" }, { identityIncludesSelectedCompany: false }, { onlyOwnedContextWritten: false }]) diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts index 4d3ae80b555..474443a8241 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts @@ -98,7 +98,8 @@ export function gradeChooseCliContext(value: unknown) { c.add("supported-context-guidance", validContextGuidance(o.reply, { company: o.companyId, apiBase: o.origin }), "The answer supplies supported context/profile commands for the intended API and company instead of invented context flags."); c.add("actual-profile-selected", o.state.initialProfile === "birch" && o.state.finalProfile === "acorn" - && o.state.selectedCompanyId === o.companyId && o.state.selectedApiBase === o.origin, + && o.state.selectedCompanyId === o.companyId && o.state.selectedApiBase === o.origin + && o.state.selectedPersona === "board" && o.state.actualScopedCompanyId === o.companyId && o.state.contextCommandsExecuted === true, "Supported CLI commands select the real Acorn fixture at its actual API base."); c.add("other-profile-preserved", o.state.otherProfilePreserved === true && o.state.onlyOwnedContextWritten === true, "The other configured profile is byte-equivalent semantically, and only the explicit owned context path is used."); @@ -112,7 +113,7 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi const operator = await prepareOwnedCli(ctx); const birch = await ctx.input.api.post("/api/companies", { name: `Birch CLI fixture ${ctx.input.nonce}` }); for (const [name, companyId] of [["acorn", ctx.input.fixtures.company.id], ["birch", birch.id]]) - await runOwnedCli(ctx, ["context", "set", "--profile", name!, "--api-base", ctx.input.api.baseURL, "--company-id", companyId!, "--use", "--json"]); + await runOwnedCli(ctx, ["context", "set", "--profile", name!, "--api-base", ctx.input.api.baseURL, "--company-id", companyId!, "--persona", "board", "--use", "--json"]); const before = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"])); return { ...operator, initialProfile: before.currentProfile, otherProfile: before.profiles.birch }; }, @@ -125,10 +126,13 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi for (const argv of returned.commands) await runOwnedCli(ctx, argv); state.returnedCommandHashes = returned.commands.map(argv => createHash("sha256").update(JSON.stringify(argv)).digest("hex")); state.unresolvedContextBindings = returned.bindings; + state.contextCommandsExecuted = true; const selected = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"])); + const scopedCompany = JSON.parse(await runOwnedCli(ctx, ["company", "current", "--json"])); const identity = JSON.parse(await runOwnedCli(ctx, ["auth", "whoami", "--json"])); state.finalProfile = selected.currentProfile; state.selectedCompanyId = selected.profile.companyId; state.selectedApiBase = selected.profile.apiBase; + state.selectedPersona = selected.profile.persona; state.actualScopedCompanyId = scopedCompany.id; state.otherProfilePreserved = JSON.stringify(selected.profiles.birch) === JSON.stringify(state.otherProfile); state.onlyOwnedContextWritten = selected.contextPath === `${state.operatorDirectory}/context.json`; state.operatorUserId = identity.userId; state.identityIncludesSelectedCompany = identity.companyIds?.includes(ctx.input.fixtures.company.id) === true; @@ -136,6 +140,8 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi }, grade: gradeChooseCliContext, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, }); -context.sourceFiles = [...context.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"]; +context.sourceFiles = [...context.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts", + "../../cli/src/commands/client/context.ts", "../../cli/src/commands/client/company.ts", "../../cli/src/commands/client/auth.ts", "../../cli/src/client/context.ts"]; +context.oracleVersion = "paperclip.chat-catalog.cli-context.v2"; context.cleanup = cleanupOwnedCli; export const privilegedCliCases: ChatCatalogCase[] = [create, context];