import { expect } from "@playwright/test"; import { createHash } from "node:crypto"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { checks, project, entity, type Entity } from "./internal-support.js"; import { fixtureWorker } from "./internal-definition.js"; import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli, websiteCreateArgv, validContextGuidance, contextGuidanceArgv } from "./privileged-cli-support.js"; export function gradeCreateFromCli(value: unknown) { const c = checks(), o = privilegedObservation(value); c.add("actual-cli-observation", !!o, "The command returned by the actual model must run against the owned human CLI actor."); if (!o) return c.result(); const cli = o.state.cliResult as Entity | undefined, saved = o.tasks.find(row => row.id === cli?.id); const worker = o.state.worker as Entity | undefined, launch = o.state.project as Entity | undefined; c.add("one-later-cli-created-task", o.state.tasksBeforeCli === 0 && o.tasks.length === 1 && !!cli?.id && saved?.id === cli.id && o.state.cliCommandExecuted === true && o.state.explicitCompany === o.companyId, "The later CLI journey creates exactly one task, rather than the agent creating work while merely asked for a command."); c.add("saved-company-project-assignee", !!saved && saved.companyId === o.companyId && saved.projectId === launch?.id && saved.assigneeAgentId === worker?.id && /website/i.test(saved.title ?? "") && saved.status === "backlog", "Fresh public state independently binds the CLI result to the requested website task, Launch and Morgan."); c.add("real-command-structured-result", Array.isArray(o.state.executedArgv) && o.state.executedArgv[0] === "issue" && o.state.executedArgv[1] === "create" && cli?.companyId === saved?.companyId, "The response's finite supported flags actually execute with an inspectable structured result; no plausible invented flag is silently repaired."); return c.result(); } const create = privilegedDefinition("cli-operations.create-from-cli", "privileged-cli", { async setup(ctx) { const operator = await prepareOwnedCli(ctx); const launch = await project(ctx), worker = await fixtureWorker(ctx); return { ...operator, project: launch, worker }; }, async act(ctx, state) { const reply = await settlePrivilegedChat(ctx); state.tasksBeforeCli = reply.tasks.length; const fixture = { company: ctx.input.fixtures.company.id, project: entity(state, "project").id, agent: entity(state, "worker").id, apiBase: ctx.input.api.baseURL }; const bindings: Array<{ flag: string; placeholder: string; fixtureValue: string }> = []; const argv = websiteCreateArgv(reply.reply, fixture, bindings); await runOwnedCli(ctx, argv); const created = await ctx.input.api.get(`/api/companies/${fixture.company}/issues`); if (reply.tasks.length || created.length !== 1) throw new Error("Actual returned CLI command did not create exactly one later task"); const result = created[0]!; state.cliResult = { id: result.id, companyId: result.companyId, title: result.title, identifier: result.identifier, projectId: result.projectId, assigneeAgentId: result.assigneeAgentId, status: result.status }; state.executedArgv = argv.filter((value, index) => index < 2 || value.startsWith("-")); state.executedArgvSha256 = createHash("sha256").update(JSON.stringify(argv)).digest("hex"); state.unresolvedIdentifierBindingPolicy = "Only explicit unresolved company/project/assignee placeholders bind to independently saved fixture identities; resolved values are never repaired."; state.unresolvedIdentifierBindings = bindings; state.explicitCompany = fixture.company; state.cliCommandExecuted = true; const saved = await ctx.input.api.get(`/api/issues/${result.id}`); await ctx.input.evidence("chat-catalog-cli-created-task.json", { commandFlags: state.executedArgv, argvSha256: state.executedArgvSha256, bindingPolicy: state.unresolvedIdentifierBindingPolicy, result: state.cliResult, saved: { id: saved.id, companyId: saved.companyId, projectId: saved.projectId, assigneeAgentId: saved.assigneeAgentId, status: saved.status } }); }, grade: gradeCreateFromCli, async verifyResult(ctx, state) { const row = state.cliResult as Entity; const source = ctx.input.page.url(); await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${row.identifier ?? row.id}`, ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("heading", { name: row.title!, exact: true }).first()).toBeVisible(); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, }); create.sourceFiles = [...create.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"]; create.cleanup = cleanupOwnedCli; export function gradeChooseCliContext(value: unknown) { const c = checks(), o = privilegedObservation(value); c.add("owned-cli-context-evidence", !!o, "Two isolated profiles and real authenticated identity readback are required."); if (!o) return c.result(); c.add("supported-context-guidance", validContextGuidance(o.reply, { company: o.companyId, apiBase: o.origin }), "The answer supplies supported context/profile commands for the intended API and company instead of invented context flags."); c.add("actual-profile-selected", o.state.initialProfile === "birch" && o.state.finalProfile === "acorn" && o.state.selectedCompanyId === o.companyId && o.state.selectedApiBase === o.origin, "Supported CLI commands select the real Acorn fixture at its actual API base."); c.add("other-profile-preserved", o.state.otherProfilePreserved === true && o.state.onlyOwnedContextWritten === true, "The other configured profile is byte-equivalent semantically, and only the explicit owned context path is used."); c.add("identity-readback", typeof o.state.operatorUserId === "string" && !!o.state.operatorUserId && o.state.identityIncludesSelectedCompany === true, "An actual authenticated CLI read confirms the human identity and membership before future writes."); return c.result(); } const context = privilegedDefinition("cli-operations.choose-cli-context", "privileged-cli", { async setup(ctx) { const operator = await prepareOwnedCli(ctx); const birch = await ctx.input.api.post("/api/companies", { name: `Birch CLI fixture ${ctx.input.nonce}` }); for (const [name, companyId] of [["acorn", ctx.input.fixtures.company.id], ["birch", birch.id]]) await runOwnedCli(ctx, ["context", "set", "--profile", name!, "--api-base", ctx.input.api.baseURL, "--company-id", companyId!, "--use", "--json"]); const before = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"])); return { ...operator, initialProfile: before.currentProfile, otherProfile: before.profiles.birch }; }, async act(ctx, state) { const reply = await settlePrivilegedChat(ctx); const returned = contextGuidanceArgv(reply.reply, { company: ctx.input.fixtures.company.id, apiBase: ctx.input.api.baseURL }); if (!returned) throw new Error("Actual model context commands are unsupported or do not select and read back the intended profile"); for (const argv of returned.commands) await runOwnedCli(ctx, argv); state.returnedCommandHashes = returned.commands.map(argv => createHash("sha256").update(JSON.stringify(argv)).digest("hex")); state.unresolvedContextBindings = returned.bindings; const selected = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"])); const identity = JSON.parse(await runOwnedCli(ctx, ["auth", "whoami", "--json"])); state.finalProfile = selected.currentProfile; state.selectedCompanyId = selected.profile.companyId; state.selectedApiBase = selected.profile.apiBase; state.otherProfilePreserved = JSON.stringify(selected.profiles.birch) === JSON.stringify(state.otherProfile); state.onlyOwnedContextWritten = selected.contextPath === `${state.operatorDirectory}/context.json`; state.operatorUserId = identity.userId; state.identityIncludesSelectedCompany = identity.companyIds?.includes(ctx.input.fixtures.company.id) === true; await ctx.input.evidence("chat-catalog-cli-context-readback.json", { ...state, persona: "authenticated-human" }); }, grade: gradeChooseCliContext, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, }); context.sourceFiles = [...context.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"]; context.cleanup = cleanupOwnedCli; export const privilegedCliCases: ChatCatalogCase[] = [create, context];