eval: author owned operator and developer reference cases

doc/plans/2026-10-26-chat-catalog-eval-progress.md

@@ -55,4 +55,4 @@ Next-wave review items: bound identity probe credentials privately across repeat
 
 Core behavior fixes require separate focused PRs that remain unmerged; their dependent passes cannot count toward the eval threshold. Eval merge requires every scenario implemented/processed, at least 196 final eligible passes, latest-head checks/reviews and actual protected merge. Per-case fingerprints, global suite hash and production/source revisions remain distinct; stale passes cannot be carried across changed execution/grader dependencies.
 
-Credential discovery (read-only): `~/.secrets` contains `AGENTMAIL_PAPERCLIP_STAGING_QA_API_KEY` and `COPILOT_GITHUB_TOKEN` references; no Notion, Slack or vault reference matched. Neither establishes an explicitly disposable mailbox/recipient or repository allowlist. Local `op` exists, but account/item reads fail because it cannot connect to the 1Password desktop app; no item contents were read, and no credentials were printed or persisted.
+Credential discovery: relevant source metadata is readable through the existing 1Password service account outside the sandbox. Notion, QA AgentMail, GitHub, Daytona and Anthropic references are available; no matching Slack reference was found. Credential validity, a disposable repository allowlist, designated mail recipient and pinned cloud image remain unverified. Automatic approval review rejected credentialed provider identity/resource discovery and metadata persistence; explicit approval for the narrow read-only AgentMail/GitHub endpoints is pending. No provider discovery request or metadata write occurred. Private item IDs and account metadata remain outside tracked documents.

tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts

@@ -0,0 +1,128 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdtemp, mkdir, realpath, rm } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+const execute = promisify(execFile);
+const privateActors = new WeakMap<ChatCatalogContext, { id: string; token: string; directory: string }>();
+export async function prepareOwnedCli(ctx: ChatCatalogContext) {
+  if (ctx.input.execution.environment.id !== "local") throw new Error("CLI companion journey requires the owned local attempt");
+  const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT;
+  if (!root) throw new Error("CLI companion requires an owned attempt directory");
+  const directory = await mkdtemp(path.join(await realpath(root), "operator-cli-"));
+  await mkdir(path.join(directory, "home"));
+  const key = await ctx.input.api.postSensitive<{ id: string; token: string }>("/api/board-api-keys", {
+    name: `catalog-operator-${ctx.input.nonce}`, requestedCompanyId: ctx.input.fixtures.company.id, expiresAt: new Date(Date.now() + 900_000).toISOString(),
+  });
+  if (!key.id || !key.token) throw new Error("Owned human CLI credential was not issued");
+  ctx.input.privateSecrets?.push(key.token);
+  privateActors.set(ctx, { ...key, directory });
+  return { operatorDirectory: directory, operatorKeyId: key.id };
+}
+export async function runOwnedCli(ctx: ChatCatalogContext, args: string[]) {
+  const actor = privateActors.get(ctx); if (!actor) throw new Error("Owned human CLI actor is unavailable");
+  // A fresh minimal environment cannot pick up another operator's context,
+  // stored auth, run ID, provider credentials or default company.
+  const env: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: path.join(actor.directory, "home"),
+    PAPERCLIP_HOME: path.join(actor.directory, "home"), PAPERCLIP_API_KEY: actor.token,
+    PAPERCLIP_API_URL: ctx.input.api.baseURL, PAPERCLIP_TELEMETRY_DISABLED: "1", NO_COLOR: "1" };
+  const loader = fileURLToPath(new URL("../../../cli/node_modules/tsx/dist/loader.mjs", import.meta.url));
+  const entry = fileURLToPath(new URL("../../../cli/src/index.ts", import.meta.url));
+  try {
+    const result = await execute(process.execPath, ["--import", loader, entry, ...args,
+      "--context", path.join(actor.directory, "context.json")], { cwd: actor.directory, env, timeout: 45_000, maxBuffer: 1_048_576 });
+    if (result.stdout.includes(actor.token) || result.stderr.includes(actor.token)) throw new Error("Private CLI credential appeared in output");
+    return result.stdout;
+  } catch { throw new Error("Owned CLI command failed; private subprocess output withheld"); }
+}
+export async function cleanupOwnedCli(ctx: ChatCatalogContext) {
+  const actor = privateActors.get(ctx); if (!actor) return;
+  await ctx.input.api.delete(`/api/board-api-keys/${actor.id}`);
+  privateActors.delete(ctx);
+  const root = await realpath(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT!);
+  const owned = await realpath(actor.directory);
+  if (!owned.startsWith(root + path.sep) || !path.basename(owned).startsWith("operator-cli-")) throw new Error("Refusing CLI cleanup outside its owned directory");
+  await rm(owned, { recursive: true });
+}
+
+/** Parse one finite supported command into argv; never evaluate a shell. */
+export function websiteCreateArgv(reply: string, fixture: { company: string; project: string; agent: string; apiBase: string }, bindings: Array<{ flag: string; placeholder: string; fixtureValue: string }> = []) {
+  const blocks = [...reply.matchAll(/```(?:bash|sh|shell)?\s*\n([\s\S]*?)```/g)].map(match => match[1]!.replace(/\\\r?\n/g, " ").trim());
+  const commands = blocks.flatMap(block => block.split(/\r?\n/)).filter(line => /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+issue\s+create\b/.test(line.trim()));
+  if (commands.length !== 1) throw new Error("One supported website creation command is required");
+  const line = commands[0]!.trim();
+  if (/[`;|&\n\r]|\$\(/.test(line)) throw new Error("Shell execution is not admitted in the owned CLI journey");
+  const tokens = line.match(/"(?:[^"\\]|\\.)*"|'[^']*'|[^\s]+/g)!;
+  const values = tokens.map(token => token.startsWith('"') ? JSON.parse(token) as string : token.startsWith("'") ? token.slice(1, -1) : token);
+  const start = values.indexOf("issue"), argv = values.slice(start);
+  const allowed = new Set(["--company-id", "-C", "--title", "--description", "--status", "--priority", "--assignee-agent-id", "--project-id", "--api-base"]);
+  const fields = new Map<string, string>();
+  for (let i = 2; i < argv.length; i++) {
+    const flag = argv[i]!; if (flag === "--json" || flag === "--standalone") continue;
+    if (!allowed.has(flag) || !argv[i + 1] || fields.has(flag)) throw new Error("Unsupported or ambiguous creation flags");
+    let value = argv[++i]!;
+    const target = ["--company-id", "-C"].includes(flag) ? fixture.company : flag === "--project-id" ? fixture.project : flag === "--assignee-agent-id" ? fixture.agent : undefined;
+    // Only unresolved identifier placeholders are bound; actual wrong IDs fail.
+    if (target && /^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(value)) { bindings.push({ flag, placeholder: value, fixtureValue: target }); value = target; argv[i] = target; }
+    fields.set(flag, value);
+  }
+  if ((fields.get("--company-id") ?? fields.get("-C")) !== fixture.company || fields.get("--project-id") !== fixture.project
+    || fields.get("--assignee-agent-id") !== fixture.agent || !/website/i.test(fields.get("--title") ?? "")
+    || fields.has("--status") && fields.get("--status") !== "backlog" || fields.has("--api-base") && fields.get("--api-base") !== fixture.apiBase)
+    throw new Error("Creation command does not target exactly the intended company, project and Morgan");
+  if (fields.has("--company-id") && fields.has("-C")) throw new Error("Duplicate company context flags are ambiguous");
+  if (!argv.includes("--json")) argv.push("--json");
+  return argv;
+}
+
+export function contextGuidanceArgv(reply: string, fixture: { company: string; apiBase: string }) {
+  const lines = [...reply.matchAll(/```(?:bash|sh|shell)?\s*\n([\s\S]*?)```/g)]
+    .flatMap(match => match[1]!.replace(/\\\r?\n/g, " ").split(/\r?\n/))
+    .map(line => line.trim()).filter(line => /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+context\b/.test(line));
+  let selected = false, readback = false;
+  const commands: string[][] = [], bindings: Array<{ commandIndex: number; flag: string; placeholder: string; fixtureValue: string }> = [];
+  for (const line of lines) {
+    if (/[`;|&\n\r]|\$\(/.test(line)) return;
+    let tokens: string[];
+    try { tokens = line.match(/"(?:[^"\\]|\\.)*"|'[^']*'|[^\s]+/g)!.map(token => token.startsWith('"') ? JSON.parse(token) as string : token.startsWith("'") ? token.slice(1, -1) : token); }
+    catch { return; }
+    const original = tokens.slice(tokens.indexOf("context"));
+    const command = original[1], args = original.slice(2);
+    if (command === "use") { if (args.shift()?.toLowerCase() !== "acorn") return; selected = true; }
+    else if (command === "show") readback ||= selected;
+    else if (command !== "set" && command !== "list") return;
+    let profile: string | undefined, company: string | undefined, apiBase: string | undefined, use = false;
+    const allowedFlags = command === "set" ? ["--profile", "--company-id", "--api-base", "--use", "--json"]
+      : command === "show" ? ["--profile", "--json"] : command === "list" ? ["--json"] : [];
+    const flags = new Set<string>();
+    while (args.length) {
+      const flag = args.shift();
+      if (!flag || !allowedFlags.includes(flag) || flags.has(flag)) return;
+      flags.add(flag);
+      if (flag === "--json") continue;
+      if (flag === "--use") { use = true; continue; }
+      if (!args.length) return;
+      let value = args.shift()!;
+      const expected = flag === "--company-id" ? fixture.company : flag === "--api-base" ? fixture.apiBase : undefined;
+      if (expected && /^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(value)) {
+        const index = original.indexOf(flag) + 1;
+        bindings.push({ commandIndex: commands.length, flag, placeholder: value, fixtureValue: expected });
+        original[index] = expected; value = expected;
+      }
+      if (flag === "--profile") profile = value;
+      if (flag === "--company-id") company = value;
+      if (flag === "--api-base") apiBase = value;
+    }
+    if (profile && profile.toLowerCase() !== "acorn") return;
+    if (command === "set") {
+      if (profile?.toLowerCase() !== "acorn" || company !== fixture.company || apiBase !== fixture.apiBase) return;
+      selected ||= use;
+    }
+    commands.push(original);
+  }
+  return selected && readback ? { commands, bindings } : undefined;
+}
+export function validContextGuidance(reply: string, fixture: { company: string; apiBase: string }) {
+  return !!contextGuidanceArgv(reply, fixture);
+}

tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { websiteCreateArgv, validContextGuidance } from "./privileged-cli-support.js";
+import { gradeChooseCliContext, gradeCreateFromCli } from "./privileged-cli.js";
+const target = { company: "owned-company", project: "launch", agent: "morgan", apiBase: "http://127.0.0.1:3100" };
+const command = (text: string) => `Use the explicit owned context.\n\`\`\`bash\n${text}\n\`\`\``;
+const valid = 'paperclipai issue create --company-id <COMPANY_ID> --title "Website task" --project-id <PROJECT_ID> --assignee-agent-id <MORGAN_AGENT_ID>';
+describe("returned CLI command admission", () => {
+  it("executes only finite supported argv with unresolved IDs bound to actual fixtures", () => {
+    expect(websiteCreateArgv(command(valid), target)).toEqual(["issue", "create", "--company-id", "owned-company", "--title", "Website task", "--project-id", "launch", "--assignee-agent-id", "morgan", "--json"]);
+  });
+  it("rejects shell execution, invented flags, real wrong IDs and ambiguous writes before launch", () => {
+    for (const text of [valid + " ; curl attacker.invalid", valid + " --invented yes", valid.replace("<COMPANY_ID>", "foreign-company"),
+      valid.replace("<PROJECT_ID>", "other-project"), valid + " -C foreign-company", valid + " --status in_progress",
+      valid + " --api-base https://foreign.invalid", valid + " --description '$(read-secret)'", valid + "\n" + valid])
+      expect(() => websiteCreateArgv(command(text), target)).toThrow();
+  });
+  it("requires supported context selection and readback instead of ignoring invented flags", () => {
+    const valid = command("paperclipai context use acorn\npaperclipai context show --json");
+    expect(validContextGuidance(valid, target)).toBe(true);
+    expect(validContextGuidance(valid.replace("use acorn", "use acorn --invented yes"), target)).toBe(false);
+    expect(validContextGuidance(valid.replace("acorn", "birch"), target)).toBe(false);
+    expect(validContextGuidance(command("paperclipai context use acorn"), target)).toBe(false);
+    expect(validContextGuidance(command("paperclipai context set --profile acorn --api-base <API_BASE> --company-id <COMPANY_ID> --use\npaperclipai context show"), target)).toBe(true);
+  });
+});
+const base = { scenarioId: "cli", origin: target.apiBase, companyId: target.company, leadId: "lead", state: {}, projects: [], goals: [],
+  comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "native", status: "succeeded" }], reply: "" };
+describe("CLI public-state oracles", () => {
+  it("requires one actual later result, requested actor/project/company, and independently saved task", () => {
+    const row = { id: "created", companyId: target.company, title: "Website task", projectId: "launch", assigneeAgentId: "morgan", status: "backlog" };
+    const o = { ...base, tasks: [row], state: { cliResult: row, worker: { id: "morgan" }, project: { id: "launch" }, tasksBeforeCli: 0,
+      cliCommandExecuted: true, explicitCompany: target.company, executedArgv: ["issue", "create"] } };
+    expect(gradeCreateFromCli(o).passed).toBe(true);
+    expect(gradeCreateFromCli({ ...o, tasks: [{ ...row, companyId: "foreign" }] }).passed).toBe(false);
+    expect(gradeCreateFromCli({ ...o, tasks: [row, { ...row, id: "duplicate" }] }).passed).toBe(false);
+    expect(gradeCreateFromCli({ ...o, state: { ...o.state, tasksBeforeCli: 1 } }).passed).toBe(false);
+    expect(gradeCreateFromCli(null).passed).toBe(false);
+  });
+  it("requires selected identity/company and preserves the other operator profile", () => {
+    const o = { ...base, tasks: [], reply: command("paperclipai context use acorn\npaperclipai context show --json"),
+      state: { initialProfile: "birch", finalProfile: "acorn", selectedCompanyId: target.company, selectedApiBase: target.apiBase,
+        otherProfilePreserved: true, onlyOwnedContextWritten: true, operatorUserId: "synthetic-person", identityIncludesSelectedCompany: true } };
+    expect(gradeChooseCliContext(o).passed).toBe(true);
+    for (const delta of [{ otherProfilePreserved: false }, { selectedCompanyId: "foreign" }, { identityIncludesSelectedCompany: false }, { onlyOwnedContextWritten: false }])
+      expect(gradeChooseCliContext({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false);
+  });
+});

tests/runner-e2e/chat-catalog-domains/privileged-cli.ts

@@ -0,0 +1,104 @@
+import { expect } from "@playwright/test";
+import { createHash } from "node:crypto";
+import type { ChatCatalogCase } from "../chat-catalog-contract.js";
+import { checks, project, entity, type Entity } from "./internal-support.js";
+import { fixtureWorker } from "./internal-definition.js";
+import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js";
+import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli, websiteCreateArgv, validContextGuidance, contextGuidanceArgv } from "./privileged-cli-support.js";
+
+export function gradeCreateFromCli(value: unknown) {
+  const c = checks(), o = privilegedObservation(value);
+  c.add("actual-cli-observation", !!o, "The command returned by the actual model must run against the owned human CLI actor."); if (!o) return c.result();
+  const cli = o.state.cliResult as Entity | undefined, saved = o.tasks.find(row => row.id === cli?.id);
+  const worker = o.state.worker as Entity | undefined, launch = o.state.project as Entity | undefined;
+  c.add("one-later-cli-created-task", o.state.tasksBeforeCli === 0 && o.tasks.length === 1 && !!cli?.id && saved?.id === cli.id
+    && o.state.cliCommandExecuted === true && o.state.explicitCompany === o.companyId,
+    "The later CLI journey creates exactly one task, rather than the agent creating work while merely asked for a command.");
+  c.add("saved-company-project-assignee", !!saved && saved.companyId === o.companyId && saved.projectId === launch?.id
+    && saved.assigneeAgentId === worker?.id && /website/i.test(saved.title ?? "") && saved.status === "backlog",
+    "Fresh public state independently binds the CLI result to the requested website task, Launch and Morgan.");
+  c.add("real-command-structured-result", Array.isArray(o.state.executedArgv) && o.state.executedArgv[0] === "issue"
+    && o.state.executedArgv[1] === "create" && cli?.companyId === saved?.companyId,
+    "The response's finite supported flags actually execute with an inspectable structured result; no plausible invented flag is silently repaired.");
+  return c.result();
+}
+const create = privilegedDefinition("cli-operations.create-from-cli", "privileged-cli", {
+  async setup(ctx) {
+    const operator = await prepareOwnedCli(ctx);
+    const launch = await project(ctx), worker = await fixtureWorker(ctx);
+    return { ...operator, project: launch, worker };
+  },
+  async act(ctx, state) {
+    const reply = await settlePrivilegedChat(ctx);
+    state.tasksBeforeCli = reply.tasks.length;
+    const fixture = { company: ctx.input.fixtures.company.id, project: entity(state, "project").id,
+      agent: entity(state, "worker").id, apiBase: ctx.input.api.baseURL };
+    const bindings: Array<{ flag: string; placeholder: string; fixtureValue: string }> = [];
+    const argv = websiteCreateArgv(reply.reply, fixture, bindings);
+    const result = JSON.parse(await runOwnedCli(ctx, argv)) as Entity;
+    state.cliResult = { id: result.id, companyId: result.companyId, title: result.title, identifier: result.identifier,
+      projectId: result.projectId, assigneeAgentId: result.assigneeAgentId, status: result.status };
+    state.executedArgv = argv.filter((value, index) => index < 2 || value.startsWith("-"));
+    state.executedArgvSha256 = createHash("sha256").update(JSON.stringify(argv)).digest("hex");
+    state.unresolvedIdentifierBindingPolicy = "Only explicit unresolved company/project/assignee placeholders bind to independently saved fixture identities; resolved values are never repaired.";
+    state.unresolvedIdentifierBindings = bindings;
+    state.explicitCompany = fixture.company; state.cliCommandExecuted = true;
+    const saved = await ctx.input.api.get<Entity>(`/api/issues/${result.id}`);
+    await ctx.input.evidence("chat-catalog-cli-created-task.json", { commandFlags: state.executedArgv, argvSha256: state.executedArgvSha256, bindingPolicy: state.unresolvedIdentifierBindingPolicy, result: state.cliResult,
+      saved: { id: saved.id, companyId: saved.companyId, projectId: saved.projectId, assigneeAgentId: saved.assigneeAgentId, status: saved.status } });
+  }, grade: gradeCreateFromCli,
+  async verifyResult(ctx, state) {
+    const row = state.cliResult as Entity;
+    const source = ctx.input.page.url();
+    await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${row.identifier ?? row.id}`, ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByRole("heading", { name: row.title!, exact: true }).first()).toBeVisible();
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+});
+create.sourceFiles = [...create.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"];
+create.cleanup = cleanupOwnedCli;
+export function gradeChooseCliContext(value: unknown) {
+  const c = checks(), o = privilegedObservation(value);
+  c.add("owned-cli-context-evidence", !!o, "Two isolated profiles and real authenticated identity readback are required."); if (!o) return c.result();
+  c.add("supported-context-guidance", validContextGuidance(o.reply, { company: o.companyId, apiBase: o.origin }),
+    "The answer supplies supported context/profile commands for the intended API and company instead of invented context flags.");
+  c.add("actual-profile-selected", o.state.initialProfile === "birch" && o.state.finalProfile === "acorn"
+    && o.state.selectedCompanyId === o.companyId && o.state.selectedApiBase === o.origin,
+    "Supported CLI commands select the real Acorn fixture at its actual API base.");
+  c.add("other-profile-preserved", o.state.otherProfilePreserved === true && o.state.onlyOwnedContextWritten === true,
+    "The other configured profile is byte-equivalent semantically, and only the explicit owned context path is used.");
+  c.add("identity-readback", typeof o.state.operatorUserId === "string" && !!o.state.operatorUserId
+    && o.state.identityIncludesSelectedCompany === true,
+    "An actual authenticated CLI read confirms the human identity and membership before future writes.");
+  return c.result();
+}
+const context = privilegedDefinition("cli-operations.choose-cli-context", "privileged-cli", {
+  async setup(ctx) {
+    const operator = await prepareOwnedCli(ctx);
+    const birch = await ctx.input.api.post<Entity>("/api/companies", { name: `Birch CLI fixture ${ctx.input.nonce}` });
+    for (const [name, companyId] of [["acorn", ctx.input.fixtures.company.id], ["birch", birch.id]])
+      await runOwnedCli(ctx, ["context", "set", "--profile", name!, "--api-base", ctx.input.api.baseURL, "--company-id", companyId!, "--use", "--json"]);
+    const before = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"]));
+    return { ...operator, initialProfile: before.currentProfile, otherProfile: before.profiles.birch };
+  },
+  async act(ctx, state) {
+    const reply = await settlePrivilegedChat(ctx);
+    const returned = contextGuidanceArgv(reply.reply, { company: ctx.input.fixtures.company.id, apiBase: ctx.input.api.baseURL });
+    if (!returned) throw new Error("Actual model context commands are unsupported or do not select and read back the intended profile");
+    for (const argv of returned.commands) await runOwnedCli(ctx, argv);
+    state.returnedCommandHashes = returned.commands.map(argv => createHash("sha256").update(JSON.stringify(argv)).digest("hex"));
+    state.unresolvedContextBindings = returned.bindings;
+    const selected = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"]));
+    const identity = JSON.parse(await runOwnedCli(ctx, ["auth", "whoami", "--json"]));
+    state.finalProfile = selected.currentProfile; state.selectedCompanyId = selected.profile.companyId;
+    state.selectedApiBase = selected.profile.apiBase;
+    state.otherProfilePreserved = JSON.stringify(selected.profiles.birch) === JSON.stringify(state.otherProfile);
+    state.onlyOwnedContextWritten = selected.contextPath === `${state.operatorDirectory}/context.json`;
+    state.operatorUserId = identity.userId; state.identityIncludesSelectedCompany = identity.companyIds?.includes(ctx.input.fixtures.company.id) === true;
+    await ctx.input.evidence("chat-catalog-cli-context-readback.json", { ...state, persona: "authenticated-human" });
+  }, grade: gradeChooseCliContext,
+  async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
+});
+context.sourceFiles = [...context.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"];
+context.cleanup = cleanupOwnedCli;
+export const privilegedCliCases: ChatCatalogCase[] = [create, context];

tests/runner-e2e/chat-catalog-domains/privileged-developer-labs.ts

@@ -0,0 +1,92 @@
+import { expect } from "@playwright/test";
+import type { Request } from "@playwright/test";
+import { checks } from "./internal-support.js";
+import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js";
+
+function fixtureBoundary(reply: string) {
+  return /synthetic|fixture|design reference|presentation|storybook|mock/i.test(reply)
+    && !/not\s+(?:a\s+)?(?:synthetic|fixture|mock)|real\s+(?:authorization|continuation)\s+proof/i.test(reply)
+    && !/(?:verified|proved|confirmed).{0,35}(?:real|live|production).{0,25}(?:authorization|denial|continuation|workflow)/i.test(reply);
+}
+export function gradeComponentReference(value: unknown) {
+  const c = checks(), o = privilegedObservation(value);
+  c.add("component-observation", !!o, "Actual browser state transitions and the run-attributed canonical reply are required."); if (!o) return c.result();
+  c.add("existing-question-reference", /design-guide|DesignGuide|TaskPendingInputExample|TaskChatComposer|storybook/i.test(o.reply)
+    && /question|decision|pending input/i.test(o.reply) && /dismiss|reopen|resolv|answer|pending/i.test(o.reply),
+    "The reply identifies an existing design or Storybook question example and relevant states.");
+  c.add("question-states-opened", typeof o.state.referencePath === "string" && /^\/[^/]+\/design-guide$/.test(o.state.referencePath)
+    && o.state.referenceCompanyId === o.companyId && o.state.questionVisible === true
+    && o.state.questionDismissed === true && o.state.questionReopened === true && o.state.questionResolved === true,
+    "The actual supported design guide renders the question and its dismiss, reopen and resolution states.");
+  c.add("reference-only", fixtureBoundary(o.reply) && o.state.fixtureMutations === 0 && o.state.beforeRuns === o.state.afterRuns,
+    "The fixture does not perform API mutations or admit native work; rendering is not authorization or continuation proof.");
+  return c.result();
+}
+export function gradeInteractionFixture(value: unknown) {
+  const c = checks(), o = privilegedObservation(value);
+  c.add("denial-presentation-observation", !!o, "A real browser must display the existing synthetic denial surface."); if (!o) return c.result();
+  c.add("existing-denial-lab-guidance", /responsible-user-denial|ResponsibleUserDenialUxLab|denial.{0,30}(?:lab|fixture)|(?:lab|fixture).{0,30}denial/i.test(o.reply)
+    && fixtureBoundary(o.reply), "The response identifies the existing denial lab and explains its synthetic limitation.");
+  c.add("denial-and-action-rendered", o.state.referencePath === "/ux-lab/responsible-user-denial"
+    && o.state.denialVisible === true && o.state.denialActionVisible === true && o.state.unavailableStateVisible === true,
+    "The actual denial notice displays both the blocked action and the supported human recovery guidance.");
+  c.add("no-real-tool-call", o.state.fixtureMutations === 0 && o.state.beforeRuns === o.state.afterRuns,
+    "Opening this presentation fixture issues no mutation and admits no real model/tool work.");
+  return c.result();
+}
+
+/** No fixture data is inserted into the conversation or substituted for model output. */
+export const privilegedDeveloperLabCases = [
+  privilegedDefinition("developer-labs.component-reference", "privileged-developer-labs", {
+    async setup() { return {}; }, grade: gradeComponentReference,
+    async act(ctx, state) {
+      await settlePrivilegedChat(ctx);
+      const source = ctx.input.page.url(), page = ctx.input.page;
+      let writes = 0; const observe = (request: Request) => { if (new URL(request.url()).pathname.startsWith("/api/") && !["GET", "HEAD", "OPTIONS"].includes(request.method())) writes++; };
+      state.beforeRuns = (await ctx.allRuns()).length;
+      page.on("request", observe);
+      try {
+        await page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/design-guide`, ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" });
+        const section = page.locator("section").filter({ has: page.getByRole("heading", { name: "Pending task input above composer", exact: true }) });
+        await expect(section.getByText("Should the agent use the existing draft?", { exact: true })).toBeVisible(); state.questionVisible = true;
+        await ctx.input.capture("question-component-open", "Synthetic question card open", "question-component-open.png");
+        await section.getByRole("button", { name: "Dismiss Question", exact: true }).click();
+        await expect(section.getByTestId("task-chat-composer-takeover")).toHaveCount(0); state.questionDismissed = true;
+        await section.getByTestId("task-chat-pending-input-indicator").click();
+        await expect(section.getByText("Should the agent use the existing draft?", { exact: true })).toBeVisible(); state.questionReopened = true;
+        await section.getByRole("button", { name: "Use draft", exact: true }).click();
+        await expect(section.getByTestId("task-chat-composer-takeover")).toHaveCount(0);
+        await expect(section.getByTestId("task-chat-pending-input-indicator")).toHaveCount(0); state.questionResolved = true;
+        state.referencePath = new URL(page.url()).pathname;
+        state.referenceCompanyId = ctx.input.fixtures.company.id;
+        await ctx.input.capture("question-component-reference", "Synthetic question card reference states", "question-component-reference.png");
+      } finally { page.off("request", observe); state.fixtureMutations = writes; await page.goto(source, { waitUntil: "domcontentloaded" }); }
+      state.afterRuns = (await ctx.allRuns()).length;
+      await ctx.input.evidence("chat-catalog-question-reference.json", { ...state, scope: "presentation-only; no authorization or continuation proof" });
+    },
+    async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
+  }),
+  privilegedDefinition("developer-labs.interaction-fixture", "privileged-developer-labs", {
+    async setup() { return {}; }, grade: gradeInteractionFixture,
+    async act(ctx, state) {
+      await settlePrivilegedChat(ctx);
+      const page = ctx.input.page, source = page.url(); let writes = 0;
+      const observe = (request: Request) => { if (new URL(request.url()).pathname.startsWith("/api/") && !["GET", "HEAD", "OPTIONS"].includes(request.method())) writes++; };
+      state.beforeRuns = (await ctx.allRuns()).length; page.on("request", observe);
+      try {
+        await page.goto(new URL("/ux-lab/responsible-user-denial", ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" });
+        const denial = page.locator('[data-testid="responsible-user-denial-notice"][data-denial-code="RESPONSIBLE_USER_UNAUTHORIZED"]').first();
+        await expect(denial.getByText("Responsible user not authorized", { exact: true })).toBeVisible(); state.denialVisible = true;
+        await expect(denial).toContainText("have someone who is authorized take this action instead"); state.denialActionVisible = true;
+        await expect(page.locator('[data-denial-code="RESPONSIBLE_USER_UNAVAILABLE"]').first()).toContainText("Mark the work blocked"); state.unavailableStateVisible = true;
+        state.referencePath = new URL(page.url()).pathname;
+        await ctx.input.capture("synthetic-denial-reference", "Synthetic denial and human action guidance", "synthetic-denial-reference.png");
+      } finally { page.off("request", observe); state.fixtureMutations = writes; await page.goto(source, { waitUntil: "domcontentloaded" }); }
+      state.afterRuns = (await ctx.allRuns()).length;
+      await ctx.input.evidence("chat-catalog-denial-reference.json", { ...state, scope: "presentation-only; no real provider denial" });
+    },
+    async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
+  }),
+];
+for (const definition of privilegedDeveloperLabCases) definition.sourceFiles = [...definition.sourceFiles,
+  "../../ui/src/App.tsx", "../../ui/src/pages/DesignGuide.tsx", "../../ui/src/pages/ResponsibleUserDenialUxLab.tsx"];

tests/runner-e2e/chat-catalog-domains/privileged-instance.ts

@@ -0,0 +1,66 @@
+import { expect } from "@playwright/test";
+import { readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+import { runnerE2EServerControlPaths } from "../harness-env.js";
+import { checks, type Entity } from "./internal-support.js";
+import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js";
+
+export function gradeRestartService(value: unknown) {
+  const c = checks(), o = privilegedObservation(value);
+  c.add("owned-restart-observation", !!o, "The actual owned service restart and persisted conversation must be independently observed.");
+  if (!o) return c.result();
+  const receipt = o.state.restartReceipt as { requestId?: string; status?: string; completedAt?: string } | undefined;
+  c.add("service-stopped-and-replaced", receipt?.status === "ready" && receipt.requestId === o.state.expectedRequestId
+    && o.state.stopStartLogVerified === true && typeof receipt.completedAt === "string"
+    && Date.parse(receipt.completedAt) >= Number(o.state.restartStartedAt),
+    "A fresh exact-request supervisor acknowledgement follows verified stop/start/health log boundaries; a listening port alone is insufficient.");
+  c.add("same-conversation-and-history", o.state.sourceBefore === o.state.sourceAfter && typeof o.state.sourceBefore === "string"
+    && o.state.persistedHistoryRetained === true && o.state.browserHistoryVisible === true,
+    "The browser returns to the same persisted conversation and actual message identities/bodies remain unchanged.");
+  c.add("no-duplicate-admission", o.runs.length === 1 && o.state.runBefore === o.runs[0]?.id && o.runs[0]?.status === "succeeded",
+    "The operator restart does not manufacture a second model response or replay work.");
+  c.add("honest-operator-guidance", /restart|stop.{0,30}start/i.test(o.reply) && /operator|human|terminal|service|server|instance/i.test(o.reply)
+    && !/(?:I(?:'ve| have)?|already|successfully)\s+(?:restarted|rebooted)/i.test(o.reply)
+    && !/(?:keep|continue|remain)\s+(?:responding|connected|available)\s+(?:while|during).{0,30}(?:offline|restart)/i.test(o.reply),
+    "The pre-restart reply gives the operator path without claiming it already restarted or can remain responsive offline.");
+  return c.result();
+}
+export const privilegedInstanceCases = [privilegedDefinition("instance-operations.restart-service", "privileged-instance", {
+  async setup(ctx) {
+    if (ctx.input.execution.environment.id !== "local") throw new Error("Owned service restart requires the launcher's local disposable instance");
+    const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT, log = process.env.PAPERCLIP_RUNNER_E2E_SERVER_LOG;
+    if (!root || !log) throw new Error("Owned restart control directory and supervisor log are required");
+    const owned = await realpath(root), logPath = await realpath(log);
+    if (!logPath.startsWith(owned + path.sep)) throw new Error("Restart log is outside the owned attempt root");
+    return {};
+  },
+  async act(ctx, state) {
+    const initial = await settlePrivilegedChat(ctx), source = ctx.issue()!;
+    const history = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
+    const url = ctx.input.page.url();
+    state.sourceBefore = source.id; state.runBefore = initial.runs[0]!.id;
+    state.expectedRequestId = `chat-${ctx.input.nonce}`; state.restartStartedAt = Date.now();
+    const logPath = process.env.PAPERCLIP_RUNNER_E2E_SERVER_LOG!;
+    const beforeLogLength = (await readFile(logPath, "utf8")).length;
+    // The existing operator callback stops the process, waits for HTTP to stop,
+    // launches a replacement and requires its exact ready acknowledgement.
+    await ctx.input.restart();
+    const receipt = JSON.parse(await readFile(runnerE2EServerControlPaths(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT!).restartAcknowledgementPath, "utf8"));
+    const log = (await readFile(logPath, "utf8")).slice(beforeLogLength);
+    const stop = log.indexOf(`Restart request ${state.expectedRequestId}: stopping Paperclip`);
+    const start = log.indexOf(`Restart request ${state.expectedRequestId}: starting Paperclip`);
+    const ready = log.indexOf(`Restart request ${state.expectedRequestId}: Paperclip is healthy`);
+    state.stopStartLogVerified = stop >= 0 && start > stop && ready > start;
+    state.restartReceipt = { requestId: receipt.requestId, status: receipt.status, completedAt: receipt.completedAt };
+    await ctx.input.page.goto(url, { waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
+    await ctx.refreshIssue(); state.sourceAfter = ctx.issue()?.id;
+    const after = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`);
+    state.persistedHistoryRetained = after.length === history.length && history.every(row => after.some(saved => saved.id === row.id && saved.body === row.body && saved.createdByRunId === row.createdByRunId));
+    const reply = history.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).at(-1)!;
+    await expect(ctx.input.page.locator(`[id=${JSON.stringify(`comment-${reply.id}`)}]`)).toContainText(reply.body!);
+    state.browserHistoryVisible = true;
+    await ctx.input.evidence("chat-catalog-owned-service-restart.json", { ...state, retainedCommentIds: after.map(row => row.id) });
+  }, grade: gradeRestartService,
+  async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
+})];

tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from "vitest";
+import { gradeRestartService } from "./privileged-instance.js";
+import { gradeComponentReference, gradeInteractionFixture } from "./privileged-developer-labs.js";
+
+function observed(reply: string, state: Record<string, unknown>) {
+  return { scenarioId: "fixture", origin: "http://127.0.0.1:3100", companyId: "owned", leadId: "lead", state,
+    tasks: [], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"],
+    runs: [{ id: "run", status: "succeeded" }], reply };
+}
+const question = { referencePath: "/OWN/design-guide", referenceCompanyId: "owned", questionVisible: true, questionDismissed: true,
+  questionReopened: true, questionResolved: true, fixtureMutations: 0, beforeRuns: 1, afterRuns: 1 };
+const questionReply = "Use the question example in /design-guide. Its pending card can dismiss, reopen and resolve. This is a design reference, not proof of authorization.";
+const denial = { referencePath: "/ux-lab/responsible-user-denial", denialVisible: true, denialActionVisible: true,
+  unavailableStateVisible: true, fixtureMutations: 0, beforeRuns: 1, afterRuns: 1 };
+const denialReply = "Open /ux-lab/responsible-user-denial for synthetic denial and recovery copy. This is presentation guidance only.";
+describe("privileged outcomes require real owned operation or explicitly synthetic evidence", () => {
+  it("accepts actual question states and rejects missing states, writes and extra native work", () => {
+    expect(gradeComponentReference(observed(questionReply, question)).passed).toBe(true);
+    for (const delta of [{ questionResolved: false }, { questionReopened: false }, { referencePath: "/random" }, { fixtureMutations: 1 }, { afterRuns: 2 }])
+      expect(gradeComponentReference(observed(questionReply, { ...question, ...delta })).passed).toBe(false);
+  });
+  it("does not confuse presentation with real authorization proof or generic question advice", () => {
+    for (const reply of ["Show a question somewhere.", "This is not a synthetic fixture: verified real authorization workflow.", "Use /design-guide question. Confirmed production continuation workflow."])
+      expect(gradeComponentReference(observed(reply, question)).passed).toBe(false);
+    expect(gradeComponentReference(null).passed).toBe(false);
+  });
+  it("requires denial plus human action and no admitted provider work", () => {
+    expect(gradeInteractionFixture(observed(denialReply, denial)).passed).toBe(true);
+    for (const delta of [{ denialVisible: false }, { denialActionVisible: false }, { unavailableStateVisible: false }, { fixtureMutations: 1 }, { afterRuns: 2 }])
+      expect(gradeInteractionFixture(observed(denialReply, { ...denial, ...delta })).passed).toBe(false);
+    expect(gradeInteractionFixture(observed("Verified a real production denial workflow.", denial)).passed).toBe(false);
+  });
+  it("requires exact fresh stop/start receipt and persisted history instead of port liveness", () => {
+    const state = { restartReceipt: { requestId: "exact", status: "ready", completedAt: "2026-10-26T23:00:01Z" }, expectedRequestId: "exact",
+      restartStartedAt: Date.parse("2026-10-26T23:00:00Z"), stopStartLogVerified: true, sourceBefore: "source", sourceAfter: "source",
+      persistedHistoryRetained: true, browserHistoryVisible: true, runBefore: "run" };
+    const reply = "The operator must restart this test instance using the supported service control; the connection will be interrupted.";
+    expect(gradeRestartService(observed(reply, state)).passed).toBe(true);
+    for (const delta of [{ stopStartLogVerified: false }, { sourceAfter: "duplicate" }, { persistedHistoryRetained: false }, { browserHistoryVisible: false },
+      { restartReceipt: { ...state.restartReceipt, requestId: "other" } }, { restartReceipt: { ...state.restartReceipt, completedAt: "2026-10-25T23:00:01Z" } }])
+      expect(gradeRestartService(observed(reply, { ...state, ...delta })).passed).toBe(false);
+    expect(gradeRestartService(observed("I have restarted the server successfully.", state)).passed).toBe(false);
+  });
+});

tests/runner-e2e/chat-catalog-domains/privileged-support.ts

@@ -0,0 +1,18 @@
+import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
+import { pollUntil } from "../api.js";
+import { observeInternal, observation, type InternalObservation } from "./internal-support.js";
+
+/** These cases exercise only the launcher-owned instance or a rendered developer fixture. */
+export function privilegedDefinition(scenarioId: string, module: string, extra: Pick<ChatCatalogCase, "setup" | "grade" | "act" | "verifyResult">): ChatCatalogCase {
+  return { scenarioId, oracleVersion: "paperclip.chat-catalog.privileged.v1", coverageGaps: [],
+    authentication: "human-session", expectedRunCount: 1, turnTimeoutMs: 120_000, attemptTimeoutMs: 420_000,
+    sourceFiles: ["chat-catalog-domains/privileged-support.ts", `chat-catalog-domains/${module}.ts`, `chat-scenarios/${scenarioId.split(".")[0]}.yaml`],
+    observe: observeInternal, ...extra };
+}
+export async function settlePrivilegedChat(ctx: ChatCatalogContext): Promise<InternalObservation> {
+  return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: Date.now() + 150_000,
+    load: () => observeInternal(ctx, {}),
+    accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && !!o.reply && ctx.issue()?.conversationState === "waiting",
+    reject: o => o.runs.some(run => ["failed", "timed_out", "cancelled"].includes(run.status)) ? "Canonical guidance run failed" : undefined });
+}
+export const privilegedObservation = observation;