diff --git a/doc/plans/2026-10-26-chat-catalog-eval-progress.md b/doc/plans/2026-10-26-chat-catalog-eval-progress.md index 6d2b6458650..e8b9b83b556 100644 --- a/doc/plans/2026-10-26-chat-catalog-eval-progress.md +++ b/doc/plans/2026-10-26-chat-catalog-eval-progress.md @@ -55,4 +55,4 @@ Next-wave review items: bound identity probe credentials privately across repeat Core behavior fixes require separate focused PRs that remain unmerged; their dependent passes cannot count toward the eval threshold. Eval merge requires every scenario implemented/processed, at least 196 final eligible passes, latest-head checks/reviews and actual protected merge. Per-case fingerprints, global suite hash and production/source revisions remain distinct; stale passes cannot be carried across changed execution/grader dependencies. -Credential discovery (read-only): `~/.secrets` contains `AGENTMAIL_PAPERCLIP_STAGING_QA_API_KEY` and `COPILOT_GITHUB_TOKEN` references; no Notion, Slack or vault reference matched. Neither establishes an explicitly disposable mailbox/recipient or repository allowlist. Local `op` exists, but account/item reads fail because it cannot connect to the 1Password desktop app; no item contents were read, and no credentials were printed or persisted. +Credential discovery: relevant source metadata is readable through the existing 1Password service account outside the sandbox. Notion, QA AgentMail, GitHub, Daytona and Anthropic references are available; no matching Slack reference was found. Credential validity, a disposable repository allowlist, designated mail recipient and pinned cloud image remain unverified. Automatic approval review rejected credentialed provider identity/resource discovery and metadata persistence; explicit approval for the narrow read-only AgentMail/GitHub endpoints is pending. No provider discovery request or metadata write occurred. Private item IDs and account metadata remain outside tracked documents. diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts new file mode 100644 index 00000000000..54ef4c3d1cd --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli-support.ts @@ -0,0 +1,128 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtemp, mkdir, realpath, rm } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const execute = promisify(execFile); +const privateActors = new WeakMap(); +export async function prepareOwnedCli(ctx: ChatCatalogContext) { + if (ctx.input.execution.environment.id !== "local") throw new Error("CLI companion journey requires the owned local attempt"); + const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; + if (!root) throw new Error("CLI companion requires an owned attempt directory"); + const directory = await mkdtemp(path.join(await realpath(root), "operator-cli-")); + await mkdir(path.join(directory, "home")); + const key = await ctx.input.api.postSensitive<{ id: string; token: string }>("/api/board-api-keys", { + name: `catalog-operator-${ctx.input.nonce}`, requestedCompanyId: ctx.input.fixtures.company.id, expiresAt: new Date(Date.now() + 900_000).toISOString(), + }); + if (!key.id || !key.token) throw new Error("Owned human CLI credential was not issued"); + ctx.input.privateSecrets?.push(key.token); + privateActors.set(ctx, { ...key, directory }); + return { operatorDirectory: directory, operatorKeyId: key.id }; +} +export async function runOwnedCli(ctx: ChatCatalogContext, args: string[]) { + const actor = privateActors.get(ctx); if (!actor) throw new Error("Owned human CLI actor is unavailable"); + // A fresh minimal environment cannot pick up another operator's context, + // stored auth, run ID, provider credentials or default company. + const env: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: path.join(actor.directory, "home"), + PAPERCLIP_HOME: path.join(actor.directory, "home"), PAPERCLIP_API_KEY: actor.token, + PAPERCLIP_API_URL: ctx.input.api.baseURL, PAPERCLIP_TELEMETRY_DISABLED: "1", NO_COLOR: "1" }; + const loader = fileURLToPath(new URL("../../../cli/node_modules/tsx/dist/loader.mjs", import.meta.url)); + const entry = fileURLToPath(new URL("../../../cli/src/index.ts", import.meta.url)); + try { + const result = await execute(process.execPath, ["--import", loader, entry, ...args, + "--context", path.join(actor.directory, "context.json")], { cwd: actor.directory, env, timeout: 45_000, maxBuffer: 1_048_576 }); + if (result.stdout.includes(actor.token) || result.stderr.includes(actor.token)) throw new Error("Private CLI credential appeared in output"); + return result.stdout; + } catch { throw new Error("Owned CLI command failed; private subprocess output withheld"); } +} +export async function cleanupOwnedCli(ctx: ChatCatalogContext) { + const actor = privateActors.get(ctx); if (!actor) return; + await ctx.input.api.delete(`/api/board-api-keys/${actor.id}`); + privateActors.delete(ctx); + const root = await realpath(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT!); + const owned = await realpath(actor.directory); + if (!owned.startsWith(root + path.sep) || !path.basename(owned).startsWith("operator-cli-")) throw new Error("Refusing CLI cleanup outside its owned directory"); + await rm(owned, { recursive: true }); +} + +/** Parse one finite supported command into argv; never evaluate a shell. */ +export function websiteCreateArgv(reply: string, fixture: { company: string; project: string; agent: string; apiBase: string }, bindings: Array<{ flag: string; placeholder: string; fixtureValue: string }> = []) { + const blocks = [...reply.matchAll(/```(?:bash|sh|shell)?\s*\n([\s\S]*?)```/g)].map(match => match[1]!.replace(/\\\r?\n/g, " ").trim()); + const commands = blocks.flatMap(block => block.split(/\r?\n/)).filter(line => /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+issue\s+create\b/.test(line.trim())); + if (commands.length !== 1) throw new Error("One supported website creation command is required"); + const line = commands[0]!.trim(); + if (/[`;|&\n\r]|\$\(/.test(line)) throw new Error("Shell execution is not admitted in the owned CLI journey"); + const tokens = line.match(/"(?:[^"\\]|\\.)*"|'[^']*'|[^\s]+/g)!; + const values = tokens.map(token => token.startsWith('"') ? JSON.parse(token) as string : token.startsWith("'") ? token.slice(1, -1) : token); + const start = values.indexOf("issue"), argv = values.slice(start); + const allowed = new Set(["--company-id", "-C", "--title", "--description", "--status", "--priority", "--assignee-agent-id", "--project-id", "--api-base"]); + const fields = new Map(); + for (let i = 2; i < argv.length; i++) { + const flag = argv[i]!; if (flag === "--json" || flag === "--standalone") continue; + if (!allowed.has(flag) || !argv[i + 1] || fields.has(flag)) throw new Error("Unsupported or ambiguous creation flags"); + let value = argv[++i]!; + const target = ["--company-id", "-C"].includes(flag) ? fixture.company : flag === "--project-id" ? fixture.project : flag === "--assignee-agent-id" ? fixture.agent : undefined; + // Only unresolved identifier placeholders are bound; actual wrong IDs fail. + if (target && /^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(value)) { bindings.push({ flag, placeholder: value, fixtureValue: target }); value = target; argv[i] = target; } + fields.set(flag, value); + } + if ((fields.get("--company-id") ?? fields.get("-C")) !== fixture.company || fields.get("--project-id") !== fixture.project + || fields.get("--assignee-agent-id") !== fixture.agent || !/website/i.test(fields.get("--title") ?? "") + || fields.has("--status") && fields.get("--status") !== "backlog" || fields.has("--api-base") && fields.get("--api-base") !== fixture.apiBase) + throw new Error("Creation command does not target exactly the intended company, project and Morgan"); + if (fields.has("--company-id") && fields.has("-C")) throw new Error("Duplicate company context flags are ambiguous"); + if (!argv.includes("--json")) argv.push("--json"); + return argv; +} + +export function contextGuidanceArgv(reply: string, fixture: { company: string; apiBase: string }) { + const lines = [...reply.matchAll(/```(?:bash|sh|shell)?\s*\n([\s\S]*?)```/g)] + .flatMap(match => match[1]!.replace(/\\\r?\n/g, " ").split(/\r?\n/)) + .map(line => line.trim()).filter(line => /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+context\b/.test(line)); + let selected = false, readback = false; + const commands: string[][] = [], bindings: Array<{ commandIndex: number; flag: string; placeholder: string; fixtureValue: string }> = []; + for (const line of lines) { + if (/[`;|&\n\r]|\$\(/.test(line)) return; + let tokens: string[]; + try { tokens = line.match(/"(?:[^"\\]|\\.)*"|'[^']*'|[^\s]+/g)!.map(token => token.startsWith('"') ? JSON.parse(token) as string : token.startsWith("'") ? token.slice(1, -1) : token); } + catch { return; } + const original = tokens.slice(tokens.indexOf("context")); + const command = original[1], args = original.slice(2); + if (command === "use") { if (args.shift()?.toLowerCase() !== "acorn") return; selected = true; } + else if (command === "show") readback ||= selected; + else if (command !== "set" && command !== "list") return; + let profile: string | undefined, company: string | undefined, apiBase: string | undefined, use = false; + const allowedFlags = command === "set" ? ["--profile", "--company-id", "--api-base", "--use", "--json"] + : command === "show" ? ["--profile", "--json"] : command === "list" ? ["--json"] : []; + const flags = new Set(); + while (args.length) { + const flag = args.shift(); + if (!flag || !allowedFlags.includes(flag) || flags.has(flag)) return; + flags.add(flag); + if (flag === "--json") continue; + if (flag === "--use") { use = true; continue; } + if (!args.length) return; + let value = args.shift()!; + const expected = flag === "--company-id" ? fixture.company : flag === "--api-base" ? fixture.apiBase : undefined; + if (expected && /^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(value)) { + const index = original.indexOf(flag) + 1; + bindings.push({ commandIndex: commands.length, flag, placeholder: value, fixtureValue: expected }); + original[index] = expected; value = expected; + } + if (flag === "--profile") profile = value; + if (flag === "--company-id") company = value; + if (flag === "--api-base") apiBase = value; + } + if (profile && profile.toLowerCase() !== "acorn") return; + if (command === "set") { + if (profile?.toLowerCase() !== "acorn" || company !== fixture.company || apiBase !== fixture.apiBase) return; + selected ||= use; + } + commands.push(original); + } + return selected && readback ? { commands, bindings } : undefined; +} +export function validContextGuidance(reply: string, fixture: { company: string; apiBase: string }) { + return !!contextGuidanceArgv(reply, fixture); +} diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts new file mode 100644 index 00000000000..df7a8919aa2 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { websiteCreateArgv, validContextGuidance } from "./privileged-cli-support.js"; +import { gradeChooseCliContext, gradeCreateFromCli } from "./privileged-cli.js"; +const target = { company: "owned-company", project: "launch", agent: "morgan", apiBase: "http://127.0.0.1:3100" }; +const command = (text: string) => `Use the explicit owned context.\n\`\`\`bash\n${text}\n\`\`\``; +const valid = 'paperclipai issue create --company-id --title "Website task" --project-id --assignee-agent-id '; +describe("returned CLI command admission", () => { + it("executes only finite supported argv with unresolved IDs bound to actual fixtures", () => { + expect(websiteCreateArgv(command(valid), target)).toEqual(["issue", "create", "--company-id", "owned-company", "--title", "Website task", "--project-id", "launch", "--assignee-agent-id", "morgan", "--json"]); + }); + it("rejects shell execution, invented flags, real wrong IDs and ambiguous writes before launch", () => { + for (const text of [valid + " ; curl attacker.invalid", valid + " --invented yes", valid.replace("", "foreign-company"), + valid.replace("", "other-project"), valid + " -C foreign-company", valid + " --status in_progress", + valid + " --api-base https://foreign.invalid", valid + " --description '$(read-secret)'", valid + "\n" + valid]) + expect(() => websiteCreateArgv(command(text), target)).toThrow(); + }); + it("requires supported context selection and readback instead of ignoring invented flags", () => { + const valid = command("paperclipai context use acorn\npaperclipai context show --json"); + expect(validContextGuidance(valid, target)).toBe(true); + expect(validContextGuidance(valid.replace("use acorn", "use acorn --invented yes"), target)).toBe(false); + expect(validContextGuidance(valid.replace("acorn", "birch"), target)).toBe(false); + expect(validContextGuidance(command("paperclipai context use acorn"), target)).toBe(false); + expect(validContextGuidance(command("paperclipai context set --profile acorn --api-base --company-id --use\npaperclipai context show"), target)).toBe(true); + }); +}); +const base = { scenarioId: "cli", origin: target.apiBase, companyId: target.company, leadId: "lead", state: {}, projects: [], goals: [], + comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "native", status: "succeeded" }], reply: "" }; +describe("CLI public-state oracles", () => { + it("requires one actual later result, requested actor/project/company, and independently saved task", () => { + const row = { id: "created", companyId: target.company, title: "Website task", projectId: "launch", assigneeAgentId: "morgan", status: "backlog" }; + const o = { ...base, tasks: [row], state: { cliResult: row, worker: { id: "morgan" }, project: { id: "launch" }, tasksBeforeCli: 0, + cliCommandExecuted: true, explicitCompany: target.company, executedArgv: ["issue", "create"] } }; + expect(gradeCreateFromCli(o).passed).toBe(true); + expect(gradeCreateFromCli({ ...o, tasks: [{ ...row, companyId: "foreign" }] }).passed).toBe(false); + expect(gradeCreateFromCli({ ...o, tasks: [row, { ...row, id: "duplicate" }] }).passed).toBe(false); + expect(gradeCreateFromCli({ ...o, state: { ...o.state, tasksBeforeCli: 1 } }).passed).toBe(false); + expect(gradeCreateFromCli(null).passed).toBe(false); + }); + it("requires selected identity/company and preserves the other operator profile", () => { + const o = { ...base, tasks: [], reply: command("paperclipai context use acorn\npaperclipai context show --json"), + state: { initialProfile: "birch", finalProfile: "acorn", selectedCompanyId: target.company, selectedApiBase: target.apiBase, + otherProfilePreserved: true, onlyOwnedContextWritten: true, operatorUserId: "synthetic-person", identityIncludesSelectedCompany: true } }; + expect(gradeChooseCliContext(o).passed).toBe(true); + for (const delta of [{ otherProfilePreserved: false }, { selectedCompanyId: "foreign" }, { identityIncludesSelectedCompany: false }, { onlyOwnedContextWritten: false }]) + expect(gradeChooseCliContext({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts new file mode 100644 index 00000000000..1b1be1cba81 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts @@ -0,0 +1,104 @@ +import { expect } from "@playwright/test"; +import { createHash } from "node:crypto"; +import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import { checks, project, entity, type Entity } from "./internal-support.js"; +import { fixtureWorker } from "./internal-definition.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli, websiteCreateArgv, validContextGuidance, contextGuidanceArgv } from "./privileged-cli-support.js"; + +export function gradeCreateFromCli(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("actual-cli-observation", !!o, "The command returned by the actual model must run against the owned human CLI actor."); if (!o) return c.result(); + const cli = o.state.cliResult as Entity | undefined, saved = o.tasks.find(row => row.id === cli?.id); + const worker = o.state.worker as Entity | undefined, launch = o.state.project as Entity | undefined; + c.add("one-later-cli-created-task", o.state.tasksBeforeCli === 0 && o.tasks.length === 1 && !!cli?.id && saved?.id === cli.id + && o.state.cliCommandExecuted === true && o.state.explicitCompany === o.companyId, + "The later CLI journey creates exactly one task, rather than the agent creating work while merely asked for a command."); + c.add("saved-company-project-assignee", !!saved && saved.companyId === o.companyId && saved.projectId === launch?.id + && saved.assigneeAgentId === worker?.id && /website/i.test(saved.title ?? "") && saved.status === "backlog", + "Fresh public state independently binds the CLI result to the requested website task, Launch and Morgan."); + c.add("real-command-structured-result", Array.isArray(o.state.executedArgv) && o.state.executedArgv[0] === "issue" + && o.state.executedArgv[1] === "create" && cli?.companyId === saved?.companyId, + "The response's finite supported flags actually execute with an inspectable structured result; no plausible invented flag is silently repaired."); + return c.result(); +} +const create = privilegedDefinition("cli-operations.create-from-cli", "privileged-cli", { + async setup(ctx) { + const operator = await prepareOwnedCli(ctx); + const launch = await project(ctx), worker = await fixtureWorker(ctx); + return { ...operator, project: launch, worker }; + }, + async act(ctx, state) { + const reply = await settlePrivilegedChat(ctx); + state.tasksBeforeCli = reply.tasks.length; + const fixture = { company: ctx.input.fixtures.company.id, project: entity(state, "project").id, + agent: entity(state, "worker").id, apiBase: ctx.input.api.baseURL }; + const bindings: Array<{ flag: string; placeholder: string; fixtureValue: string }> = []; + const argv = websiteCreateArgv(reply.reply, fixture, bindings); + const result = JSON.parse(await runOwnedCli(ctx, argv)) as Entity; + state.cliResult = { id: result.id, companyId: result.companyId, title: result.title, identifier: result.identifier, + projectId: result.projectId, assigneeAgentId: result.assigneeAgentId, status: result.status }; + state.executedArgv = argv.filter((value, index) => index < 2 || value.startsWith("-")); + state.executedArgvSha256 = createHash("sha256").update(JSON.stringify(argv)).digest("hex"); + state.unresolvedIdentifierBindingPolicy = "Only explicit unresolved company/project/assignee placeholders bind to independently saved fixture identities; resolved values are never repaired."; + state.unresolvedIdentifierBindings = bindings; + state.explicitCompany = fixture.company; state.cliCommandExecuted = true; + const saved = await ctx.input.api.get(`/api/issues/${result.id}`); + await ctx.input.evidence("chat-catalog-cli-created-task.json", { commandFlags: state.executedArgv, argvSha256: state.executedArgvSha256, bindingPolicy: state.unresolvedIdentifierBindingPolicy, result: state.cliResult, + saved: { id: saved.id, companyId: saved.companyId, projectId: saved.projectId, assigneeAgentId: saved.assigneeAgentId, status: saved.status } }); + }, grade: gradeCreateFromCli, + async verifyResult(ctx, state) { + const row = state.cliResult as Entity; + const source = ctx.input.page.url(); + await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${row.identifier ?? row.id}`, ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("heading", { name: row.title!, exact: true }).first()).toBeVisible(); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, +}); +create.sourceFiles = [...create.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"]; +create.cleanup = cleanupOwnedCli; +export function gradeChooseCliContext(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("owned-cli-context-evidence", !!o, "Two isolated profiles and real authenticated identity readback are required."); if (!o) return c.result(); + c.add("supported-context-guidance", validContextGuidance(o.reply, { company: o.companyId, apiBase: o.origin }), + "The answer supplies supported context/profile commands for the intended API and company instead of invented context flags."); + c.add("actual-profile-selected", o.state.initialProfile === "birch" && o.state.finalProfile === "acorn" + && o.state.selectedCompanyId === o.companyId && o.state.selectedApiBase === o.origin, + "Supported CLI commands select the real Acorn fixture at its actual API base."); + c.add("other-profile-preserved", o.state.otherProfilePreserved === true && o.state.onlyOwnedContextWritten === true, + "The other configured profile is byte-equivalent semantically, and only the explicit owned context path is used."); + c.add("identity-readback", typeof o.state.operatorUserId === "string" && !!o.state.operatorUserId + && o.state.identityIncludesSelectedCompany === true, + "An actual authenticated CLI read confirms the human identity and membership before future writes."); + return c.result(); +} +const context = privilegedDefinition("cli-operations.choose-cli-context", "privileged-cli", { + async setup(ctx) { + const operator = await prepareOwnedCli(ctx); + const birch = await ctx.input.api.post("/api/companies", { name: `Birch CLI fixture ${ctx.input.nonce}` }); + for (const [name, companyId] of [["acorn", ctx.input.fixtures.company.id], ["birch", birch.id]]) + await runOwnedCli(ctx, ["context", "set", "--profile", name!, "--api-base", ctx.input.api.baseURL, "--company-id", companyId!, "--use", "--json"]); + const before = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"])); + return { ...operator, initialProfile: before.currentProfile, otherProfile: before.profiles.birch }; + }, + async act(ctx, state) { + const reply = await settlePrivilegedChat(ctx); + const returned = contextGuidanceArgv(reply.reply, { company: ctx.input.fixtures.company.id, apiBase: ctx.input.api.baseURL }); + if (!returned) throw new Error("Actual model context commands are unsupported or do not select and read back the intended profile"); + for (const argv of returned.commands) await runOwnedCli(ctx, argv); + state.returnedCommandHashes = returned.commands.map(argv => createHash("sha256").update(JSON.stringify(argv)).digest("hex")); + state.unresolvedContextBindings = returned.bindings; + const selected = JSON.parse(await runOwnedCli(ctx, ["context", "show", "--json"])); + const identity = JSON.parse(await runOwnedCli(ctx, ["auth", "whoami", "--json"])); + state.finalProfile = selected.currentProfile; state.selectedCompanyId = selected.profile.companyId; + state.selectedApiBase = selected.profile.apiBase; + state.otherProfilePreserved = JSON.stringify(selected.profiles.birch) === JSON.stringify(state.otherProfile); + state.onlyOwnedContextWritten = selected.contextPath === `${state.operatorDirectory}/context.json`; + state.operatorUserId = identity.userId; state.identityIncludesSelectedCompany = identity.companyIds?.includes(ctx.input.fixtures.company.id) === true; + await ctx.input.evidence("chat-catalog-cli-context-readback.json", { ...state, persona: "authenticated-human" }); + }, grade: gradeChooseCliContext, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +context.sourceFiles = [...context.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"]; +context.cleanup = cleanupOwnedCli; +export const privilegedCliCases: ChatCatalogCase[] = [create, context]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-developer-labs.ts b/tests/runner-e2e/chat-catalog-domains/privileged-developer-labs.ts new file mode 100644 index 00000000000..b80ed292d1a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-developer-labs.ts @@ -0,0 +1,92 @@ +import { expect } from "@playwright/test"; +import type { Request } from "@playwright/test"; +import { checks } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; + +function fixtureBoundary(reply: string) { + return /synthetic|fixture|design reference|presentation|storybook|mock/i.test(reply) + && !/not\s+(?:a\s+)?(?:synthetic|fixture|mock)|real\s+(?:authorization|continuation)\s+proof/i.test(reply) + && !/(?:verified|proved|confirmed).{0,35}(?:real|live|production).{0,25}(?:authorization|denial|continuation|workflow)/i.test(reply); +} +export function gradeComponentReference(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("component-observation", !!o, "Actual browser state transitions and the run-attributed canonical reply are required."); if (!o) return c.result(); + c.add("existing-question-reference", /design-guide|DesignGuide|TaskPendingInputExample|TaskChatComposer|storybook/i.test(o.reply) + && /question|decision|pending input/i.test(o.reply) && /dismiss|reopen|resolv|answer|pending/i.test(o.reply), + "The reply identifies an existing design or Storybook question example and relevant states."); + c.add("question-states-opened", typeof o.state.referencePath === "string" && /^\/[^/]+\/design-guide$/.test(o.state.referencePath) + && o.state.referenceCompanyId === o.companyId && o.state.questionVisible === true + && o.state.questionDismissed === true && o.state.questionReopened === true && o.state.questionResolved === true, + "The actual supported design guide renders the question and its dismiss, reopen and resolution states."); + c.add("reference-only", fixtureBoundary(o.reply) && o.state.fixtureMutations === 0 && o.state.beforeRuns === o.state.afterRuns, + "The fixture does not perform API mutations or admit native work; rendering is not authorization or continuation proof."); + return c.result(); +} +export function gradeInteractionFixture(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("denial-presentation-observation", !!o, "A real browser must display the existing synthetic denial surface."); if (!o) return c.result(); + c.add("existing-denial-lab-guidance", /responsible-user-denial|ResponsibleUserDenialUxLab|denial.{0,30}(?:lab|fixture)|(?:lab|fixture).{0,30}denial/i.test(o.reply) + && fixtureBoundary(o.reply), "The response identifies the existing denial lab and explains its synthetic limitation."); + c.add("denial-and-action-rendered", o.state.referencePath === "/ux-lab/responsible-user-denial" + && o.state.denialVisible === true && o.state.denialActionVisible === true && o.state.unavailableStateVisible === true, + "The actual denial notice displays both the blocked action and the supported human recovery guidance."); + c.add("no-real-tool-call", o.state.fixtureMutations === 0 && o.state.beforeRuns === o.state.afterRuns, + "Opening this presentation fixture issues no mutation and admits no real model/tool work."); + return c.result(); +} + +/** No fixture data is inserted into the conversation or substituted for model output. */ +export const privilegedDeveloperLabCases = [ + privilegedDefinition("developer-labs.component-reference", "privileged-developer-labs", { + async setup() { return {}; }, grade: gradeComponentReference, + async act(ctx, state) { + await settlePrivilegedChat(ctx); + const source = ctx.input.page.url(), page = ctx.input.page; + let writes = 0; const observe = (request: Request) => { if (new URL(request.url()).pathname.startsWith("/api/") && !["GET", "HEAD", "OPTIONS"].includes(request.method())) writes++; }; + state.beforeRuns = (await ctx.allRuns()).length; + page.on("request", observe); + try { + await page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/design-guide`, ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" }); + const section = page.locator("section").filter({ has: page.getByRole("heading", { name: "Pending task input above composer", exact: true }) }); + await expect(section.getByText("Should the agent use the existing draft?", { exact: true })).toBeVisible(); state.questionVisible = true; + await ctx.input.capture("question-component-open", "Synthetic question card open", "question-component-open.png"); + await section.getByRole("button", { name: "Dismiss Question", exact: true }).click(); + await expect(section.getByTestId("task-chat-composer-takeover")).toHaveCount(0); state.questionDismissed = true; + await section.getByTestId("task-chat-pending-input-indicator").click(); + await expect(section.getByText("Should the agent use the existing draft?", { exact: true })).toBeVisible(); state.questionReopened = true; + await section.getByRole("button", { name: "Use draft", exact: true }).click(); + await expect(section.getByTestId("task-chat-composer-takeover")).toHaveCount(0); + await expect(section.getByTestId("task-chat-pending-input-indicator")).toHaveCount(0); state.questionResolved = true; + state.referencePath = new URL(page.url()).pathname; + state.referenceCompanyId = ctx.input.fixtures.company.id; + await ctx.input.capture("question-component-reference", "Synthetic question card reference states", "question-component-reference.png"); + } finally { page.off("request", observe); state.fixtureMutations = writes; await page.goto(source, { waitUntil: "domcontentloaded" }); } + state.afterRuns = (await ctx.allRuns()).length; + await ctx.input.evidence("chat-catalog-question-reference.json", { ...state, scope: "presentation-only; no authorization or continuation proof" }); + }, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, + }), + privilegedDefinition("developer-labs.interaction-fixture", "privileged-developer-labs", { + async setup() { return {}; }, grade: gradeInteractionFixture, + async act(ctx, state) { + await settlePrivilegedChat(ctx); + const page = ctx.input.page, source = page.url(); let writes = 0; + const observe = (request: Request) => { if (new URL(request.url()).pathname.startsWith("/api/") && !["GET", "HEAD", "OPTIONS"].includes(request.method())) writes++; }; + state.beforeRuns = (await ctx.allRuns()).length; page.on("request", observe); + try { + await page.goto(new URL("/ux-lab/responsible-user-denial", ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" }); + const denial = page.locator('[data-testid="responsible-user-denial-notice"][data-denial-code="RESPONSIBLE_USER_UNAUTHORIZED"]').first(); + await expect(denial.getByText("Responsible user not authorized", { exact: true })).toBeVisible(); state.denialVisible = true; + await expect(denial).toContainText("have someone who is authorized take this action instead"); state.denialActionVisible = true; + await expect(page.locator('[data-denial-code="RESPONSIBLE_USER_UNAVAILABLE"]').first()).toContainText("Mark the work blocked"); state.unavailableStateVisible = true; + state.referencePath = new URL(page.url()).pathname; + await ctx.input.capture("synthetic-denial-reference", "Synthetic denial and human action guidance", "synthetic-denial-reference.png"); + } finally { page.off("request", observe); state.fixtureMutations = writes; await page.goto(source, { waitUntil: "domcontentloaded" }); } + state.afterRuns = (await ctx.allRuns()).length; + await ctx.input.evidence("chat-catalog-denial-reference.json", { ...state, scope: "presentation-only; no real provider denial" }); + }, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, + }), +]; +for (const definition of privilegedDeveloperLabCases) definition.sourceFiles = [...definition.sourceFiles, + "../../ui/src/App.tsx", "../../ui/src/pages/DesignGuide.tsx", "../../ui/src/pages/ResponsibleUserDenialUxLab.tsx"]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts b/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts new file mode 100644 index 00000000000..1cbf8a9c359 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts @@ -0,0 +1,66 @@ +import { expect } from "@playwright/test"; +import { readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { runnerE2EServerControlPaths } from "../harness-env.js"; +import { checks, type Entity } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; + +export function gradeRestartService(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("owned-restart-observation", !!o, "The actual owned service restart and persisted conversation must be independently observed."); + if (!o) return c.result(); + const receipt = o.state.restartReceipt as { requestId?: string; status?: string; completedAt?: string } | undefined; + c.add("service-stopped-and-replaced", receipt?.status === "ready" && receipt.requestId === o.state.expectedRequestId + && o.state.stopStartLogVerified === true && typeof receipt.completedAt === "string" + && Date.parse(receipt.completedAt) >= Number(o.state.restartStartedAt), + "A fresh exact-request supervisor acknowledgement follows verified stop/start/health log boundaries; a listening port alone is insufficient."); + c.add("same-conversation-and-history", o.state.sourceBefore === o.state.sourceAfter && typeof o.state.sourceBefore === "string" + && o.state.persistedHistoryRetained === true && o.state.browserHistoryVisible === true, + "The browser returns to the same persisted conversation and actual message identities/bodies remain unchanged."); + c.add("no-duplicate-admission", o.runs.length === 1 && o.state.runBefore === o.runs[0]?.id && o.runs[0]?.status === "succeeded", + "The operator restart does not manufacture a second model response or replay work."); + c.add("honest-operator-guidance", /restart|stop.{0,30}start/i.test(o.reply) && /operator|human|terminal|service|server|instance/i.test(o.reply) + && !/(?:I(?:'ve| have)?|already|successfully)\s+(?:restarted|rebooted)/i.test(o.reply) + && !/(?:keep|continue|remain)\s+(?:responding|connected|available)\s+(?:while|during).{0,30}(?:offline|restart)/i.test(o.reply), + "The pre-restart reply gives the operator path without claiming it already restarted or can remain responsive offline."); + return c.result(); +} +export const privilegedInstanceCases = [privilegedDefinition("instance-operations.restart-service", "privileged-instance", { + async setup(ctx) { + if (ctx.input.execution.environment.id !== "local") throw new Error("Owned service restart requires the launcher's local disposable instance"); + const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT, log = process.env.PAPERCLIP_RUNNER_E2E_SERVER_LOG; + if (!root || !log) throw new Error("Owned restart control directory and supervisor log are required"); + const owned = await realpath(root), logPath = await realpath(log); + if (!logPath.startsWith(owned + path.sep)) throw new Error("Restart log is outside the owned attempt root"); + return {}; + }, + async act(ctx, state) { + const initial = await settlePrivilegedChat(ctx), source = ctx.issue()!; + const history = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); + const url = ctx.input.page.url(); + state.sourceBefore = source.id; state.runBefore = initial.runs[0]!.id; + state.expectedRequestId = `chat-${ctx.input.nonce}`; state.restartStartedAt = Date.now(); + const logPath = process.env.PAPERCLIP_RUNNER_E2E_SERVER_LOG!; + const beforeLogLength = (await readFile(logPath, "utf8")).length; + // The existing operator callback stops the process, waits for HTTP to stop, + // launches a replacement and requires its exact ready acknowledgement. + await ctx.input.restart(); + const receipt = JSON.parse(await readFile(runnerE2EServerControlPaths(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT!).restartAcknowledgementPath, "utf8")); + const log = (await readFile(logPath, "utf8")).slice(beforeLogLength); + const stop = log.indexOf(`Restart request ${state.expectedRequestId}: stopping Paperclip`); + const start = log.indexOf(`Restart request ${state.expectedRequestId}: starting Paperclip`); + const ready = log.indexOf(`Restart request ${state.expectedRequestId}: Paperclip is healthy`); + state.stopStartLogVerified = stop >= 0 && start > stop && ready > start; + state.restartReceipt = { requestId: receipt.requestId, status: receipt.status, completedAt: receipt.completedAt }; + await ctx.input.page.goto(url, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); + await ctx.refreshIssue(); state.sourceAfter = ctx.issue()?.id; + const after = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); + state.persistedHistoryRetained = after.length === history.length && history.every(row => after.some(saved => saved.id === row.id && saved.body === row.body && saved.createdByRunId === row.createdByRunId)); + const reply = history.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).at(-1)!; + await expect(ctx.input.page.locator(`[id=${JSON.stringify(`comment-${reply.id}`)}]`)).toContainText(reply.body!); + state.browserHistoryVisible = true; + await ctx.input.evidence("chat-catalog-owned-service-restart.json", { ...state, retainedCommentIds: after.map(row => row.id) }); + }, grade: gradeRestartService, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts new file mode 100644 index 00000000000..9e267f60996 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { gradeRestartService } from "./privileged-instance.js"; +import { gradeComponentReference, gradeInteractionFixture } from "./privileged-developer-labs.js"; + +function observed(reply: string, state: Record) { + return { scenarioId: "fixture", origin: "http://127.0.0.1:3100", companyId: "owned", leadId: "lead", state, + tasks: [], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], + runs: [{ id: "run", status: "succeeded" }], reply }; +} +const question = { referencePath: "/OWN/design-guide", referenceCompanyId: "owned", questionVisible: true, questionDismissed: true, + questionReopened: true, questionResolved: true, fixtureMutations: 0, beforeRuns: 1, afterRuns: 1 }; +const questionReply = "Use the question example in /design-guide. Its pending card can dismiss, reopen and resolve. This is a design reference, not proof of authorization."; +const denial = { referencePath: "/ux-lab/responsible-user-denial", denialVisible: true, denialActionVisible: true, + unavailableStateVisible: true, fixtureMutations: 0, beforeRuns: 1, afterRuns: 1 }; +const denialReply = "Open /ux-lab/responsible-user-denial for synthetic denial and recovery copy. This is presentation guidance only."; +describe("privileged outcomes require real owned operation or explicitly synthetic evidence", () => { + it("accepts actual question states and rejects missing states, writes and extra native work", () => { + expect(gradeComponentReference(observed(questionReply, question)).passed).toBe(true); + for (const delta of [{ questionResolved: false }, { questionReopened: false }, { referencePath: "/random" }, { fixtureMutations: 1 }, { afterRuns: 2 }]) + expect(gradeComponentReference(observed(questionReply, { ...question, ...delta })).passed).toBe(false); + }); + it("does not confuse presentation with real authorization proof or generic question advice", () => { + for (const reply of ["Show a question somewhere.", "This is not a synthetic fixture: verified real authorization workflow.", "Use /design-guide question. Confirmed production continuation workflow."]) + expect(gradeComponentReference(observed(reply, question)).passed).toBe(false); + expect(gradeComponentReference(null).passed).toBe(false); + }); + it("requires denial plus human action and no admitted provider work", () => { + expect(gradeInteractionFixture(observed(denialReply, denial)).passed).toBe(true); + for (const delta of [{ denialVisible: false }, { denialActionVisible: false }, { unavailableStateVisible: false }, { fixtureMutations: 1 }, { afterRuns: 2 }]) + expect(gradeInteractionFixture(observed(denialReply, { ...denial, ...delta })).passed).toBe(false); + expect(gradeInteractionFixture(observed("Verified a real production denial workflow.", denial)).passed).toBe(false); + }); + it("requires exact fresh stop/start receipt and persisted history instead of port liveness", () => { + const state = { restartReceipt: { requestId: "exact", status: "ready", completedAt: "2026-10-26T23:00:01Z" }, expectedRequestId: "exact", + restartStartedAt: Date.parse("2026-10-26T23:00:00Z"), stopStartLogVerified: true, sourceBefore: "source", sourceAfter: "source", + persistedHistoryRetained: true, browserHistoryVisible: true, runBefore: "run" }; + const reply = "The operator must restart this test instance using the supported service control; the connection will be interrupted."; + expect(gradeRestartService(observed(reply, state)).passed).toBe(true); + for (const delta of [{ stopStartLogVerified: false }, { sourceAfter: "duplicate" }, { persistedHistoryRetained: false }, { browserHistoryVisible: false }, + { restartReceipt: { ...state.restartReceipt, requestId: "other" } }, { restartReceipt: { ...state.restartReceipt, completedAt: "2026-10-25T23:00:01Z" } }]) + expect(gradeRestartService(observed(reply, { ...state, ...delta })).passed).toBe(false); + expect(gradeRestartService(observed("I have restarted the server successfully.", state)).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-support.ts b/tests/runner-e2e/chat-catalog-domains/privileged-support.ts new file mode 100644 index 00000000000..4c23608d541 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-support.ts @@ -0,0 +1,18 @@ +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { observeInternal, observation, type InternalObservation } from "./internal-support.js"; + +/** These cases exercise only the launcher-owned instance or a rendered developer fixture. */ +export function privilegedDefinition(scenarioId: string, module: string, extra: Pick): ChatCatalogCase { + return { scenarioId, oracleVersion: "paperclip.chat-catalog.privileged.v1", coverageGaps: [], + authentication: "human-session", expectedRunCount: 1, turnTimeoutMs: 120_000, attemptTimeoutMs: 420_000, + sourceFiles: ["chat-catalog-domains/privileged-support.ts", `chat-catalog-domains/${module}.ts`, `chat-scenarios/${scenarioId.split(".")[0]}.yaml`], + observe: observeInternal, ...extra }; +} +export async function settlePrivilegedChat(ctx: ChatCatalogContext): Promise { + return pollUntil({ label: "Canonical privileged guidance settled", deadlineAt: Date.now() + 150_000, + load: () => observeInternal(ctx, {}), + accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && !!o.reply && ctx.issue()?.conversationState === "waiting", + reject: o => o.runs.some(run => ["failed", "timed_out", "cancelled"].includes(run.status)) ? "Canonical guidance run failed" : undefined }); +} +export const privilegedObservation = observation;