From b1f30d7d5187cba2076255fd0b0ff61f6b3fe1ac Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 18:39:51 -0500 Subject: [PATCH] eval: add real operator companions and finite disabled-chat boundary --- tests/runner-e2e/chat-boundary-flow.test.ts | 9 ++ .../external-assistant-connections.ts | 1 + .../identity-boundary-contract.ts | 2 +- .../privileged-backup.test.ts | 23 ++++ .../chat-catalog-domains/privileged-backup.ts | 106 ++++++++++++++++++ .../privileged-companions.test.ts | 60 ++++++++++ .../privileged-configuration.ts | 74 ++++++++++++ .../privileged-developer-labs.ts | 5 +- .../privileged-document-cli.ts | 71 ++++++++++++ .../privileged-gated-lab.ts | 44 ++++++++ .../privileged-oracles.test.ts | 1 + .../privileged-performance.test.ts | 31 +++++ .../privileged-performance.ts | 94 ++++++++++++++++ .../privileged-production-ui.ts | 62 ++++++++++ .../privileged-run-cli.ts | 87 ++++++++++++++ tests/runner-e2e/chat-catalog-registry.ts | 5 +- 16 files changed, 671 insertions(+), 4 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-backup.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-document-cli.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-gated-lab.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-performance.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-production-ui.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts diff --git a/tests/runner-e2e/chat-boundary-flow.test.ts b/tests/runner-e2e/chat-boundary-flow.test.ts index d57474bc7d2..fde302a5efe 100644 --- a/tests/runner-e2e/chat-boundary-flow.test.ts +++ b/tests/runner-e2e/chat-boundary-flow.test.ts @@ -21,6 +21,15 @@ describe("separate browser boundary dispatch", () => { await expect(loadChatCatalogImplementation(descriptor, async () => { imported = true; return {}; })).rejects.toThrow("ordinary Agent Chat"); expect(imported).toBe(false); }); + it("admits disabled Agent Chat only on the real owned authenticated target with zero dispatch", async () => { + const disabled: ChatCatalogDescriptor = { ...descriptor, scenarioId: "agent-chat.chat-disabled", boundaryTarget: "authenticated-harness", authentication: "human-session" }; + const implementation = { ...runtime(), ...disabled }; + const importer = async () => ({ boundaryCases: [implementation] }); + await expect(loadIdentityBoundaryImplementation(disabled, importer)).resolves.toMatchObject({ scenarioId: "agent-chat.chat-disabled" }); + await expect(loadIdentityBoundaryImplementation({ ...disabled, boundaryTarget: "empty-installed-target" }, importer)).rejects.toThrow("owned authenticated company"); + await expect(loadIdentityBoundaryImplementation({ ...disabled, authentication: "local-trusted" }, importer)).rejects.toThrow("owned authenticated company"); + await expect(loadIdentityBoundaryImplementation({ ...disabled, expectedRunCount: 1 }, importer)).rejects.toThrow("zero agent admissions"); + }); it("requires actual installed identity and keeps the existing chat-only lane closed", () => { const execution = { id: "boundary", suite: { id: "chat-features" }, environment: { id: "local" } } as MatrixExecution; expect(() => chatBoundaryFingerprint(descriptor, execution, {})).toThrow("immutable installed CLI and server identity pins"); diff --git a/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts b/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts index e6c4435b804..21c39d9ca85 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts @@ -45,6 +45,7 @@ export const assistantConnectionExternalCases = [definition("assistant-connectio return { setupServerUrl: setup.serverUrl, unauthenticatedMcpStatus, resultPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/assistant-connection`, resultLabel: "Assistant Connection (MCP)" }; }, gradeAssistantSetup, { authentication: "human-session", + coverageGaps: ["named-chatgpt-human-setup"], sourceFiles: ["chat-catalog-domains/external-assistant-connections.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "public-mcp-client.ts", "chat-scenarios/assistant-connections.yaml"], async act(ctx, state) { await waitForReply(ctx); const source = ctx.input.page.url(); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-boundary-contract.ts b/tests/runner-e2e/chat-catalog-domains/identity-boundary-contract.ts index 6c6ac346fc6..b32badf8ef9 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-boundary-contract.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-boundary-contract.ts @@ -4,7 +4,7 @@ import type { ChatCatalogCase, ChatCatalogGrade } from "../chat-catalog-contract import type { ChatScenario } from "../chat-feature-cases.js"; import type { CredentialName, MatrixExecution } from "../types.js"; -export const identityBoundaryScenarioIds = ["access.sign-in", "onboarding.new-instance", "onboarding.choose-first-runtime", "onboarding.resume-wizard"] as const; +export const identityBoundaryScenarioIds = ["access.sign-in", "onboarding.new-instance", "onboarding.choose-first-runtime", "onboarding.resume-wizard", "agent-chat.chat-disabled"] as const; export interface IdentityBoundaryContext { journeyKind: "browser-boundary"; boundaryTarget: "authenticated-harness" | "empty-installed-target"; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts new file mode 100644 index 00000000000..ae1c6debe62 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts @@ -0,0 +1,23 @@ +import { expect, it } from "vitest"; +import { gradeBackupRestore } from "./privileged-backup.js"; +function valid() { + const marker = { id: "canary", companyId: "owned", title: "Backup restore canary", description: "exact recovery marker", projectId: "project", status: "backlog", priority: "medium", assigneeAgentId: null, goalId: null }; + return { scenarioId: "instance-operations.backup-and-restore", origin: "http://owned.invalid", companyId: "owned", leadId: "lead", + tasks: [marker], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "run", agentId: "lead", status: "succeeded" }], + reply: "The human instance admin must create a backup, then restore only into a separate empty target to verify recovery.", + state: { task: marker, sourceOwned: true, agentBackupStatus: 403, archiveSha256: "a".repeat(64), archiveBytes: 1024, + archiveWithinOwnedRoot: true, archivePublished: false, separateDatabase: true, targetWasEmpty: true, + restoreCompleted: true, restoredFixtureMatches: true, sourceFixtureUnchanged: true, targetDropped: true } }; +} +it("requires a real private archive and separately verified restoration", () => expect(gradeBackupRestore(valid()).passed).toBe(true)); +it("rejects backup-only success, source overwrite, nonempty target and secret publication", () => { + for (const delta of [{ restoreCompleted: false }, { restoredFixtureMatches: false }, { sourceFixtureUnchanged: false }, { targetWasEmpty: false }, + { separateDatabase: false }, { sourceOwned: false }, { agentBackupStatus: 200 }, { archivePublished: true }, { targetDropped: false }]) { + const o = valid(); expect(gradeBackupRestore({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); + } +}); +it("rejects a changed canary or premature restored claim", () => { + const o = valid(); expect(gradeBackupRestore({ ...o, tasks: [{ ...o.tasks[0]!, description: "changed" }] }).passed).toBe(false); + expect(gradeBackupRestore({ ...o, reply: "I have restored the backup successfully." }).passed).toBe(false); + expect(gradeBackupRestore(null).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-backup.ts b/tests/runner-e2e/chat-catalog-domains/privileged-backup.ts new file mode 100644 index 00000000000..21162d8c46e --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-backup.ts @@ -0,0 +1,106 @@ +import { createHash } from "node:crypto"; +import { readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createDb, getPostgresDataDirectory, runDatabaseRestore } from "../../../packages/db/src/index.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { checks, project, putDocument, task, sameTask, type Entity } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +const privateSources = new WeakMap(); +const pool = (url: string) => createDb(url, { maxConnections: 1, prepare: false, connectTimeoutSeconds: 5, applicationName: "owned-catalog-backup-proof" }).$client; +export function gradeBackupRestore(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("backup-restoration-observation", !!o, "An actual board backup and actual separate empty database restoration are required."); if (!o) return c.result(); + const marker = o.state.task as Entity | undefined; + c.add("owned-source-and-board-authority", o.state.sourceOwned === true && o.state.agentBackupStatus === 403, + "The source belongs to the disposable launcher and the actual agent bearer cannot trigger instance backup."); + c.add("real-private-archive", typeof o.state.archiveSha256 === "string" && o.state.archiveSha256.length === 64 + && Number(o.state.archiveBytes) > 0 && o.state.archiveWithinOwnedRoot === true && o.state.archivePublished === false, + "A real inspectable backup archive exists privately in the owned root; its contents or secrets are never published as chat attachments."); + c.add("separate-empty-target-restored", o.state.separateDatabase === true && o.state.targetWasEmpty === true && o.state.restoreCompleted === true + && o.state.restoredFixtureMatches === true && o.state.sourceFixtureUnchanged === true && o.state.targetDropped === true + && !!marker && sameTask(marker, o.tasks.find(row => row.id === marker.id)), + "The supported restore engine runs only against a newly created empty target and verifies actual task/document rows without overwriting the source."); + c.add("truthful-backup-guidance", /backup|back up/i.test(o.reply) && /restor|recover/i.test(o.reply) && /operator|admin|human|board/i.test(o.reply) + && !/(?:I(?:'ve| have)?|already|successfully)\s+(?:backed up|restored|verified recovery)/i.test(o.reply), + "The initial reply distinguishes privileged backup from the later verified restoration instead of claiming either already occurred."); + return c.result(); +} +const backup = privilegedDefinition("instance-operations.backup-and-restore", "privileged-backup", { + async setup(ctx) { + const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT, configPath = process.env.PAPERCLIP_CONFIG; + if (!root || !configPath || !ctx.input.privateSecrets) throw new Error("Owned backup fixture config and private redaction sink are required"); + const owned = await realpath(root), configFile = await realpath(configPath); + if (!configFile.startsWith(owned + path.sep)) throw new Error("Backup refuses an operator config outside the owned attempt"); + const config = JSON.parse(await readFile(configFile, "utf8")); + if (config.database?.mode !== "embedded-postgres" || !Number.isSafeInteger(config.database.embeddedPostgresPort)) throw new Error("Backup proof accepts only the launcher's owned embedded PostgreSQL cluster"); + const url = `postgres://paperclip:paperclip@127.0.0.1:${config.database.embeddedPostgresPort}/paperclip`; + const data = await getPostgresDataDirectory(url); + if (!data || !(await realpath(data)).startsWith(owned + path.sep)) throw new Error("Backup source PostgreSQL data directory is not owned by this attempt"); + const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `catalog-backup-probe-${ctx.input.nonce}` }); + ctx.input.privateSecrets.push(key.token); privateSources.set(ctx, { url, keyId: key.id, token: key.token, root: owned }); + const response = await fetch(new URL("/api/instance/database-backups", ctx.input.api.baseURL), { method: "POST", redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { Authorization: `Bearer ${key.token}`, "Content-Type": "application/json" }, body: "{}" }); + const agentBackupStatus = response.status; await response.body?.cancel(); + const launch = await project(ctx), marker = await task(ctx, "Backup restore canary", launch.id, { description: `Owned recovery marker ${ctx.input.nonce}` }); + const body = `# Retained plan\n\nExact recovery bytes ${ctx.input.nonce}`; + const plan = await putDocument(ctx, marker.id, body); + return { sourceOwned: true, agentBackupStatus, project: launch, task: marker, documentRevisionId: plan.latestRevisionId, + expectedBodySha256: createHash("sha256").update(body).digest("hex") }; + }, + async act(ctx, state) { + await settlePrivilegedChat(ctx); + const source = privateSources.get(ctx); if (!source) throw new Error("Private owned backup source unavailable"); + const sql = pool(source.url), targetName = `catalog_restore_${createHash("sha256").update(ctx.input.nonce).digest("hex").slice(0, 24)}`; + const targetUrl = new URL(source.url); targetUrl.pathname = `/${targetName}`; + let targetCreated = false; + const markerId = (state.task as { id: string }).id; + const readMarker = async (client: typeof sql) => { + const tasks = await client.unsafe("select id, company_id, project_id, title, description, status from issues where id = $1", [markerId]); + const docs = await client.unsafe("select d.id, d.issue_id, d.key, doc.latest_revision_id from issue_documents d join documents doc on doc.id = d.document_id where d.issue_id = $1 and d.key = 'plan'", [markerId]); + const revisions = await client.unsafe("select id, document_id, body from document_revisions where id = $1", [String(state.documentRevisionId)]); + return { tasks: [...tasks], docs: [...docs], revisions: [...revisions] }; + }; + try { + const before = await readMarker(sql); + const receipt = await ctx.input.api.post<{ backupFile: string; sizeBytes: number }>("/api/instance/database-backups", {}); + const archive = await realpath(receipt.backupFile); + if (!archive.startsWith(source.root + path.sep)) throw new Error("Backup archive escapes the disposable root"); + const bytes = await readFile(archive); + state.archiveSha256 = createHash("sha256").update(bytes).digest("hex"); state.archiveBytes = bytes.length; + state.archiveWithinOwnedRoot = true; state.archivePublished = false; + // The name is generated here, checked absent, and never read from model output. + const existing = await sql.unsafe("select datname from pg_database where datname = $1", [targetName]); + if (existing.length) throw new Error("Owned restore target unexpectedly exists"); + await sql.unsafe(`create database "${targetName}"`); targetCreated = true; state.separateDatabase = targetName !== "paperclip"; + const target = pool(targetUrl.href); + try { + const tables = await target.unsafe("select table_name from information_schema.tables where table_schema = 'public'"); + state.targetWasEmpty = tables.length === 0; + if (!state.targetWasEmpty) throw new Error("Refusing restore into a nonempty target"); + await target.end({ timeout: 5 }); + await runDatabaseRestore({ connectionString: targetUrl.href, backupFile: archive, connectTimeoutSeconds: 5 }); state.restoreCompleted = true; + } finally { await target.end({ timeout: 5 }); } + const restored = pool(targetUrl.href); + try { + state.restoredFixtureMatches = JSON.stringify(await readMarker(restored)) === JSON.stringify(before) + && before.docs.length === 1 && before.docs[0]!.latest_revision_id === state.documentRevisionId + && before.revisions.length === 1 && createHash("sha256").update(String(before.revisions[0]!.body)).digest("hex") === state.expectedBodySha256; + } finally { await restored.end({ timeout: 5 }); } + state.sourceFixtureUnchanged = JSON.stringify(await readMarker(sql)) === JSON.stringify(before); + } catch { throw new Error("Owned backup/restore verification failed; connection, archive contents and private diagnostics withheld"); } + finally { + try { if (targetCreated) { await sql.unsafe(`drop database "${targetName}"`); state.targetDropped = true; } } + finally { await sql.end({ timeout: 5 }); } + } + await ctx.input.evidence("chat-catalog-private-backup-restore.json", { archiveSha256: state.archiveSha256, archiveBytes: state.archiveBytes, + targetWasEmpty: state.targetWasEmpty, separateDatabase: state.separateDatabase, restoredFixtureMatches: state.restoredFixtureMatches, + sourceFixtureUnchanged: state.sourceFixtureUnchanged, targetDropped: state.targetDropped, archivePublished: false, + archiveRetention: "private owned attempt only; secret-bearing archive excluded from ordinary chat artifacts" }); + }, grade: gradeBackupRestore, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +backup.attemptTimeoutMs = 600_000; +backup.cleanup = async ctx => { const source = privateSources.get(ctx); if (source) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${source.keyId}`); privateSources.delete(ctx); } }; +backup.sourceFiles = [...backup.sourceFiles, "../../packages/db/src/backup-lib.ts", "../../server/src/routes/instance-database-backups.ts"]; +export const privilegedBackupCases = [backup]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts new file mode 100644 index 00000000000..b2bd69b72ce --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { gradeInstanceSettings } from "./privileged-configuration.js"; +import { documentExportCommand, gradeExportDocumentCli } from "./privileged-document-cli.js"; +import { inspectRunCommands, gradeInspectRunCli } from "./privileged-run-cli.js"; +import { gradeGatedLab } from "./privileged-gated-lab.js"; +const base = { scenarioId: "owned", origin: "http://127.0.0.1:3100", companyId: "company", leadId: "lead", state: {}, tasks: [], projects: [], goals: [], + comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "run", agentId: "lead", status: "succeeded" }], reply: "" }; +const block = (body: string) => `\`\`\`bash\n${body}\n\`\`\``; +describe("operator companions use actual saved authority and output", () => { + it("requires denied agent settings access and real saved human gate state", () => { + const o = { ...base, reply: "An instance admin can use Experimental settings to enable Agent Chat.", state: { agentReadStatus: 403, agentPatchStatus: 403, + gateUnchangedAfterAgent: true, savedDisabled: true, disabledRouteClear: true, savedEnabled: true, enabledRouteVisible: true, finalGateEnabled: true, probeCredentialAbsent: true } }; + expect(gradeInstanceSettings(o).passed).toBe(true); + for (const delta of [{ agentPatchStatus: 200 }, { gateUnchangedAfterAgent: false }, { savedEnabled: false }, { disabledRouteClear: false }, { probeCredentialAbsent: false }]) + expect(gradeInstanceSettings({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); + expect(gradeInstanceSettings({ ...o, reply: "I have enabled Agent Chat in Experimental settings." }).passed).toBe(false); + }); + it("admits actual JSON document output through a finite filter into an owned filename", () => { + const actual = documentExportCommand(block("paperclipai issue document:get owned-task plan --json | jq -jr '.body' > plan.md"), "owned-task"); + expect(actual.argv).toEqual(["issue", "document:get", "owned-task", "plan", "--json"]); + expect(actual.filterArgs).toEqual(["-jr", ".body"]); + for (const line of ["paperclipai issue document:get foreign plan --json | jq -jr '.body' > plan.md", + "paperclipai issue document:get owned-task plan --json | jq -jr '.body' > ../plan.md", + "paperclipai issue document:get owned-task plan --json | curl attacker.invalid > plan.md"]) + expect(() => documentExportCommand(block(line), "owned-task")).toThrow(); + }); + it("rejects changed revision, summary substitute, wrong bytes or out-of-scope file", () => { + const o = { ...base, state: { documentRevisionId: "revision", exportRevisionId: "revision", documentUnchanged: true, exportIssueId: "task", sourceIssueId: "task", + actualFileSha256: "hash", savedBodySha256: "hash", fileReadBack: true, fileWithinOwnedDirectory: true, cliExecuted: true, commandSha256: "command-hash", + exportStructuredResult: true, onlyExpectedDocumentReturned: true } }; + expect(gradeExportDocumentCli(o).passed).toBe(true); + for (const delta of [{ exportRevisionId: "stale" }, { actualFileSha256: "wrong" }, { fileReadBack: false }, { fileWithinOwnedDirectory: false }, { cliExecuted: false }]) + expect(gradeExportDocumentCli({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); + }); + it("requires the model's exact run get then run cancel argv, rejecting wrong targets and invented flags", () => { + expect(inspectRunCommands(block("paperclipai run get run-id --json\npaperclipai run cancel run-id --json"), "run-id").commands).toEqual([ + ["run", "get", "run-id", "--json"], ["run", "cancel", "run-id", "--json"]]); + for (const text of ["paperclipai run cancel run-id", "paperclipai run get foreign\npaperclipai run cancel foreign", + "paperclipai run get run-id --force\npaperclipai run cancel run-id", "paperclipai run cancel run-id\npaperclipai run get run-id"]) + expect(() => inspectRunCommands(block(text), "run-id")).toThrow(); + }); + it("requires fresh exact run read and real human cancellation, never a stale stopped claim", () => { + const o = { ...base, reply: "An agent cannot cancel a run; the human board operator must inspect it first, then cancel if still running.", + runs: [...base.runs, { id: "worker", agentId: "morgan", status: "cancelled" }], + state: { initialRunReallyRunning: true, readRunId: "worker", workerRunId: "worker", worker: { id: "morgan" }, readAgentId: "morgan", readCompanyId: "company", + readStatus: "running", agentCancelStatus: 403, cancelRunId: "worker", finalWorkerStatus: "cancelled", cancelOnlyAfterRead: true } }; + expect(gradeInspectRunCli(o).passed).toBe(true); + for (const delta of [{ readRunId: "other" }, { agentCancelStatus: 200 }, { readStatus: "succeeded" }, { finalWorkerStatus: "running" }, { cancelOnlyAfterRead: false }]) + expect(gradeInspectRunCli({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); + }); + it("requires an actual source-built production not-found page without unrelated redirect", () => { + const o = { ...base, reply: "The task-chat-lab is development-only and is not available in production. Use pnpm dev in your owned checkout, then /dev/task-chat-lab.", + state: { productionBuildSha256: "a".repeat(64), productionHeaderMatched: true, labAbsent: true, notFoundVisible: true, + historicalPath: "/OWN/dev/task-chat-lab", finalProductionPath: "/OWN/dev/task-chat-lab", unrelatedTaskAbsent: true } }; + expect(gradeGatedLab(o).passed).toBe(true); + for (const delta of [{ productionHeaderMatched: false }, { labAbsent: false }, { notFoundVisible: false }, { finalProductionPath: "/OWN/issues/other" }]) + expect(gradeGatedLab({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); + expect(gradeGatedLab({ ...o, reply: "The lab is available in production; it is not a development-only task-chat-lab." }).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts b/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts new file mode 100644 index 00000000000..c18e01f713a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts @@ -0,0 +1,74 @@ +import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { checks, observeInternal } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +const probeKeys = new WeakMap(); +const endpoint = "/api/instance/settings/experimental"; +async function probe(ctx: ChatCatalogContext, method: "GET" | "PATCH") { + const key = probeKeys.get(ctx); if (!key) throw new Error("Private instance-permission probe actor is missing"); + const response = await fetch(new URL(endpoint, ctx.input.api.baseURL), { method, redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { Authorization: `Bearer ${key.token}`, "Content-Type": "application/json" }, + ...(method === "PATCH" ? { body: JSON.stringify({ enableAgentChat: false }) } : {}) }); + await response.body?.cancel(); return response.status; +} +export function gradeInstanceSettings(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("actual-setting-evidence", !!o, "Actual denied agent access and independently saved operator changes are required."); if (!o) return c.result(); + c.add("agent-not-instance-admin", o.state.agentReadStatus === 403 && o.state.agentPatchStatus === 403 && o.state.gateUnchangedAfterAgent === true, + "The actual agent bearer cannot read or patch experimental instance administration settings."); + c.add("supported-human-path", /experimental|enableAgentChat/i.test(o.reply) && /operator|admin|human|settings/i.test(o.reply) + && !/(?:I(?:'ve| have)?|successfully|already)\s+(?:enabled|turned on|activated)\s+(?:Agent Chat|it)/i.test(o.reply), + "The reply identifies the supported setting and operator path without claiming an unauthorized setting write."); + c.add("saved-gate-and-route", o.state.savedDisabled === true && o.state.disabledRouteClear === true + && o.state.savedEnabled === true && o.state.enabledRouteVisible === true && o.state.finalGateEnabled === true, + "The human toggles the actual owned setting, observes the disabled route explanation, persists enablement and opens the real chat route."); + c.add("credential-not-disclosed", o.state.probeCredentialAbsent === true, + "The bounded private permission-probe credential never enters the model reply or evidence."); + return c.result(); +} +const settings = privilegedDefinition("instance-operations.instance-settings", "privileged-configuration", { + async setup(ctx) { + if (!ctx.input.privateSecrets) throw new Error("Private instance-permission probe requires the attempt redaction sink"); + const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `catalog-instance-probe-${ctx.input.nonce}` }); + ctx.input.privateSecrets.push(key.token); probeKeys.set(ctx, key); + const before = await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint); + const agentReadStatus = await probe(ctx, "GET"), agentPatchStatus = await probe(ctx, "PATCH"); + const after = await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint); + return { agentReadStatus, agentPatchStatus, gateUnchangedAfterAgent: before.enableAgentChat === after.enableAgentChat }; + }, + async act(ctx, state) { + await settlePrivilegedChat(ctx); + const page = ctx.input.page, source = page.url(); + const settingsUrl = new URL(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/instance/experimental`, ctx.input.api.baseURL).href; + await page.goto(settingsUrl, { waitUntil: "domcontentloaded" }); + const toggle = page.getByRole("switch", { name: "Toggle agent chat experimental setting", exact: true }); + await expect(toggle).toHaveAttribute("aria-checked", "true"); + await toggle.click(); + await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint)).enableAgentChat).toBe(false); + state.savedDisabled = true; + await page.goto(source, { waitUntil: "domcontentloaded" }); + await expect(page.getByText(/Agent Chat is disabled(?:\.| in Experimental settings)/).first()).toBeVisible(); + state.disabledRouteClear = true; + await page.goto(settingsUrl, { waitUntil: "domcontentloaded" }); + await expect(toggle).toHaveAttribute("aria-checked", "false"); await toggle.click(); + await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint)).enableAgentChat).toBe(true); + state.savedEnabled = true; + await page.goto(source, { waitUntil: "domcontentloaded" }); await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); + state.enabledRouteVisible = true; + await ctx.input.evidence("chat-catalog-instance-gate-ui.json", { savedDisabled: true, disabledRouteClear: true, savedEnabled: true, enabledRouteVisible: true }); + }, grade: gradeInstanceSettings, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +settings.observe = async (ctx, state) => { + const o = await observeInternal(ctx, state), key = probeKeys.get(ctx); + state.probeCredentialAbsent = !!key && !o.reply.includes(key.token); + state.finalGateEnabled = (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint)).enableAgentChat; + return { ...o, state }; +}; +settings.cleanup = async ctx => { + const key = probeKeys.get(ctx); if (key) { + await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`); probeKeys.delete(ctx); + } +}; +settings.sourceFiles = [...settings.sourceFiles, "../../ui/src/pages/InstanceExperimentalSettings.tsx", "../../ui/src/pages/AgentChat.tsx"]; +export const privilegedConfigurationCases = [settings]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-developer-labs.ts b/tests/runner-e2e/chat-catalog-domains/privileged-developer-labs.ts index b80ed292d1a..6b63604fb02 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-developer-labs.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-developer-labs.ts @@ -4,9 +4,10 @@ import { checks } from "./internal-support.js"; import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; function fixtureBoundary(reply: string) { + const positive = reply.replace(/\b(?:not|isn't|does not|cannot|can't)\b[^.!?\n]*/gi, ""); return /synthetic|fixture|design reference|presentation|storybook|mock/i.test(reply) - && !/not\s+(?:a\s+)?(?:synthetic|fixture|mock)|real\s+(?:authorization|continuation)\s+proof/i.test(reply) - && !/(?:verified|proved|confirmed).{0,35}(?:real|live|production).{0,25}(?:authorization|denial|continuation|workflow)/i.test(reply); + && !/not\s+(?:a\s+)?(?:synthetic|fixture|mock)/i.test(reply) + && !/real\s+(?:authorization|continuation)\s+proof|(?:verified|proved|confirmed).{0,35}(?:real|live|production).{0,25}(?:authorization|denial|continuation|workflow)/i.test(positive); } export function gradeComponentReference(value: unknown) { const c = checks(), o = privilegedObservation(value); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-document-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-document-cli.ts new file mode 100644 index 00000000000..cd3af4b58c6 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-document-cli.ts @@ -0,0 +1,71 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { checks, putDocument, type Entity } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli } from "./privileged-cli-support.js"; + +export function documentExportCommand(reply: string, sourceId: string) { + const blocks = [...reply.matchAll(/```(?:bash|sh|shell)?\s*\n([\s\S]*?)```/g)].map(match => match[1]!.replace(/\\\r?\n/g, " ").trim()); + const matches = blocks.flatMap(block => block.split(/\r?\n/)).map(line => line.match(/^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+issue\s+document:get\s+(\S+)\s+plan\s+--json\s*\|\s*jq\s+(-[rj]+)\s+['"]?\.body['"]?\s*>\s*([A-Za-z0-9][A-Za-z0-9_.-]*)$/)).filter(Boolean); + if (matches.length !== 1) throw new Error("One supported plan JSON-to-file command is required; arbitrary shell is not admitted"); + const command = matches[0]!; + const original = command[1]!, id = /^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(original) ? sourceId : original; + if (id !== sourceId || path.basename(command[3]!) !== command[3]!) throw new Error("Document export targets a different task or destination"); + return { argv: ["issue", "document:get", id, "plan", "--json"], filterArgs: [command[2]!, ".body"], filename: command[3]!, + unresolvedIssueBinding: original === id ? undefined : { placeholder: original, fixtureValue: id } }; +} +export function gradeExportDocumentCli(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("saved-document-export-evidence", !!o, "An actual supported CLI read, structured result and owned file bytes are required."); if (!o) return c.result(); + c.add("correct-saved-revision", typeof o.state.documentRevisionId === "string" && o.state.exportRevisionId === o.state.documentRevisionId + && o.state.documentUnchanged === true && o.state.exportIssueId === o.state.sourceIssueId, + "The downloaded structured document is the exact selected saved plan revision on the authorized source task."); + c.add("actual-file-bytes", o.state.actualFileSha256 === o.state.savedBodySha256 && typeof o.state.actualFileSha256 === "string" + && o.state.fileReadBack === true && o.state.fileWithinOwnedDirectory === true, + "The actual terminal output file is read back and matches the saved document body byte-for-byte."); + c.add("no-summary-substitute", o.state.cliExecuted === true && typeof o.state.commandSha256 === "string" + && o.state.exportStructuredResult === true && o.state.onlyExpectedDocumentReturned === true, + "A chat summary or unrelated document cannot stand in for the real authorized file download."); + return c.result(); +} +const exportPlan = privilegedDefinition("cli-operations.export-document-cli", "privileged-document-cli", { + async setup(ctx) { + const operator = await prepareOwnedCli(ctx); + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/chats/${ctx.input.fixtures.agent.id}`); + await ctx.refreshIssue(); const issue = ctx.issue(); if (!issue) throw new Error("Owned standing task did not persist"); + const body = `# Website plan\n\nA saved synthetic revision for Acorn.\n\nReference ${ctx.input.nonce}.`; + const document = await putDocument(ctx, issue.id, body); + return { ...operator, sourceIssueId: issue.id, documentRevisionId: document.latestRevisionId, + savedBodySha256: createHash("sha256").update(body).digest("hex") }; + }, + async act(ctx, state) { + const reply = await settlePrivilegedChat(ctx), sourceId = String(state.sourceIssueId); + const command = documentExportCommand(reply.reply, sourceId); + const json = await runOwnedCli(ctx, command.argv), document = JSON.parse(json) as Entity; + // Execute the actual finite jq filter from the returned command; never a shell. + const bytes = execFileSync("jq", command.filterArgs, { input: json, timeout: 5_000, maxBuffer: 1_048_576 }); + const destination = path.join(String(state.operatorDirectory), command.filename); + await writeFile(destination, bytes, { flag: "wx", mode: 0o600 }); + const actual = await readFile(destination); + state.actualFileSha256 = createHash("sha256").update(actual).digest("hex"); state.fileReadBack = true; + state.fileWithinOwnedDirectory = path.dirname(destination) === state.operatorDirectory; + state.exportRevisionId = document.latestRevisionId; state.exportIssueId = document.issueId; + const saved = await ctx.input.api.get(`/api/issues/${sourceId}/documents/plan`); + state.documentUnchanged = saved.latestRevisionId === state.documentRevisionId && createHash("sha256").update(saved.body ?? "").digest("hex") === state.savedBodySha256; + state.exportStructuredResult = typeof document.body === "string" && typeof document.latestRevisionId === "string"; + state.onlyExpectedDocumentReturned = document.key === "plan" && document.issueId === sourceId; + state.cliExecuted = true; + state.commandSha256 = createHash("sha256").update(JSON.stringify(command)).digest("hex"); + state.unresolvedIssueBinding = command.unresolvedIssueBinding; + await ctx.input.evidence("chat-catalog-cli-plan-file.json", { sourceIssueId: sourceId, revisionId: state.documentRevisionId, + fileSha256: state.actualFileSha256, expectedSha256: state.savedBodySha256, bytes: actual.length, + filename: command.filename, commandSha256: state.commandSha256, unresolvedIssueBinding: command.unresolvedIssueBinding }); + }, grade: gradeExportDocumentCli, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +exportPlan.sourceFiles = [...exportPlan.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "../../cli/src/index.ts"]; +exportPlan.cleanup = cleanupOwnedCli; +export const privilegedDocumentCliCases = [exportPlan]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-gated-lab.ts b/tests/runner-e2e/chat-catalog-domains/privileged-gated-lab.ts new file mode 100644 index 00000000000..a14995301b5 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-gated-lab.ts @@ -0,0 +1,44 @@ +import { expect } from "@playwright/test"; +import { checks } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +import { startOwnedProductionUi } from "./privileged-production-ui.js"; + +export function gradeGatedLab(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("production-route-observation", !!o, "A real production build must deny the historical developer route."); if (!o) return c.result(); + c.add("clear-development-only-boundary", /dev(?:elopment)?[- ]only|local.{0,30}(?:checkout|development)|owned checkout/i.test(o.reply) + && /task-chat-lab|TaskChatLab/i.test(o.reply) && /pnpm dev|dev\/task-chat-lab|checkout/i.test(o.reply) + && !/not\s+(?:a\s+)?(?:dev|development)[- ]only/i.test(o.reply) + && !/\b(?:I(?:'ve| have)? (?:opened|enabled)|it is enabled|the lab is available)\b[^.\n]{0,35}(?:in|on) production/i.test(o.reply), + "The actual reply identifies the development-only limitation and the appropriate checkout path without claiming the old route is a production feature."); + c.add("actual-production-build-gate", typeof o.state.productionBuildSha256 === "string" && o.state.productionBuildSha256.length === 64 + && o.state.productionHeaderMatched === true && o.state.labAbsent === true && o.state.notFoundVisible === true, + "A fresh source-built production bundle serves the browser; it displays the real not-found state instead of the disabled lab."); + c.add("historical-link-not-redirected", o.state.historicalPath === o.state.finalProductionPath && o.state.unrelatedTaskAbsent === true, + "The historical route remains explicit and does not silently open unrelated production work."); + return c.result(); +} +const gated = privilegedDefinition("developer-labs.gated-lab", "privileged-gated-lab", { + async setup() { return {}; }, + async act(ctx, state) { + await settlePrivilegedChat(ctx); + const target = await startOwnedProductionUi(ctx), page = ctx.input.page, source = page.url(); + try { + const historicalPath = `/${ctx.input.fixtures.company.issuePrefix}/dev/task-chat-lab`; + const response = await page.goto(target.origin + historicalPath, { waitUntil: "domcontentloaded" }); + state.productionHeaderMatched = response?.headers()["x-eval-production-ui-build"] === target.buildSha256; + await expect(page.getByRole("heading", { name: /not found|page.*exist|couldn.*find/i }).first()).toBeVisible(); + await expect(page.getByRole("heading", { name: "Task Chat Lab", exact: true })).toHaveCount(0); + state.productionBuildSha256 = target.buildSha256; state.notFoundVisible = true; state.labAbsent = true; + state.productionSourceRevision = target.sourceRevision; + state.historicalPath = historicalPath; state.finalProductionPath = new URL(page.url()).pathname; + state.unrelatedTaskAbsent = await page.getByTestId("task-chat-composer-input").count() === 0; + await ctx.input.evidence("chat-catalog-production-lab-gate.json", { ...state, files: target.files, proof: "production build; no second database or provider" }); + await ctx.input.capture("production-lab-gate", "Real production build excludes developer lab", "production-lab-gate.png"); + } finally { try { await page.goto(source, { waitUntil: "domcontentloaded" }); } finally { await target.stop(); } } + }, grade: gradeGatedLab, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +gated.attemptTimeoutMs = 600_000; +gated.sourceFiles = [...gated.sourceFiles, "chat-catalog-domains/privileged-production-ui.ts", "../../ui/vite.config.ts", "../../ui/src/App.tsx", "../../ui/src/pages/NotFound.tsx"]; +export const privilegedGatedLabCases = [gated]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts index 9e267f60996..c945f70cf82 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-oracles.test.ts @@ -20,6 +20,7 @@ describe("privileged outcomes require real owned operation or explicitly synthet expect(gradeComponentReference(observed(questionReply, { ...question, ...delta })).passed).toBe(false); }); it("does not confuse presentation with real authorization proof or generic question advice", () => { + expect(gradeComponentReference(observed(questionReply + " This is not real authorization proof.", question)).passed).toBe(true); for (const reply of ["Show a question somewhere.", "This is not a synthetic fixture: verified real authorization workflow.", "Use /design-guide question. Confirmed production continuation workflow."]) expect(gradeComponentReference(observed(reply, question)).passed).toBe(false); expect(gradeComponentReference(null).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts new file mode 100644 index 00000000000..fe27fada76c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-performance.test.ts @@ -0,0 +1,31 @@ +import { expect, it } from "vitest"; +import { gradeLongThreadPerformance } from "./privileged-performance.js"; +function valid() { + return { scenarioId: "developer-labs.long-thread-performance", origin: "http://owned.invalid", companyId: "owned", leadId: "lead", + tasks: [], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical", "actual report"], + runs: [{ id: "initial", agentId: "lead", status: "succeeded" }, { id: "interpret", agentId: "lead", status: "succeeded" }], reply: "initial guidance", + state: { performanceReport: { scope: "fixed synthetic UI", commentCount: 469, rowCount: 487, markdownCount: 150, + mountMs: 12.3, latestMs: 8.1, maximumMs: 12.3, totalMs: 20.4, commitCount: 2, fixtureSha256: "a".repeat(64), + environment: { browser: "real browser", measuredAt: "2026-10-26T23:00:00Z" } }, + interpretation: { attributed: true, humanCommentId: "real-human-report", runId: "interpret", + reply: "The synthetic fixture rendered 487 rows with 469 comments. Mount was 12.3 ms; total was 20.4 ms. This measures render cost, not production throughput or model latency." } } }; +} +it("accepts actual fixed-fixture measurements and attributed limited interpretation", () => expect(gradeLongThreadPerformance(valid()).passed).toBe(true)); +it("accepts ordinary display rounding and the reversed phrasing of a production limitation", () => { + const o = valid(); o.state.interpretation.reply = "The synthetic fixture mount was 12 ms. Production throughput cannot be inferred from these render timings."; + expect(gradeLongThreadPerformance(o).passed).toBe(true); +}); +it("rejects invented timings and swaps between named metrics", () => { + for (const text of ["The synthetic fixture mount was 500 ms. It does not prove production throughput.", + "The synthetic fixture mount was 20.4 ms. It does not prove production throughput."]) { + const o = valid(); o.state.interpretation.reply = text; expect(gradeLongThreadPerformance(o).passed).toBe(false); + } +}); +it("rejects wrong data size, unrelated phase and unmeasured production throughput", () => { + const o = valid(); o.state.interpretation.reply += " Production throughput is 100 requests per second."; + expect(gradeLongThreadPerformance(o).passed).toBe(false); + const wrong = valid(); wrong.state.interpretation.reply = wrong.state.interpretation.reply.replace("487 rows", "900 rows"); + expect(gradeLongThreadPerformance(wrong).passed).toBe(false); + const phase = valid(); phase.state.interpretation.attributed = false; expect(gradeLongThreadPerformance(phase).passed).toBe(false); + expect(gradeLongThreadPerformance(null).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts b/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts new file mode 100644 index 00000000000..b959f6c435c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-performance.ts @@ -0,0 +1,94 @@ +import { expect } from "@playwright/test"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import os from "node:os"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, type Entity } from "./internal-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +interface FixtureReport { + scope: string; commentCount: number; rowCount: number; markdownCount: number; + mountMs: number; latestMs: number; maximumMs: number; totalMs: number; commitCount: number; + environment: { platform: string; architecture: string; node: string; cpuModel: string; cpuCount: number; browser: string; viewport: unknown; measuredAt: string }; + fixtureSha256: string; +} +export function gradeLongThreadPerformance(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("actual-performance-observation", !!o, "A documented fixed-fixture measurement and attributed interpretation turn are required."); if (!o) return c.result(); + const report = o.state.performanceReport as FixtureReport | undefined; + c.add("fixed-fixture-measured", !!report && report.commentCount === 469 && report.rowCount === 487 && report.markdownCount === 150 + && [report.mountMs, report.latestMs, report.maximumMs, report.totalMs].every(value => Number.isFinite(value) && value >= 0) + && report.commitCount >= 1 && report.fixtureSha256.length === 64 && !!report.environment.browser && !!report.environment.measuredAt, + "Actual browser profiler values, fixed data size, source hash and measurement environment are retained together."); + const phase = o.state.interpretation as { attributed?: boolean; reply?: string; humanCommentId?: string; runId?: string } | undefined; + const reply = phase?.reply ?? ""; + c.add("report-bound-native-reply", phase?.attributed === true && !!phase.humanCommentId && !!phase.runId && !!reply + && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded" && run.agentId === o.leadId), + "The interpretation is the actual second native reply to the exact persisted human report, not initial guidance or a fabricated result."); + const timings = [...reply.matchAll(/\b(\d+(?:\.\d+)?)\s*(?:ms|milliseconds)\b/gi)]; + const expected = report ? [report.mountMs, report.latestMs, report.maximumMs, report.totalMs] : []; + c.add("timing-claims-grounded", !!report && timings.every(match => { + const value = Number(match[1]), decimals = match[1]!.split(".")[1]?.length ?? 0; + const prefix = reply.slice(Math.max(0, match.index! - 60), match.index); + const label = [...prefix.matchAll(/\b(mount|latest|maximum|max|total)\b/gi)].at(-1)?.[1]?.toLowerCase(); + const candidates = label === "mount" ? [report!.mountMs] : label === "latest" ? [report!.latestMs] + : label === "maximum" || label === "max" ? [report!.maximumMs] : label === "total" ? [report!.totalMs] : expected; + return candidates.some(actual => Math.abs(actual - value) <= Math.max(0.5 * 10 ** -decimals + 0.001, Math.abs(actual) * 0.01)); + }), + "Any timing repeated in the reply matches an actual displayed measurement, allowing ordinary display rounding."); + const sizes = [...reply.matchAll(/\b(\d+)\s+(comments|rows|messages)\b/gi)]; + c.add("data-size-grounded", !!report && sizes.every(match => Number(match[1]) === (match[2]!.toLowerCase() === "rows" ? report.rowCount : report.commentCount)), + "Any stated row or comment count matches the actual fixed fixture."); + c.add("measured-scope-explained", /synthetic|fixture/i.test(reply) && /mount|render|DOM|profil|commit/i.test(reply) + && reply.split(/[.!?;\n]/).some(clause => /\b(?:not|cannot|can't|doesn't|does not|no)\b/i.test(clause) + && /production|real.user|model.latency|throughput/i.test(clause)), + "Chat describes this measured render behavior and its limits without treating one synthetic run as production throughput or model latency."); + const positive = reply.split(/[.!?;\n]|\bbut\b|\bhowever\b/i).filter(clause => !/\b(?:not|cannot|can't|doesn't|does not|no)\b/i.test(clause)).join("."); + c.add("no-invented-production-performance", !/\b(?:production|real.user|model)[^.\n]{0,50}\b(?:latency|throughput|requests? per second|tokens? per second|users? per second)\b/i.test(positive), + "The interpretation makes no unsupported production or provider performance claim."); + return c.result(); +} +const performance = privilegedDefinition("developer-labs.long-thread-performance", "privileged-performance", { + async setup() { return {}; }, + async act(ctx, state) { + const first = await settlePrivilegedChat(ctx), source = ctx.input.page.url(), page = ctx.input.page; + state.initialGuidanceRunId = first.runs[0]!.id; + const numeric = async (id: string) => Number((await page.getByTestId(id).innerText()).replace(/\s*ms$/, "")); + let report: FixtureReport; + try { + await page.goto(new URL("/tests/perf/long-thread", ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" }); + await expect(page.getByTestId("issue-chat-long-thread-perf")).toBeVisible(); + await expect.poll(() => numeric("perf-mount-duration")).toBeGreaterThanOrEqual(0); + const browser = await page.evaluate(() => navigator.userAgent); + report = { scope: "one fixed synthetic long-thread UI fixture; direct-render mount and profiler timings", commentCount: 469, + rowCount: await numeric("perf-fixture-row-target"), markdownCount: await numeric("perf-fixture-markdown-rows"), + mountMs: await numeric("perf-mount-duration"), latestMs: await numeric("perf-latest-duration"), maximumMs: await numeric("perf-max-duration"), + totalMs: await numeric("perf-total-duration"), commitCount: await numeric("perf-commit-count"), + fixtureSha256: createHash("sha256").update(await readFile(new URL("../../../ui/src/fixtures/issueChatLongThreadFixture.ts", import.meta.url))).digest("hex"), + environment: { platform: os.platform(), architecture: os.arch(), node: process.versions.node, cpuModel: os.cpus()[0]?.model ?? "unavailable", + cpuCount: os.cpus().length, browser, viewport: page.viewportSize(), measuredAt: new Date().toISOString() } }; + state.performanceReport = report; + await ctx.input.evidence("chat-catalog-long-thread-measurement.json", report); + await ctx.input.capture("long-thread-performance", "Actual fixed-fixture render measurements", "long-thread-performance.png"); + } finally { await page.goto(source, { waitUntil: "domcontentloaded" }); } + // Declared operator companion step: facts only, no desired conclusion or threshold. + const message = `Here is the report from my owned documented long-thread fixture. Please interpret the measured behavior and its limits:\n\n${JSON.stringify(report!)}`; + await sendChatMessage(page, message); + const final = await pollUntil({ label: "Actual native interpretation of the measured report", deadlineAt: Date.now() + 150_000, + load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: o => o.issue?.conversationState === "waiting" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") }); + const run = final.runs.find(run => run.id !== first.runs[0]!.id)!; + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + const human = comments.filter(comment => !comment.authorAgentId && comment.body === message); + const replies = comments.filter(comment => comment.authorAgentId === ctx.input.fixtures.agent.id && comment.createdByRunId === run.id); + const profile = await ctx.input.api.get<{ runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`); + state.interpretation = { humanCommentId: human[0]?.id, runId: run.id, reply: replies.map(row => row.body).join("\n"), + attributed: human.length === 1 && replies.length > 0 && profile.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(human[0]!.id) === true + && replies.every(reply => Date.parse(reply.createdAt!) >= Date.parse(human[0]!.createdAt!)) }; + await ctx.input.evidence("chat-catalog-long-thread-interpretation.json", state.interpretation); + }, grade: gradeLongThreadPerformance, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +performance.expectedRunCount = 2; performance.expectedHumanMessageCount = 2; performance.attemptTimeoutMs = 600_000; +performance.sourceFiles = [...performance.sourceFiles, "../../ui/src/pages/IssueChatLongThreadPerf.tsx", "../../ui/src/fixtures/issueChatLongThreadFixture.ts"]; +export const privilegedPerformanceCases = [performance]; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-production-ui.ts b/tests/runner-e2e/chat-catalog-domains/privileged-production-ui.ts new file mode 100644 index 00000000000..6be7b546ca1 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-production-ui.ts @@ -0,0 +1,62 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { createServer } from "node:http"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, realpath, readdir, rm } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const execute = promisify(execFile); + +/** A real production UI build, isolated static listener, and read-only API proxy. + * It shares only the already-owned test API; it never starts another database. */ +export async function startOwnedProductionUi(ctx: ChatCatalogContext) { + const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; + if (!root) throw new Error("Production route proof requires the owned attempt directory"); + const sourceRevision = process.env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA; + if (!sourceRevision || !/^[a-f0-9]{40}$/.test(sourceRevision)) throw new Error("Production UI proof requires the pinned full candidate source revision"); + const directory = await mkdtemp(path.join(await realpath(root), "production-ui-")); + const repository = fileURLToPath(new URL("../../../", import.meta.url)), out = path.join(directory, "dist"); + try { + await execute(process.execPath, [path.join(repository, "ui/node_modules/vite/bin/vite.js"), "build", "--mode", "production", "--outDir", out], { + cwd: path.join(repository, "ui"), timeout: 180_000, maxBuffer: 2_097_152, + env: { PATH: process.env.PATH, HOME: directory, NODE_ENV: "production", PAPERCLIP_TELEMETRY_DISABLED: "1", PAPERCLIP_BUILD_COMMIT: sourceRevision }, + }); + } catch { await rm(directory, { recursive: true }); throw new Error("Owned production UI build failed; private build output withheld"); } + async function inventory(current: string): Promise> { + const rows: Array<{ path: string; sha256: string }> = []; + for (const entry of await readdir(current, { withFileTypes: true })) { + const file = path.join(current, entry.name); + if (entry.isDirectory()) rows.push(...await inventory(file)); + else if (entry.isFile()) rows.push({ path: path.relative(out, file), sha256: createHash("sha256").update(await readFile(file)).digest("hex") }); + } + return rows; + } + const files = (await inventory(out)).sort((a, b) => a.path.localeCompare(b.path)); + const buildSha256 = createHash("sha256").update(JSON.stringify(files)).digest("hex"); + const mime: Record = { ".html": "text/html", ".js": "application/javascript", ".css": "text/css", ".json": "application/json", ".svg": "image/svg+xml", ".woff2": "font/woff2", ".png": "image/png", ".ico": "image/x-icon" }; + const server = createServer(async (request, response) => { + try { + const route = new URL(request.url ?? "/", "http://owned.invalid"); + if (route.pathname.startsWith("/api/")) { + if (request.method !== "GET" && request.method !== "HEAD") { response.writeHead(403).end(); return; } + const upstream = await fetch(new URL(route.pathname + route.search, ctx.input.api.baseURL), { redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { ...(request.headers.cookie ? { cookie: request.headers.cookie } : {}) } }); + response.writeHead(upstream.status, { "content-type": upstream.headers.get("content-type") ?? "application/json", "cache-control": "no-store" }); + response.end(Buffer.from(await upstream.arrayBuffer())); return; + } + const candidate = path.resolve(out, "." + decodeURIComponent(route.pathname)); + if (!candidate.startsWith(out + path.sep)) { response.writeHead(404).end(); return; } + let file = candidate; + let body: Buffer; + try { body = await readFile(file); } + catch { file = path.join(out, "index.html"); body = await readFile(file); } + response.writeHead(200, { "content-type": mime[path.extname(file)] ?? "application/octet-stream", "cache-control": "no-store", "x-eval-production-ui-build": buildSha256 }); + response.end(body); + } catch { response.writeHead(502).end(); } + }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Owned production UI listener has no address"); + return { origin: `http://127.0.0.1:${address.port}`, buildSha256, sourceRevision, files, + stop: async () => { server.closeAllConnections(); await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); await rm(directory, { recursive: true }); } }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts new file mode 100644 index 00000000000..806c94c2e81 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-run-cli.ts @@ -0,0 +1,87 @@ +import { expect } from "@playwright/test"; +import { createHash } from "node:crypto"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { checks, observeInternal } from "./internal-support.js"; +import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { privilegedDefinition, privilegedObservation } from "./privileged-support.js"; +import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli } from "./privileged-cli-support.js"; + +export function inspectRunCommands(reply: string, runId: string) { + const matches = [...reply.matchAll(/```(?:bash|sh|shell)?\s*\n([\s\S]*?)```/g)] + .flatMap(match => match[1]!.replace(/\\\r?\n/g, " ").split(/\r?\n/)) + .map(line => line.trim()).filter(line => /^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+run\s+(?:get|cancel)\b/.test(line)); + const commands: string[][] = [], bindings: Array<{ placeholder: string; fixtureValue: string }> = []; + for (const line of matches) { + const parsed = line.match(/^(?:paperclipai|npx(?: -y)? paperclipai|pnpm (?:exec )?paperclipai)\s+run\s+(get|cancel)\s+(\S+)(?:\s+(--json))?$/); + if (!parsed) throw new Error("Unsupported run command flags or shell syntax"); + let id = parsed[2]!; + if (/^(?:<[A-Z_ -]+>|\$\{?[A-Z_]+\}?)$/.test(id)) { bindings.push({ placeholder: id, fixtureValue: runId }); id = runId; } + if (id !== runId) throw new Error("Run command names a different native run"); + commands.push(["run", parsed[1]!, id, ...(parsed[3] ? [parsed[3]] : [])]); + } + if (commands.length !== 2 || commands[0]?.[1] !== "get" || commands[1]?.[1] !== "cancel") throw new Error("Read the exact run before the separate authorized cancellation command"); + return { commands, bindings }; +} +export function gradeInspectRunCli(value: unknown) { + const c = checks(), o = privilegedObservation(value); + c.add("run-companion-observation", !!o, "Actual read-before-cancel state and the owned native worker run are required."); if (!o) return c.result(); + c.add("read-exact-last-morgan-run", o.state.initialRunReallyRunning === true && o.state.readRunId === o.state.workerRunId + && o.state.readAgentId === (o.state.worker as { id?: string } | undefined)?.id && o.state.readCompanyId === o.companyId + && o.state.readStatus === "running", "The actual CLI read selects Morgan's exact current run, rather than a stale or other-company run."); + c.add("board-only-control", o.state.agentCancelStatus === 403 && /board|human|operator/i.test(o.reply) + && /agent.{0,60}(?:cannot|can't|not|insufficient)|(?:cannot|can't|not|insufficient).{0,60}agent/i.test(o.reply), + "A real agent bearer is denied cancellation; guidance distinguishes the read command and required human control actor."); + c.add("actual-cancelled-result", o.state.cancelRunId === o.state.workerRunId && o.state.finalWorkerStatus === "cancelled" + && o.state.cancelOnlyAfterRead === true && o.runs.length === 2 + && o.runs.filter(run => run.status === "cancelled" && run.id === o.state.workerRunId).length === 1 + && o.runs.filter(run => run.status === "succeeded" && run.agentId === o.leadId).length === 1, + "The owned human CLI actually cancels the same running worker, with one successful canonical guidance run and no duplicate admission."); + c.add("no-premature-success-claim", !/(?:I(?:'ve| have)?|already|successfully)\s+(?:stopped|cancelled|canceled)\s+(?:Morgan|the run|it)/i.test(o.reply), + "The earlier guidance cannot claim the later authorized cancellation already happened."); + return c.result(); +} +const probeKeys = new WeakMap(); +const inspect = privilegedDefinition("cli-operations.inspect-run-cli", "privileged-run-cli", { + async setup(ctx) { + const operator = await prepareOwnedCli(ctx), busy = await busyRuntimeWorker(ctx); + if (!ctx.input.privateSecrets) throw new Error("Private actor probe redaction is unavailable"); + const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `catalog-run-control-${ctx.input.nonce}` }); + ctx.input.privateSecrets.push(key.token); probeKeys.set(ctx, key); + const response = await fetch(new URL(`/api/heartbeat-runs/${busy.workerRunId}/cancel`, ctx.input.api.baseURL), { + method: "POST", redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { Authorization: `Bearer ${key.token}`, "Content-Type": "application/json" }, body: "{}" }); + const agentCancelStatus = response.status; await response.body?.cancel(); + return { ...operator, ...busy, agentCancelStatus, initialRunReallyRunning: busy.workerWasReallyWaiting }; + }, + async act(ctx, state) { + const initial = await pollUntil({ label: "Canonical run-control guidance", deadlineAt: Date.now() + 150_000, + load: () => observeInternal(ctx, state), accept: o => !!o.reply && ctx.issue()?.conversationState === "waiting" + && o.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded").length === 1 }); + const returned = inspectRunCommands(initial.reply, String(state.workerRunId)); + const read = JSON.parse(await runOwnedCli(ctx, [...returned.commands[0]!, ...(!returned.commands[0]!.includes("--json") ? ["--json"] : [])])); + state.readRunId = read.id; state.readAgentId = read.agentId; state.readCompanyId = read.companyId; state.readStatus = read.status; + if (read.id !== state.workerRunId || read.companyId !== ctx.input.fixtures.company.id || read.agentId !== (state.worker as { id: string }).id) throw new Error("CLI read did not verify the exact owned Morgan run"); + if (read.status === "running") { + const result = JSON.parse(await runOwnedCli(ctx, [...returned.commands[1]!, ...(!returned.commands[1]!.includes("--json") ? ["--json"] : [])])); + state.cancelRunId = result.id; state.cancelOnlyAfterRead = true; + } + const final = await pollUntil({ label: "Actual owned run cancellation", deadlineAt: Date.now() + 60_000, + load: () => ctx.input.api.get<{ id: string; status: string }>(`/api/heartbeat-runs/${state.workerRunId}`), accept: row => row.status === "cancelled" }); + state.finalWorkerStatus = final.status; + state.commandHashes = returned.commands.map(argv => createHash("sha256").update(JSON.stringify(argv)).digest("hex")); + state.unresolvedRunBindings = returned.bindings; + await ctx.input.evidence("chat-catalog-cli-run-control.json", { workerRunId: state.workerRunId, readRunId: state.readRunId, + readStatus: state.readStatus, finalStatus: final.status, actor: "authenticated-human", commandHashes: state.commandHashes }); + }, grade: gradeInspectRunCli, + async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, +}); +inspect.expectedRunCount = 2; inspect.expectedHumanMessageCount = 1; inspect.attemptTimeoutMs = 600_000; +inspect.runPolicy = { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 1 } }; +inspect.sourceFiles = [...inspect.sourceFiles, "chat-catalog-domains/privileged-cli-support.ts", "chat-catalog-domains/runtime-busy-worker.ts", "../../cli/src/index.ts"]; +inspect.cleanup = async (ctx, state) => { + await releaseRuntimeWorker(state); + try { const key = probeKeys.get(ctx); if (key) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`); probeKeys.delete(ctx); } } + finally { await cleanupOwnedCli(ctx); } +}; +export const privilegedRunCliCases = [inspect]; diff --git a/tests/runner-e2e/chat-catalog-registry.ts b/tests/runner-e2e/chat-catalog-registry.ts index b2ac91e26b4..e38e2cc0ac4 100644 --- a/tests/runner-e2e/chat-catalog-registry.ts +++ b/tests/runner-e2e/chat-catalog-registry.ts @@ -3,6 +3,7 @@ import { validChatCatalogHumanCounts } from "./chat-catalog-contract.js"; import type { ChatCatalogCase } from "./chat-catalog-contract.js"; import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; import type { IdentityBoundaryCase } from "./chat-catalog-domains/identity-boundary-contract.js"; +import { identityBoundaryScenarioIds } from "./chat-catalog-domains/identity-boundary-contract.js"; export type ChatCatalogDescriptor = Omit & { journeyKind?: "agent-chat" | "browser-boundary"; @@ -60,8 +61,10 @@ export async function loadChatCatalogImplementation( /** Separate typed dispatch: no company, agent, or ChatFlowInput is fabricated. */ export async function loadIdentityBoundaryImplementation(descriptor: ChatCatalogDescriptor, importer: (specifier: string) => Promise> = specifier => import(specifier)): Promise { - if (descriptor.journeyKind !== "browser-boundary" || !["access.sign-in", "onboarding.new-instance", "onboarding.choose-first-runtime", "onboarding.resume-wizard"].includes(descriptor.scenarioId)) + if (descriptor.journeyKind !== "browser-boundary" || !(identityBoundaryScenarioIds as readonly string[]).includes(descriptor.scenarioId)) throw new Error("Only explicit catalog browser boundaries may bypass Agent Chat prerequisites"); + if (descriptor.scenarioId === "agent-chat.chat-disabled" && (descriptor.boundaryTarget !== "authenticated-harness" || descriptor.authentication !== "human-session")) + throw new Error("Disabled Agent Chat requires the actual owned authenticated company target"); if (descriptor.expectedRunCount !== 0 || (descriptor.minimumRunCount ?? 0) !== 0 || descriptor.expectedHumanMessageCount !== 0 || (descriptor.setupHumanMessageCount ?? 0) !== 0) throw new Error("Pre-chat boundaries require zero agent admissions and zero persisted chat queries"); const module = await importer(chatCatalogModuleUrl(descriptor.modulePath));