From e2b337b330f2a8ea7f7ffb915cf19ca5faf712cf Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 12:16:41 -0600 Subject: [PATCH 1/2] fix(authz): name the Experimental settings page when an agent is denied an instance experiment An agent asked to turn on an instance experiment (for example Cases) reads or patches /api/instance/settings/experimental and gets the generic board-only denial, which names no page. Turning Cases off also denies the case API with a bare "Cases are disabled". In both cases the agent cannot tell the person where to act, so it cannot give a usable next step. - The board-only denial for the experimental settings route now links [Open Experimental settings](//company/settings/instance/experimental), unless the operator hides that page. - The case API gate keeps its message and adds a final remediation that names the same page (guarded the same way). - The API catalog entries for the experimental settings route name the page, say agents cannot read or change it, and give Cases/enableCases as an example. Co-Authored-By: Claude Opus 5.5 --- server/src/__tests__/board-access-denial.test.ts | 15 +++++++++++++++ server/src/__tests__/cases-routes.test.ts | 6 +++++- server/src/__tests__/openapi-routes.test.ts | 10 ++++++++++ server/src/routes/authz.ts | 3 +++ server/src/routes/cases.ts | 12 +++++++++++- server/src/routes/openapi.ts | 9 +++++++-- 6 files changed, 51 insertions(+), 4 deletions(-) diff --git a/server/src/__tests__/board-access-denial.test.ts b/server/src/__tests__/board-access-denial.test.ts index 04da1f29281..603dc21fb9c 100644 --- a/server/src/__tests__/board-access-denial.test.ts +++ b/server/src/__tests__/board-access-denial.test.ts @@ -48,6 +48,8 @@ describe("board-only denial", () => { ["get", "/api/plugins/ui-contributions", "[Open plugin settings](//company/settings/instance/plugins)"], ["post", `/api/projects/${AGENT_ID}/access-members`, `[Open the project's access settings](//projects/${AGENT_ID}/configuration)`], ["delete", `/api/projects/${AGENT_ID}/access-members/m-1`, `[Open the project's access settings](//projects/${AGENT_ID}/configuration)`], + ["get", "/api/instance/settings/experimental", "[Open Experimental settings](//company/settings/instance/experimental)"], + ["patch", "/api/instance/settings/experimental", "[Open Experimental settings](//company/settings/instance/experimental)"], ] as const)("tells an agent %s %s is final and names the board page", async (method, path, link) => { const res = await request(createApp(agentActor))[method](path).expect(403); expect(res.body.error).toBe("Board access required"); @@ -122,6 +124,19 @@ describe("board-only denial", () => { } }); + it("does not link Experimental settings while the operator hides that page", async () => { + const previous = process.env.PAPERCLIP_HIDDEN_SETTINGS; + process.env.PAPERCLIP_HIDDEN_SETTINGS = "instance.experimental"; + try { + const res = await request(createApp(agentActor)).patch("/api/instance/settings/experimental").expect(403); + expect(res.body.remediation).toContain("this denial is final"); + expect(res.body.remediation).not.toContain("/instance/experimental"); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_HIDDEN_SETTINGS; + else process.env.PAPERCLIP_HIDDEN_SETTINGS = previous; + } + }); + it("applies through the org-access guard", async () => { const res = await request(createApp(agentActor, assertBoardOrgAccess)).delete(`/api/companies/${COMPANY_ID}`).expect(403); expect(res.body.code).toBe(BOARD_ACCESS_REQUIRED_CODE); diff --git a/server/src/__tests__/cases-routes.test.ts b/server/src/__tests__/cases-routes.test.ts index 8aefa71e864..4877bb06567 100644 --- a/server/src/__tests__/cases-routes.test.ts +++ b/server/src/__tests__/cases-routes.test.ts @@ -202,7 +202,11 @@ describeEmbeddedPostgres("cases routes", () => { }).returning(); const http = request(app(boardActor)); - await http.get(`/api/companies/${company.id}/cases`).expect(403); + const denied = await http.get(`/api/companies/${company.id}/cases`).expect(403); + // The denial is final and names the page where an instance admin turns Cases on. + expect(denied.body).toMatchObject({ error: "Cases are disabled", code: "cases_disabled" }); + expect(denied.body.remediation).toContain("[Experimental settings](//company/settings/instance/experimental)"); + expect(denied.body.remediation).toContain("Agent credentials cannot change that setting"); await http.get(`/api/companies/${company.id}/cases/types`).expect(403); await http.post(`/api/companies/${company.id}/cases`).send({ caseType: "bug", title: "Bug" }).expect(403); await http.get(`/api/cases/${caseRow!.id}`).expect(403); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 1678e78207e..fa4d72949ac 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -653,6 +653,16 @@ describe("openapi routes", () => { } }); + it("names the Experimental settings page where an instance admin turns a feature on", async () => { + const res = await request(createApp()).get("/api/openapi.json"); + const experimental = res.body.paths["/api/instance/settings/experimental"]; + for (const operation of [experimental.get, experimental.patch]) { + expect(operation.description).toContain("[Experimental settings](//company/settings/instance/experimental)"); + expect(operation.description).toContain("Agent credentials cannot read or change these settings"); + expect(operation.description).toContain("Cases via enableCases"); + } + }); + it("tells agents that archiving keeps a status card and where a person restores it", async () => { const res = await request(createApp()).get("/api/openapi.json"); const patch = res.body.paths["/api/status-cards/{id}"].patch; diff --git a/server/src/routes/authz.ts b/server/src/routes/authz.ts index 4fab0967fa7..117a384b436 100644 --- a/server/src/routes/authz.ts +++ b/server/src/routes/authz.ts @@ -73,6 +73,9 @@ const BOARD_CONTROL_PAGES: Array<[RegExp, (match: RegExpExecArray, req: Request) return tab ? [`the agent's ${tab[0]}`, `//agents/${match[1]}/${tab[1]}`] : ["the agent's profile", `//agents/${match[1]}`]; }], [/^\/companies\/[^/]+(?:\/(?:archive|branding|settings))?\/?$/, () => ["company settings", "//company/settings"]], + // An operator can hide the Experimental page, so link it only while it is visible. + [/^\/instance\/settings\/experimental\/?$/, () => getHiddenSettings().has("instance.experimental") + ? null : ["Experimental settings", "//company/settings/instance/experimental"]], ]; // Plugin management pages. A plugin id or key segment opens that plugin's settings diff --git a/server/src/routes/cases.ts b/server/src/routes/cases.ts index e6897b69a57..070be9828bb 100644 --- a/server/src/routes/cases.ts +++ b/server/src/routes/cases.ts @@ -49,6 +49,7 @@ import { } from "@paperclipai/shared"; import { formatAttachmentSize, MAX_ATTACHMENT_BYTES, normalizeContentType } from "../attachment-types.js"; import { HttpError, badRequest, conflict, forbidden, notFound, unprocessable } from "../errors.js"; +import { getHiddenSettings } from "../services/settings-visibility.js"; import { validate } from "../middleware/validate.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { documentAnnotationService, logActivity } from "../services/index.js"; @@ -80,7 +81,16 @@ function eventActorValues(actor: CaseActor) { async function assertCasesEnabled(db: Db) { const experimental = await instanceSettingsService(db).getExperimental(); if (!experimental.enableCases) { - throw forbidden("Cases are disabled"); + // An operator can hide the Experimental page, so link it only while it is visible. + const where = getHiddenSettings().has("instance.experimental") + ? "An instance operator controls it." + : "An instance admin turns them on with Cases in [Experimental settings](//company/settings/instance/experimental), " + + "with this conversation's company prefix as ."; + throw forbidden("Cases are disabled", { + code: "cases_disabled", + remediation: "Cases are an experimental feature that is turned off on this instance, so this denial is final and no other route reaches cases. " + + `${where} Agent credentials cannot change that setting. Reply now.`, + }); } } diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 76d45918345..a44ebb223ab 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -7261,6 +7261,11 @@ registry.registerPath({ // ─── Instance settings ──────────────────────────────────────────────────────── +// Agents cannot read or write experimental settings; the person turns a feature on in the UI. +const EXPERIMENTAL_SETTINGS_PAGE = "Instance experimental features (for example Cases via enableCases) are turned on or off by an instance admin in " + + "Experimental settings at //company/settings/instance/experimental. Agent credentials cannot read or change these settings, " + + "so do not claim a change: link that page as [Experimental settings](//company/settings/instance/experimental), name the feature's setting, and reply."; + registry.registerPath({ method: "get", path: "/api/instance/settings", @@ -7300,7 +7305,7 @@ registry.registerPath({ path: "/api/instance/settings/experimental", tags: ["instance"], summary: "Get experimental instance settings", - description: "Agent Chat is always on for every person on every instance and needs no setup. enableAgentChat is a retired compatibility key: it always reads true and written values are ignored. If someone asks to enable Agent Chat, tell them it is already on, since they are using it now; do not send them to a settings toggle.", + description: `${EXPERIMENTAL_SETTINGS_PAGE} Agent Chat is always on for every person on every instance and needs no setup. enableAgentChat is a retired compatibility key: it always reads true and written values are ignored. If someone asks to enable Agent Chat, tell them it is already on, since they are using it now; do not send them to a settings toggle.`, responses: { 200: r.ok(), 401: r.unauthorized }, }); @@ -7309,7 +7314,7 @@ registry.registerPath({ path: "/api/instance/settings/experimental", tags: ["instance"], summary: "Update experimental instance settings", - description: "Requires instance-admin access or a trusted Cloud stack admin. Operator-hidden settings and Cloud-managed feature restrictions still apply. This does not grant access to other instance administration operations. Agent Chat is always on for every person on every instance and needs no setup. enableAgentChat is a retired compatibility key: it always reads true and written values are ignored. If someone asks to enable Agent Chat, tell them it is already on, since they are using it now; do not send them to a settings toggle.", + description: `${EXPERIMENTAL_SETTINGS_PAGE} Requires instance-admin access or a trusted Cloud stack admin. Operator-hidden settings and Cloud-managed feature restrictions still apply. This does not grant access to other instance administration operations. Agent Chat is always on for every person on every instance and needs no setup. enableAgentChat is a retired compatibility key: it always reads true and written values are ignored. If someone asks to enable Agent Chat, tell them it is already on, since they are using it now; do not send them to a settings toggle.`, request: { body: jsonBody(patchInstanceExperimentalSettingsSchema) }, responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden }, }); From 086af6dd8bfb6cbb246201b1ebb292f291dd084d Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 12:22:26 -0600 Subject: [PATCH 2/2] test(evals): retarget instance-settings from the removed Agent Chat toggle to the Cases experiment Agent Chat is always on since #19360, so "Enable Agent Chat on this instance." has no setting to change and the scripted human's "Toggle agent chat experimental setting" switch no longer exists. The case keeps its promise with an experiment that still exists and has a real route gate: Cases (enableCases), whose //cases route redirects to the dashboard while off. - YAML: new query/context/expected outcome for Cases; inventory digest, manifest rows and changedFields updated and pinned. - Setup starts Cases off (restored in cleanup), refuses a hidden setting, and probes GET/PATCH experimental settings with the agent's own key (403 board_access_required, setting unchanged). - The human checks Cases is still off after the agent's turn and its route is unavailable, then follows the Experimental settings link the reply rendered (no typed-URL fallback), turns Cases on, sees it persist after reload, and opens the Cases page. - The reply-wording regex is replaced by structural checks: a rendered link to Experimental settings that the person followed, and saved setting state. - oracleVersion privileged.instance-settings.v2; sourceFiles follow the code. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-contract.test.ts | 3 +- .../runner-e2e/chat-catalog-descriptors.json | 6 +- .../privileged-budget.test.ts | 17 +- .../privileged-companions.test.ts | 20 ++- .../privileged-configuration.ts | 147 +++++++++++++----- .../chat-full-catalog-testops-manifest.json | 24 ++- ...hat-full-catalog-testops-manifest.test.mjs | 13 ++ .../chat-scenarios/instance-operations.yaml | 15 +- 8 files changed, 181 insertions(+), 64 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-contract.test.ts b/tests/runner-e2e/chat-catalog-contract.test.ts index 428515dcdca..aa3d2b131b8 100644 --- a/tests/runner-e2e/chat-catalog-contract.test.ts +++ b/tests/runner-e2e/chat-catalog-contract.test.ts @@ -32,7 +32,8 @@ describe("catalog oracle contract", () => { // agents.restore-instructions retargeted to no saved history (instruction saves keep none since #14420). // access.manage-membership retargeted to read-and-route; activity.routine-audit builds real same-morning history. // gateways-profiles.create-gateway and rotate-token: agents keep delegated gateway/token authority and never receive token values. - expect(chatCatalogInventoryDigest).toBe("48fdd6e646117851eee6e309c914fadbc80f2dd7ac4f90eb1538de70e18c4c6c"); + // instance-operations.instance-settings retargeted from the removed Agent Chat toggle (always on since #19360) to the Cases experiment. + expect(chatCatalogInventoryDigest).toBe("a39eb193d579c18be6e5f6fae8061428c9f7996549cdca6537aa31baecfdd03e"); for (const implementation of chatCatalogCases) expect(chatCatalogScenarios.some(row => row.id === implementation.scenarioId)).toBe(true); }); }); diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 98e7b3817a4..d6065fa141f 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -3505,10 +3505,12 @@ "chat-catalog-domains/turn-response-card.ts", "chat-catalog-domains/privileged-configuration.ts", "chat-scenarios/instance-operations.yaml", + "chat-catalog-domains/internal-comment-reference.ts", "../../ui/src/pages/InstanceExperimentalSettings.tsx", - "../../ui/src/pages/AgentChat.tsx" + "../../ui/src/components/CasesExperimentalGate.tsx", + "../../ui/src/pages/Cases.tsx" ], - "oracleVersion": "paperclip.chat-catalog.privileged.v1", + "oracleVersion": "paperclip.chat-catalog.privileged.instance-settings.v2", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts index 662eefd394c..402a1517e68 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-budget.test.ts @@ -52,16 +52,23 @@ describe("instance settings consume the original observation clock", () => { it("actual settings act opts in and forwards only the remaining original budget", async () => { vi.useFakeTimers(); vi.setSystemTime(0); const ctx = context(); const goto = vi.fn(async () => { throw new Error("actual navigation boundary"); }); - ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", goto }, fixtures: { company: { issuePrefix: "RUN" } }, api: { baseURL: "https://fixture.test" } } as never; - const pending = privilegedConfigurationCases[0]!.act!(ctx, {}).catch(String); - await vi.advanceTimersByTimeAsync(151_000); expect(goto).not.toHaveBeenCalled(); + const get = vi.fn(async (_path: string, _options?: unknown) => ({ enableCases: false })); + const settingReads = () => get.mock.calls.filter(([path]) => path === "/api/instance/settings/experimental"); + ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", goto }, fixtures: { company: { issuePrefix: "RUN" } }, api: { baseURL: "https://fixture.test", get } } as never; + const state: Record = {}; + const pending = privilegedConfigurationCases[0]!.act!(ctx, state).catch(String); + await vi.advanceTimersByTimeAsync(151_000); expect(goto).not.toHaveBeenCalled(); expect(settingReads()).toEqual([]); await vi.advanceTimersByTimeAsync(29_000); expect(await pending).toContain("actual navigation boundary"); - expect(goto).toHaveBeenCalledExactlyOnceWith("https://fixture.test/RUN/company/settings/instance/experimental", { waitUntil: "domcontentloaded", timeout: 210_000 }); + // After the settled turn the setting is read once, then the person checks the gated Cases route while it is off. + expect(settingReads()).toEqual([["/api/instance/settings/experimental", { timeout: 210_000 }]]); + expect(state.settingOffAfterAgent).toBe(true); + expect(goto).toHaveBeenCalledExactlyOnceWith("https://fixture.test/RUN/cases", { waitUntil: "domcontentloaded", timeout: 210_000 }); }); it("slow navigation cannot start the next mutation or reset the original budget", async () => { vi.useFakeTimers(); vi.setSystemTime(389_000); const ctx = context(0); const click = vi.fn(); const goto = vi.fn(async () => { vi.setSystemTime(391_000); }); - ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", goto, getByRole: () => ({ click }) }, fixtures: { company: { issuePrefix: "RUN" } }, api: { baseURL: "https://fixture.test" } } as never; + ctx.input = { page: { url: () => "https://fixture.test/RUN/chats/ceo", goto, getByRole: () => ({ click }), getByTestId: () => ({ count: async () => 0 }) }, + fixtures: { company: { issuePrefix: "RUN" } }, api: { baseURL: "https://fixture.test", get: async () => ({ enableCases: false }) } } as never; await expect(privilegedConfigurationCases[0]!.act!(ctx, {})).rejects.toThrow(/exhausted/); expect(goto).toHaveBeenCalledOnce(); expect(click).not.toHaveBeenCalled(); }); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts index 550ce68aa05..03dc74edf24 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-companions.test.ts @@ -7,13 +7,23 @@ const base = { scenarioId: "owned", origin: "http://127.0.0.1:3100", companyId: comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: ["canonical"], runs: [{ id: "run", agentId: "lead", status: "succeeded" }], reply: "" }; const block = (body: string) => `\`\`\`bash\n${body}\n\`\`\``; describe("operator companions use actual saved authority and output", () => { - it("requires denied agent settings access and real saved human gate state", () => { - const o = { ...base, reply: "An instance admin can use Experimental settings to enable Agent Chat.", state: { agentReadStatus: 403, agentPatchStatus: 403, - gateUnchangedAfterAgent: true, savedDisabled: true, disabledRouteClear: true, savedEnabled: true, enabledRouteVisible: true, finalGateEnabled: true, probeCredentialAbsent: true } }; + it("requires denied agent settings access, a followed settings link and real saved human gate state", () => { + const settingsLink = "http://127.0.0.1:3100/RUN/company/settings/instance/experimental"; + const o = { ...base, reply: `I can't change instance settings. An instance admin can turn on **Cases** in [Experimental settings](/RUN/company/settings/instance/experimental).`, + links: [settingsLink], state: { companyPrefix: "RUN", agentReadStatus: 403, agentPatchStatus: 403, agentDenialCode: "board_access_required", + gateUnchangedAfterProbe: true, settingOffAfterAgent: true, followedSettingsLink: true, disabledRouteRedirects: true, savedEnabled: true, + persistedAfterReload: true, enabledRouteVisible: true, finalGateEnabled: true, probeCredentialAbsent: true } }; expect(gradeInstanceSettings(o).passed).toBe(true); - for (const delta of [{ agentPatchStatus: 200 }, { gateUnchangedAfterAgent: false }, { savedEnabled: false }, { disabledRouteClear: false }, { probeCredentialAbsent: false }]) + // Wording is not graded: a reworded reply with the same rendered link passes, and an unprefixed link is the same page. + expect(gradeInstanceSettings({ ...o, reply: "That's an admin-only switch. Open the experimental page and flip Cases.", links: ["http://127.0.0.1:3100/company/settings/instance/experimental/"] }).passed).toBe(true); + for (const delta of [{ agentPatchStatus: 200 }, { agentDenialCode: "cases_disabled" }, { gateUnchangedAfterProbe: false }, { settingOffAfterAgent: false }, + { followedSettingsLink: false }, { disabledRouteRedirects: false }, { savedEnabled: false }, { persistedAfterReload: false }, + { enabledRouteVisible: false }, { finalGateEnabled: false }, { probeCredentialAbsent: false }]) expect(gradeInstanceSettings({ ...o, state: { ...o.state, ...delta } }).passed).toBe(false); - expect(gradeInstanceSettings({ ...o, reply: "I have enabled Agent Chat in Experimental settings." }).passed).toBe(false); + // A reply without a usable link to the settings page fails even when the person still enabled Cases. + for (const links of [[], ["http://127.0.0.1:3100/RUN/company/settings/instance/general"], ["https://elsewhere.test/RUN/company/settings/instance/experimental"], + ["http://127.0.0.1:3100/%3Cprefix%3E/company/settings/instance/experimental"]]) + expect(gradeInstanceSettings({ ...o, links }).passed).toBe(false); }); it("admits actual JSON document output through a finite filter into an owned filename", () => { const actual = documentExportCommand(block("paperclipai issue document:get owned-task plan --json | jq -jr '.body' > plan.md"), "owned-task"); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts b/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts index fa3a654e34e..fc011ee798d 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts @@ -1,82 +1,147 @@ -import { expect } from "@playwright/test"; +import { expect, type Page } from "@playwright/test"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; -import { checks, observeInternal } from "./internal-support.js"; +import { checks, observeInternal, type Entity } from "./internal-support.js"; +import { RENDERED_REPLY_WAIT_MS, waitForRenderedReply } from "./internal-comment-reference.js"; import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, remainingPrivilegedObservationBudget } from "./privileged-support.js"; -const probeKeys = new WeakMap(); + +/** + * The person asks the chat agent to turn on an instance experiment (Cases). The agent has no + * instance authority, so the supported outcome is a reply that links Experimental settings; the + * person follows that rendered link, turns Cases on there, and the Cases route opens. Agent Chat + * itself is always on (#19360), so it is no longer an experiment a person can ask to enable. + */ const endpoint = "/api/instance/settings/experimental"; +const SETTINGS_PATH = "/company/settings/instance/experimental"; +const CASES_TOGGLE = "Toggle cases experimental setting"; +const probeKeys = new WeakMap(); +const restoreValues = new WeakMap(); +type Experimental = { enableCases: boolean }; + async function probe(ctx: ChatCatalogContext, method: "GET" | "PATCH") { const key = probeKeys.get(ctx); if (!key) throw new Error("Private instance-permission probe actor is missing"); const response = await fetch(new URL(endpoint, ctx.input.api.baseURL), { method, redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}`, "Content-Type": "application/json" }, - ...(method === "PATCH" ? { body: JSON.stringify({ enableAgentChat: false }) } : {}) }); - await response.body?.cancel(); return response.status; + ...(method === "PATCH" ? { body: JSON.stringify({ enableCases: true }) } : {}) }); + const body = await response.json().catch(() => ({})) as { code?: unknown }; + return { status: response.status, code: typeof body.code === "string" ? body.code : null }; } +async function casesEnabled(ctx: ChatCatalogContext, timeout?: number) { + return (await ctx.input.api.get(endpoint, timeout === undefined ? undefined : { timeout })).enableCases === true; +} +/** The Experimental settings page, with or without the company prefix, on the fixture origin. */ +export function isExperimentalSettingsLink(href: string, origin: string, prefix: string): boolean { + try { + const url = new URL(href, origin); + return url.origin === new URL(origin).origin && [SETTINGS_PATH, `/${prefix}${SETTINGS_PATH}`].includes(url.pathname.replace(/\/+$/, "")); + } catch { return false; } +} +/** The Experimental settings link rendered inside one of the agent's saved replies, if any. */ +async function renderedSettingsLink(page: Page, replyIds: string[], origin: string, prefix: string) { + for (const id of replyIds) { + const anchors = page.locator(`[id=${JSON.stringify(`comment-${id}`)}] a[href]`); + for (let index = 0; index < await anchors.count(); index++) { + const anchor = anchors.nth(index), href = await anchor.getAttribute("href"); + if (href && isExperimentalSettingsLink(href, origin, prefix)) return anchor; + } + } + return null; +} + export function gradeInstanceSettings(value: unknown) { const c = checks(), o = privilegedObservation(value); c.add("actual-setting-evidence", !!o, "Actual denied agent access and independently saved operator changes are required."); if (!o) return c.result(); - c.add("agent-not-instance-admin", o.state.agentReadStatus === 403 && o.state.agentPatchStatus === 403 && o.state.gateUnchangedAfterAgent === true, - "The actual agent bearer cannot read or patch experimental instance administration settings."); - c.add("supported-human-path", /experimental|enableAgentChat/i.test(o.reply) && /operator|admin|human|settings/i.test(o.reply) - && !/(?:I(?:'ve| have)?|successfully|already)\s+(?:enabled|turned on|activated)\s+(?:Agent Chat|it)/i.test(o.reply), - "The reply identifies the supported setting and operator path without claiming an unauthorized setting write."); - c.add("saved-gate-and-route", o.state.savedDisabled === true && o.state.disabledRouteClear === true - && o.state.savedEnabled === true && o.state.enabledRouteVisible === true && o.state.finalGateEnabled === true, - "The human toggles the actual owned setting, observes the disabled route explanation, persists enablement and opens the real chat route."); + const prefix = typeof o.state.companyPrefix === "string" ? o.state.companyPrefix : ""; + c.add("agent-not-instance-admin", o.state.agentReadStatus === 403 && o.state.agentPatchStatus === 403 && o.state.agentDenialCode === "board_access_required" + && o.state.gateUnchangedAfterProbe === true && o.state.settingOffAfterAgent === true, + "The actual agent bearer is denied reading and patching experimental settings, and Cases is still off after the agent's turn."); + c.add("agent-links-experimental-settings", !!prefix && o.links.some(href => isExperimentalSettingsLink(href, o.origin, prefix)) && o.state.followedSettingsLink === true, + "The reply renders a link to Experimental settings, and the person reaches the setting by following that rendered link."); + c.add("saved-gate-and-route", o.state.disabledRouteRedirects === true && o.state.savedEnabled === true && o.state.persistedAfterReload === true + && o.state.enabledRouteVisible === true && o.state.finalGateEnabled === true, + "Cases is unavailable while off; the person's toggle saves it on, it stays on after reload, and the Cases page then opens."); c.add("credential-not-disclosed", o.state.probeCredentialAbsent === true, "The bounded private permission-probe credential never enters the model reply or evidence."); return c.result(); } + const settings = privilegedDefinition("instance-operations.instance-settings", "privileged-configuration", { async setup(ctx) { if (!ctx.input.privateSecrets) throw new Error("Private instance-permission probe requires the attempt redaction sink"); + const health = await ctx.input.api.get<{ hiddenSettings?: string[] }>("/api/health"); + const hidden = health.hiddenSettings ?? []; + if (hidden.includes("instance.experimental") || hidden.includes("instance.experimental.enableCases")) throw new Error("The Cases experimental setting is hidden on this target"); + // The person asks to turn Cases on, so it starts off; restore the target's own value afterwards. + const initial = await casesEnabled(ctx); + restoreValues.set(ctx, initial); + if (initial) await ctx.input.api.patch(endpoint, { enableCases: false }); + if (await casesEnabled(ctx)) throw new Error("Cases did not start off"); const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `catalog-instance-probe-${ctx.input.nonce}` }); ctx.input.privateSecrets.push(key.token); probeKeys.set(ctx, key); - const before = await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint); - const agentReadStatus = await probe(ctx, "GET"), agentPatchStatus = await probe(ctx, "PATCH"); - const after = await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint); - return { agentReadStatus, agentPatchStatus, gateUnchangedAfterAgent: before.enableAgentChat === after.enableAgentChat }; + const read = await probe(ctx, "GET"), patch = await probe(ctx, "PATCH"); + return { companyPrefix: ctx.input.fixtures.company.issuePrefix, agentReadStatus: read.status, agentPatchStatus: patch.status, + agentDenialCode: read.code === patch.code ? read.code : null, gateUnchangedAfterProbe: !(await casesEnabled(ctx)) }; }, async act(ctx, state) { const remaining = () => remainingPrivilegedObservationBudget(ctx); remaining(); - await settlePrivilegedChat(ctx, ctx.observationDeadlineAt); - remaining(); - const page = ctx.input.page, source = page.url(); - const settingsUrl = new URL(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/instance/experimental`, ctx.input.api.baseURL).href; - await page.goto(settingsUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); - const toggle = page.getByRole("switch", { name: "Toggle agent chat experimental setting", exact: true }); - await expect(toggle).toHaveAttribute("aria-checked", "true", { timeout: remaining() }); - await toggle.click({ timeout: remaining() }); - await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint, { timeout: remaining() })).enableAgentChat, { timeout: remaining() }).toBe(false); - state.savedDisabled = true; + const settled = await settlePrivilegedChat(ctx, ctx.observationDeadlineAt); + const page = ctx.input.page, source = page.url(), origin = ctx.input.api.baseURL, prefix = ctx.input.fixtures.company.issuePrefix ?? ""; + if (!prefix) throw new Error("Instance settings journey requires the fixture company prefix"); + // Only the agent acted so far: Cases must still be off. + state.settingOffAfterAgent = !(await casesEnabled(ctx, remaining())); + // While Cases is off, its page is unavailable: the route gate sends the person to the dashboard. + const casesUrl = new URL(`/${prefix}/cases`, origin).href; + await page.goto(casesUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(page).toHaveURL(/\/dashboard(?:[?#]|$)/, { timeout: remaining() }); + state.disabledRouteRedirects = await page.getByTestId("case-pages").count() === 0; await page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); - await expect(page.getByText(/Agent Chat is disabled(?:\.| in Experimental settings| for this instance)/).first()).toBeVisible({ timeout: remaining() }); - state.disabledRouteClear = true; - await page.goto(settingsUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); - await expect(toggle).toHaveAttribute("aria-checked", "false", { timeout: remaining() }); await toggle.click({ timeout: remaining() }); - await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint, { timeout: remaining() })).enableAgentChat, { timeout: remaining() }).toBe(true); + await waitForRenderedReply(ctx, settled, Math.min(RENDERED_REPLY_WAIT_MS, remaining())); + // The person follows the Experimental settings link the reply rendered; there is no typed-URL fallback. + const chat = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`, { timeout: remaining() }); + const replyIds = chat.filter(row => row.authorAgentId === settled.leadId && settled.runs.some(run => run.id === row.createdByRunId)).map(row => row.id).filter((id): id is string => typeof id === "string"); + const link = await renderedSettingsLink(page, replyIds, origin, prefix); + state.followedSettingsLink = !!link; + if (!link) return; + await link.click({ timeout: remaining() }); + await expect(page).toHaveURL(new RegExp(`${SETTINGS_PATH}/?(?:[?#]|$)`), { timeout: remaining() }); + const toggle = page.getByRole("switch", { name: CASES_TOGGLE, exact: true }); + await expect(toggle).toHaveAttribute("aria-checked", "false", { timeout: remaining() }); + await toggle.click({ timeout: remaining() }); + await expect.poll(async () => casesEnabled(ctx, remaining()), { timeout: remaining() }).toBe(true); state.savedEnabled = true; - await page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); await expect(page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); - remaining(); - state.enabledRouteVisible = true; - await ctx.input.evidence("chat-catalog-instance-gate-ui.json", { savedDisabled: true, disabledRouteClear: true, savedEnabled: true, enabledRouteVisible: true }); - remaining(); + await page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(toggle).toHaveAttribute("aria-checked", "true", { timeout: remaining() }); + state.persistedAfterReload = true; + await page.goto(casesUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(page.getByTestId("case-pages")).toBeVisible({ timeout: remaining() }); + state.enabledRouteVisible = new URL(page.url()).pathname.replace(/\/+$/, "") === `/${prefix}/cases`; + await ctx.input.evidence("chat-catalog-instance-gate-ui.json", { disabledRouteRedirects: state.disabledRouteRedirects, followedSettingsLink: true, + savedEnabled: true, persistedAfterReload: true, enabledRouteVisible: state.enabledRouteVisible }); + await page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); + await waitForRenderedReply(ctx, settled, Math.min(RENDERED_REPLY_WAIT_MS, remaining())); }, grade: gradeInstanceSettings, async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remainingPrivilegedObservationBudget(ctx) }); remainingPrivilegedObservationBudget(ctx); }, }); +settings.oracleVersion = "paperclip.chat-catalog.privileged.instance-settings.v2"; settings.observe = async (ctx, state) => { const o = await observeInternal(ctx, state), key = probeKeys.get(ctx); state.probeCredentialAbsent = !!key && !o.reply.includes(key.token); - state.finalGateEnabled = (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint)).enableAgentChat; + state.finalGateEnabled = await casesEnabled(ctx); return { ...o, state }; }; settings.cleanup = async ctx => { - const key = probeKeys.get(ctx); if (key) { - await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`); probeKeys.delete(ctx); + try { + const key = probeKeys.get(ctx); if (key) { + await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`); probeKeys.delete(ctx); + } + } finally { + const restore = restoreValues.get(ctx); + if (restore !== undefined) { await ctx.input.api.patch(endpoint, { enableCases: restore }); restoreValues.delete(ctx); } } }; -settings.sourceFiles = [...settings.sourceFiles, "../../ui/src/pages/InstanceExperimentalSettings.tsx", "../../ui/src/pages/AgentChat.tsx"]; +settings.sourceFiles = [...settings.sourceFiles, "chat-catalog-domains/internal-comment-reference.ts", "../../ui/src/pages/InstanceExperimentalSettings.tsx", + "../../ui/src/components/CasesExperimentalGate.tsx", "../../ui/src/pages/Cases.tsx"]; export const privilegedConfigurationCases = [settings]; diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.json b/tests/runner-e2e/chat-full-catalog-testops-manifest.json index 5af047affea..fcdbd616db8 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.json +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.json @@ -1146,6 +1146,24 @@ "field": "expectedOutcome", "previous": "The agent directs the human through scoped token management and secure one-time delivery. The test client uses the new credential and the old one is revoked at the intended point. Token bytes never enter chat, and unrelated clients' credentials retain their existing access.", "current": "The agent creates the replacement under delegated authority or leaves that step to the person, and the reply or a question card links the gateway's Tokens page, where the person reveals or issues the new value. The test client uses the new credential before the old one is revoked, and unrelated clients keep their existing access. Token values never appear in chat, run events or tool output." + }, + { + "key": "instance-operations.instance-settings", + "field": "originalQuery", + "previous": "Enable Agent Chat on this instance.", + "current": "Turn on Cases for this instance." + }, + { + "key": "instance-operations.instance-settings", + "field": "originalContext", + "previous": "Experimental settings, binding and storage configuration require instance authority that the chat agent lacks.", + "current": "Cases is an experimental instance feature and is off on this target. Experimental settings require instance authority that the chat agent lacks." + }, + { + "key": "instance-operations.instance-settings", + "field": "expectedOutcome", + "previous": "The response identifies the supported instance setting and operator path without granting the agent administration rights. The human journey verifies saved configuration and actual route availability. Unsupported deployment choices or a disabled gate produce clear state instead of an agent claiming a successful setting change.", + "current": "The agent cannot change instance settings, so it leaves the setting unchanged and does not claim it turned Cases on. The reply identifies the Cases experimental setting and links the Experimental settings page where an instance admin turns it on. The person follows that link and turns Cases on. While the setting is off the Cases page is unavailable; once it is on, the saved setting persists and the Cases page opens." } ], "candidateDescriptorSnapshot": "fixtures/chat-repair-candidate-descriptors.json", @@ -8667,9 +8685,9 @@ "requiredCredentials": [], "externalInput": null, "installedSource": null, - "originalQuery": "Enable Agent Chat on this instance.", - "expectedOutcome": "The response identifies the supported instance setting and operator path without granting the agent administration rights. The human journey verifies saved configuration and actual route availability. Unsupported deployment choices or a disabled gate produce clear state instead of an agent claiming a successful setting change.", - "originalContext": "Experimental settings, binding and storage configuration require instance authority that the chat agent lacks.", + "originalQuery": "Turn on Cases for this instance.", + "expectedOutcome": "The agent cannot change instance settings, so it leaves the setting unchanged and does not claim it turned Cases on. The reply identifies the Cases experimental setting and links the Experimental settings page where an instance admin turns it on. The person follows that link and turns Cases on. While the setting is off the Cases page is unavailable; once it is on, the saved setting persists and the Cases page opens.", + "originalContext": "Cases is an experimental instance feature and is off on this target. Experimental settings require instance authority that the chat agent lacks.", "capabilities": [ "instance-operations/configuration" ], diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index afdff982799..005325bdc9e 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -418,6 +418,19 @@ test('repair source metadata binds the corrected candidate and explicit changed [ "gateways-profiles.rotate-token", "expectedOutcome" + ], + // Retargeted removed-feature case: Agent Chat is always on (#19360), so the instance experiment is Cases. + [ + "instance-operations.instance-settings", + "originalQuery" + ], + [ + "instance-operations.instance-settings", + "originalContext" + ], + [ + "instance-operations.instance-settings", + "expectedOutcome" ] ]); for (const change of repair.changedFields) { diff --git a/tests/runner-e2e/chat-scenarios/instance-operations.yaml b/tests/runner-e2e/chat-scenarios/instance-operations.yaml index 5cece50c0ab..f6dcaa01e57 100644 --- a/tests/runner-e2e/chat-scenarios/instance-operations.yaml +++ b/tests/runner-e2e/chat-scenarios/instance-operations.yaml @@ -34,15 +34,16 @@ scenarios: capabilities: - instance-operations/configuration context: >- - Experimental settings, binding and storage configuration require instance authority that - the chat agent lacks. + Cases is an experimental instance feature and is off on this target. Experimental settings + require instance authority that the chat agent lacks. query: >- - Enable Agent Chat on this instance. + Turn on Cases for this instance. expected_outcome: >- - The response identifies the supported instance setting and operator path without granting - the agent administration rights. The human journey verifies saved configuration and actual - route availability. Unsupported deployment choices or a disabled gate produce clear state - instead of an agent claiming a successful setting change. + The agent cannot change instance settings, so it leaves the setting unchanged and does not + claim it turned Cases on. The reply identifies the Cases experimental setting and links the + Experimental settings page where an instance admin turns it on. The person follows that + link and turns Cases on. While the setting is off the Cases page is unavailable; once it is + on, the saved setting persists and the Cases page opens. - id: instance-operations.diagnose-health capabilities: - instance-operations/diagnostics