Merge remote-tracking branch 'origin/chat-eval/fix-capable-target' into chat-eval/b3-combined

* origin/chat-eval/fix-capable-target:
  fix(evals): give choose-capable-target a saveable computer, grade it structurally and surface setup validation reasons
  fix(authz): name the environment settings page when an agent is denied an environment route

server/src/__tests__/board-access-denial.test.ts

@@ -48,6 +48,10 @@ describe("board-only denial", () => {
     ["get", "/api/plugins/ui-contributions", "[Open plugin settings](/<prefix>/company/settings/instance/plugins)"],
     ["post", `/api/projects/${AGENT_ID}/access-members`, `[Open the project's access settings](/<prefix>/projects/${AGENT_ID}/configuration)`],
     ["delete", `/api/projects/${AGENT_ID}/access-members/m-1`, `[Open the project's access settings](/<prefix>/projects/${AGENT_ID}/configuration)`],
+    ["get", `/api/companies/${COMPANY_ID}/environments`, "[Open environment settings](/<prefix>/company/settings/instance/environments)"],
+    ["get", `/api/companies/${COMPANY_ID}/environments/capabilities`, "[Open environment settings](/<prefix>/company/settings/instance/environments)"],
+    ["post", `/api/environments/${AGENT_ID}/probe`, "[Open environment settings](/<prefix>/company/settings/instance/environments)"],
+    ["get", `/api/environment-leases/${AGENT_ID}`, "[Open environment settings](/<prefix>/company/settings/instance/environments)"],
   ] as const)("tells an agent %s %s is final and names the board page", async (method, path, link) => {
     const res = await request(createApp(agentActor))[method](path).expect(403);
     expect(res.body.error).toBe("Board access required");
@@ -122,6 +126,31 @@ describe("board-only denial", () => {
     }
   });
 
+  it("tells an agent denied an environment route that testing and changing computers are board-only too", async () => {
+    // An agent asked which computer can run a harness reads the capabilities first. That
+    // denial must also settle probing, or the agent keeps searching for a route to test one.
+    const res = await request(createApp(agentActor, assertBoardOrgAccess)).get(`/api/companies/${COMPANY_ID}/environments/capabilities`).expect(403);
+    expect(res.body.remediation).toContain("Execution environments are board-only throughout");
+    expect(res.body.remediation).toContain("testing (probing) them");
+    // The agent-readable project environment options are a different route.
+    const other = await request(createApp(agentActor)).get(`/api/companies/${COMPANY_ID}/project-environments`).expect(403);
+    expect(other.body.remediation).not.toContain("Execution environments");
+    expect(other.body.remediation).not.toContain("/instance/environments");
+  });
+
+  it("does not link environment settings while the operator hides them", async () => {
+    const previous = process.env.PAPERCLIP_HIDDEN_SETTINGS;
+    process.env.PAPERCLIP_HIDDEN_SETTINGS = "instance.environments";
+    try {
+      const res = await request(createApp(agentActor)).get(`/api/companies/${COMPANY_ID}/environments`).expect(403);
+      expect(res.body.remediation).toContain("this denial is final");
+      expect(res.body.remediation).not.toContain("/instance/environments");
+    } finally {
+      if (previous === undefined) delete process.env.PAPERCLIP_HIDDEN_SETTINGS;
+      else process.env.PAPERCLIP_HIDDEN_SETTINGS = previous;
+    }
+  });
+
   it("applies through the org-access guard", async () => {
     const res = await request(createApp(agentActor, assertBoardOrgAccess)).delete(`/api/companies/${COMPANY_ID}`).expect(403);
     expect(res.body.code).toBe(BOARD_ACCESS_REQUIRED_CODE);

server/src/routes/authz.ts

@@ -95,6 +95,17 @@ function pluginControlPage(path: string): [string, string] | null {
 const PLUGIN_ROUTE = /^\/plugins(?:\/|$)/;
 const PLUGIN_ADMINISTRATION_SCOPE = "Plugin management is board-only throughout: listing, installing, configuring (including choosing a secret for a plugin setting), enabling, disabling, upgrading and removing plugins have no agent route either.";
 
+// Execution environments (the computers agents run on) are managed on one settings
+// page. Every environment route is board-only, so a denial on the first one an agent
+// reads (usually the list or the capabilities) also answers testing and changing them.
+// An operator can hide the page, so link it only while it is visible.
+const ENVIRONMENT_ROUTE = /^\/(?:companies\/[^/]+\/)?environments(?:\/|$)|^\/environment-leases(?:\/|$)/;
+const ENVIRONMENT_ADMINISTRATION_SCOPE = "Execution environments are board-only throughout: listing computers, reading which coding harnesses they support, testing (probing) them and changing them have no agent route either.";
+function environmentControlPage(path: string): [string, string] | null {
+  if (!ENVIRONMENT_ROUTE.test(path) || getHiddenSettings().has("instance.environments")) return null;
+  return ["environment settings", "/<prefix>/company/settings/instance/environments"];
+}
+
 function deniedApiPath(req: Request) {
   return (req.originalUrl ?? "").split("?")[0]!.replace(/^\/api(?=\/)/, "");
 }
@@ -103,6 +114,8 @@ function boardControlPage(req: Request) {
   const path = deniedApiPath(req);
   const pluginPage = pluginControlPage(path);
   if (pluginPage) return pluginPage;
+  const environmentPage = environmentControlPage(path);
+  if (environmentPage) return environmentPage;
   for (const [pattern, page] of BOARD_CONTROL_PAGES) {
     const match = pattern.exec(path);
     if (match) return page(match, req);
@@ -117,7 +130,9 @@ function boardControlPage(req: Request) {
 export function boardAccessRequired(req: Request, controlPage?: BoardControlPage) {
   if (req.actor.type !== "agent") return forbidden("Board access required");
   const page = controlPage ?? boardControlPage(req);
-  const scope = PLUGIN_ROUTE.test(deniedApiPath(req)) ? `${PLUGIN_ADMINISTRATION_SCOPE} ` : "";
+  const deniedPath = deniedApiPath(req);
+  const scope = PLUGIN_ROUTE.test(deniedPath) ? `${PLUGIN_ADMINISTRATION_SCOPE} `
+    : ENVIRONMENT_ROUTE.test(deniedPath) ? `${ENVIRONMENT_ADMINISTRATION_SCOPE} ` : "";
   const where = page
     ? `They can do it in the Paperclip web app: link [Open ${page[0]}](${page[1]}), with this conversation's company prefix as <prefix>.`
     : "They can do it in the Paperclip web app or with their own board credentials. Link a page only if you already know its route; do not search for one.";