test(evals): add finite protected Grok routine form entry

tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.test.ts

@@ -0,0 +1,51 @@
+import { expect, it } from "vitest";
+import {
+  validGrokRoutinePrivateInput,
+  matchesConfiguredGrokRoutine,
+} from "./external-grok-private-contract.js";
+
+const companyId = "11111111-1111-4111-8111-111111111111";
+const input = {
+  agentId: "22222222-2222-4222-8222-222222222222",
+  bindingId: "33333333-3333-4333-8333-333333333333",
+  expectedRevision: 0,
+  botName: "Owned Grok Bot",
+  webhookUrl: "https://fixture.invalid/owned-webhook",
+};
+const setupURL = `http://127.0.0.1:3100/ACORN/agents/${input.agentId}/runtime`;
+
+it("admits only the exact saved-agent protected routine form", () => {
+  expect(validGrokRoutinePrivateInput(companyId, input, setupURL)).toBe(true);
+  for (const bad of [
+    { ...input, expectedRevision: -1 },
+    { ...input, expectedRevision: 0.5 },
+    { ...input, agentId: "wrong-agent" },
+    { ...input, botName: "Bot\r\nInjected" },
+    { ...input, webhookUrl: "http://fixture.invalid/webhook" },
+    { ...input, webhookUrl: "https://user:key@fixture.invalid/webhook" },
+    { ...input, webhookUrl: "https://fixture.invalid/webhook#key" },
+  ])
+    expect(validGrokRoutinePrivateInput(companyId, bad, setupURL)).toBe(false);
+  expect(
+    validGrokRoutinePrivateInput(companyId, input, setupURL + "/other"),
+  ).toBe(false);
+  expect(validGrokRoutinePrivateInput(undefined, input, setupURL)).toBe(false);
+});
+
+it("requires the exact persisted binding/revision and configuration without a false verification claim", () => {
+  const saved = {
+    bindingId: input.bindingId,
+    revision: 1,
+    configured: true,
+    verified: false,
+  };
+  expect(matchesConfiguredGrokRoutine(saved, input.bindingId, 0)).toBe(true);
+  for (const bad of [
+    { ...saved, bindingId: companyId },
+    { ...saved, revision: 0 },
+    { ...saved, configured: false },
+    { ...saved, verified: true },
+    undefined,
+  ])
+    expect(matchesConfiguredGrokRoutine(bad, input.bindingId, 0)).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.ts

@@ -0,0 +1,65 @@
+export interface GrokRoutinePrivateInput {
+  agentId: string;
+  bindingId: string;
+  expectedRevision: number;
+  botName: string;
+  webhookUrl: string;
+}
+
+/** Only the saved agent's exact routine form is admitted by this finite mode.
+ * These checks neither connect an account nor qualify a routine roundtrip. */
+export function validGrokRoutinePrivateInput(
+  companyId: unknown,
+  input: GrokRoutinePrivateInput | undefined,
+  setupURL: string,
+) {
+  try {
+    const setup = new URL(setupURL);
+    if (
+      typeof companyId !== "string" ||
+      !/^[a-f0-9-]{36}$/i.test(companyId) ||
+      !input ||
+      !/^[a-f0-9-]{36}$/i.test(input.agentId) ||
+      !/^[a-f0-9-]{36}$/i.test(input.bindingId) ||
+      !Number.isSafeInteger(input.expectedRevision) ||
+      input.expectedRevision < 0 ||
+      !input.botName.trim() ||
+      input.botName.length > 100 ||
+      /[\r\n]/.test(input.botName) ||
+      input.webhookUrl.length > 2048 ||
+      /[\r\n]/.test(input.webhookUrl) ||
+      !setup.pathname.endsWith(`/agents/${input.agentId}/runtime`)
+    )
+      return false;
+    const webhook = new URL(input.webhookUrl);
+    return (
+      webhook.protocol === "https:" &&
+      !webhook.username &&
+      !webhook.password &&
+      !webhook.hash
+    );
+  } catch {
+    return false;
+  }
+}
+
+export function matchesConfiguredGrokRoutine(
+  value: unknown,
+  bindingId: string,
+  expectedRevision: number,
+) {
+  const saved = value as
+    | {
+        bindingId?: unknown;
+        configured?: unknown;
+        verified?: unknown;
+        revision?: unknown;
+      }
+    | undefined;
+  return (
+    saved?.bindingId === bindingId &&
+    saved.configured === true &&
+    saved.verified === false &&
+    saved.revision === expectedRevision + 1
+  );
+}

tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts

@@ -1,14 +1,15 @@
 import { readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { chromium, test, type Cookie } from "@playwright/test";
-interface Request { mode: "secure-key" | "canary" | "gateway-token-rotation" | "company-secret-create" | "personal-secret-create"; companyId?:string; personal?:{definitionId:string;userId:string}; gateway?: { id: string; oldTokenId: string; otherToken: string; endpointPath: string; readName: string }; baseURL: string; setupURL: string; providerURL: string; credential: string; cookies: Cookie[]; executablePath?: string; channel?: string }
+import { validGrokRoutinePrivateInput, matchesConfiguredGrokRoutine, type GrokRoutinePrivateInput } from "./external-grok-private-contract.js";
+interface Request { mode: "secure-key" | "canary" | "gateway-token-rotation" | "company-secret-create" | "personal-secret-create" | "grok-routine-configure"; grok?:GrokRoutinePrivateInput; companyId?:string; personal?:{definitionId:string;userId:string}; gateway?: { id: string; oldTokenId: string; otherToken: string; endpointPath: string; readName: string }; baseURL: string; setupURL: string; providerURL: string; credential: string; cookies: Cookie[]; executablePath?: string; channel?: string }
 /** Separate Node process: no Playwright Test worker, API-step listener or artifact
  * recorder exists. This module never enables tracing, video or screenshots. */
 async function main() {
   let attached = false; try { test.info(); attached = true; } catch { /* expected outside Test */ }
   if (attached) throw new Error("Private browser worker cannot run inside Playwright Test");
   const input = JSON.parse(await readFile(process.argv[2]!, "utf8")) as Request;
-  if (!input.credential || input.credential.length > 8192 || !["secure-key", "canary", "gateway-token-rotation", "company-secret-create", "personal-secret-create"].includes(input.mode)) throw new Error("Invalid private browser request");
+  if (!input.credential || input.credential.length > (input.mode === "grok-routine-configure" ? 16384 : 8192) || !["secure-key", "canary", "gateway-token-rotation", "company-secret-create", "personal-secret-create", "grok-routine-configure"].includes(input.mode)) throw new Error("Invalid private browser request");
   const base = new URL(input.baseURL);
   if (base.hostname !== "127.0.0.1" || base.protocol !== "http:" || new URL(input.setupURL).origin !== base.origin) throw new Error("Private browser target is outside the disposable loopback instance");
   const browser = await chromium.launch({ headless: true, ...(input.executablePath ? { executablePath: input.executablePath } : {}), ...(input.channel ? { channel: input.channel } : {}) });
@@ -21,6 +22,61 @@ async function main() {
       if (await field.inputValue() !== input.credential) throw new Error("Canary entry failed");
       return { processIsolation: true, testRunnerAttached: false, credentialEntered: true, tracingEnabled: false, videoEnabled: false, apiStepRecorder: false };
     }
+    if (input.mode === "grok-routine-configure") {
+      const grok = input.grok;
+      if (!validGrokRoutinePrivateInput(input.companyId, grok, input.setupURL) || !grok) {
+        throw new Error("Invalid finite Grok routine operation");
+      }
+      const endpoint = `/api/companies/${input.companyId}/agents/${grok.agentId}/grok-binding`;
+      const beforeResponse = await context.request.get(endpoint);
+      if (!beforeResponse.ok()) throw new Error("Existing Grok binding is unavailable");
+      const before = await beforeResponse.json() as { enabled?: boolean; canConfigureConnection?: boolean; binding?: { id?: string; connected?: boolean; routine?: { revision?: number; configured?: boolean } } };
+      if (!before.enabled || !before.canConfigureConnection || before.binding?.id !== grok.bindingId ||
+        before.binding.connected !== true || (before.binding.routine?.revision ?? 0) !== grok.expectedRevision) {
+        throw new Error("Genuine connected Grok binding and exact revision are required");
+      }
+      await page.goto(input.setupURL, { waitUntil: "domcontentloaded" });
+      const section = page.getByRole("region", { name: "Grok Bot connection", exact: true });
+      await section.waitFor();
+      if (before.binding.routine?.configured) {
+        await section.getByRole("button", { name: "Replace routine credentials", exact: true }).click();
+      }
+      const urlField = section.getByLabel("Webhook URL", { exact: true });
+      const keyField = section.getByLabel("Webhook key", { exact: true });
+      if (await urlField.getAttribute("type") !== "password" || await keyField.getAttribute("type") !== "password") {
+        throw new Error("Grok credential fields are not protected");
+      }
+      await section.getByLabel("Bot name in Grok", { exact: true }).fill(grok.botName);
+      await urlField.fill(grok.webhookUrl);
+      await keyField.fill(input.credential);
+      const savedResponse = page.waitForResponse(response => new URL(response.url()).pathname === `${endpoint}/routine` &&
+        response.request().method() === "PUT" && response.ok());
+      await section.getByRole("button", { name: "Save routine", exact: true }).click();
+      const response = await savedResponse;
+      const submitted = response.request().postDataJSON() as { bindingId?: string; expectedRevision?: number; botName?: string; webhookUrl?: string; webhookKey?: string };
+      if (submitted.bindingId !== grok.bindingId || submitted.expectedRevision !== grok.expectedRevision ||
+        submitted.botName !== grok.botName || submitted.webhookUrl !== grok.webhookUrl || submitted.webhookKey !== input.credential) {
+        throw new Error("Grok form did not submit the intended scoped private configuration");
+      }
+      const saved = await response.json() as { bindingId?: string; configured?: boolean; verified?: boolean; revision?: number };
+      if (!matchesConfiguredGrokRoutine(saved, grok.bindingId, grok.expectedRevision)) {
+        throw new Error("Grok routine response does not match the configured unverified binding");
+      }
+      await section.getByRole("button", { name: "Verify connection", exact: true }).waitFor();
+      await keyField.waitFor({ state: "detached" });
+      const afterResponse = await context.request.get(endpoint);
+      if (!afterResponse.ok()) throw new Error("Saved Grok binding could not be independently read");
+      const after = await afterResponse.json() as { binding?: { id?: string; connected?: boolean; routine?: { configured?: boolean; revision?: number } } };
+      if (after.binding?.id !== grok.bindingId || after.binding.connected !== true ||
+        after.binding.routine?.configured !== true || after.binding.routine.revision !== saved.revision) {
+        throw new Error("Saved Grok configuration did not persist on the same connected binding");
+      }
+      const storageClean = await page.evaluate(values => !values.some(value =>
+        JSON.stringify([Object.entries(localStorage), Object.entries(sessionStorage), document.body.innerText, location.href]).includes(value)),
+      [input.credential, grok.webhookUrl]);
+      return { processIsolation: true, testRunnerAttached: false, tracingEnabled: false, videoEnabled: false, apiStepRecorder: false,
+        bindingId: grok.bindingId, routineConfigured: true, routineUiVisible: true, humanSubmittedSecurely: true, storageClean };
+    }
     if(input.mode === "personal-secret-create"){
       const personal=input.personal,setup=new URL(input.setupURL);
       if(!input.companyId||!/^[a-f0-9-]{36}$/i.test(input.companyId)||!personal||!/^[a-f0-9-]{36}$/i.test(personal.definitionId)||!personal.userId||personal.userId.length>160||!setup.pathname.endsWith('/company/settings/secrets')||setup.searchParams.get('definition')!==personal.definitionId)throw new Error('Invalid finite personal secret operation');

tests/runner-e2e/chat-catalog-domains/external-private-browser.ts

@@ -5,8 +5,9 @@ import { fileURLToPath } from "node:url";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { chromium, type Cookie } from "@playwright/test";
-export interface PrivateBrowserReceipt { processIsolation: true; testRunnerAttached: false; tracingEnabled: false; videoEnabled: false; apiStepRecorder: false; credentialEntered?: boolean; connectionId?: string; challengeVisible?: boolean; passwordInput?: boolean; autocompleteOff?: boolean; humanSubmittedSecurely?: boolean; visibleReady?: boolean; storageClean?: boolean; newTokenId?: string; tokenUiVisible?: boolean; newWorkedBeforeRevocation?: boolean; oldWorkedUntilRevocation?: boolean; oldRevoked?: boolean; newWorksAfter?: boolean; oldStatusAfter?: number; otherWorksAfter?: boolean; secretId?:string; secretUiVisible?:boolean }
-export async function privateBrowserEntry(input: { mode: "secure-key" | "canary" | "gateway-token-rotation" | "company-secret-create" | "personal-secret-create"; companyId?:string; personal?:{definitionId:string;userId:string}; gateway?: { id: string; oldTokenId: string; otherToken: string; endpointPath: string; readName: string }; baseURL: string; setupURL: string; providerURL: string; credential: string; cookies: Cookie[] }, onToken?: (token: { id: string; token: string }) => void) {
+import type { GrokRoutinePrivateInput } from "./external-grok-private-contract.js";
+export interface PrivateBrowserReceipt { processIsolation: true; testRunnerAttached: false; tracingEnabled: false; videoEnabled: false; apiStepRecorder: false; credentialEntered?: boolean; connectionId?: string; challengeVisible?: boolean; passwordInput?: boolean; autocompleteOff?: boolean; humanSubmittedSecurely?: boolean; visibleReady?: boolean; storageClean?: boolean; newTokenId?: string; tokenUiVisible?: boolean; newWorkedBeforeRevocation?: boolean; oldWorkedUntilRevocation?: boolean; oldRevoked?: boolean; newWorksAfter?: boolean; oldStatusAfter?: number; otherWorksAfter?: boolean; secretId?:string; secretUiVisible?:boolean; bindingId?:string; routineConfigured?:boolean; routineUiVisible?:boolean }
+export async function privateBrowserEntry(input: { mode: "secure-key" | "canary" | "gateway-token-rotation" | "company-secret-create" | "personal-secret-create" | "grok-routine-configure"; grok?:GrokRoutinePrivateInput; companyId?:string; personal?:{definitionId:string;userId:string}; gateway?: { id: string; oldTokenId: string; otherToken: string; endpointPath: string; readName: string }; baseURL: string; setupURL: string; providerURL: string; credential: string; cookies: Cookie[] }, onToken?: (token: { id: string; token: string }) => void) {
   const directory = await mkdtemp(path.join(tmpdir(), "paperclip-private-browser-"));
   try {
     const request = path.join(directory, "request.json");
@@ -21,7 +22,8 @@ export async function privateBrowserEntry(input: { mode: "secure-key" | "canary"
     } catch { throw new Error("Private browser entry failed; child output and credential-bearing request withheld"); }
     let receipt: PrivateBrowserReceipt; try { receipt = JSON.parse(output) as PrivateBrowserReceipt; } catch { throw new Error("Private browser returned invalid closed receipt"); }
     if (receipt.processIsolation !== true || receipt.testRunnerAttached !== false || receipt.tracingEnabled !== false || receipt.videoEnabled !== false || receipt.apiStepRecorder !== false
-      || Object.keys(receipt).some(key => !["processIsolation", "testRunnerAttached", "tracingEnabled", "videoEnabled", "apiStepRecorder", "credentialEntered", "connectionId", "challengeVisible", "passwordInput", "autocompleteOff", "humanSubmittedSecurely", "visibleReady", "storageClean", "newTokenId", "tokenUiVisible", "newWorkedBeforeRevocation", "oldWorkedUntilRevocation", "oldRevoked", "newWorksAfter", "oldStatusAfter", "otherWorksAfter", "secretId", "secretUiVisible"].includes(key))) throw new Error("Private browser receipt exceeds its closed contract");
+      || Object.keys(receipt).some(key => !["processIsolation", "testRunnerAttached", "tracingEnabled", "videoEnabled", "apiStepRecorder", "credentialEntered", "connectionId", "challengeVisible", "passwordInput", "autocompleteOff", "humanSubmittedSecurely", "visibleReady", "storageClean", "newTokenId", "tokenUiVisible", "newWorkedBeforeRevocation", "oldWorkedUntilRevocation", "oldRevoked", "newWorksAfter", "oldStatusAfter", "otherWorksAfter", "secretId", "secretUiVisible", "bindingId", "routineConfigured", "routineUiVisible"].includes(key))) throw new Error("Private browser receipt exceeds its closed contract");
+    if (input.mode === "grok-routine-configure" && (receipt.bindingId !== input.grok?.bindingId || receipt.routineConfigured !== true || receipt.routineUiVisible !== true || receipt.visibleReady !== undefined)) throw new Error("Invalid closed Grok routine configuration receipt");
     if (input.mode === "gateway-token-rotation") {
       if (!onToken) throw new Error("Gateway rotation requires its private credential sink");
       const token = JSON.parse(await readFile(path.join(directory, "token.json"), "utf8")) as { id: string; token: string };