From 1bb90f12cf1fefa1f5d3be27a5784ee7fb7a2c71 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 01:01:18 -0500 Subject: [PATCH] test(evals): add finite protected Grok routine form entry --- .../external-grok-private-contract.test.ts | 51 +++++++++++++++ .../external-grok-private-contract.ts | 65 +++++++++++++++++++ .../external-private-browser-worker.ts | 60 ++++++++++++++++- .../external-private-browser.ts | 8 ++- 4 files changed, 179 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.ts diff --git a/tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.test.ts b/tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.test.ts new file mode 100644 index 00000000000..2c31dfd3e0e --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.test.ts @@ -0,0 +1,51 @@ +import { expect, it } from "vitest"; +import { + validGrokRoutinePrivateInput, + matchesConfiguredGrokRoutine, +} from "./external-grok-private-contract.js"; + +const companyId = "11111111-1111-4111-8111-111111111111"; +const input = { + agentId: "22222222-2222-4222-8222-222222222222", + bindingId: "33333333-3333-4333-8333-333333333333", + expectedRevision: 0, + botName: "Owned Grok Bot", + webhookUrl: "https://fixture.invalid/owned-webhook", +}; +const setupURL = `http://127.0.0.1:3100/ACORN/agents/${input.agentId}/runtime`; + +it("admits only the exact saved-agent protected routine form", () => { + expect(validGrokRoutinePrivateInput(companyId, input, setupURL)).toBe(true); + for (const bad of [ + { ...input, expectedRevision: -1 }, + { ...input, expectedRevision: 0.5 }, + { ...input, agentId: "wrong-agent" }, + { ...input, botName: "Bot\r\nInjected" }, + { ...input, webhookUrl: "http://fixture.invalid/webhook" }, + { ...input, webhookUrl: "https://user:key@fixture.invalid/webhook" }, + { ...input, webhookUrl: "https://fixture.invalid/webhook#key" }, + ]) + expect(validGrokRoutinePrivateInput(companyId, bad, setupURL)).toBe(false); + expect( + validGrokRoutinePrivateInput(companyId, input, setupURL + "/other"), + ).toBe(false); + expect(validGrokRoutinePrivateInput(undefined, input, setupURL)).toBe(false); +}); + +it("requires the exact persisted binding/revision and configuration without a false verification claim", () => { + const saved = { + bindingId: input.bindingId, + revision: 1, + configured: true, + verified: false, + }; + expect(matchesConfiguredGrokRoutine(saved, input.bindingId, 0)).toBe(true); + for (const bad of [ + { ...saved, bindingId: companyId }, + { ...saved, revision: 0 }, + { ...saved, configured: false }, + { ...saved, verified: true }, + undefined, + ]) + expect(matchesConfiguredGrokRoutine(bad, input.bindingId, 0)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.ts b/tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.ts new file mode 100644 index 00000000000..dfd5989fa68 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-grok-private-contract.ts @@ -0,0 +1,65 @@ +export interface GrokRoutinePrivateInput { + agentId: string; + bindingId: string; + expectedRevision: number; + botName: string; + webhookUrl: string; +} + +/** Only the saved agent's exact routine form is admitted by this finite mode. + * These checks neither connect an account nor qualify a routine roundtrip. */ +export function validGrokRoutinePrivateInput( + companyId: unknown, + input: GrokRoutinePrivateInput | undefined, + setupURL: string, +) { + try { + const setup = new URL(setupURL); + if ( + typeof companyId !== "string" || + !/^[a-f0-9-]{36}$/i.test(companyId) || + !input || + !/^[a-f0-9-]{36}$/i.test(input.agentId) || + !/^[a-f0-9-]{36}$/i.test(input.bindingId) || + !Number.isSafeInteger(input.expectedRevision) || + input.expectedRevision < 0 || + !input.botName.trim() || + input.botName.length > 100 || + /[\r\n]/.test(input.botName) || + input.webhookUrl.length > 2048 || + /[\r\n]/.test(input.webhookUrl) || + !setup.pathname.endsWith(`/agents/${input.agentId}/runtime`) + ) + return false; + const webhook = new URL(input.webhookUrl); + return ( + webhook.protocol === "https:" && + !webhook.username && + !webhook.password && + !webhook.hash + ); + } catch { + return false; + } +} + +export function matchesConfiguredGrokRoutine( + value: unknown, + bindingId: string, + expectedRevision: number, +) { + const saved = value as + | { + bindingId?: unknown; + configured?: unknown; + verified?: unknown; + revision?: unknown; + } + | undefined; + return ( + saved?.bindingId === bindingId && + saved.configured === true && + saved.verified === false && + saved.revision === expectedRevision + 1 + ); +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts b/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts index 617d3edb1d0..db0ca15feab 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-private-browser-worker.ts @@ -1,14 +1,15 @@ import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { chromium, test, type Cookie } from "@playwright/test"; -interface Request { mode: "secure-key" | "canary" | "gateway-token-rotation" | "company-secret-create" | "personal-secret-create"; companyId?:string; personal?:{definitionId:string;userId:string}; gateway?: { id: string; oldTokenId: string; otherToken: string; endpointPath: string; readName: string }; baseURL: string; setupURL: string; providerURL: string; credential: string; cookies: Cookie[]; executablePath?: string; channel?: string } +import { validGrokRoutinePrivateInput, matchesConfiguredGrokRoutine, type GrokRoutinePrivateInput } from "./external-grok-private-contract.js"; +interface Request { mode: "secure-key" | "canary" | "gateway-token-rotation" | "company-secret-create" | "personal-secret-create" | "grok-routine-configure"; grok?:GrokRoutinePrivateInput; companyId?:string; personal?:{definitionId:string;userId:string}; gateway?: { id: string; oldTokenId: string; otherToken: string; endpointPath: string; readName: string }; baseURL: string; setupURL: string; providerURL: string; credential: string; cookies: Cookie[]; executablePath?: string; channel?: string } /** Separate Node process: no Playwright Test worker, API-step listener or artifact * recorder exists. This module never enables tracing, video or screenshots. */ async function main() { let attached = false; try { test.info(); attached = true; } catch { /* expected outside Test */ } if (attached) throw new Error("Private browser worker cannot run inside Playwright Test"); const input = JSON.parse(await readFile(process.argv[2]!, "utf8")) as Request; - if (!input.credential || input.credential.length > 8192 || !["secure-key", "canary", "gateway-token-rotation", "company-secret-create", "personal-secret-create"].includes(input.mode)) throw new Error("Invalid private browser request"); + if (!input.credential || input.credential.length > (input.mode === "grok-routine-configure" ? 16384 : 8192) || !["secure-key", "canary", "gateway-token-rotation", "company-secret-create", "personal-secret-create", "grok-routine-configure"].includes(input.mode)) throw new Error("Invalid private browser request"); const base = new URL(input.baseURL); if (base.hostname !== "127.0.0.1" || base.protocol !== "http:" || new URL(input.setupURL).origin !== base.origin) throw new Error("Private browser target is outside the disposable loopback instance"); const browser = await chromium.launch({ headless: true, ...(input.executablePath ? { executablePath: input.executablePath } : {}), ...(input.channel ? { channel: input.channel } : {}) }); @@ -21,6 +22,61 @@ async function main() { if (await field.inputValue() !== input.credential) throw new Error("Canary entry failed"); return { processIsolation: true, testRunnerAttached: false, credentialEntered: true, tracingEnabled: false, videoEnabled: false, apiStepRecorder: false }; } + if (input.mode === "grok-routine-configure") { + const grok = input.grok; + if (!validGrokRoutinePrivateInput(input.companyId, grok, input.setupURL) || !grok) { + throw new Error("Invalid finite Grok routine operation"); + } + const endpoint = `/api/companies/${input.companyId}/agents/${grok.agentId}/grok-binding`; + const beforeResponse = await context.request.get(endpoint); + if (!beforeResponse.ok()) throw new Error("Existing Grok binding is unavailable"); + const before = await beforeResponse.json() as { enabled?: boolean; canConfigureConnection?: boolean; binding?: { id?: string; connected?: boolean; routine?: { revision?: number; configured?: boolean } } }; + if (!before.enabled || !before.canConfigureConnection || before.binding?.id !== grok.bindingId || + before.binding.connected !== true || (before.binding.routine?.revision ?? 0) !== grok.expectedRevision) { + throw new Error("Genuine connected Grok binding and exact revision are required"); + } + await page.goto(input.setupURL, { waitUntil: "domcontentloaded" }); + const section = page.getByRole("region", { name: "Grok Bot connection", exact: true }); + await section.waitFor(); + if (before.binding.routine?.configured) { + await section.getByRole("button", { name: "Replace routine credentials", exact: true }).click(); + } + const urlField = section.getByLabel("Webhook URL", { exact: true }); + const keyField = section.getByLabel("Webhook key", { exact: true }); + if (await urlField.getAttribute("type") !== "password" || await keyField.getAttribute("type") !== "password") { + throw new Error("Grok credential fields are not protected"); + } + await section.getByLabel("Bot name in Grok", { exact: true }).fill(grok.botName); + await urlField.fill(grok.webhookUrl); + await keyField.fill(input.credential); + const savedResponse = page.waitForResponse(response => new URL(response.url()).pathname === `${endpoint}/routine` && + response.request().method() === "PUT" && response.ok()); + await section.getByRole("button", { name: "Save routine", exact: true }).click(); + const response = await savedResponse; + const submitted = response.request().postDataJSON() as { bindingId?: string; expectedRevision?: number; botName?: string; webhookUrl?: string; webhookKey?: string }; + if (submitted.bindingId !== grok.bindingId || submitted.expectedRevision !== grok.expectedRevision || + submitted.botName !== grok.botName || submitted.webhookUrl !== grok.webhookUrl || submitted.webhookKey !== input.credential) { + throw new Error("Grok form did not submit the intended scoped private configuration"); + } + const saved = await response.json() as { bindingId?: string; configured?: boolean; verified?: boolean; revision?: number }; + if (!matchesConfiguredGrokRoutine(saved, grok.bindingId, grok.expectedRevision)) { + throw new Error("Grok routine response does not match the configured unverified binding"); + } + await section.getByRole("button", { name: "Verify connection", exact: true }).waitFor(); + await keyField.waitFor({ state: "detached" }); + const afterResponse = await context.request.get(endpoint); + if (!afterResponse.ok()) throw new Error("Saved Grok binding could not be independently read"); + const after = await afterResponse.json() as { binding?: { id?: string; connected?: boolean; routine?: { configured?: boolean; revision?: number } } }; + if (after.binding?.id !== grok.bindingId || after.binding.connected !== true || + after.binding.routine?.configured !== true || after.binding.routine.revision !== saved.revision) { + throw new Error("Saved Grok configuration did not persist on the same connected binding"); + } + const storageClean = await page.evaluate(values => !values.some(value => + JSON.stringify([Object.entries(localStorage), Object.entries(sessionStorage), document.body.innerText, location.href]).includes(value)), + [input.credential, grok.webhookUrl]); + return { processIsolation: true, testRunnerAttached: false, tracingEnabled: false, videoEnabled: false, apiStepRecorder: false, + bindingId: grok.bindingId, routineConfigured: true, routineUiVisible: true, humanSubmittedSecurely: true, storageClean }; + } if(input.mode === "personal-secret-create"){ const personal=input.personal,setup=new URL(input.setupURL); if(!input.companyId||!/^[a-f0-9-]{36}$/i.test(input.companyId)||!personal||!/^[a-f0-9-]{36}$/i.test(personal.definitionId)||!personal.userId||personal.userId.length>160||!setup.pathname.endsWith('/company/settings/secrets')||setup.searchParams.get('definition')!==personal.definitionId)throw new Error('Invalid finite personal secret operation'); diff --git a/tests/runner-e2e/chat-catalog-domains/external-private-browser.ts b/tests/runner-e2e/chat-catalog-domains/external-private-browser.ts index 8e79db6c3ee..8a09ad7609b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-private-browser.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-private-browser.ts @@ -5,8 +5,9 @@ import { fileURLToPath } from "node:url"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { chromium, type Cookie } from "@playwright/test"; -export interface PrivateBrowserReceipt { processIsolation: true; testRunnerAttached: false; tracingEnabled: false; videoEnabled: false; apiStepRecorder: false; credentialEntered?: boolean; connectionId?: string; challengeVisible?: boolean; passwordInput?: boolean; autocompleteOff?: boolean; humanSubmittedSecurely?: boolean; visibleReady?: boolean; storageClean?: boolean; newTokenId?: string; tokenUiVisible?: boolean; newWorkedBeforeRevocation?: boolean; oldWorkedUntilRevocation?: boolean; oldRevoked?: boolean; newWorksAfter?: boolean; oldStatusAfter?: number; otherWorksAfter?: boolean; secretId?:string; secretUiVisible?:boolean } -export async function privateBrowserEntry(input: { mode: "secure-key" | "canary" | "gateway-token-rotation" | "company-secret-create" | "personal-secret-create"; companyId?:string; personal?:{definitionId:string;userId:string}; gateway?: { id: string; oldTokenId: string; otherToken: string; endpointPath: string; readName: string }; baseURL: string; setupURL: string; providerURL: string; credential: string; cookies: Cookie[] }, onToken?: (token: { id: string; token: string }) => void) { +import type { GrokRoutinePrivateInput } from "./external-grok-private-contract.js"; +export interface PrivateBrowserReceipt { processIsolation: true; testRunnerAttached: false; tracingEnabled: false; videoEnabled: false; apiStepRecorder: false; credentialEntered?: boolean; connectionId?: string; challengeVisible?: boolean; passwordInput?: boolean; autocompleteOff?: boolean; humanSubmittedSecurely?: boolean; visibleReady?: boolean; storageClean?: boolean; newTokenId?: string; tokenUiVisible?: boolean; newWorkedBeforeRevocation?: boolean; oldWorkedUntilRevocation?: boolean; oldRevoked?: boolean; newWorksAfter?: boolean; oldStatusAfter?: number; otherWorksAfter?: boolean; secretId?:string; secretUiVisible?:boolean; bindingId?:string; routineConfigured?:boolean; routineUiVisible?:boolean } +export async function privateBrowserEntry(input: { mode: "secure-key" | "canary" | "gateway-token-rotation" | "company-secret-create" | "personal-secret-create" | "grok-routine-configure"; grok?:GrokRoutinePrivateInput; companyId?:string; personal?:{definitionId:string;userId:string}; gateway?: { id: string; oldTokenId: string; otherToken: string; endpointPath: string; readName: string }; baseURL: string; setupURL: string; providerURL: string; credential: string; cookies: Cookie[] }, onToken?: (token: { id: string; token: string }) => void) { const directory = await mkdtemp(path.join(tmpdir(), "paperclip-private-browser-")); try { const request = path.join(directory, "request.json"); @@ -21,7 +22,8 @@ export async function privateBrowserEntry(input: { mode: "secure-key" | "canary" } catch { throw new Error("Private browser entry failed; child output and credential-bearing request withheld"); } let receipt: PrivateBrowserReceipt; try { receipt = JSON.parse(output) as PrivateBrowserReceipt; } catch { throw new Error("Private browser returned invalid closed receipt"); } if (receipt.processIsolation !== true || receipt.testRunnerAttached !== false || receipt.tracingEnabled !== false || receipt.videoEnabled !== false || receipt.apiStepRecorder !== false - || Object.keys(receipt).some(key => !["processIsolation", "testRunnerAttached", "tracingEnabled", "videoEnabled", "apiStepRecorder", "credentialEntered", "connectionId", "challengeVisible", "passwordInput", "autocompleteOff", "humanSubmittedSecurely", "visibleReady", "storageClean", "newTokenId", "tokenUiVisible", "newWorkedBeforeRevocation", "oldWorkedUntilRevocation", "oldRevoked", "newWorksAfter", "oldStatusAfter", "otherWorksAfter", "secretId", "secretUiVisible"].includes(key))) throw new Error("Private browser receipt exceeds its closed contract"); + || Object.keys(receipt).some(key => !["processIsolation", "testRunnerAttached", "tracingEnabled", "videoEnabled", "apiStepRecorder", "credentialEntered", "connectionId", "challengeVisible", "passwordInput", "autocompleteOff", "humanSubmittedSecurely", "visibleReady", "storageClean", "newTokenId", "tokenUiVisible", "newWorkedBeforeRevocation", "oldWorkedUntilRevocation", "oldRevoked", "newWorksAfter", "oldStatusAfter", "otherWorksAfter", "secretId", "secretUiVisible", "bindingId", "routineConfigured", "routineUiVisible"].includes(key))) throw new Error("Private browser receipt exceeds its closed contract"); + if (input.mode === "grok-routine-configure" && (receipt.bindingId !== input.grok?.bindingId || receipt.routineConfigured !== true || receipt.routineUiVisible !== true || receipt.visibleReady !== undefined)) throw new Error("Invalid closed Grok routine configuration receipt"); if (input.mode === "gateway-token-rotation") { if (!onToken) throw new Error("Gateway rotation requires its private credential sink"); const token = JSON.parse(await readFile(path.join(directory, "token.json"), "utf8")) as { id: string; token: string };