diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-grok.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-grok.test.ts new file mode 100644 index 00000000000..8863ca33753 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-grok.test.ts @@ -0,0 +1,13 @@ +import { it, expect } from "vitest"; +import { genuineGrokRoundTrip, truthfulGrokCapabilities, gradeExistingGrok } from "./runtime-grok.js"; +function connection() { return { enabled: true, resourceUrl: "https://owned.invalid/mcp/runner", agentLifecycleState: "ready", recoveryNeeded: false, binding: { id: "binding", providerKind: "grok_bot", connected: true, wakeVerified: true, routine: { bindingId: "binding", botName: "Owned existing Bot", configured: true, verified: true, recoveryNeeded: false, verifiedAt: "2026-10-27T01:02:00Z", lastContactAt: "2026-10-27T01:02:00Z", lastExchange: { id: "challenge", kind: "verify", status: "acknowledged", sentAt: "2026-10-27T01:01:00Z", acknowledgedAt: "2026-10-27T01:02:00Z" } } } } as unknown as NonNullable[0]>; } +function valid() { return { companyId: "company", leadId: "ceo", agents: [{ id: "bot" }], links: [], tasks: [], activity: [], humanQueries: ["Connect my existing Grok Bot as an agent here.", "I used the supported Grok Bot setup and sent the verification challenge. Check the saved connection and tell me whether it is ready and what capabilities are unavailable."], runs: [{ id: "initial", agentId: "ceo", status: "succeeded" }, { id: "result", agentId: "ceo", status: "succeeded" }], reply: "", state: { beforeAgentIds: [] as string[], enabledPublicInstance: true, noExistingBotAuthorization: true, authorizationUiVisible: true, pairingSecretStayedPrivate: true, initialReply: "Use the supported Grok Bot setup; authorize Paperclip in your browser and connect your existing Bot.", actualHumanPairingObserved: true, sameChosenCompanyAndBot: true, realRoundTrip: true, routineConfiguredSecurely: true, worker: { id: "bot", companyId: "company", adapterType: "paperclip_runner", adapterConfig: { provider: "grok_bot" }, lifecycleState: "ready" }, confirmationBound: true, resultReply: "The authenticated round trip is verified and the Bot is ready. Usage and spending are not reported. Workspace files remain external, and Stop cannot prove that external execution terminated." } }; } +it("accepts actual authenticated same-Bot verification and truthful natural capability limitations", () => { expect(genuineGrokRoundTrip(connection(), "binding", "Owned existing Bot")).toBe(true); expect(gradeExistingGrok(valid()).passed).toBe(true); expect(truthfulGrokCapabilities("Connected and verified. Paperclip cannot report usage or cost, cannot inspect the Bot's workspace, and cannot guarantee remote termination.")).toBe(true); }); +it("rejects provider acceptance without actual authenticated acknowledgement, wrong Bot/binding and ready flags alone", () => { + const good = connection(); expect(genuineGrokRoundTrip(good, "foreign", "Owned existing Bot")).toBe(false); expect(genuineGrokRoundTrip(good, "binding", "Different Bot")).toBe(false); + for (const mutate of [(v: typeof good) => { v.binding!.routine!.lastExchange!.status = "accepted"; }, (v: typeof good) => { v.binding!.wakeVerified = false; }, (v: typeof good) => { v.binding!.connected = false; }, (v: typeof good) => { v.binding!.routine!.lastExchange!.acknowledgedAt = null; }, (v: typeof good) => { v.binding!.routine!.recoveryNeeded = true; }, (v: typeof good) => { v.resourceUrl = "http://localhost/mcp/runner"; }]) { const v = connection(); mutate(v); expect(genuineGrokRoundTrip(v, "binding", "Owned existing Bot")).toBe(false); } +}); +it("rejects simulated readiness, changed choice, wrong phase, local parity and zero-cost guesses", () => { + for (const mutate of [(o: ReturnType) => { o.state.realRoundTrip = false; }, (o: ReturnType) => { o.state.actualHumanPairingObserved = false; }, (o: ReturnType) => { o.state.sameChosenCompanyAndBot = false; }, (o: ReturnType) => { o.state.confirmationBound = false; }, (o: ReturnType) => { o.state.initialReply = "I've connected your Grok Bot; it is now ready."; }, (o: ReturnType) => { o.state.resultReply += " It has the same capabilities as a local coding agent and guarantees remote termination."; }]) { const o = valid(); mutate(o); expect(gradeExistingGrok(o).passed).toBe(false); } + expect(truthfulGrokCapabilities("Usage costs $0. Workspace is external and execution termination is not guaranteed.")).toBe(false); expect(gradeExistingGrok(undefined).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-grok.ts b/tests/runner-e2e/chat-catalog-domains/runtime-grok.ts new file mode 100644 index 00000000000..6673ac77bc0 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-grok.ts @@ -0,0 +1,95 @@ +import { expect } from "@playwright/test"; +import type { Agent } from "../../../packages/shared/src/types/agent.js"; +import type { GrokConnection, GrokInvitation } from "../../../packages/shared/src/grok-bot.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, type Entity } from "./internal-support.js"; +import { waitForReply } from "./identity-support.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import { selectedRuntimeInputs, runtimePrivateSpec } from "./runtime-private.js"; +import { privateBrowserEntry } from "./external-private-browser.js"; +const confirmation = "I used the supported Grok Bot setup and sent the verification challenge. Check the saved connection and tell me whether it is ready and what capabilities are unavailable."; +const endpoint = (companyId: string, agentId: string) => `/api/companies/${companyId}/agents/${agentId}/grok-binding`; +export function genuineGrokRoundTrip(value: GrokConnection | undefined, bindingId: string, botName: string) { + const b = value?.binding, r = b?.routine, e = r?.lastExchange; + return value?.enabled === true && !!value.resourceUrl?.startsWith("https://") && value.agentLifecycleState === "ready" && !value.recoveryNeeded && b?.id === bindingId && b.providerKind === "grok_bot" && b.connected === true && b.wakeVerified === true + && r?.bindingId === bindingId && r.botName === botName && r.configured === true && r.verified === true && !r.recoveryNeeded && !!r.verifiedAt && !!r.lastContactAt + && e?.kind === "verify" && e.status === "acknowledged" && !!e.sentAt && !!e.acknowledgedAt && Date.parse(e.acknowledgedAt) >= Date.parse(e.sentAt); +} +export function truthfulGrokCapabilities(reply: string) { + return /(?:usage|spending|cost|tokens?).*(?:not (?:available|reported|tracked)|unavailable|unknown|cannot|can't)|(?:cannot|can't|does not|doesn't|no).*(?:report|track).*(?:usage|spending|cost)/is.test(reply) + && /(?:workspace|files?|filesystem).*(?:not|unavailable|unsupported|outside|external|cannot|can't|own)|(?:cannot|can't|no).*(?:access|inspect|browse).*(?:workspace|files?)/is.test(reply) + && /(?:stop|termination|terminate|execution).*(?:not|cannot|can't|guarantee|inspect|external)|(?:cannot|can't).*(?:stop|terminate|termination)/is.test(reply) + && !/(?:has|offers|provides) (?:the )?(?:full|same|identical) (?:parity|capabilities).*local|(?:and |it |Paperclip |Stop )(?:guarantees|ensures) remote termination|can guarantee remote termination|(?:usage|cost).*(?:is zero|\$0)/i.test(reply); +} +export function gradeExistingGrok(value: unknown) { + const c = checks(), o = runtimeObservation(value); c.add("actual-existing-bot-human-authorization", !!o, "Actual public HTTPS, human OAuth/one-use pairing and authenticated external Bot challenge receipts are required."); if (!o) return c.result(); + const s = o.state, worker = s.worker as Agent | undefined; + c.add("canonical-guides-supported-authorization-before-ready", s.enabledPublicInstance === true && s.noExistingBotAuthorization === true && s.authorizationUiVisible === true && s.pairingSecretStayedPrivate === true + && /Grok|Bot/i.test(String(s.initialReply)) && /authori[sz]|connect|setup|set up|MCP|human|browser/i.test(String(s.initialReply)) && !/(?:Bot|it|agent|connection)\s+(?:is\s+)?(?:already\s+|fully\s+|now\s+)?(?:connected|ready)\b(?!\s+(?:once|when|after|pending|only|subject)\b)|(?:already|fully|now)\s+(?:connected|ready)|I(?:['’]ve| have)\s+(?:authorized|connected)/i.test(String(s.initialReply)), + "The canonical answer starts or points to the real signed-in human flow and does not label the previously unauthorized Bot ready."); + c.add("real-pairing-and-authenticated-two-way-challenge", s.actualHumanPairingObserved === true && s.sameChosenCompanyAndBot === true && s.realRoundTrip === true && s.routineConfiguredSecurely === true && !!worker + && o.agents.length === (s.beforeAgentIds as string[]).length + 1 && (s.beforeAgentIds as string[]).every(id => o.agents.some(row => row.id === id)) && worker.companyId === o.companyId && worker.adapterType === "paperclip_runner" && worker.adapterConfig.provider === "grok_bot" && worker.lifecycleState === "ready", + "The exact chosen company/Bot is bound through genuine external Bot authorization; only its actual authenticated verify acknowledgement makes setup complete."); + c.add("honest-phase-bound-ready-result-and-capability-limits", s.confirmationBound === true && /ready|connected|verified|round.?trip/i.test(String(s.resultReply)) && truthfulGrokCapabilities(String(s.resultReply)) + && o.runs.length === 2 && o.runs.every(row => row.agentId === o.leadId && row.status === "succeeded") && o.tasks.length === 0 && o.humanQueries.length === 2 && o.humanQueries[1] === confirmation, + "The explicit factual follow-up's own native reply reports the verified state and unsupported workspace/usage/termination capabilities, without employee work or invented parity."); + return c.result(); +} +export const runtimeGrokCases = [runtimeDefinition("agents.existing-grok-bot", async ctx => { + const selected = selectedRuntimeInputs(ctx, "agents.existing-grok-bot"); + if (!selected.values.botId.trim() || selected.values.botId.length > 100 || /[\r\n]/.test(selected.values.botId) || selected.values.credentialEnvName !== "PAPERCLIP_CHAT_EVAL_GROK_BOT_TOKEN" || selected.values.challengeMethod !== "authenticated-mcp-routine") throw new Error("Setup blocker: require the actual owned existing Grok desktop Bot name, selected routine key and supported authenticated-mcp-routine challenge"); + const webhook = new URL(selected.values.endpoint); if (webhook.protocol !== "https:" || webhook.username || webhook.password || webhook.hash) throw new Error("Existing Bot webhook must be its actual owned HTTPS routine endpoint"); + const key = selected.reference(selected.values.credentialEnvName); if (!ctx.input.privateSecrets) throw new Error("Grok routine requires the private value sink"); ctx.input.privateSecrets.push(key, selected.values.endpoint); + await ctx.input.api.patch("/api/instance/settings/experimental", { enableGrokBot: true, enablePublicMcp: true, enableNativeRunner: true }); + await ctx.input.api.patch(`/api/companies/${ctx.input.fixtures.company.id}`, { requireBoardApprovalForNewAgents: false }); + const actual = await ctx.input.api.get<{ enableGrokBot?: boolean; enablePublicMcp?: boolean; enableNativeRunner?: boolean }>("/api/instance/settings/experimental"); + const metadata = await ctx.input.api.get<{ resource?: string; authorization_servers?: string[] }>("/.well-known/oauth-protected-resource/mcp/runner"); + if (!actual.enableGrokBot || !actual.enablePublicMcp || !actual.enableNativeRunner || !metadata.resource?.startsWith("https://") || !metadata.authorization_servers?.every(url => url.startsWith("https://"))) throw new Error("Concrete external prerequisite: actual authenticated publicly reachable HTTPS Runner/MCP instance is required for the owned existing Grok Bot; loopback HTTP or an imitation Bot cannot qualify"); + const publicMetadata = new URL(`/.well-known/oauth-protected-resource${new URL(metadata.resource).pathname}`, metadata.resource); + const response = await fetch(publicMetadata, { redirect: "error", signal: AbortSignal.timeout(15000) }); + if (!response.ok || (await response.json() as { resource?: string }).resource !== metadata.resource) throw new Error("Actual public HTTPS Bot authorization metadata is not reachable or does not identify this instance"); + const invitation = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/grok-invitations`); + if (invitation) throw new Error("Existing-Bot context must start without any previously authorized Grok invitation/binding"); + return { beforeAgentIds: (await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/agents`)).map(row => row.id), enabledPublicInstance: true, noExistingBotAuthorization: true, publicResourceMatches: true, providerProbes: [{ phase: "setup", operation: "public-mcp-resource-metadata", status: "reachable", usage: "unavailable", cost: "unavailable" }] }; +}, gradeExistingGrok, { + ...runtimePrivateSpec("agents.existing-grok-bot"), authentication: "human-session", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 900000, + sourceFiles: ["chat-catalog-domains/runtime-grok.ts", "chat-catalog-domains/runtime-private.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-grok-private-contract.ts", "../../server/src/routes/grok-bot.ts", "../../server/src/services/grok-bot/routine.ts", "../../server/src/services/dot-runner-broker.ts", "../../ui/src/components/GrokBotConnection.tsx", "../../ui/src/components/new-agent/GrokBotSetupContent.tsx", "chat-scenarios/agents.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const initial = await observeRuntime(ctx, state), source = ctx.input.page.url(), selected = selectedRuntimeInputs(ctx, "agents.existing-grok-bot"); state.initialReply = initial.reply; + let invitation = await ctx.input.api.get(`/api/companies/${initial.companyId}/grok-invitations`); + if (!invitation) invitation = await ctx.input.api.post(`/api/companies/${initial.companyId}/grok-invitations`, { name: "Existing Grok Bot", botName: selected.values.botId, role: "engineer", reportsTo: initial.leadId }); + if (invitation.agent.botName !== selected.values.botId || invitation.binding?.connected) throw new Error("The supported human setup switched the chosen existing Bot or was already authorized"); + const agentId = invitation.agent.id, route = endpoint(initial.companyId, agentId); state.grokAgentId = agentId; + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/agents/${agentId}/runtime`, { waitUntil: "domcontentloaded" }); + const section = ctx.input.page.getByRole("region", { name: "Grok Bot connection", exact: true }); await expect(section).toBeVisible(); + await section.getByRole("button", { name: "Reconnect agent access", exact: true }).click(); + await expect(section.getByRole("button", { name: "Copy connection instructions", exact: true })).toBeVisible(); + await expect(section.getByLabel("One-use pairing code", { exact: true })).toHaveAttribute("type", "password"); state.authorizationUiVisible = true; state.pairingSecretStayedPrivate = true; + const pending = await ctx.input.api.get(route); state.bindingId = pending.binding?.id; + if (!state.bindingId || pending.binding?.connected || pending.agentLifecycleState === "ready") throw new Error("Pairing instructions alone incorrectly imply authorized or ready Grok access"); + // This waits for the ACTUAL existing Bot to install MCP and its responsible + // person to complete the real browser OAuth + one-use pairing. The harness + // never registers a replacement OAuth client or impersonates the Bot. + const connected = await pollUntil({ label: "Genuine existing Bot human OAuth and one-use pairing prerequisite", deadlineAt: Date.now() + 120000, load: () => ctx.input.api.get(route), accept: row => !!row.binding && row.binding.id === state.bindingId && row.binding.connected, + reject: row => row.binding && row.binding.id !== state.bindingId ? "Existing Bot authorization changed the original pairing identity" : undefined }); + state.actualHumanPairingObserved = connected.binding?.connected === true; + const receipt = await privateBrowserEntry({ mode: "grok-routine-configure", companyId: initial.companyId, grok: { agentId, bindingId: String(state.bindingId), expectedRevision: connected.binding?.routine?.revision ?? 0, botName: selected.values.botId, webhookUrl: selected.values.endpoint }, baseURL: ctx.input.api.baseURL, setupURL: ctx.input.page.url(), providerURL: ctx.input.api.baseURL, credential: selected.reference(selected.values.credentialEnvName), cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }); + state.routineConfiguredSecurely = receipt.bindingId === state.bindingId && receipt.routineConfigured === true && receipt.routineUiVisible === true && receipt.humanSubmittedSecurely === true && receipt.storageClean === true; + const configured = await ctx.input.api.get(route); + if (configured.binding?.id !== state.bindingId || !configured.binding.connected || configured.binding.routine?.botName !== selected.values.botId || configured.binding.routine.verified || !configured.binding.routine.configured) throw new Error("The actual saved same-Bot routine configuration did not remain unverified before the real challenge"); + const priorExchangeId = configured.binding.routine.lastExchange?.id; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await section.getByRole("button", { name: "Verify connection", exact: true }).click(); + const verified = await pollUntil({ label: "Actual authenticated external Bot two-way verification", deadlineAt: Date.now() + 120000, load: () => ctx.input.api.get(route), accept: row => genuineGrokRoundTrip(row, String(state.bindingId), selected.values.botId), reject: row => row.binding?.routine?.lastExchange?.status === "rejected" || row.recoveryNeeded || row.binding?.routine?.recoveryNeeded ? "Real Bot challenge rejected or became uncertain; setup is not complete" : undefined }); + state.realRoundTrip = genuineGrokRoundTrip(verified, String(state.bindingId), selected.values.botId) && verified.binding?.routine?.lastExchange?.id !== priorExchangeId; state.sameChosenCompanyAndBot = verified.binding?.routine?.botName === selected.values.botId && verified.binding.id === state.bindingId; + (state.providerProbes as unknown[]).push({ phase: "actual-existing-bot", operation: "authenticated-routine-verify", status: verified.binding?.routine?.lastExchange?.status, usage: "unavailable", cost: "unavailable" }); + await expect(section.getByText("Routine round trip verified.", { exact: false })).toBeVisible(); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, confirmation); + const result = await pollUntil({ label: "Native CEO explains real Bot readiness and limits", deadlineAt: Date.now() + 180000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.every(row => row.status === "succeeded") }); + const run = result.runs.find(row => row.id !== initial.runs[0]!.id)!; + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = comments.filter(row => !row.authorAgentId && row.body === confirmation), wake = run.contextSnapshot?.wakeCommentIds; + state.confirmationBound = human.length === 1 && Array.isArray(wake) && wake.length === 1 && wake[0] === human[0]!.id; state.resultReply = comments.filter(row => row.authorAgentId === initial.leadId && row.createdByRunId === run.id).map(row => row.body).join("\n"); + state.worker = await ctx.input.api.get(`/api/agents/${agentId}`); + }, + async cleanup(ctx, state) { if (state.grokAgentId) await ctx.input.api.delete(endpoint(ctx.input.fixtures.company.id, String(state.grokAgentId))); }, +})];