From 010e5c51a7ea90751de4526c17a44f8b26759d9e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 19:48:23 -0500 Subject: [PATCH] test(evals): verify operator synthetic plugin installation --- .../external-definitions.ts | 1 + .../external-plugin-install.ts | 69 +++++++++++++++++++ .../chat-catalog-domains/external.test.ts | 10 ++- 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-plugin-install.ts diff --git a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts index e34de3e36df..e43fc7268a3 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts @@ -11,3 +11,4 @@ export { appCatalogExternalCases } from "./external-app-catalog.js"; export { pluginExternalCases } from "./external-plugins.js"; export { assistantDeviceExternalCases } from "./external-assistant-device.js"; export { gatewayClientExternalCases } from "./external-gateway-client.js"; +export { pluginInstallExternalCases } from "./external-plugin-install.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-install.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-install.ts new file mode 100644 index 00000000000..6fb12b8577c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-install.ts @@ -0,0 +1,69 @@ +import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { PluginRecord } from "../../../packages/shared/src/types/plugin.js"; +import { pollUntil } from "../api.js"; +import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; +import { createPreviewFixture } from "./external-plugin-fixture.js"; +import { probeAgentPermission, waitForReply } from "./identity-support.js"; +const fixtures = new WeakMap>>(); +export function gradePluginInstall(value: unknown) { + const c = checks(), o = externalObservation(value); + c.add("complete-plugin-install-evidence", !!o, "Vetted package, administrator admission, real activation and visible installed state are required."); if (!o) return c.result(); + const plugin = o.state.pluginAfter as PluginRecord | undefined; + c.add("exact-vetted-package-installed", o.state.syntheticPackage === true && plugin?.id === o.state.pluginId && plugin?.pluginKey === o.state.pluginKey + && plugin?.status === "ready" && plugin?.manifestJson?.displayName === "Website Preview Eval", "The exact CLI-scaffolded package is loaded and activated by the real product loader."); + c.add("administrator-installation-only", o.state.operatorChannel === "board-api-local-path" && o.state.agentInstallStatus === 403 && o.state.absentBeforeHuman === true && o.state.humanInstallStatus === 200 + && !!plugin && o.activity.some(row => row.entityId === plugin.id && row.actorType === "user" && row.action === "plugin.installed"), + "The ordinary agent cannot install the package; the authenticated operator management request has independent mutation attribution."); + c.add("actual-contribution-and-visible-state", o.state.previewReadback === true && o.state.humanSawActivation === true, + "The human sees the installed activation state and the real company-scoped contributed route answers its fixture marker."); + c.add("truthful-installation-handoff", /Website Preview Eval|@paperclip-eval\/website-preview/i.test(o.reply) && /admin|operator|instance.*(?:setting|plugin)/i.test(o.reply) + && /install/i.test(o.reply) && !/\bI (?:have )?(?:installed|activated)\b|\b(?:is|now|already) (?:installed|activated|ready)\b/i.test(o.reply), + "The model identifies the exact package and administrator installation path without claiming the later human action already occurred."); + return c.result(); +} +export const pluginInstallExternalCases = [definition("plugins.install-plugin", async ctx => { + const fixture = await createPreviewFixture(ctx); fixtures.set(ctx, fixture); + try { + const installed = await ctx.input.api.get("/api/plugins"); + if (installed.some(row => row.pluginKey === fixture.key)) throw new Error("Disposable plugin already installed before its installation journey"); + const agentInstallStatus = await probeAgentPermission(ctx, "/api/plugins/install", "POST", { packageName: fixture.packageRoot, isLocalPath: true }); + await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue(); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The selected vetted synthetic package is Website Preview Eval (@paperclip-eval/website-preview), key ${fixture.key}, available as a local package at ${fixture.packageRoot}. It is not installed. Local-path installation uses the administrator plugin-management API. No external package needs downloading.` }); + return { pluginKey: fixture.key, syntheticPackage: true, agentInstallStatus }; + } catch (error) { fixtures.delete(ctx); await fixture.close(); throw error; } +}, gradePluginInstall, { + sourceFiles: ["chat-catalog-domains/external-plugin-install.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../cli/dist/index.js", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/plugins.yaml"], + async act(ctx, state) { + await waitForReply(ctx); + const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Vetted package fixture is missing"); + const installed = await ctx.input.api.get("/api/plugins"); state.absentBeforeHuman = !installed.some(row => row.pluginKey === fixture.key); + if (!state.absentBeforeHuman) throw new Error("Plugin installed before the authorized human management step"); + // The visible npm dialog does not accept localPath; the public board API is + // the supported local-package installation path, under the same human session. + const response = await ctx.input.api.request.post("/api/plugins/install", { data: { packageName: fixture.packageRoot, isLocalPath: true } }); + state.operatorChannel = "board-api-local-path"; state.humanInstallStatus = response.status(); if (!response.ok()) throw new Error("Operator package installation failed; error details withheld"); + const plugin = await response.json() as PluginRecord; state.pluginId = plugin.id; + await pollUntil({ label: "operator-installed synthetic plugin activation", deadlineAt: Date.now() + 45_000, + load: () => ctx.input.api.get(`/api/plugins/${plugin.id}`), accept: row => row.status === "ready", reject: row => row.status === "error" ? "Synthetic plugin activation failed; inspect public plugin status" : undefined }); + const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/instance/plugins`, { waitUntil: "domcontentloaded" }); + const row = ctx.input.page.locator("li").filter({ hasText: "Website Preview Eval" }); + await expect(row.getByRole("link", { name: "Website Preview Eval", exact: true })).toBeVisible(); await expect(row.getByRole("button", { name: "Disable", exact: true })).toBeVisible(); state.humanSawActivation = true; + await ctx.input.evidence("chat-catalog-preview-plugin-install.json", { operatorChannel: state.operatorChannel, pluginId: plugin.id, pluginKey: fixture.key, status: "ready", visibleManagerState: "Disable" }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { + const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Vetted package fixture is missing"); + state.pluginAfter = await ctx.input.api.get(`/api/plugins/${state.pluginId}`); + const route = await ctx.input.api.request.get(`/api/plugins/${state.pluginId}/api/preview?companyId=${ctx.input.fixtures.company.id}`); + const result = route.ok() ? await route.json() as { companyId?: string; marker?: string } : undefined; + state.previewReadback = route.ok() && result?.companyId === ctx.input.fixtures.company.id && result?.marker === fixture.marker; + return observeExternal(ctx, state); + }, + async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); }, + async cleanup(ctx, state) { + const fixture = fixtures.get(ctx); fixtures.delete(ctx); + try { if (typeof state.pluginId === "string") await ctx.input.api.delete(`/api/plugins/${state.pluginId}`, { allowNotFound: true }); } + finally { await fixture?.close(); } + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/external.test.ts b/tests/runner-e2e/chat-catalog-domains/external.test.ts index 6b85dd1cdc2..e2a10eca4ce 100644 --- a/tests/runner-e2e/chat-catalog-domains/external.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { assistantConnectionExternalCases } from "./external-assistant-connections.js"; import { chatChannelExternalCases } from "./external-chat-channels.js"; import { gatewayClientExternalCases } from "./external-gateway-client.js"; +import { pluginInstallExternalCases } from "./external-plugin-install.js"; import { assistantDeviceExternalCases } from "./external-assistant-device.js"; import { pluginExternalCases } from "./external-plugins.js"; import { appCatalogExternalCases } from "./external-app-catalog.js"; @@ -13,7 +14,7 @@ import { connectionSetupExternalCases } from "./external-connection-setup.js"; import { externalPrerequisiteReceipt, externalRequirements, validateExternalConfig, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js"; import type { ExternalObservation } from "./external-support.js"; -const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases]; +const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases]; function base(scenarioId: string): ExternalObservation { return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { initialRunId: "initial" }, phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], @@ -21,7 +22,11 @@ function base(scenarioId: string): ExternalObservation { } function valid(id: string) { const o = base(id); - if (id === "gateways-profiles.connect-client") { + if (id === "plugins.install-plugin") { + Object.assign(o.state, { pluginId: "plugin", pluginKey: "synthetic.preview", syntheticPackage: true, pluginAfter: { id: "plugin", pluginKey: "synthetic.preview", status: "ready", manifestJson: { displayName: "Website Preview Eval" } }, operatorChannel: "board-api-local-path", agentInstallStatus: 403, absentBeforeHuman: true, humanInstallStatus: 200, previewReadback: true, humanSawActivation: true }); + o.reply = "An instance administrator can install Website Preview Eval through plugin management."; + o.activity = [{ id: "activity", entityId: "plugin", actorType: "user", action: "plugin.installed" }]; + } else if (id === "gateways-profiles.connect-client") { Object.assign(o.state, { profileId: "profile", readEntryId: "read", writeEntryId: "write", connectionId: "connection", sourceIssueId: "source", probeRunId: "initial", effective: { agentId: "ceo", profiles: [{ id: "profile" }], allowedTools: [{ id: "read" }] }, sessionCreated: true, discoveredRead: true, discoveredWrite: false, decisions: [{ runId: "initial", decisions: [{ invocation: { runId: "initial", agentId: "ceo", issueId: "source", connectionId: "connection", catalogEntryId: "read", status: "succeeded" } }] }], @@ -85,6 +90,7 @@ function valid(id: string) { return o; } const wrong: Record void> = { + "plugins.install-plugin": o => { o.state.pluginAfter = { id: "plugin", pluginKey: "synthetic.preview", status: "error" }; }, "gateways-profiles.connect-client": o => { o.state.providerWriteCalls = 1; }, "assistant-connections.device-login": o => { o.state.elapsedBeforeExpiryProbe = 500; }, "plugins.configure-plugin": o => { o.state.configuredReadback = false; },