From 25914805ac7243d6fe005df39ecf0bad4c1ab8ed Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 22:41:06 -0500 Subject: [PATCH] test(evals): author real plugin preview tool journey --- .../external-definitions.ts | 1 + .../external-plugin-tool.test.ts | 42 +++++++++++++ .../external-plugin-tool.ts | 60 +++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts diff --git a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts index 2166ffe41d6..cfb13a498a9 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts @@ -24,3 +24,4 @@ export { appTestExternalCases } from "./external-app-test.js"; export { notionAudienceExternalCases } from "./external-notion-audience.js"; export { notionRevocationExternalCases } from "./external-notion-revocation.js"; export { scopedAssistantExternalCases } from "./external-assistant-scoped.js"; +export { pluginToolExternalCases } from "./external-plugin-tool.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts new file mode 100644 index 00000000000..83983691265 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { gradePluginTool, previewToolManifest } from "./external-plugin-tool.js"; + +function valid() { + return { + scenarioId: "plugins.use-plugin-tool", companyId: "company", leadId: "ceo", reply: "The website artifact is missing, so I cannot produce a new preview.", + phaseReplies: [{ runId: "first", body: "Open the website preview." }, { runId: "second", body: "The website artifact is missing, so I cannot produce a new preview." }], + runs: [{ id: "first", status: "succeeded" }, { id: "second", status: "succeeded" }], + connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], activity: [], links: [], humanQueries: [], + state: { pluginReady: true, toolVisibleToAgent: true, artifactExistsBefore: true, artifactSha256: "actual-sha", projectId: "project", initialRunId: "first", initialPreviewLinkMatched: true, initialPreviewVisible: true, artifactRemovedBeforeRepeat: true, + receipts: [{ runId: "first", companyId: "company", agentId: "ceo", projectId: "project", failed: false, sha256: "actual-sha" }, { runId: "second", companyId: "company", agentId: "ceo", projectId: "project", failed: true }], + }, + }; +} + +describe("real admitted website preview contribution", () => { + it("validates a supported page/tool package with no external credential requirement", () => { + const manifest = previewToolManifest("paperclip.eval-preview-owned"); + expect(manifest.entrypoints.ui).toBe("./dist/ui"); + expect(manifest.ui?.slots?.[0]?.routePath).toMatch(/^preview-/); + expect(manifest.tools?.[0]?.name).toBe("preview"); + expect(manifest.instanceConfigSchema).toBeUndefined(); + expect(manifest.jobs).toBeUndefined(); + }); + it("accepts the actual successful view and separately observed missing-artifact phase", () => expect(gradePluginTool(valid()).passed).toBe(true)); + it("rejects installed-only, wrong scope/phase/artifact and cached success evidence", () => { + for (const mutate of [ + (o: ReturnType) => { o.state.toolVisibleToAgent = false; }, + (o: ReturnType) => { o.state.initialPreviewVisible = false; }, + (o: ReturnType) => { o.state.receipts[0]!.companyId = "another-company"; }, + (o: ReturnType) => { o.state.receipts[0]!.projectId = "another-project"; }, + (o: ReturnType) => { o.state.receipts[0]!.sha256 = "stale-artifact"; }, + (o: ReturnType) => { o.state.receipts[1]!.runId = "first"; }, + (o: ReturnType) => { o.state.receipts[1]!.failed = false; }, + (o: ReturnType) => { o.phaseReplies[1]!.body = "The website is now ready despite a missing optional icon."; }, + (o: ReturnType) => { o.phaseReplies[1]!.body = "Here is the website preview."; }, + (o: ReturnType) => { o.runs[1]!.status = "failed"; }, + ]) { const o = valid(); mutate(o); expect(gradePluginTool(o).passed).toBe(false); } + expect(gradePluginTool(undefined).passed).toBe(false); + expect(gradePluginTool({ reply: "Preview ready." }).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts new file mode 100644 index 00000000000..e19122605a9 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts @@ -0,0 +1,60 @@ +import { createHash } from "node:crypto"; +import { mkdir,writeFile,readFile,rename } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { PluginRecord,PaperclipPluginManifestV1 } from "../../../packages/shared/src/types/plugin.js"; +import { pluginManifestV1Schema } from "../../../packages/shared/src/validators/plugin.js"; +import { createPreviewFixture,previewFixtureManifest } from "./external-plugin-fixture.js"; +import { definition,root,checks,externalObservation,observeExternal,verifyRenderedReply } from "./external-support.js"; +import { waitForReply,readAgentResource } from "./identity-support.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +interface Receipt {companyId:string;agentId:string;runId:string;projectId:string;failed:boolean;sha256?:string} +interface Fixture {close:()=>Promise;artifactPath:string;unavailablePath:string} +const fixtures=new WeakMap(); +const hash=(bytes:string|Buffer)=>createHash("sha256").update(bytes).digest("hex"); +export function previewToolManifest(key:string):PaperclipPluginManifestV1{ + const base=previewFixtureManifest(key);const manifest:PaperclipPluginManifestV1={...base,capabilities:["api.routes.register","agent.tools.register","ui.page.register"],entrypoints:{worker:"./dist/worker.js",ui:"./dist/ui"},instanceConfigSchema:undefined,jobs:undefined,ui:{slots:[{type:"page",id:"website-preview",displayName:"Synthetic website preview",exportName:"WebsitePreview",routePath:`preview-${key.replace(/[^a-z0-9-]/g,"-").slice(-24)}`}]}};pluginManifestV1Schema.parse(manifest);return manifest; +} +function previewLink(links:string[],origin:string,prefix:unknown,route:unknown){return links.find(link=>{try{const url=new URL(link,origin);return url.origin===origin&&url.pathname===`/${prefix}/${route}`;}catch{return false;}});} +export function gradePluginTool(value:unknown){ + const c=checks(),o=externalObservation(value);c.add("complete-real-plugin-tool-evidence",!!o,"Actual vetted plugin admission, assigned tool, independently scoped invocations, website bytes and both phase outcomes are required.");if(!o)return c.result(); + const receipts=o.state.receipts as Receipt[]|undefined; + c.add("real-plugin-and-admitted-tool-prerequisite",o.state.pluginReady===true&&o.state.toolVisibleToAgent===true&&o.state.artifactExistsBefore===true&&typeof o.state.artifactSha256==="string", "The real public loader activates the owned package, the actual agent catalog exposes its scoped tool and the synthetic website bytes exist before chat."); + c.add("actual-initial-model-preview",Array.isArray(receipts)&&receipts.some(row=>row.runId===o.state.initialRunId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&!row.failed&&row.sha256===o.state.artifactSha256)&&o.state.initialPreviewLinkMatched===true&&o.state.initialPreviewVisible===true,"The first actual source model run invokes the plugin in its admitted company/project context and links a contributed resource view that really renders the website."); + const second=o.phaseReplies.filter(row=>row.runId!==o.state.initialRunId); + c.add("independent-artifact-failure-not-cached-success",o.state.artifactRemovedBeforeRepeat===true&&Array.isArray(receipts)&&second.some(reply=>o.runs.some(run=>run.id===reply.runId&&run.status==="succeeded")&&receipts.some(row=>row.runId===reply.runId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&row.failed))&&/unavailable|missing|failed|could not|couldn.t|cannot|can.t/i.test(second.map(row=>row.body).join("\n"))&&!/\b(?:preview|website)\s+(?:is|looks)\s+(?:now\s+)?ready\b/i.test(second.map(row=>row.body).join("\n")),"The declared repeat phase hits a real missing artifact and reports that failure, rather than recycling the earlier successful preview as fresh output."); + c.add("no-cross-context-plugin-execution",Array.isArray(receipts)&&receipts.length>=2&&receipts.every(row=>row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&o.runs.some(run=>run.id===row.runId)),"Every actual contributed tool call remains bound to this source agent/company/project and a real measured run.");return c.result(); +} +export const pluginToolExternalCases=[definition("plugins.use-plugin-tool",async ctx=>{ + const scaffold=await createPreviewFixture(ctx);const artifactPath=path.join(scaffold.packageRoot,"dist/website.html"),unavailablePath=`${artifactPath}.withdrawn`;fixtures.set(ctx,{close:scaffold.close,artifactPath,unavailablePath}); + let installedId:string|undefined,profileId:string|undefined; + try{ + const project=await ctx.input.api.post<{id:string}>(`${root(ctx)}/projects`,{name:"Synthetic website preview",description:"Owned website preview fixture",status:"in_progress"}); + const manifest=previewToolManifest(scaffold.key),route=manifest.ui!.slots![0]!.routePath!,html=`Synthetic website

Synthetic website

Website artifact ${ctx.input.nonce}

`; + const url=new URL(`/${ctx.input.fixtures.company.issuePrefix}/${route}`,ctx.input.api.baseURL).href,sdkUrl=new URL("../../../packages/plugins/sdk/dist/index.js",import.meta.url).href; + const worker=`import {definePlugin,runWorker} from ${JSON.stringify(sdkUrl)};import {readFile} from 'node:fs/promises';import {createHash} from 'node:crypto';const receipts=[];const company=${JSON.stringify(ctx.input.fixtures.company.id)},agent=${JSON.stringify(ctx.input.fixtures.agent.id)},project=${JSON.stringify(project.id)};async function artifact(){const html=await readFile(new URL('./website.html',import.meta.url),'utf8');return {html,sha256:createHash('sha256').update(html).digest('hex')}}const plugin=definePlugin({async setup(ctx){ctx.tools.register('preview',{displayName:'Preview website',description:'Read the actual owned synthetic website artifact and return its contributed preview view',parametersSchema:{type:'object',properties:{},additionalProperties:false}},async(_params,run)=>{if(run.companyId!==company||run.agentId!==agent||run.projectId!==project)throw new Error('Preview context is not authorized');try{const value=await artifact();receipts.push({...run,failed:false,sha256:value.sha256});return {content:'Website preview: '+${JSON.stringify(url)},data:{previewUrl:${JSON.stringify(url)},sha256:value.sha256}}}catch{receipts.push({...run,failed:true});return {error:'The synthetic website preview is unavailable because its artifact is missing.'}}});ctx.data.register('website-preview',async params=>{if(params.companyId!==company)throw new Error('Wrong company');return artifact()});},async onHealth(){return {status:'ok'}},async onApiRequest(input){if(input.companyId!==company)throw new Error('Wrong company');return {body:{receipts}}}});export default plugin;runWorker(plugin,import.meta.url);\n`; + const ui=`import React from 'react';import {usePluginData} from '@paperclipai/plugin-sdk/ui';export function WebsitePreview({context}){const result=usePluginData('website-preview',{companyId:context.companyId});if(result.error)return React.createElement('p',{role:'alert'},'The website artifact is unavailable.');if(!result.data)return React.createElement('p',null,'Loading website preview');return React.createElement('section',null,React.createElement('h1',null,'Synthetic website preview'),React.createElement('iframe',{title:'Website preview',sandbox:'',srcDoc:result.data.html}));}\n`; + await mkdir(path.join(scaffold.packageRoot,"dist/ui"),{recursive:true});await writeFile(artifactPath,html);await writeFile(path.join(scaffold.packageRoot,"dist/manifest.js"),`export default ${JSON.stringify(manifest)};\n`);await writeFile(path.join(scaffold.packageRoot,"dist/worker.js"),worker);await writeFile(path.join(scaffold.packageRoot,"dist/ui/index.js"),ui); + const packageFile=path.join(scaffold.packageRoot,"package.json"),pkg=JSON.parse(await readFile(packageFile,"utf8"));pkg.paperclipPlugin.ui="./dist/ui";await writeFile(packageFile,JSON.stringify(pkg)); + await ctx.input.evidence("chat-catalog-vetted-preview-tool-package.json",{synthetic:true,manifestSha256:hash(JSON.stringify(manifest)),workerSha256:hash(worker),uiSha256:hash(ui),artifactSha256:hash(html)}); + const installed=await ctx.input.api.post("/api/plugins/install",{packageName:scaffold.packageRoot,isLocalPath:true});installedId=installed.id;const plugin=await pollUntil({label:"real contributed preview tool",deadlineAt:Date.now()+45000,load:()=>ctx.input.api.get(`/api/plugins/${installed.id}`),accept:row=>row.status==="ready",reject:row=>row.status==="error"?"Preview fixture loader failed; details withheld":undefined}); + const tools=await ctx.input.api.get>(`/api/plugins/tools?pluginId=${plugin.id}`);if(tools.length!==1)throw new Error("Setup blocker: actual plugin loader must register exactly one preview tool"); + const profile=await ctx.input.api.post<{id:string}>(`${root(ctx)}/tools/profiles`,{profileKey:"owned-website-preview",name:"Owned website preview",defaultAction:"deny",entries:[{selectorType:"tool_name",effect:"include",toolName:tools[0]!.name}]});profileId=profile.id;await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profile.id}/bind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id}); + const visible=await readAgentResource(ctx,`/api/plugins/tools?pluginId=${plugin.id}`),toolVisibleToAgent=visible.status===200&&Array.isArray(visible.data)&&visible.data.some((row:{name?:string})=>row.name===tools[0]!.name);if(!toolVisibleToAgent)throw new Error("Setup blocker: installed plugin tool is not actually admitted to the selected agent"); + await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`);await ctx.refreshIssue();await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`,{projectId:project.id,description:`The selected active plugin is Website Preview Eval (${plugin.id}). Its owned synthetic website artifact is website.html and this task belongs to the Synthetic website preview project. Use its admitted preview contribution to show the website.`}); + return {pluginId:plugin.id,pluginReady:true,toolVisibleToAgent,artifactExistsBefore:true,artifactSha256:hash(html),projectId:project.id,profileId:profile.id,companyPrefix:ctx.input.fixtures.company.issuePrefix,route}; + }catch(error){fixtures.delete(ctx);try{if(profileId)await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profileId}/unbind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});if(installedId)await ctx.input.api.delete(`/api/plugins/${installedId}`,{allowNotFound:true});}finally{await scaffold.close();}throw error;} +},gradePluginTool,{ + expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, + sourceFiles:["chat-catalog-domains/external-plugin-tool.ts","chat-catalog-domains/external-plugin-fixture.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","chat-flow.ts","../../cli/dist/index.js","../../cli/src/index.ts","../../packages/plugins/create-paperclip-plugin/src/index.ts","../../packages/plugins/sdk/src/index.ts","../../packages/plugins/sdk/dist/index.js","chat-scenarios/plugins.yaml"], + async act(ctx,state){ + await waitForReply(ctx);const first=await observeExternal(ctx,state),reply=first.phaseReplies.at(-1);if(!reply)throw new Error("Initial source preview reply missing");state.initialRunId=reply.runId; + const href=previewLink(first.links,first.origin,state.companyPrefix,state.route);state.initialPreviewLinkMatched=!!href;if(!href)throw new Error("Actual plugin preview link missing from model reply");const source=ctx.input.page.url();await ctx.input.page.goto(new URL(href,first.origin).href,{waitUntil:"domcontentloaded"});await expect(ctx.input.page.getByRole("heading",{name:"Synthetic website preview",exact:true})).toBeVisible();await expect(ctx.input.page.frameLocator('iframe[title="Website preview"]').getByRole("heading",{name:"Synthetic website",exact:true})).toBeVisible();state.initialPreviewVisible=true;await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"}); + const fixture=fixtures.get(ctx)!;await rename(fixture.artifactPath,fixture.unavailablePath);state.artifactRemovedBeforeRepeat=true;await sendChatMessage(ctx.input.page,ctx.scenario.query); + }, + async observe(ctx,state){const response=await ctx.input.api.get<{receipts:Receipt[]}>(`/api/plugins/${state.pluginId}/api/preview?companyId=${ctx.input.fixtures.company.id}`);state.receipts=response.receipts;return observeExternal(ctx,state);}, + async verifyResult(ctx,state,value){await verifyRenderedReply(ctx,value,String(state.initialRunId));}, + async cleanup(ctx,state){const fixture=fixtures.get(ctx);fixtures.delete(ctx);try{if(typeof state.profileId==="string")await ctx.input.api.post(`${root(ctx)}/tools/profiles/${state.profileId}/unbind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});if(typeof state.pluginId==="string")await ctx.input.api.delete(`/api/plugins/${state.pluginId}`,{allowNotFound:true});}finally{await fixture?.close();}}, +})];