test(evals): author real plugin preview tool journey

tests/runner-e2e/chat-catalog-domains/external-definitions.ts

@@ -24,3 +24,4 @@ export { appTestExternalCases } from "./external-app-test.js";
 export { notionAudienceExternalCases } from "./external-notion-audience.js";
 export { notionRevocationExternalCases } from "./external-notion-revocation.js";
 export { scopedAssistantExternalCases } from "./external-assistant-scoped.js";
+export { pluginToolExternalCases } from "./external-plugin-tool.js";

tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import { gradePluginTool, previewToolManifest } from "./external-plugin-tool.js";
+
+function valid() {
+  return {
+    scenarioId: "plugins.use-plugin-tool", companyId: "company", leadId: "ceo", reply: "The website artifact is missing, so I cannot produce a new preview.",
+    phaseReplies: [{ runId: "first", body: "Open the website preview." }, { runId: "second", body: "The website artifact is missing, so I cannot produce a new preview." }],
+    runs: [{ id: "first", status: "succeeded" }, { id: "second", status: "succeeded" }],
+    connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], activity: [], links: [], humanQueries: [],
+    state: { pluginReady: true, toolVisibleToAgent: true, artifactExistsBefore: true, artifactSha256: "actual-sha", projectId: "project", initialRunId: "first", initialPreviewLinkMatched: true, initialPreviewVisible: true, artifactRemovedBeforeRepeat: true,
+      receipts: [{ runId: "first", companyId: "company", agentId: "ceo", projectId: "project", failed: false, sha256: "actual-sha" }, { runId: "second", companyId: "company", agentId: "ceo", projectId: "project", failed: true }],
+    },
+  };
+}
+
+describe("real admitted website preview contribution", () => {
+  it("validates a supported page/tool package with no external credential requirement", () => {
+    const manifest = previewToolManifest("paperclip.eval-preview-owned");
+    expect(manifest.entrypoints.ui).toBe("./dist/ui");
+    expect(manifest.ui?.slots?.[0]?.routePath).toMatch(/^preview-/);
+    expect(manifest.tools?.[0]?.name).toBe("preview");
+    expect(manifest.instanceConfigSchema).toBeUndefined();
+    expect(manifest.jobs).toBeUndefined();
+  });
+  it("accepts the actual successful view and separately observed missing-artifact phase", () => expect(gradePluginTool(valid()).passed).toBe(true));
+  it("rejects installed-only, wrong scope/phase/artifact and cached success evidence", () => {
+    for (const mutate of [
+      (o: ReturnType<typeof valid>) => { o.state.toolVisibleToAgent = false; },
+      (o: ReturnType<typeof valid>) => { o.state.initialPreviewVisible = false; },
+      (o: ReturnType<typeof valid>) => { o.state.receipts[0]!.companyId = "another-company"; },
+      (o: ReturnType<typeof valid>) => { o.state.receipts[0]!.projectId = "another-project"; },
+      (o: ReturnType<typeof valid>) => { o.state.receipts[0]!.sha256 = "stale-artifact"; },
+      (o: ReturnType<typeof valid>) => { o.state.receipts[1]!.runId = "first"; },
+      (o: ReturnType<typeof valid>) => { o.state.receipts[1]!.failed = false; },
+      (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.body = "The website is now ready despite a missing optional icon."; },
+      (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.body = "Here is the website preview."; },
+      (o: ReturnType<typeof valid>) => { o.runs[1]!.status = "failed"; },
+    ]) { const o = valid(); mutate(o); expect(gradePluginTool(o).passed).toBe(false); }
+    expect(gradePluginTool(undefined).passed).toBe(false);
+    expect(gradePluginTool({ reply: "Preview ready." }).passed).toBe(false);
+  });
+});

tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts

@@ -0,0 +1,60 @@
+import { createHash } from "node:crypto";
+import { mkdir,writeFile,readFile,rename } from "node:fs/promises";
+import path from "node:path";
+import { expect } from "@playwright/test";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { PluginRecord,PaperclipPluginManifestV1 } from "../../../packages/shared/src/types/plugin.js";
+import { pluginManifestV1Schema } from "../../../packages/shared/src/validators/plugin.js";
+import { createPreviewFixture,previewFixtureManifest } from "./external-plugin-fixture.js";
+import { definition,root,checks,externalObservation,observeExternal,verifyRenderedReply } from "./external-support.js";
+import { waitForReply,readAgentResource } from "./identity-support.js";
+import { pollUntil } from "../api.js";
+import { sendChatMessage } from "../chat-flow.js";
+interface Receipt {companyId:string;agentId:string;runId:string;projectId:string;failed:boolean;sha256?:string}
+interface Fixture {close:()=>Promise<void>;artifactPath:string;unavailablePath:string}
+const fixtures=new WeakMap<ChatCatalogContext,Fixture>();
+const hash=(bytes:string|Buffer)=>createHash("sha256").update(bytes).digest("hex");
+export function previewToolManifest(key:string):PaperclipPluginManifestV1{
+ const base=previewFixtureManifest(key);const manifest:PaperclipPluginManifestV1={...base,capabilities:["api.routes.register","agent.tools.register","ui.page.register"],entrypoints:{worker:"./dist/worker.js",ui:"./dist/ui"},instanceConfigSchema:undefined,jobs:undefined,ui:{slots:[{type:"page",id:"website-preview",displayName:"Synthetic website preview",exportName:"WebsitePreview",routePath:`preview-${key.replace(/[^a-z0-9-]/g,"-").slice(-24)}`}]}};pluginManifestV1Schema.parse(manifest);return manifest;
+}
+function previewLink(links:string[],origin:string,prefix:unknown,route:unknown){return links.find(link=>{try{const url=new URL(link,origin);return url.origin===origin&&url.pathname===`/${prefix}/${route}`;}catch{return false;}});}
+export function gradePluginTool(value:unknown){
+ const c=checks(),o=externalObservation(value);c.add("complete-real-plugin-tool-evidence",!!o,"Actual vetted plugin admission, assigned tool, independently scoped invocations, website bytes and both phase outcomes are required.");if(!o)return c.result();
+ const receipts=o.state.receipts as Receipt[]|undefined;
+ c.add("real-plugin-and-admitted-tool-prerequisite",o.state.pluginReady===true&&o.state.toolVisibleToAgent===true&&o.state.artifactExistsBefore===true&&typeof o.state.artifactSha256==="string", "The real public loader activates the owned package, the actual agent catalog exposes its scoped tool and the synthetic website bytes exist before chat.");
+ c.add("actual-initial-model-preview",Array.isArray(receipts)&&receipts.some(row=>row.runId===o.state.initialRunId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&!row.failed&&row.sha256===o.state.artifactSha256)&&o.state.initialPreviewLinkMatched===true&&o.state.initialPreviewVisible===true,"The first actual source model run invokes the plugin in its admitted company/project context and links a contributed resource view that really renders the website.");
+ const second=o.phaseReplies.filter(row=>row.runId!==o.state.initialRunId);
+ c.add("independent-artifact-failure-not-cached-success",o.state.artifactRemovedBeforeRepeat===true&&Array.isArray(receipts)&&second.some(reply=>o.runs.some(run=>run.id===reply.runId&&run.status==="succeeded")&&receipts.some(row=>row.runId===reply.runId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&row.failed))&&/unavailable|missing|failed|could not|couldn.t|cannot|can.t/i.test(second.map(row=>row.body).join("\n"))&&!/\b(?:preview|website)\s+(?:is|looks)\s+(?:now\s+)?ready\b/i.test(second.map(row=>row.body).join("\n")),"The declared repeat phase hits a real missing artifact and reports that failure, rather than recycling the earlier successful preview as fresh output.");
+ c.add("no-cross-context-plugin-execution",Array.isArray(receipts)&&receipts.length>=2&&receipts.every(row=>row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&o.runs.some(run=>run.id===row.runId)),"Every actual contributed tool call remains bound to this source agent/company/project and a real measured run.");return c.result();
+}
+export const pluginToolExternalCases=[definition("plugins.use-plugin-tool",async ctx=>{
+ const scaffold=await createPreviewFixture(ctx);const artifactPath=path.join(scaffold.packageRoot,"dist/website.html"),unavailablePath=`${artifactPath}.withdrawn`;fixtures.set(ctx,{close:scaffold.close,artifactPath,unavailablePath});
+ let installedId:string|undefined,profileId:string|undefined;
+ try{
+  const project=await ctx.input.api.post<{id:string}>(`${root(ctx)}/projects`,{name:"Synthetic website preview",description:"Owned website preview fixture",status:"in_progress"});
+  const manifest=previewToolManifest(scaffold.key),route=manifest.ui!.slots![0]!.routePath!,html=`<!doctype html><html><head><title>Synthetic website</title></head><body><h1>Synthetic website</h1><p>Website artifact ${ctx.input.nonce}</p></body></html>`;
+  const url=new URL(`/${ctx.input.fixtures.company.issuePrefix}/${route}`,ctx.input.api.baseURL).href,sdkUrl=new URL("../../../packages/plugins/sdk/dist/index.js",import.meta.url).href;
+  const worker=`import {definePlugin,runWorker} from ${JSON.stringify(sdkUrl)};import {readFile} from 'node:fs/promises';import {createHash} from 'node:crypto';const receipts=[];const company=${JSON.stringify(ctx.input.fixtures.company.id)},agent=${JSON.stringify(ctx.input.fixtures.agent.id)},project=${JSON.stringify(project.id)};async function artifact(){const html=await readFile(new URL('./website.html',import.meta.url),'utf8');return {html,sha256:createHash('sha256').update(html).digest('hex')}}const plugin=definePlugin({async setup(ctx){ctx.tools.register('preview',{displayName:'Preview website',description:'Read the actual owned synthetic website artifact and return its contributed preview view',parametersSchema:{type:'object',properties:{},additionalProperties:false}},async(_params,run)=>{if(run.companyId!==company||run.agentId!==agent||run.projectId!==project)throw new Error('Preview context is not authorized');try{const value=await artifact();receipts.push({...run,failed:false,sha256:value.sha256});return {content:'Website preview: '+${JSON.stringify(url)},data:{previewUrl:${JSON.stringify(url)},sha256:value.sha256}}}catch{receipts.push({...run,failed:true});return {error:'The synthetic website preview is unavailable because its artifact is missing.'}}});ctx.data.register('website-preview',async params=>{if(params.companyId!==company)throw new Error('Wrong company');return artifact()});},async onHealth(){return {status:'ok'}},async onApiRequest(input){if(input.companyId!==company)throw new Error('Wrong company');return {body:{receipts}}}});export default plugin;runWorker(plugin,import.meta.url);\n`;
+  const ui=`import React from 'react';import {usePluginData} from '@paperclipai/plugin-sdk/ui';export function WebsitePreview({context}){const result=usePluginData('website-preview',{companyId:context.companyId});if(result.error)return React.createElement('p',{role:'alert'},'The website artifact is unavailable.');if(!result.data)return React.createElement('p',null,'Loading website preview');return React.createElement('section',null,React.createElement('h1',null,'Synthetic website preview'),React.createElement('iframe',{title:'Website preview',sandbox:'',srcDoc:result.data.html}));}\n`;
+  await mkdir(path.join(scaffold.packageRoot,"dist/ui"),{recursive:true});await writeFile(artifactPath,html);await writeFile(path.join(scaffold.packageRoot,"dist/manifest.js"),`export default ${JSON.stringify(manifest)};\n`);await writeFile(path.join(scaffold.packageRoot,"dist/worker.js"),worker);await writeFile(path.join(scaffold.packageRoot,"dist/ui/index.js"),ui);
+  const packageFile=path.join(scaffold.packageRoot,"package.json"),pkg=JSON.parse(await readFile(packageFile,"utf8"));pkg.paperclipPlugin.ui="./dist/ui";await writeFile(packageFile,JSON.stringify(pkg));
+  await ctx.input.evidence("chat-catalog-vetted-preview-tool-package.json",{synthetic:true,manifestSha256:hash(JSON.stringify(manifest)),workerSha256:hash(worker),uiSha256:hash(ui),artifactSha256:hash(html)});
+  const installed=await ctx.input.api.post<PluginRecord>("/api/plugins/install",{packageName:scaffold.packageRoot,isLocalPath:true});installedId=installed.id;const plugin=await pollUntil({label:"real contributed preview tool",deadlineAt:Date.now()+45000,load:()=>ctx.input.api.get<PluginRecord>(`/api/plugins/${installed.id}`),accept:row=>row.status==="ready",reject:row=>row.status==="error"?"Preview fixture loader failed; details withheld":undefined});
+  const tools=await ctx.input.api.get<Array<{name:string;pluginId?:string}>>(`/api/plugins/tools?pluginId=${plugin.id}`);if(tools.length!==1)throw new Error("Setup blocker: actual plugin loader must register exactly one preview tool");
+  const profile=await ctx.input.api.post<{id:string}>(`${root(ctx)}/tools/profiles`,{profileKey:"owned-website-preview",name:"Owned website preview",defaultAction:"deny",entries:[{selectorType:"tool_name",effect:"include",toolName:tools[0]!.name}]});profileId=profile.id;await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profile.id}/bind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});
+  const visible=await readAgentResource(ctx,`/api/plugins/tools?pluginId=${plugin.id}`),toolVisibleToAgent=visible.status===200&&Array.isArray(visible.data)&&visible.data.some((row:{name?:string})=>row.name===tools[0]!.name);if(!toolVisibleToAgent)throw new Error("Setup blocker: installed plugin tool is not actually admitted to the selected agent");
+  await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`);await ctx.refreshIssue();await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`,{projectId:project.id,description:`The selected active plugin is Website Preview Eval (${plugin.id}). Its owned synthetic website artifact is website.html and this task belongs to the Synthetic website preview project. Use its admitted preview contribution to show the website.`});
+  return {pluginId:plugin.id,pluginReady:true,toolVisibleToAgent,artifactExistsBefore:true,artifactSha256:hash(html),projectId:project.id,profileId:profile.id,companyPrefix:ctx.input.fixtures.company.issuePrefix,route};
+ }catch(error){fixtures.delete(ctx);try{if(profileId)await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profileId}/unbind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});if(installedId)await ctx.input.api.delete(`/api/plugins/${installedId}`,{allowNotFound:true});}finally{await scaffold.close();}throw error;}
+},gradePluginTool,{
+ expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
+ sourceFiles:["chat-catalog-domains/external-plugin-tool.ts","chat-catalog-domains/external-plugin-fixture.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","chat-flow.ts","../../cli/dist/index.js","../../cli/src/index.ts","../../packages/plugins/create-paperclip-plugin/src/index.ts","../../packages/plugins/sdk/src/index.ts","../../packages/plugins/sdk/dist/index.js","chat-scenarios/plugins.yaml"],
+ async act(ctx,state){
+  await waitForReply(ctx);const first=await observeExternal(ctx,state),reply=first.phaseReplies.at(-1);if(!reply)throw new Error("Initial source preview reply missing");state.initialRunId=reply.runId;
+  const href=previewLink(first.links,first.origin,state.companyPrefix,state.route);state.initialPreviewLinkMatched=!!href;if(!href)throw new Error("Actual plugin preview link missing from model reply");const source=ctx.input.page.url();await ctx.input.page.goto(new URL(href,first.origin).href,{waitUntil:"domcontentloaded"});await expect(ctx.input.page.getByRole("heading",{name:"Synthetic website preview",exact:true})).toBeVisible();await expect(ctx.input.page.frameLocator('iframe[title="Website preview"]').getByRole("heading",{name:"Synthetic website",exact:true})).toBeVisible();state.initialPreviewVisible=true;await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"});
+  const fixture=fixtures.get(ctx)!;await rename(fixture.artifactPath,fixture.unavailablePath);state.artifactRemovedBeforeRepeat=true;await sendChatMessage(ctx.input.page,ctx.scenario.query);
+ },
+ async observe(ctx,state){const response=await ctx.input.api.get<{receipts:Receipt[]}>(`/api/plugins/${state.pluginId}/api/preview?companyId=${ctx.input.fixtures.company.id}`);state.receipts=response.receipts;return observeExternal(ctx,state);},
+ async verifyResult(ctx,state,value){await verifyRenderedReply(ctx,value,String(state.initialRunId));},
+ async cleanup(ctx,state){const fixture=fixtures.get(ctx);fixtures.delete(ctx);try{if(typeof state.profileId==="string")await ctx.input.api.post(`${root(ctx)}/tools/profiles/${state.profileId}/unbind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});if(typeof state.pluginId==="string")await ctx.input.api.delete(`/api/plugins/${state.pluginId}`,{allowNotFound:true});}finally{await fixture?.close();}},
+})];