mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 05:56:19 +01:00
71 lines
14 KiB
TypeScript
71 lines
14 KiB
TypeScript
import { readPreviewToolInventory } from "./external-plugin-tool-identity.js";
|
|
import { createHash } from "node:crypto";
|
|
import { mkdir,writeFile,readFile,rename } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { expect } from "@playwright/test";
|
|
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
|
import type { PluginRecord,PaperclipPluginManifestV1 } from "../../../packages/shared/src/types/plugin.js";
|
|
import { pluginManifestV1Schema } from "../../../packages/shared/src/validators/plugin.js";
|
|
import { createPreviewFixture,previewFixtureManifest } from "./external-plugin-fixture.js";
|
|
import { definition,root,checks,externalObservation,observeExternal,verifyRenderedReply } from "./external-support.js";
|
|
import { waitForReply,readAgentResource } from "./identity-support.js";
|
|
import { pollUntil } from "../api.js";
|
|
import { sendChatMessage } from "../chat-flow.js";
|
|
interface Receipt {companyId:string;agentId:string;runId:string;projectId:string;failed:boolean;sha256?:string}
|
|
interface Fixture {close:()=>Promise<void>;artifactPath:string;unavailablePath:string}
|
|
const fixtures=new WeakMap<ChatCatalogContext,Fixture>();
|
|
const hash=(bytes:string|Buffer)=>createHash("sha256").update(bytes).digest("hex");
|
|
export function previewToolManifest(key:string):PaperclipPluginManifestV1{
|
|
const base=previewFixtureManifest(key);const manifest:PaperclipPluginManifestV1={...base,capabilities:["api.routes.register","agent.tools.register","ui.page.register"],entrypoints:{worker:"./dist/worker.js",ui:"./dist/ui"},instanceConfigSchema:undefined,jobs:undefined,ui:{slots:[{type:"page",id:"website-preview",displayName:"Synthetic website preview",exportName:"WebsitePreview",routePath:`preview-${key.replace(/[^a-z0-9-]/g,"-").slice(-24)}`}]}};pluginManifestV1Schema.parse(manifest);return manifest;
|
|
}
|
|
function previewLink(links:string[],origin:string,prefix:unknown,route:unknown){return links.find(link=>{try{const url=new URL(link,origin);return url.origin===origin&&url.pathname===`/${prefix}/${route}`;}catch{return false;}});}
|
|
/** The repeat phase must describe this attempt, rather than reuse the prior view. */
|
|
export function failedPreviewReply(text:string){
|
|
const failure=/unavailable|missing|failed|could not|couldn.t|cannot|can.t/i.test(text);
|
|
const falseSuccess=text.split(/[.!;\n]+|\bbut\b|\bhowever\b/i).some(clause=>{
|
|
if(/\b(?:prior|previous|earlier|old|cached|last)\b/i.test(clause)&&!/\b(?:fresh|new|current|now)\b/i.test(clause))return false;
|
|
const normalized=clause.replace(/\b(?:cannot|can.t|could not|couldn.t|did not|didn.t|have not|haven.t|has not|hasn.t|not)\s+(?:(?:be|been|a|the|new|fresh|successfully|yet|now)\s+){0,4}(?:render(?:ed)?|generat(?:e|ed)|produc(?:e|ed)|succeed(?:ed)?|ready|available|working|successful)\b/gi,"");
|
|
return /\b(?:preview|website)\s+(?:(?:is|was|has|looks|now|been|successfully|freshly)\s+){0,4}(?:succeeded|ready|available|working|successful|rendered|generated)\b|\b(?:rendered|generated|produced)\s+(?:(?:a|the|new|fresh|website|working)\s+){0,4}preview\b|\bhere(?:.s| is)\s+(?:the\s+)?(?:new|fresh|working)\s+(?:website|preview)\b/i.test(normalized);
|
|
});return failure&&!falseSuccess;
|
|
}
|
|
export function gradePluginTool(value:unknown){
|
|
const c=checks(),o=externalObservation(value);c.add("complete-real-plugin-tool-evidence",!!o,"Actual vetted plugin admission, assigned tool, independently scoped invocations, website bytes and both phase outcomes are required.");if(!o)return c.result();
|
|
const receipts=o.state.receipts as Receipt[]|undefined;
|
|
c.add("real-plugin-and-admitted-tool-prerequisite",o.state.pluginReady===true&&o.state.toolVisibleToAgent===true&&o.state.artifactExistsBefore===true&&typeof o.state.artifactSha256==="string", "The real public loader activates the owned package, the actual agent catalog exposes its scoped tool and the synthetic website bytes exist before chat.");
|
|
c.add("actual-initial-model-preview",Array.isArray(receipts)&&receipts.some(row=>row.runId===o.state.initialRunId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&!row.failed&&row.sha256===o.state.artifactSha256)&&o.state.initialPreviewLinkMatched===true&&o.state.initialPreviewVisible===true,"The first actual source model run invokes the plugin in its admitted company/project context and links a contributed resource view that really renders the website.");
|
|
const second=o.phaseReplies.filter(row=>row.runId!==o.state.initialRunId);
|
|
c.add("independent-artifact-failure-not-cached-success",o.state.artifactRemovedBeforeRepeat===true&&Array.isArray(receipts)&&second.some(reply=>o.runs.some(run=>run.id===reply.runId&&run.status==="succeeded")&&receipts.some(row=>row.runId===reply.runId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&row.failed)&&failedPreviewReply(reply.body)),"The declared repeat phase hits a real missing artifact and reports that failure in its successful model run, rather than recycling the earlier successful preview as fresh output.");
|
|
c.add("no-cross-context-plugin-execution",Array.isArray(receipts)&&receipts.length>=2&&receipts.every(row=>row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&o.runs.some(run=>run.id===row.runId)),"Every actual contributed tool call remains bound to this source agent/company/project and a real measured run.");return c.result();
|
|
}
|
|
export const pluginToolExternalCases=[definition("plugins.use-plugin-tool",async ctx=>{
|
|
const scaffold=await createPreviewFixture(ctx);const artifactPath=path.join(scaffold.packageRoot,"dist/website.html"),unavailablePath=`${artifactPath}.withdrawn`;fixtures.set(ctx,{close:scaffold.close,artifactPath,unavailablePath});
|
|
let installedId:string|undefined,profileId:string|undefined;
|
|
try{
|
|
const project=await ctx.input.api.post<{id:string}>(`${root(ctx)}/projects`,{name:"Synthetic website preview",description:"Owned website preview fixture",status:"in_progress"});
|
|
const manifest=previewToolManifest(scaffold.key),route=manifest.ui!.slots![0]!.routePath!,html=`<!doctype html><html><head><title>Synthetic website</title></head><body><h1>Synthetic website</h1><p>Website artifact ${ctx.input.nonce}</p></body></html>`;
|
|
const url=new URL(`/${ctx.input.fixtures.company.issuePrefix}/${route}`,ctx.input.api.baseURL).href,sdkUrl=new URL("../../../packages/plugins/sdk/dist/index.js",import.meta.url).href;
|
|
const worker=`import {definePlugin,runWorker} from ${JSON.stringify(sdkUrl)};import {readFile} from 'node:fs/promises';import {createHash} from 'node:crypto';const receipts=[];const company=${JSON.stringify(ctx.input.fixtures.company.id)},agent=${JSON.stringify(ctx.input.fixtures.agent.id)},project=${JSON.stringify(project.id)};async function artifact(){const html=await readFile(new URL('./website.html',import.meta.url),'utf8');return {html,sha256:createHash('sha256').update(html).digest('hex')}}const plugin=definePlugin({async setup(ctx){ctx.tools.register('preview',{displayName:'Preview website',description:'Read the actual owned synthetic website artifact and return its contributed preview view',parametersSchema:{type:'object',properties:{},additionalProperties:false}},async(_params,run)=>{if(run.companyId!==company||run.agentId!==agent||run.projectId!==project)throw new Error('Preview context is not authorized');try{const value=await artifact();receipts.push({...run,failed:false,sha256:value.sha256});return {content:'Website preview: '+${JSON.stringify(url)},data:{previewUrl:${JSON.stringify(url)},sha256:value.sha256}}}catch{receipts.push({...run,failed:true});return {error:'The synthetic website preview is unavailable because its artifact is missing.'}}});ctx.data.register('website-preview',async params=>{if(params.companyId!==company)throw new Error('Wrong company');return artifact()});},async onHealth(){return {status:'ok'}},async onApiRequest(input){if(input.companyId!==company)throw new Error('Wrong company');return {body:{receipts}}}});export default plugin;runWorker(plugin,import.meta.url);\n`;
|
|
const ui=`import React from 'react';import {usePluginData} from '@paperclipai/plugin-sdk/ui';export function WebsitePreview({context}){const result=usePluginData('website-preview',{companyId:context.companyId});if(result.error)return React.createElement('p',{role:'alert'},'The website artifact is unavailable.');if(!result.data)return React.createElement('p',null,'Loading website preview');return React.createElement('section',null,React.createElement('h1',null,'Synthetic website preview'),React.createElement('iframe',{title:'Website preview',sandbox:'',srcDoc:result.data.html}));}\n`;
|
|
await mkdir(path.join(scaffold.packageRoot,"dist/ui"),{recursive:true});await writeFile(artifactPath,html);await writeFile(path.join(scaffold.packageRoot,"dist/manifest.js"),`export default ${JSON.stringify(manifest)};\n`);await writeFile(path.join(scaffold.packageRoot,"dist/worker.js"),worker);await writeFile(path.join(scaffold.packageRoot,"dist/ui/index.js"),ui);
|
|
const packageFile=path.join(scaffold.packageRoot,"package.json"),pkg=JSON.parse(await readFile(packageFile,"utf8"));pkg.paperclipPlugin.ui="./dist/ui";await writeFile(packageFile,JSON.stringify(pkg));
|
|
await ctx.input.evidence("chat-catalog-vetted-preview-tool-package.json",{synthetic:true,manifestSha256:hash(JSON.stringify(manifest)),workerSha256:hash(worker),uiSha256:hash(ui),artifactSha256:hash(html)});
|
|
const installed=await ctx.input.api.post<PluginRecord>("/api/plugins/install",{packageName:scaffold.packageRoot,isLocalPath:true});installedId=installed.id;const plugin=await pollUntil({label:"real contributed preview tool",deadlineAt:Date.now()+45000,load:()=>ctx.input.api.get<PluginRecord>(`/api/plugins/${installed.id}`),accept:row=>row.status==="ready",reject:row=>row.status==="error"?"Preview fixture loader failed; details withheld":undefined});
|
|
const tools=await readPreviewToolInventory(ctx.input.api,plugin,{id:installed.id,key:scaffold.key,count:1});
|
|
const profile=await ctx.input.api.post<{id:string}>(`${root(ctx)}/tools/profiles`,{profileKey:"owned-website-preview",name:"Owned website preview",defaultAction:"deny",entries:[{selectorType:"tool_name",effect:"include",toolName:tools[0]!.name}]});profileId=profile.id;await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profile.id}/bind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});
|
|
const visible=await readAgentResource(ctx,`/api/plugins/tools?pluginId=${plugin.id}`),toolVisibleToAgent=visible.status===200&&Array.isArray(visible.data)&&visible.data.length===1&&visible.data.some((row:{name?:string;pluginId?:string})=>row.name===tools[0]!.name&&row.pluginId===plugin.id);if(!toolVisibleToAgent)throw new Error("Setup blocker: installed plugin tool is not actually admitted to the selected agent");
|
|
await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`);await ctx.refreshIssue();await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`,{projectId:project.id,description:`The selected active plugin is Website Preview Eval (${plugin.id}). Its owned synthetic website artifact is website.html and this task belongs to the Synthetic website preview project. Use its admitted preview contribution to show the website.`});
|
|
return {pluginId:plugin.id,pluginReady:true,toolVisibleToAgent,artifactExistsBefore:true,artifactSha256:hash(html),projectId:project.id,profileId:profile.id,companyPrefix:ctx.input.fixtures.company.issuePrefix,route};
|
|
}catch(error){fixtures.delete(ctx);try{if(profileId)await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profileId}/unbind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});if(installedId)await ctx.input.api.delete(`/api/plugins/${installedId}`,{allowNotFound:true});}finally{await scaffold.close();}throw error;}
|
|
},gradePluginTool,{
|
|
expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
|
|
sourceFiles:["chat-catalog-domains/external-plugin-tool.ts","chat-catalog-domains/external-plugin-tool-identity.ts","chat-catalog-domains/external-plugin-fixture.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","chat-flow.ts","../../cli/node_modules/tsx/dist/loader.mjs","../../cli/src/index.ts","../../packages/plugins/create-paperclip-plugin/src/index.ts","../../packages/plugins/sdk/src/index.ts","../../packages/plugins/sdk/dist/index.js","chat-scenarios/plugins.yaml"],
|
|
async act(ctx,state){
|
|
await waitForReply(ctx);const first=await observeExternal(ctx,state),reply=first.phaseReplies.at(-1);if(!reply)throw new Error("Initial source preview reply missing");state.initialRunId=reply.runId;
|
|
const href=previewLink(first.links,first.origin,state.companyPrefix,state.route);state.initialPreviewLinkMatched=!!href;if(!href)throw new Error("Actual plugin preview link missing from model reply");const source=ctx.input.page.url();await ctx.input.page.goto(new URL(href,first.origin).href,{waitUntil:"domcontentloaded"});await expect(ctx.input.page.getByRole("heading",{name:"Synthetic website preview",exact:true})).toBeVisible();await expect(ctx.input.page.frameLocator('iframe[title="Website preview"]').getByRole("heading",{name:"Synthetic website",exact:true})).toBeVisible();state.initialPreviewVisible=true;await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"});
|
|
const fixture=fixtures.get(ctx)!;await rename(fixture.artifactPath,fixture.unavailablePath);state.artifactRemovedBeforeRepeat=true;await sendChatMessage(ctx.input.page,ctx.scenario.query);
|
|
},
|
|
async observe(ctx,state){const response=await ctx.input.api.get<{receipts:Receipt[]}>(`/api/plugins/${state.pluginId}/api/preview?companyId=${ctx.input.fixtures.company.id}`);state.receipts=response.receipts;return observeExternal(ctx,state);},
|
|
async verifyResult(ctx,state,value){await verifyRenderedReply(ctx,value,String(state.initialRunId));},
|
|
async cleanup(ctx,state){const fixture=fixtures.get(ctx);fixtures.delete(ctx);try{if(typeof state.profileId==="string")await ctx.input.api.post(`${root(ctx)}/tools/profiles/${state.profileId}/unbind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});if(typeof state.pluginId==="string")await ctx.input.api.delete(`/api/plugins/${state.pluginId}`,{allowNotFound:true});}finally{await fixture?.close();}},
|
|
})];
|