From 509fcf35144c0d659fa399d8a3bdf1788a25de2b Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 22:44:00 -0500 Subject: [PATCH] test(evals): reject contradictory fresh preview success claims --- .../chat-catalog-domains/external-plugin-tool.test.ts | 9 ++++++++- .../chat-catalog-domains/external-plugin-tool.ts | 11 ++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts index 83983691265..01e7fd1af56 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { gradePluginTool, previewToolManifest } from "./external-plugin-tool.js"; +import { failedPreviewReply, gradePluginTool, previewToolManifest } from "./external-plugin-tool.js"; function valid() { return { @@ -23,6 +23,12 @@ describe("real admitted website preview contribution", () => { expect(manifest.jobs).toBeUndefined(); }); it("accepts the actual successful view and separately observed missing-artifact phase", () => expect(gradePluginTool(valid()).passed).toBe(true)); + it("distinguishes a prior preview from a contradictory fresh success claim", () => { + const o=valid();o.phaseReplies[1]!.body="The earlier preview succeeded, but the artifact is now unavailable, so I cannot render a fresh preview."; + expect(gradePluginTool(o).passed).toBe(true); + for(const text of ["Preview succeeded. The artifact is unavailable.","I rendered a fresh preview, but the artifact is unavailable.","Here is the working preview. One artifact is missing."])expect(failedPreviewReply(text)).toBe(false); + expect(failedPreviewReply("The website is not ready; the artifact is missing.")).toBe(true); + }); it("rejects installed-only, wrong scope/phase/artifact and cached success evidence", () => { for (const mutate of [ (o: ReturnType) => { o.state.toolVisibleToAgent = false; }, @@ -34,6 +40,7 @@ describe("real admitted website preview contribution", () => { (o: ReturnType) => { o.state.receipts[1]!.failed = false; }, (o: ReturnType) => { o.phaseReplies[1]!.body = "The website is now ready despite a missing optional icon."; }, (o: ReturnType) => { o.phaseReplies[1]!.body = "Here is the website preview."; }, + (o: ReturnType) => { o.phaseReplies[1]!.body = "Preview succeeded. The artifact is unavailable."; }, (o: ReturnType) => { o.runs[1]!.status = "failed"; }, ]) { const o = valid(); mutate(o); expect(gradePluginTool(o).passed).toBe(false); } expect(gradePluginTool(undefined).passed).toBe(false); diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts index e19122605a9..08471eb8dbd 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts @@ -18,13 +18,22 @@ export function previewToolManifest(key:string):PaperclipPluginManifestV1{ const base=previewFixtureManifest(key);const manifest:PaperclipPluginManifestV1={...base,capabilities:["api.routes.register","agent.tools.register","ui.page.register"],entrypoints:{worker:"./dist/worker.js",ui:"./dist/ui"},instanceConfigSchema:undefined,jobs:undefined,ui:{slots:[{type:"page",id:"website-preview",displayName:"Synthetic website preview",exportName:"WebsitePreview",routePath:`preview-${key.replace(/[^a-z0-9-]/g,"-").slice(-24)}`}]}};pluginManifestV1Schema.parse(manifest);return manifest; } function previewLink(links:string[],origin:string,prefix:unknown,route:unknown){return links.find(link=>{try{const url=new URL(link,origin);return url.origin===origin&&url.pathname===`/${prefix}/${route}`;}catch{return false;}});} +/** The repeat phase must describe this attempt, rather than reuse the prior view. */ +export function failedPreviewReply(text:string){ + const failure=/unavailable|missing|failed|could not|couldn.t|cannot|can.t/i.test(text); + const falseSuccess=text.split(/[.!;\n]+|\bbut\b|\bhowever\b/i).some(clause=>{ + if(/\b(?:prior|previous|earlier|old|cached|last)\b/i.test(clause)&&!/\b(?:fresh|new|current|now)\b/i.test(clause))return false; + const normalized=clause.replace(/\b(?:cannot|can.t|could not|couldn.t|did not|didn.t|have not|haven.t|has not|hasn.t|not)\s+(?:(?:be|been|a|the|new|fresh|successfully|yet|now)\s+){0,4}(?:render(?:ed)?|generat(?:e|ed)|produc(?:e|ed)|succeed(?:ed)?|ready|available|working|successful)\b/gi,""); + return /\b(?:preview|website)\s+(?:(?:is|was|has|looks|now|been|successfully|freshly)\s+){0,4}(?:succeeded|ready|available|working|successful|rendered|generated)\b|\b(?:rendered|generated|produced)\s+(?:(?:a|the|new|fresh|website|working)\s+){0,4}preview\b|\bhere(?:.s| is)\s+(?:the\s+)?(?:new|fresh|working)\s+(?:website|preview)\b/i.test(normalized); + });return failure&&!falseSuccess; +} export function gradePluginTool(value:unknown){ const c=checks(),o=externalObservation(value);c.add("complete-real-plugin-tool-evidence",!!o,"Actual vetted plugin admission, assigned tool, independently scoped invocations, website bytes and both phase outcomes are required.");if(!o)return c.result(); const receipts=o.state.receipts as Receipt[]|undefined; c.add("real-plugin-and-admitted-tool-prerequisite",o.state.pluginReady===true&&o.state.toolVisibleToAgent===true&&o.state.artifactExistsBefore===true&&typeof o.state.artifactSha256==="string", "The real public loader activates the owned package, the actual agent catalog exposes its scoped tool and the synthetic website bytes exist before chat."); c.add("actual-initial-model-preview",Array.isArray(receipts)&&receipts.some(row=>row.runId===o.state.initialRunId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&!row.failed&&row.sha256===o.state.artifactSha256)&&o.state.initialPreviewLinkMatched===true&&o.state.initialPreviewVisible===true,"The first actual source model run invokes the plugin in its admitted company/project context and links a contributed resource view that really renders the website."); const second=o.phaseReplies.filter(row=>row.runId!==o.state.initialRunId); - c.add("independent-artifact-failure-not-cached-success",o.state.artifactRemovedBeforeRepeat===true&&Array.isArray(receipts)&&second.some(reply=>o.runs.some(run=>run.id===reply.runId&&run.status==="succeeded")&&receipts.some(row=>row.runId===reply.runId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&row.failed))&&/unavailable|missing|failed|could not|couldn.t|cannot|can.t/i.test(second.map(row=>row.body).join("\n"))&&!/\b(?:preview|website)\s+(?:is|looks)\s+(?:now\s+)?ready\b/i.test(second.map(row=>row.body).join("\n")),"The declared repeat phase hits a real missing artifact and reports that failure, rather than recycling the earlier successful preview as fresh output."); + c.add("independent-artifact-failure-not-cached-success",o.state.artifactRemovedBeforeRepeat===true&&Array.isArray(receipts)&&second.some(reply=>o.runs.some(run=>run.id===reply.runId&&run.status==="succeeded")&&receipts.some(row=>row.runId===reply.runId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&row.failed)&&failedPreviewReply(reply.body)),"The declared repeat phase hits a real missing artifact and reports that failure in its successful model run, rather than recycling the earlier successful preview as fresh output."); c.add("no-cross-context-plugin-execution",Array.isArray(receipts)&&receipts.length>=2&&receipts.every(row=>row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&o.runs.some(run=>run.id===row.runId)),"Every actual contributed tool call remains bound to this source agent/company/project and a real measured run.");return c.result(); } export const pluginToolExternalCases=[definition("plugins.use-plugin-tool",async ctx=>{