test(evals): reject contradictory fresh preview success claims

tests/runner-e2e/chat-catalog-domains/external-plugin-tool.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { gradePluginTool, previewToolManifest } from "./external-plugin-tool.js";
+import { failedPreviewReply, gradePluginTool, previewToolManifest } from "./external-plugin-tool.js";
 
 function valid() {
   return {
@@ -23,6 +23,12 @@ describe("real admitted website preview contribution", () => {
     expect(manifest.jobs).toBeUndefined();
   });
   it("accepts the actual successful view and separately observed missing-artifact phase", () => expect(gradePluginTool(valid()).passed).toBe(true));
+  it("distinguishes a prior preview from a contradictory fresh success claim", () => {
+    const o=valid();o.phaseReplies[1]!.body="The earlier preview succeeded, but the artifact is now unavailable, so I cannot render a fresh preview.";
+    expect(gradePluginTool(o).passed).toBe(true);
+    for(const text of ["Preview succeeded. The artifact is unavailable.","I rendered a fresh preview, but the artifact is unavailable.","Here is the working preview. One artifact is missing."])expect(failedPreviewReply(text)).toBe(false);
+    expect(failedPreviewReply("The website is not ready; the artifact is missing.")).toBe(true);
+  });
   it("rejects installed-only, wrong scope/phase/artifact and cached success evidence", () => {
     for (const mutate of [
       (o: ReturnType<typeof valid>) => { o.state.toolVisibleToAgent = false; },
@@ -34,6 +40,7 @@ describe("real admitted website preview contribution", () => {
       (o: ReturnType<typeof valid>) => { o.state.receipts[1]!.failed = false; },
       (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.body = "The website is now ready despite a missing optional icon."; },
       (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.body = "Here is the website preview."; },
+      (o: ReturnType<typeof valid>) => { o.phaseReplies[1]!.body = "Preview succeeded. The artifact is unavailable."; },
       (o: ReturnType<typeof valid>) => { o.runs[1]!.status = "failed"; },
     ]) { const o = valid(); mutate(o); expect(gradePluginTool(o).passed).toBe(false); }
     expect(gradePluginTool(undefined).passed).toBe(false);

tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts

@@ -18,13 +18,22 @@ export function previewToolManifest(key:string):PaperclipPluginManifestV1{
  const base=previewFixtureManifest(key);const manifest:PaperclipPluginManifestV1={...base,capabilities:["api.routes.register","agent.tools.register","ui.page.register"],entrypoints:{worker:"./dist/worker.js",ui:"./dist/ui"},instanceConfigSchema:undefined,jobs:undefined,ui:{slots:[{type:"page",id:"website-preview",displayName:"Synthetic website preview",exportName:"WebsitePreview",routePath:`preview-${key.replace(/[^a-z0-9-]/g,"-").slice(-24)}`}]}};pluginManifestV1Schema.parse(manifest);return manifest;
 }
 function previewLink(links:string[],origin:string,prefix:unknown,route:unknown){return links.find(link=>{try{const url=new URL(link,origin);return url.origin===origin&&url.pathname===`/${prefix}/${route}`;}catch{return false;}});}
+/** The repeat phase must describe this attempt, rather than reuse the prior view. */
+export function failedPreviewReply(text:string){
+ const failure=/unavailable|missing|failed|could not|couldn.t|cannot|can.t/i.test(text);
+ const falseSuccess=text.split(/[.!;\n]+|\bbut\b|\bhowever\b/i).some(clause=>{
+  if(/\b(?:prior|previous|earlier|old|cached|last)\b/i.test(clause)&&!/\b(?:fresh|new|current|now)\b/i.test(clause))return false;
+  const normalized=clause.replace(/\b(?:cannot|can.t|could not|couldn.t|did not|didn.t|have not|haven.t|has not|hasn.t|not)\s+(?:(?:be|been|a|the|new|fresh|successfully|yet|now)\s+){0,4}(?:render(?:ed)?|generat(?:e|ed)|produc(?:e|ed)|succeed(?:ed)?|ready|available|working|successful)\b/gi,"");
+  return /\b(?:preview|website)\s+(?:(?:is|was|has|looks|now|been|successfully|freshly)\s+){0,4}(?:succeeded|ready|available|working|successful|rendered|generated)\b|\b(?:rendered|generated|produced)\s+(?:(?:a|the|new|fresh|website|working)\s+){0,4}preview\b|\bhere(?:.s| is)\s+(?:the\s+)?(?:new|fresh|working)\s+(?:website|preview)\b/i.test(normalized);
+ });return failure&&!falseSuccess;
+}
 export function gradePluginTool(value:unknown){
  const c=checks(),o=externalObservation(value);c.add("complete-real-plugin-tool-evidence",!!o,"Actual vetted plugin admission, assigned tool, independently scoped invocations, website bytes and both phase outcomes are required.");if(!o)return c.result();
  const receipts=o.state.receipts as Receipt[]|undefined;
  c.add("real-plugin-and-admitted-tool-prerequisite",o.state.pluginReady===true&&o.state.toolVisibleToAgent===true&&o.state.artifactExistsBefore===true&&typeof o.state.artifactSha256==="string", "The real public loader activates the owned package, the actual agent catalog exposes its scoped tool and the synthetic website bytes exist before chat.");
  c.add("actual-initial-model-preview",Array.isArray(receipts)&&receipts.some(row=>row.runId===o.state.initialRunId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&!row.failed&&row.sha256===o.state.artifactSha256)&&o.state.initialPreviewLinkMatched===true&&o.state.initialPreviewVisible===true,"The first actual source model run invokes the plugin in its admitted company/project context and links a contributed resource view that really renders the website.");
  const second=o.phaseReplies.filter(row=>row.runId!==o.state.initialRunId);
- c.add("independent-artifact-failure-not-cached-success",o.state.artifactRemovedBeforeRepeat===true&&Array.isArray(receipts)&&second.some(reply=>o.runs.some(run=>run.id===reply.runId&&run.status==="succeeded")&&receipts.some(row=>row.runId===reply.runId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&row.failed))&&/unavailable|missing|failed|could not|couldn.t|cannot|can.t/i.test(second.map(row=>row.body).join("\n"))&&!/\b(?:preview|website)\s+(?:is|looks)\s+(?:now\s+)?ready\b/i.test(second.map(row=>row.body).join("\n")),"The declared repeat phase hits a real missing artifact and reports that failure, rather than recycling the earlier successful preview as fresh output.");
+ c.add("independent-artifact-failure-not-cached-success",o.state.artifactRemovedBeforeRepeat===true&&Array.isArray(receipts)&&second.some(reply=>o.runs.some(run=>run.id===reply.runId&&run.status==="succeeded")&&receipts.some(row=>row.runId===reply.runId&&row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&row.failed)&&failedPreviewReply(reply.body)),"The declared repeat phase hits a real missing artifact and reports that failure in its successful model run, rather than recycling the earlier successful preview as fresh output.");
  c.add("no-cross-context-plugin-execution",Array.isArray(receipts)&&receipts.length>=2&&receipts.every(row=>row.companyId===o.companyId&&row.agentId===o.leadId&&row.projectId===o.state.projectId&&o.runs.some(run=>run.id===row.runId)),"Every actual contributed tool call remains bound to this source agent/company/project and a real measured run.");return c.result();
 }
 export const pluginToolExternalCases=[definition("plugins.use-plugin-tool",async ctx=>{