test(evals): independently render retrieved Slack launch artifacts

tests/runner-e2e/chat-catalog-domains/external-slack-render.test.ts

@@ -0,0 +1,17 @@
+import {expect,it} from "vitest";
+import {chromium} from "@playwright/test";
+import type {ChatCatalogContext} from "../chat-catalog-contract.js";
+import {hasSlackLaunchRendering,renderSlackLaunchArtifact,type SlackLaunchRendering} from "./external-slack-render.js";
+const valid=():SlackLaunchRendering=>({sha256:"actual-downloaded-hash",usableSignup:true,nodes:[{tag:"H1",role:null,text:"Acorn Notes",href:null,inNavigation:false,color:"rgb(0, 0, 0)",background:"rgb(255, 255, 255)"},{tag:"P",role:null,text:"A calm shared notebook for the whole research team.",href:null,inNavigation:false,color:"rgb(0, 0, 0)",background:"rgb(255, 255, 255)"}]});
+it("requires an actually visible product heading, substantive content and usable signup",()=>{
+ expect(hasSlackLaunchRendering(valid())).toBe(true);
+ expect(hasSlackLaunchRendering({...valid(),nodes:[]})).toBe(false);
+ expect(hasSlackLaunchRendering({...valid(),usableSignup:false})).toBe(false);
+ expect(hasSlackLaunchRendering({...valid(),nodes:valid().nodes.filter(node=>node.tag!=="H1")})).toBe(false);
+});
+it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SLACK_RENDER_CANARY!=="1")("rejects actual hidden/blank bytes and inert signup controls in isolated offline Chromium",async()=>{
+ const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),context=await browser.newContext(),page=await context.newPage(),ctx={input:{page}} as unknown as ChatCatalogContext;
+ const body='<html><body><main><h1>Acorn Notes</h1><p>A calm shared notebook for the whole research team.</p><a href="#signup">Join the list</a><section id="signup"><p>Join our research notebook early access list.</p></section></main></body></html>';
+ try{for(const [html,expected] of [[body,true],[body.replace('<main>','<main style="display:none">'),false],[body.replace('<h1>','<h1 style="opacity:0">'),false],[body.replace('href="#signup"','href="javascript:void(0)"'),false],[body.replace('id="signup"','id="unrelated"'),false]] as const)expect(hasSlackLaunchRendering(await renderSlackLaunchArtifact(ctx,html))).toBe(expected);}
+ finally{await context.close();await browser.close();}
+},30000);

tests/runner-e2e/chat-catalog-domains/external-slack-render.ts

@@ -0,0 +1,26 @@
+import {createHash} from "node:crypto";
+import type {ChatCatalogContext} from "../chat-catalog-contract.js";
+import {renderRuntimeArtifact,type RuntimeRenderedNode} from "./runtime-rendered-artifact.js";
+export interface SlackLaunchRendering {sha256:string;nodes:RuntimeRenderedNode[];usableSignup:boolean}
+export function hasSlackLaunchRendering(value:SlackLaunchRendering){
+ return value.usableSignup&&value.nodes.some(node=>(node.tag==="H1"||node.role==="heading")&&/\bAcorn\s+Notes\b/i.test(node.text))&&value.nodes.some(node=>node.tag==="P"&&node.text.length>=20);
+}
+/** Independently retrieved bytes only. Both contexts have no authenticated
+ * storage, scripts or network; no generated site can call back into the app. */
+export async function renderSlackLaunchArtifact(ctx:ChatCatalogContext,html:string):Promise<SlackLaunchRendering>{
+ const nodes=await renderRuntimeArtifact(ctx,html),browser=ctx.input.page.context().browser();if(!browser)throw new Error("Slack result rendering requires the existing browser");
+ const preview=await browser.newContext({javaScriptEnabled:false});
+ try{
+  const page=await preview.newPage();await page.route("**/*",route=>route.abort());await page.setContent(html,{waitUntil:"domcontentloaded"});
+  const usableSignup=await page.locator("a[href]").evaluateAll(elements=>{
+   const visible=(element:Element)=>{const box=element.getBoundingClientRect();if(box.width<2||box.height<2||box.right<=0||box.left>=innerWidth)return false;for(let node:Element|null=element;node;node=node.parentElement){const css=getComputedStyle(node);if(css.display==="none"||css.visibility==="hidden"||Number(css.opacity)===0||css.pointerEvents==="none")return false;}return true;};
+   if(elements.length>500)throw new Error("Slack launch navigation exceeds finite inspection bound");
+   return elements.some(element=>{if(!visible(element)||element.getAttribute("aria-disabled")==="true"||!/join|sign.?up|subscribe|early access/i.test(element.textContent??""))return false;
+    const href=element.getAttribute("href")??"";
+    if(href.startsWith("#")){const target=document.getElementById(href.slice(1));return !!target&&visible(target)&&(target.textContent?.trim().length??0)>=15;}
+    try{const url=new URL(href);return url.protocol==="https:"&&!url.username&&!url.password&&!!url.hostname;}catch{return false;}
+   });
+  });
+  return {sha256:createHash("sha256").update(html).digest("hex"),nodes,usableSignup};
+ }finally{await preview.close();}
+}

tests/runner-e2e/chat-catalog-domains/external-slack-thread.test.ts

@@ -7,7 +7,7 @@ function valid(){
  const other={...receipt,query:"Other status query",conversationId:"other-conversation",issueId:"other",runId:"other-run",provider:{...receipt.provider,threadId:"1234567889.000001"}};
  const run=(id:string,agentId:string,issueId:string)=>({id,companyId:"company",agentId,runtimeMode:"native",status:"succeeded",contextSnapshot:{issueId}});
  const message={id:"1234567889.000002",user:"UBOT",text:"No projects are blocked.",files:[] as string[]};
- return {scenarioId:"chat-channels.reply-in-thread",companyId:"company",leadId:"ceo",origin:"http://127.0.0.1:3399",reply:"Morgan finished. ACORN-3 is ready.",state:{worker:{id:"morgan"},project:{id:"project"},originalQuery:receipt.query,botUserId:"UBOT"},phaseReplies:[{runId:"completion",body:"Morgan finished. ACORN-3 is ready."}],tasks:[{id:"build",companyId:"company",identifier:"ACORN-3",assigneeAgentId:"morgan",projectId:"project",status:"done"}],runs:[run("other-run","ceo","other"),run("source-run","ceo","source"),run("build-run","morgan","build"),run("completion","ceo","source")],connections:[],profiles:[],secrets:[],endpoints:[],interactions:[],activity:[{action:"issue.created",entityId:"build",actorType:"agent",agentId:"ceo"}],links:[],humanQueries:[receipt.query],thread:{source:receipt,other,before:[message],after:[structuredClone(message)],messages:[{id:"1234567890.000003",user:"UBOT",text:"Morgan finished. ACORN-3 is ready.",files:[] as string[]}],duplicate:{canonicalDeliveryCaptured:true,duplicateReplayed:true},activity:[{kind:"delivery",status:"processed",summary:"message processed · 1 duplicate ignored"},{kind:"publication",status:"published",summary:"Response to external conversation"}],attachments:[{id:"attachment",issueId:"build",originatingRunId:"build-run",sha256}],products:[{issueId:"build",createdByRunId:"build-run",type:"artifact",status:"active",metadata:{attachmentId:"attachment"}}],files:[{id:"attachment",hash:sha256,body}]}};
+ return {scenarioId:"chat-channels.reply-in-thread",companyId:"company",leadId:"ceo",origin:"http://127.0.0.1:3399",reply:"Morgan finished. ACORN-3 is ready.",state:{worker:{id:"morgan"},project:{id:"project"},originalQuery:receipt.query,botUserId:"UBOT"},phaseReplies:[{runId:"completion",body:"Morgan finished. ACORN-3 is ready."}],tasks:[{id:"build",companyId:"company",identifier:"ACORN-3",assigneeAgentId:"morgan",projectId:"project",status:"done"}],runs:[run("other-run","ceo","other"),run("source-run","ceo","source"),run("build-run","morgan","build"),run("completion","ceo","source")],connections:[],profiles:[],secrets:[],endpoints:[],interactions:[],activity:[{action:"issue.created",entityId:"build",actorType:"agent",agentId:"ceo"}],links:[],humanQueries:[receipt.query],thread:{source:receipt,other,before:[message],after:[structuredClone(message)],messages:[{id:"1234567890.000003",user:"UBOT",text:"Morgan finished. ACORN-3 is ready.",files:[] as string[]}],duplicate:{canonicalDeliveryCaptured:true,duplicateReplayed:true},activity:[{kind:"delivery",status:"processed",summary:"message processed · 1 duplicate ignored"},{kind:"publication",status:"published",summary:"Response to external conversation"}],attachments:[{id:"attachment",issueId:"build",originatingRunId:"build-run",sha256}],products:[{issueId:"build",createdByRunId:"build-run",type:"artifact",status:"active",metadata:{attachmentId:"attachment"}}],files:[{id:"attachment",hash:sha256,body,rendered:{sha256,usableSignup:true,nodes:[{tag:"H1",text:"Acorn Notes",role:null,href:null,inNavigation:false,color:"rgb(0, 0, 0)",background:"rgb(255, 255, 255)"},{tag:"P",text:"A calm notebook for the whole research team.",role:null,href:null,inNavigation:false,color:"rgb(0, 0, 0)",background:"rgb(255, 255, 255)"}]}}]}};
 }
 describe("real Slack native handoff, result and duplicate isolation",()=>{
  it("accepts independently attributed build bytes and one actual original-thread completion",()=>expect(gradeSlackThread(valid()).passed).toBe(true));
@@ -23,6 +23,8 @@ describe("real Slack native handoff, result and duplicate isolation",()=>{
    (o:ReturnType<typeof valid>)=>{o.thread.attachments[0]!.originatingRunId="older-run";},
    (o:ReturnType<typeof valid>)=>{o.thread.messages.push({...o.thread.messages[0]!,id:"duplicate-response"});},
    (o:ReturnType<typeof valid>)=>{o.thread.duplicate.duplicateReplayed=false;},
+   (o:ReturnType<typeof valid>)=>{o.thread.files[0]!.rendered.nodes=[];},
+   (o:ReturnType<typeof valid>)=>{o.thread.files[0]!.rendered.usableSignup=false;},
    (o:ReturnType<typeof valid>)=>{o.thread.messages[0]!.text="I'll let you know here when ACORN-3 is ready.";},
   ]){const o=valid();mutate(o);expect(gradeSlackThread(o).passed).toBe(false);}
  });

tests/runner-e2e/chat-catalog-domains/external-slack-thread.ts

@@ -10,13 +10,14 @@ import {definition,observeExternal,externalObservation,checks,root,verifyRendere
 import {project,type Entity} from "./internal-support.js";
 import {identityWorker} from "./identity-support.js";
 import {prepareSlackFixture,slackCompanionFields,type SlackMessage} from "./external-slack-fixture.js";
+import {renderSlackLaunchArtifact,hasSlackLaunchRendering,type SlackLaunchRendering} from "./external-slack-render.js";
 const hash=(value:string|Buffer)=>createHash("sha256").update(value).digest("hex");
 interface Message {id:string;user:string;text:string;files:string[]}
 const projection=(rows:SlackMessage[]):Message[]=>rows.map(row=>({id:row.ts,user:row.user??"",text:row.text??"",files:(row.files??[]).map(file=>file.id).sort()}));
-interface ThreadEvidence {source:ChatSlackIngressReceipt;other:ChatSlackIngressReceipt&{runId:string};before:Message[];after:Message[];messages:Message[];duplicate:{canonicalDeliveryCaptured:boolean;duplicateReplayed:boolean};activity:ChatActivityItem[];attachments:IssueAttachment[];products:IssueWorkProduct[];files:Array<{id:string;hash:string;body:string}>}
+interface ThreadEvidence {source:ChatSlackIngressReceipt;other:ChatSlackIngressReceipt&{runId:string};before:Message[];after:Message[];messages:Message[];duplicate:{canonicalDeliveryCaptured:boolean;duplicateReplayed:boolean};activity:ChatActivityItem[];attachments:IssueAttachment[];products:IssueWorkProduct[];files:Array<{id:string;hash:string;body:string;rendered?:SlackLaunchRendering}>}
 interface Observation extends ExternalObservation {thread:ThreadEvidence}
 type Fixture=Awaited<ReturnType<typeof prepareSlackFixture>>;
-const privateFixtures=new WeakMap<ChatCatalogContext,{fixture:Fixture;otherRoot?:string;before?:Message[];lastRead?:number;lastThreads?:{after:Message[];messages:Message[]}}>();
+const privateFixtures=new WeakMap<ChatCatalogContext,{fixture:Fixture;otherRoot?:string;before?:Message[];lastRead?:number;rendered?:Map<string,SlackLaunchRendering>;lastThreads?:{after:Message[];messages:Message[]}}>();
 function owned(ctx:ChatCatalogContext){const value=privateFixtures.get(ctx);if(!value)throw new Error("Actual Slack fixture is not prepared");return value;}
 function completed(text:string){return !/\b(?:when|once|until|after)\b[^.!?\n]{0,100}\b(?:ready|completed|finished|built|delivered|done)\b/i.test(text)&&/\b(?:ready|completed|finished|built|delivered|done)\b/i.test(text)&&!/\b(?:not|isn['’]t|is not|hasn['’]t|have not)\s+(?:yet\s+)?(?:ready|completed|finished|built|delivered|done)\b|\b(?:will|would|should)\s+(?:be\s+)?(?:ready|completed|finished|built|delivered|done)\b/i.test(text);}
 export function gradeSlackThread(value:unknown){
@@ -28,7 +29,7 @@ export function gradeSlackThread(value:unknown){
  const sourceRuns=o.runs.filter(run=>run.contextSnapshot?.issueId===source.issueId),otherRuns=o.runs.filter(run=>run.contextSnapshot?.issueId===other.issueId);
  c.add("exact-real-human-two-thread-context",source.query===o.state.originalQuery&&source.endpointId===other.endpointId&&source.issueId!==other.issueId&&source.provider.threadId!==other.provider.threadId&&source.provider.workspaceId===other.provider.workspaceId&&source.provider.channelId===other.provider.channelId&&source.linkedPaperclipUserId===other.linkedPaperclipUserId&&source.provider.senderUserId===other.provider.senderUserId&&otherRuns.length===1&&otherRuns[0]?.id===other.runId&&otherRuns[0]?.agentId===o.leadId&&otherRuns[0]?.status==="succeeded","The genuine linked sender has two distinct conversations under the same actual assigned agent; the setup source remains separately attributed.");
  c.add("one-attributed-Morgan-build",!!worker&&tasks.length===1&&!!target&&target.status==="done"&&target.projectId===(o.state.project as Entity|undefined)?.id&&sourceRuns.length>=1&&workerRuns.length===1&&workerRuns[0]?.status==="succeeded"&&o.runs.every(run=>run.companyId===o.companyId&&run.runtimeMode==="native"&&run.status==="succeeded"&&(run.contextSnapshot?.issueId===source.issueId&&run.agentId===o.leadId||run.id===other.runId||run.contextSnapshot?.issueId===target.id&&run.agentId===worker.id))&&o.activity.some(row=>row.action==="issue.created"&&row.entityId===target.id&&row.actorType==="agent"&&row.agentId===o.leadId),"Exactly one actual Morgan task is built under normal policy, with independently attributed task creation and source/worker native executions; no unrelated run is accepted.");
- const file=t.attachments.find(file=>file.issueId===target?.id&&file.originatingRunId===workerRuns[0]?.id&&t.products.some(product=>product.issueId===target?.id&&product.createdByRunId===workerRuns[0]?.id&&product.type==="artifact"&&["active","ready_for_review","approved"].includes(product.status)&&product.metadata?.attachmentId===file.id)&&t.files.some(actual=>actual.id===file.id&&actual.hash===file.sha256&&hash(actual.body)===file.sha256&&/<html\b/i.test(actual.body)&&/<h1\b/i.test(actual.body)&&/Acorn\s+Notes/i.test(actual.body)));
+ const file=t.attachments.find(file=>file.issueId===target?.id&&file.originatingRunId===workerRuns[0]?.id&&t.products.some(product=>product.issueId===target?.id&&product.createdByRunId===workerRuns[0]?.id&&product.type==="artifact"&&["active","ready_for_review","approved"].includes(product.status)&&product.metadata?.attachmentId===file.id)&&t.files.some(actual=>actual.id===file.id&&actual.hash===file.sha256&&hash(actual.body)===file.sha256&&/<html\b/i.test(actual.body)&&/<h1\b/i.test(actual.body)&&/Acorn\s+Notes/i.test(actual.body)&&!!actual.rendered&&actual.rendered.sha256===file.sha256&&hasSlackLaunchRendering(actual.rendered)));
  c.add("inspectable-real-launch-page",!!file,"A real worker-attributed registered HTML result has independently retrieved matching bytes and the requested launch identity.");
  const reference=target?.identifier,completionReplies=o.phaseReplies.filter(reply=>sourceRuns.some(run=>run.id===reply.runId)&&!!reference&&reply.body.includes(reference)&&completed(reply.body));
  const delivered=t.messages.filter(message=>message.user===o.state.botUserId&&!!reference&&message.text.includes(reference)&&completed(message.text));
@@ -43,7 +44,7 @@ export const slackThreadExternalCases=[definition("chat-channels.reply-in-thread
 },gradeSlackThread,{
  authentication:"human-session",journeyKind:"native-slack-thread",slackIngress:{kind:"native-slack-thread",maximumProviderCalls:20,ingressTimeoutMs:120000,setupThread:"same-agent-distractor"},expectedRunCount:4,minimumRunCount:2,expectedHumanMessageCount:1,turnTimeoutMs:120000,attemptTimeoutMs:600000,
  externalInput:{provider:"slack",fields:slackCompanionFields},
- sourceFiles:["chat-catalog-domains/external-slack-thread.ts","chat-catalog-domains/external-slack-fixture.ts","chat-catalog-domains/external-slack-provider.ts","chat-catalog-domains/external-slack-relay.ts","chat-catalog-domains/external-fixtures.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","chat-slack-ingress.ts","../../packages/shared/src/types/chat-channels.ts","../../server/src/routes/chat-channels.ts","../../server/src/services/chat-channels.ts","../../server/src/services/chat-channels/inbound/authorization.ts","../../server/src/services/chat-channels/inbound/wakeups.ts","chat-scenarios/chat-channels.yaml"],
+ sourceFiles:["chat-catalog-domains/external-slack-thread.ts","chat-catalog-domains/external-slack-fixture.ts","chat-catalog-domains/external-slack-provider.ts","chat-catalog-domains/external-slack-relay.ts","chat-catalog-domains/external-slack-render.ts","chat-catalog-domains/runtime-rendered-artifact.ts","chat-catalog-domains/external-fixtures.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","chat-slack-ingress.ts","../../packages/shared/src/types/chat-channels.ts","../../server/src/routes/chat-channels.ts","../../server/src/services/chat-channels.ts","../../server/src/services/chat-channels/inbound/authorization.ts","../../server/src/services/chat-channels/inbound/wakeups.ts","chat-scenarios/chat-channels.yaml"],
  async dispatchSlackOtherThread(ctx,_state,{query,signal}){
   const local=owned(ctx),f=local.fixture,start=f.provider.receipts.length;signal.throwIfAborted();
   const ts=await f.createDistractor(query);local.otherRoot=ts;
@@ -70,7 +71,7 @@ export const slackThreadExternalCases=[definition("chat-channels.reply-in-thread
    local.lastThreads={after:projection(await f.readThread(local.otherRoot!)),messages:projection(await f.readThread(f.threadId))};local.lastRead=Date.now();
   }
   const attachments=target?await ctx.input.api.get<IssueAttachment[]>(`/api/issues/${target.id}/attachments`):[],products=target?await ctx.input.api.get<IssueWorkProduct[]>(`/api/issues/${target.id}/work-products`):[],files:ThreadEvidence["files"]=[];
-  for(const file of attachments.filter(row=>/\.html?$/i.test(row.originalFilename??"")).slice(0,10)){if(file.byteSize>262144)throw new Error("Slack launch result exceeds bounded artifact inspection");const bytes=await ctx.input.api.getAttachmentContent(file.id);files.push({id:file.id,hash:hash(bytes),body:bytes.toString("utf8")});}
+  for(const file of attachments.filter(row=>/\.html?$/i.test(row.originalFilename??"")).slice(0,10)){if(file.byteSize>262144)throw new Error("Slack launch result exceeds bounded artifact inspection");const bytes=await ctx.input.api.getAttachmentContent(file.id);const body=bytes.toString("utf8"),digest=hash(bytes);local.rendered??=new Map();let rendered=local.rendered.get(digest);if(!rendered){rendered=await renderSlackLaunchArtifact(ctx,body);local.rendered.set(digest,rendered);}files.push({id:file.id,hash:digest,body,rendered});}
   const activity=await ctx.input.api.get<ChatActivityItem[]>(`/api/chat-endpoints/${f.endpoint.id}/activity`);
   return {...o,thread:{source,other,before:local.before??[],after:local.lastThreads?.after??[],messages:local.lastThreads?.messages??[],duplicate:f.relay.receipt(),activity,attachments,products,files}} satisfies Observation;
  },
@@ -80,11 +81,12 @@ export const slackThreadExternalCases=[definition("chat-channels.reply-in-thread
   if(!target||!href)throw new Error("Actual source completion must link its built Morgan task");
   await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();await expect(ctx.input.page.getByText(target.title!,{exact:true}).first()).toBeVisible();
   const file=o.thread.attachments.find(row=>o.thread.files.some(actual=>actual.id===row.id&&actual.hash===row.sha256));if(!file?.contentPath)throw new Error("Actual launch result lacks a supported public content route");
-  const page=await ctx.input.page.context().newPage();try{
+  let retrieved:Buffer|undefined;const page=await ctx.input.page.context().newPage();try{
    const pending=page.waitForEvent("download",{timeout:5000}).catch(()=>undefined);
    await page.goto(new URL(file.contentPath,o.origin).href,{waitUntil:"domcontentloaded"}).catch(error=>{if(!/download|ERR_ABORTED/i.test(String(error)))throw error;});
-   const download=await pending;if(download){const path=await download.path();if(!path||hash(await readFile(path))!==file.sha256)throw new Error("Actual browser launch-page download differs from the registered bytes");}else await expect(page.locator("h1")).toBeVisible();
+   const download=await pending;if(download){const path=await download.path();if(path)retrieved=await readFile(path);if(!retrieved||hash(retrieved)!==file.sha256)throw new Error("Actual browser launch-page download differs from the registered bytes");}else{await expect(page.locator("h1")).toBeVisible();retrieved=await ctx.input.api.getAttachmentContent(file.id);if(hash(retrieved)!==file.sha256)throw new Error("Public rendered result changed its registered bytes");}
   }finally{await page.close();}
+  if(!retrieved||!hasSlackLaunchRendering(await renderSlackLaunchArtifact(ctx,retrieved.toString("utf8"))))throw new Error("Actual publicly retrieved launch result is not visibly usable");
  },
  async cleanup(ctx){const local=privateFixtures.get(ctx);try{await local?.fixture.cleanup();}finally{privateFixtures.delete(ctx);}},
 })];