From f097a9da02eabf64a80b1d4d8f36e5fab3435ba0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 21:45:07 -0500 Subject: [PATCH] test(evals): accept independently accessible delegation briefs --- .../runtime-delegation.test.ts | 16 +++++- .../runtime-delegation.ts | 52 +++++++++++++++++-- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-delegation.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-delegation.test.ts index e56645ce14e..3dcbd9f9d70 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-delegation.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-delegation.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeDelegation, hasLaunchHeadline } from "./runtime-delegation.js"; +import { gradeDelegation, hasLaunchHeadline, completeLaunchBrief, launchPageReflectsBrief } from "./runtime-delegation.js"; function valid() { const task = { id: "website", parentId: "launch", companyId: "company", assigneeAgentId: "morgan", status: "done", description: "Implement the approved design brief and report the completed result back." }; return { scenarioId: "task-coordination.delegate-with-context", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", @@ -8,7 +8,7 @@ function valid() { { id: "implement", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "website" } }, { id: "report", agentId: "ceo", status: "succeeded", contextSnapshot: { issueId: "chat" } }], state: { parent: { id: "launch" }, task, worker: { id: "morgan" }, humanId: "human", briefCommentId: "human-brief", briefHash: "original-hash", briefDownloadedHash: "original-hash", briefAttachment: { id: "brief", issueId: "launch", companyId: "company", createdByAgentId: null, createdByUserId: "human", issueCommentId: "human-brief", sha256: "original-hash" }, - parentOwnedByCeo: true, parentContextRetained: true, createdFromSource: true, childHasBrief: true, linkedToLaunch: false, workerBriefReadStatus: 200, workerBriefHash: "original-hash", + parentOwnedByCeo: true, parentContextRetained: true, createdFromSource: true, childHasBrief: true, linkedToLaunch: false, workerBriefReadStatus: 200, workerBriefHash: "original-hash", briefContextKind: "direct-attachment-reference", workerTaskReadStatus: 200, workerParentReadStatus: 200, parentLinkInspectable: true, durableBriefComplete: false, workerWasReallyWaiting: true, actualWebsiteArtifact: true, delivery: { checks: [{ id: "correlated", passed: true }], response: { body: "Morgan completed Website; the actual saved page is available on the child task." } } } }; } @@ -27,3 +27,15 @@ it("rejects a result reported on an uncorrelated source", () => { const o = vali it("rejects saying the parent executed Morgan's work", () => { const o = valid(); o.state.delivery.response.body = "I ran the parent Launch task and built the site myself. It is complete."; expect(gradeDelegation(o).passed).toBe(false); }); it("rejects a hidden actual parent execution", () => { const o = valid(); o.runs[2]!.contextSnapshot.issueId = "launch"; expect(gradeDelegation(o).passed).toBe(false); }); it("rejects duplicate work tasks", () => { const o = valid(); o.tasks.push({ id: "duplicate" }); expect(gradeDelegation(o).passed).toBe(false); }); + +it("accepts an actual worker-readable parent reference and its exact human attachment", () => { const o = valid(); o.state.briefContextKind = "readable-parent-reference"; expect(gradeDelegation(o).passed).toBe(true); }); +it("rejects an inaccessible parent even when the board can read its attachment", () => { const o = valid(); o.state.briefContextKind = "readable-parent-reference"; o.state.workerParentReadStatus = 403; expect(gradeDelegation(o).passed).toBe(false); }); +it("rejects a parent reference with the wrong brief bytes", () => { const o = valid(); o.state.briefContextKind = "readable-parent-reference"; o.state.workerBriefHash = "other-design"; expect(gradeDelegation(o).passed).toBe(false); }); +it("accepts a complete durable brief with actual grounded delivery without copying its attachment", () => { const o = valid(); o.state.briefContextKind = "faithful-durable-copy"; o.state.durableBriefComplete = true; o.state.workerBriefReadStatus = 404; o.state.workerBriefHash = ""; expect(gradeDelegation(o).passed).toBe(true); }); +it("rejects an incomplete durable brief", () => { const o = valid(); o.state.briefContextKind = "faithful-durable-copy"; expect(gradeDelegation(o).passed).toBe(false); }); +it("rejects a durable copy without the worker's actual access to its task", () => { const o = valid(); o.state.briefContextKind = "faithful-durable-copy"; o.state.durableBriefComplete = true; o.state.workerTaskReadStatus = 403; expect(gradeDelegation(o).passed).toBe(false); }); +it.each(["Notify the CEO in this conversation when the page is ready", "Implement the brief and let me know when done"])("accepts a clear natural completion request: %s", description => { const o = valid(); o.state.task.description = description; expect(gradeDelegation(o).passed).toBe(true); }); +it("recognizes complete substantive durable brief content", () => expect(completeLaunchBrief("Reference: Launch brief nonce. Build semantic markup with accessible input labels, one primary join button, and the headline Research without the busywork.", "Launch brief nonce")).toBe(true)); +it.each(["Unrelated headline; semantic HTML, accessible labels, one primary signup action. Launch brief nonce", "Research without the busywork. Launch brief nonce", "Research without the busywork; semantic HTML, accessible labels, one primary signup action. Other reference"])("rejects wrong or incomplete durable content", text => expect(completeLaunchBrief(text, "Launch brief nonce")).toBe(false)); +it("verifies actual semantic output and its labelled signup control", () => expect(launchPageReflectsBrief('

Research without the busywork

')).toBe(true)); +it("rejects output whose signup input lost its accessible label", () => expect(launchPageReflectsBrief('

Research without the busywork

')).toBe(false)); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-delegation.ts b/tests/runner-e2e/chat-catalog-domains/runtime-delegation.ts index 29f6c6e7815..f1deb4b702b 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-delegation.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-delegation.ts @@ -18,18 +18,32 @@ export function hasLaunchHeadline(html: string) { const heading = html.match(/]*>([\s\S]*?)<\/h1>/i)?.[1]; return heading?.replace(/<[^>]+>/g, " ").replace(/ | |�*a0;/gi, " ").replace(/\s+/g, " ").trim().toLowerCase() === "research without the busywork"; } +export function completeLaunchBrief(context: string, marker: string) { + return context.includes(marker) && /Research without the busywork/i.test(context) && /\bsemantic\b.{0,30}\b(?:html|markup|structure)\b/i.test(context) + && /\b(?:accessible|labelled|labeled)\b.{0,30}\b(?:label|input|control|form)|\blabel.{0,30}accessib/i.test(context) + && /\b(?:primary|one|single)\b.{0,35}\b(?:sign.?up|signup|join|subscribe)\b.{0,30}\b(?:action|cta|button|link)\b/i.test(context); +} +export function launchPageReflectsBrief(html: string) { + if (!hasLaunchHeadline(html) || !/<(?:main|section|header)\b/i.test(html) + || !/<(?:a|button)\b[^>]*>[\s\S]{0,400}?(?:sign.?up|join|subscribe|early access)[\s\S]{0,100}?<\/(?:a|button)>/i.test(html)) return false; + return [...html.matchAll(/]*>/gi)].every(([input]) => /type\s*=\s*["']?hidden\b|aria-label(?:ledby)?\s*=\s*["'][^"']+/i.test(input) + || !!input.match(/\bid\s*=\s*["']([^"']+)["']/)?.[1] && [...html.matchAll(/]*\bfor\s*=\s*["']([^"']+)["']/gi)].some(label => label[1] === input.match(/\bid\s*=\s*["']([^"']+)["']/)?.[1])); +} export function gradeDelegation(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("actual-launch-brief-and-native-handoff-evidence", !!o, "A real human-attached Launch brief, exactly one CEO-created Morgan child, native result and source completion are required."); if (!o) return c.result(); const s = o.state, parent = s.parent as Entity, task = s.task as Entity, receipt = s.briefAttachment as ChatQueryAttachmentReceipt; + const availableBrief = ["copied-attachment", "direct-attachment-reference"].includes(String(s.briefContextKind)) ? s.workerBriefReadStatus === 200 && s.workerBriefHash === s.briefHash + : s.briefContextKind === "readable-parent-reference" ? s.workerParentReadStatus === 200 && s.parentLinkInspectable === true && s.workerBriefReadStatus === 200 && s.workerBriefHash === s.briefHash + : s.briefContextKind === "faithful-durable-copy" && s.durableBriefComplete === true && s.actualWebsiteArtifact === true; c.add("real-parent-owner-and-human-brief", s.parentOwnedByCeo === true && !!receipt && receipt.issueId === parent.id && receipt.companyId === o.companyId && receipt.createdByAgentId == null && receipt.createdByUserId === s.humanId && receipt.issueCommentId === s.briefCommentId && receipt.sha256 === s.briefHash && s.briefDownloadedHash === s.briefHash && s.parentContextRetained === true, "The real Launch task remains owned by the CEO and its human comment retains the exact uploaded design brief."); c.add("one-authorized-child-with-durable-context", !!task && (task.parentId === parent.id || s.linkedToLaunch === true) && task.companyId === o.companyId && task.assigneeAgentId === (s.worker as Entity).id - && s.createdFromSource === true && o.tasks.length === 2 && s.childHasBrief === true && s.workerBriefReadStatus === 200 && s.workerBriefHash === s.briefHash + && s.createdFromSource === true && o.tasks.length === 2 && s.childHasBrief === true && availableBrief && s.workerTaskReadStatus === 200 && o.activity.some(row => row.action === "issue.created" && row.entityId === task.id && row.agentId === o.leadId) - && /report|return|complete|finish|deliver/i.test(task.description ?? ""), + && /report|return|complete|finish|deliver|notify.{0,60}(?:CEO|here|conversation|ready)|let (?:me|us|the CEO) know.{0,60}(?:done|ready|page|result)/i.test(task.description ?? ""), "The canonical request creates one real Morgan child linked to Launch, with accessible durable brief bytes and an explicit completion request."); const delivery = s.delivery as ReturnType | undefined; c.add("actual-morgan-result-incorporated-back-here", s.workerWasReallyWaiting === true && task.status === "done" && s.actualWebsiteArtifact === true @@ -91,9 +105,37 @@ export const runtimeDelegationCases = [runtimeDefinition("task-coordination.dele return url.origin === ctx.input.api.baseURL && i >= 0 && [(state.parent as Entity).id, (state.parent as Entity).identifier].includes(parts[i + 1]); } catch { return false; } }); - const copied = attachments.find(row => row.sha256 === state.briefHash), path = copied?.contentPath ?? (context.includes(original.contentPath) ? original.contentPath : undefined); - state.childHasBrief = !!path; + const copied = attachments.find(row => row.sha256 === state.briefHash); + let path = copied?.contentPath ?? (context.includes(original.contentPath) ? original.contentPath : undefined); + state.parentLinkInspectable = task.parentId === (state.parent as Entity).id || state.linkedToLaunch === true; const key = keys.get(ctx); if (!key || key.workerId !== (state.worker as Entity).id) throw new Error("Actual Morgan brief probe identity is missing"); + async function workerRead(path: string) { + const target = new URL(path, ctx.input.api.baseURL); + if (target.origin !== ctx.input.api.baseURL || !/^\/api\/(?:issues|attachments)\//.test(target.pathname)) throw new Error("Worker brief read escaped fixture resources"); + return fetch(target, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key!.token}` } }); + } + const ownRead = await workerRead(`/api/issues/${task.id}`); state.workerTaskReadStatus = ownRead.status; + let workerContext = ""; + if (ownRead.ok) { const own = await ownRead.json() as Entity; + if (own.id !== task.id || own.companyId !== ctx.input.fixtures.company.id || own.assigneeAgentId !== key.workerId) throw new Error("Worker task read returned a foreign owner or scope"); + workerContext = own.description ?? ""; + for (const doc of documents) { const response = await workerRead(`/api/issues/${task.id}/documents/${encodeURIComponent(doc.key!)}`); + if (response.ok) workerContext += `\n${(await response.json() as Entity).body ?? ""}`; else await response.body?.cancel(); } + } else await ownRead.body?.cancel(); + state.durableBriefComplete = completeLaunchBrief(workerContext, String(state.marker)); + if (path) state.briefContextKind = copied ? "copied-attachment" : "direct-attachment-reference"; + else if (state.parentLinkInspectable) { + const parent = await workerRead(`/api/issues/${(state.parent as Entity).id}`); state.workerParentReadStatus = parent.status; + if (parent.ok) { const row = await parent.json() as Entity; + if (row.id !== (state.parent as Entity).id || row.companyId !== ctx.input.fixtures.company.id) throw new Error("Worker parent reference escaped its actual Launch scope"); + const receipts = await workerRead(`/api/issues/${row.id}/attachments`); + if (receipts.ok && (await receipts.json() as ChatQueryAttachmentReceipt[]).some(item => item.id === original.id && item.sha256 === original.sha256 && item.issueCommentId === state.briefCommentId)) { + path = original.contentPath; state.briefContextKind = "readable-parent-reference"; + } else if (!receipts.ok) await receipts.body?.cancel(); + } else await parent.body?.cancel(); + } + if (!path && state.durableBriefComplete) state.briefContextKind = "faithful-durable-copy"; + state.childHasBrief = !!path || state.durableBriefComplete === true; if (path) { const target = new URL(path, ctx.input.api.baseURL); if (target.origin !== ctx.input.api.baseURL || !target.pathname.startsWith("/api/attachments/")) throw new Error("Delegated brief escaped the fixture API"); const response = await fetch(target, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}` } }); @@ -116,7 +158,7 @@ export const runtimeDelegationCases = [runtimeDefinition("task-coordination.dele state.task = await ctx.input.api.get(`/api/issues/${task.id}`); const workerRun = o.runs.find(row => row.agentId === (state.worker as Entity).id); if (workerRun) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, workerRun.id, ["website.html"]); - state.actualWebsiteArtifact = artifact?.contentVerified === true && hasLaunchHeadline(artifact.content) + state.actualWebsiteArtifact = artifact?.contentVerified === true && launchPageReflectsBrief(artifact.content) && o.revisions.some(row => row.createdByRunId === workerRun.id && row.createdByAgentId === (state.worker as Entity).id && (row.body ?? "").includes(String(state.marker)) && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); } }