evals: carry real launch brief through one Morgan handoff and result

tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json

@@ -246,8 +246,8 @@
     },
     {
       "scenarioId": "task-coordination.delegate-with-context",
-      "readinessClass": "local",
-      "needs": "CEO-owned source launch task and actual attached brief; single Morgan child/linked work, real report incorporated without false parent attribution."
+      "readinessClass": "implemented",
+      "needs": "Real human-authored Launch task brief comment+uploaded attachment bytes, actual CEO ownership and genuine pre-query source acknowledgement. Canonical creates one Morgan child or actually linked work task, durable original/copied brief and independent Morgan bearer download hash. Real gated native implementation produces registered HTML and latest non-plan delivery revision; correlated automatic source completion incorporates actual result with no parent execution or duplicate task. No live qualification claimed."
     },
     {
       "scenarioId": "task-coordination.block-on-design",

tests/runner-e2e/chat-catalog-domains/runtime-delegation.test.ts

@@ -0,0 +1,29 @@
+import { expect, it } from "vitest";
+import { gradeDelegation, hasLaunchHeadline } from "./runtime-delegation.js";
+function valid() {
+  const task = { id: "website", parentId: "launch", companyId: "company", assigneeAgentId: "morgan", status: "done", description: "Implement the approved design brief and report the completed result back." };
+  return { scenarioId: "task-coordination.delegate-with-context", companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "",
+    tasks: [{ id: "launch" }, task], activity: [{ id: "created", action: "issue.created", entityId: "website", agentId: "ceo" }],
+    runs: [{ id: "context", agentId: "ceo", status: "succeeded", contextSnapshot: { issueId: "chat" } }, { id: "delegate", agentId: "ceo", status: "succeeded", contextSnapshot: { issueId: "chat" } },
+      { id: "implement", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "website" } }, { id: "report", agentId: "ceo", status: "succeeded", contextSnapshot: { issueId: "chat" } }],
+    state: { parent: { id: "launch" }, task, worker: { id: "morgan" }, humanId: "human", briefCommentId: "human-brief", briefHash: "original-hash", briefDownloadedHash: "original-hash",
+      briefAttachment: { id: "brief", issueId: "launch", companyId: "company", createdByAgentId: null, createdByUserId: "human", issueCommentId: "human-brief", sha256: "original-hash" },
+      parentOwnedByCeo: true, parentContextRetained: true, createdFromSource: true, childHasBrief: true, linkedToLaunch: false, workerBriefReadStatus: 200, workerBriefHash: "original-hash",
+      workerWasReallyWaiting: true, actualWebsiteArtifact: true,
+      delivery: { checks: [{ id: "correlated", passed: true }], response: { body: "Morgan completed Website; the actual saved page is available on the child task." } } } };
+}
+it("accepts one genuinely contextual child and correlated actual result", () => expect(gradeDelegation(valid()).passed).toBe(true));
+it("accepts natural correct-worker result reporting", () => { const o = valid(); o.state.delivery.response.body = "The page Morgan built is ready. Open Website for the delivered result from that implementation."; expect(gradeDelegation(o).passed).toBe(true); });
+it("accepts an independently verified inspectable link as the supported alternative to a child edge", () => { const o = valid(); o.state.task.parentId = ""; o.state.linkedToLaunch = true; expect(gradeDelegation(o).passed).toBe(true); });
+it("accepts substantive equivalent headline markup", () => expect(hasLaunchHeadline('<html><h1><span>Research</span> without&nbsp;the busywork</h1></html>')).toBe(true));
+it("rejects the right phrase elsewhere with a wrong actual headline", () => expect(hasLaunchHeadline('<h1>Unrelated result</h1><p>Research without the busywork</p>')).toBe(false));
+it("rejects missing actual delegation evidence", () => expect(gradeDelegation(undefined).passed).toBe(false));
+it("rejects a delegated task unrelated to the original Launch work", () => { const o = valid(); o.state.task.parentId = "other"; expect(gradeDelegation(o).passed).toBe(false); });
+it("rejects a brief still available only in source chat", () => { const o = valid(); o.state.childHasBrief = false; expect(gradeDelegation(o).passed).toBe(false); });
+it("rejects a worker unable to fetch the actual brief", () => { const o = valid(); o.state.workerBriefReadStatus = 403; expect(gradeDelegation(o).passed).toBe(false); });
+it("rejects copying a similarly named brief with different bytes", () => { const o = valid(); o.state.workerBriefHash = "wrong"; expect(gradeDelegation(o).passed).toBe(false); });
+it("rejects completion with no real delivered page", () => { const o = valid(); o.state.actualWebsiteArtifact = false; expect(gradeDelegation(o).passed).toBe(false); });
+it("rejects a result reported on an uncorrelated source", () => { const o = valid(); o.state.delivery.checks[0]!.passed = false; expect(gradeDelegation(o).passed).toBe(false); });
+it("rejects saying the parent executed Morgan's work", () => { const o = valid(); o.state.delivery.response.body = "I ran the parent Launch task and built the site myself. It is complete."; expect(gradeDelegation(o).passed).toBe(false); });
+it("rejects a hidden actual parent execution", () => { const o = valid(); o.runs[2]!.contextSnapshot.issueId = "launch"; expect(gradeDelegation(o).passed).toBe(false); });
+it("rejects duplicate work tasks", () => { const o = valid(); o.tasks.push({ id: "duplicate" }); expect(gradeDelegation(o).passed).toBe(false); });

tests/runner-e2e/chat-catalog-domains/runtime-delegation.ts

@@ -0,0 +1,133 @@
+import { createHash } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+import { expect } from "@playwright/test";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { ChatQueryAttachmentReceipt } from "../chat-query-attachments.js";
+import { pollUntil } from "../api.js";
+import { sendChatMessage } from "../chat-flow.js";
+import { readRegisteredArtifacts } from "../registered-artifact.js";
+import { observeCompletionUpdate } from "../completion-update-flow.js";
+import { completionDelivery } from "../completion-updates.js";
+import { checks, type Entity } from "./internal-support.js";
+import { openStandingChat } from "./identity-support.js";
+import { prepareRuntimeWorkerGate, releaseRuntimeWorker } from "./runtime-busy-worker.js";
+import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js";
+const keys = new WeakMap<ChatCatalogContext, { id: string; token: string; workerId: string }>();
+const hash = (value: Uint8Array) => createHash("sha256").update(value).digest("hex");
+export function hasLaunchHeadline(html: string) {
+  const heading = html.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1];
+  return heading?.replace(/<[^>]+>/g, " ").replace(/&nbsp;|&#160;|&#x0*a0;/gi, " ").replace(/\s+/g, " ").trim().toLowerCase() === "research without the busywork";
+}
+export function gradeDelegation(value: unknown) {
+  const c = checks(), o = runtimeObservation(value);
+  c.add("actual-launch-brief-and-native-handoff-evidence", !!o, "A real human-attached Launch brief, exactly one CEO-created Morgan child, native result and source completion are required."); if (!o) return c.result();
+  const s = o.state, parent = s.parent as Entity, task = s.task as Entity, receipt = s.briefAttachment as ChatQueryAttachmentReceipt;
+  c.add("real-parent-owner-and-human-brief", s.parentOwnedByCeo === true && !!receipt && receipt.issueId === parent.id && receipt.companyId === o.companyId
+    && receipt.createdByAgentId == null && receipt.createdByUserId === s.humanId && receipt.issueCommentId === s.briefCommentId && receipt.sha256 === s.briefHash
+    && s.briefDownloadedHash === s.briefHash && s.parentContextRetained === true,
+  "The real Launch task remains owned by the CEO and its human comment retains the exact uploaded design brief.");
+  c.add("one-authorized-child-with-durable-context", !!task && (task.parentId === parent.id || s.linkedToLaunch === true) && task.companyId === o.companyId && task.assigneeAgentId === (s.worker as Entity).id
+    && s.createdFromSource === true && o.tasks.length === 2 && s.childHasBrief === true && s.workerBriefReadStatus === 200 && s.workerBriefHash === s.briefHash
+    && o.activity.some(row => row.action === "issue.created" && row.entityId === task.id && row.agentId === o.leadId)
+    && /report|return|complete|finish|deliver/i.test(task.description ?? ""),
+  "The canonical request creates one real Morgan child linked to Launch, with accessible durable brief bytes and an explicit completion request.");
+  const delivery = s.delivery as ReturnType<typeof completionDelivery> | undefined;
+  c.add("actual-morgan-result-incorporated-back-here", s.workerWasReallyWaiting === true && task.status === "done" && s.actualWebsiteArtifact === true
+    && !!delivery && delivery.checks.every(row => row.passed) && /done|complete|finish|ready|delivered/i.test(delivery.response?.body ?? "")
+    && !/I (?:ran|executed|built|implemented).{0,40}(?:Launch|parent)|built (?:it|the site) myself/i.test(delivery.response?.body ?? ""),
+  "Morgan produces the actual brief-based page and the same source receives its correlated accessible result, without pretending the parent executed another agent's work.");
+  c.add("separate-native-ownership-and-no-duplicate-work", o.runs.length === 4 && o.runs.every(row => row.status === "succeeded")
+    && o.runs.filter(row => row.agentId === (s.worker as Entity).id).length === 1
+    && !o.runs.some(row => row.contextSnapshot?.issueId === parent.id || (row as typeof row & { nativeIssueId?: string }).nativeIssueId === parent.id),
+  "The declared setup acknowledgement, canonical CEO turn, single Morgan child execution and one correlated completion are the entire native ledger; Launch has no masquerading execution.");
+  return c.result();
+}
+export const runtimeDelegationCases = [runtimeDefinition("task-coordination.delegate-with-context", async ctx => {
+  const prepared = await prepareRuntimeWorkerGate(ctx), root = `/api/companies/${ctx.input.fixtures.company.id}`, marker = `Launch brief ${ctx.input.nonce}`;
+  const parent = await ctx.input.api.post<Entity>(`${root}/issues`, { title: "Launch", status: "backlog", description: "The CEO owns the launch. The attached human design brief defines the page; implementation should be delegated to Morgan when requested." });
+  const briefComment = await ctx.input.api.post<Entity>(`/api/issues/${parent.id}/comments`, { body: "Approved design brief for later authorized implementation; no work is assigned yet." });
+  const brief = `# Launch design\nHeadline: Research without the busywork\nUse semantic HTML, accessible labels and one primary signup action.\nReference: ${marker}\n`;
+  const uploaded = await ctx.input.page.request.post(`${root}/issues/${parent.id}/attachments`, { multipart: { issueCommentId: briefComment.id,
+    file: { name: "design-brief.md", mimeType: "text/markdown", buffer: Buffer.from(brief) } } });
+  if (uploaded.status() !== 201) { await uploaded.dispose(); throw new Error("Real human Launch brief upload did not succeed"); }
+  const attachment = await uploaded.json() as ChatQueryAttachmentReceipt; await uploaded.dispose();
+  const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session");
+  if (attachment.createdByUserId !== session.user.id || attachment.issueCommentId !== briefComment.id || attachment.sha256 !== hash(Buffer.from(brief))) throw new Error("Launch brief lacks actual human comment/byte provenance");
+  await ctx.input.api.patch(`/api/issues/${parent.id}`, { assigneeAgentId: ctx.input.fixtures.agent.id });
+  if ((await ctx.allRuns()).length) throw new Error("Backlog Launch prerequisite unexpectedly admitted work");
+  await ctx.input.api.saveAgentInstructions(prepared.worker.id, `When an authorized Website task arrives, first run node ${JSON.stringify(prepared.gate.scriptPath)} once and wait for the real brief-release output. Do not replace the script or invent the brief. Then read the actual attached design brief referenced by the task. Implement it as website.html and register that real file as a downloadable artifact work product titled website.html. Save a non-plan delivery document with key website-delivery containing the design's reference and the actual result summary. Complete the same task without creating another task or publishing externally.`);
+  if (!ctx.input.privateSecrets) throw new Error("Actual worker brief read requires the private probe sink");
+  const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${prepared.worker.id}/keys`, { name: "delegated-launch-brief-read" });
+  ctx.input.privateSecrets.push(key.token); keys.set(ctx, { ...key, workerId: prepared.worker.id });
+  await openStandingChat(ctx);
+  const setupMessage = `This conversation tracks Launch ${parent.identifier ?? parent.id}, which you own. It has the actual approved design brief attached to human comment ${briefComment.id}; Morgan is available for implementation. Remember this work context and acknowledge it here, but do not start, modify, delegate or complete work until my next request.`;
+  await sendChatMessage(ctx.input.page, setupMessage);
+  await pollUntil({ label: "Genuine source context acknowledgement before delegation", deadlineAt: Date.now() + 150_000,
+    load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
+    accept: v => v.source?.conversationState === "waiting" && v.runs.length === 1 && v.runs[0]?.agentId === ctx.input.fixtures.agent.id && v.runs[0].status === "succeeded" });
+  return { worker: prepared.worker, gate: prepared.gate, parent: await ctx.input.api.get<Entity>(`/api/issues/${parent.id}`), marker,
+    humanId: session.user.id, briefCommentId: briefComment.id, briefAttachment: attachment, briefHash: hash(Buffer.from(brief)),
+    briefDownloadedHash: hash(await ctx.input.api.getAttachmentContent(attachment.id)), sourceId: ctx.issue()!.id };
+}, gradeDelegation, {
+  authentication: "human-session", setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 4, attemptTimeoutMs: 900_000,
+  sourceFiles: ["chat-catalog-domains/runtime-delegation.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "completion-update-flow.ts", "completion-updates.ts", "registered-artifact.ts", "chat-scenarios/task-coordination.yaml"],
+  async act(ctx, state) {
+    const phase = await pollUntil({ label: "Canonical source creates exactly one actual Morgan child with Launch context", deadlineAt: Date.now() + 180_000,
+      load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), waiting: await readFile((state.gate as { ready: string }).ready, "utf8").catch(() => ""),
+        tasks: await ctx.input.api.get<Entity[]>(`/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true&limit=1000`) }; },
+      accept: v => v.source?.conversationState === "waiting" && v.runs.length === 3 && v.waiting === "waiting"
+        && v.runs.some(row => row.agentId === (state.worker as Entity).id && row.status === "running")
+        && v.runs.filter(row => row.agentId === ctx.input.fixtures.agent.id).every(row => row.status === "succeeded")
+        && v.tasks.filter(row => row.id !== state.sourceId && row.id !== (state.parent as Entity).id).length === 1 });
+    const task = phase.tasks.find(row => row.id !== state.sourceId && row.id !== (state.parent as Entity).id)!; state.task = task; state.workerWasReallyWaiting = true;
+    const fromSource = await ctx.input.api.get<Entity[]>(`/api/companies/${ctx.input.fixtures.company.id}/issues?createdFromIssueId=${state.sourceId}&includeAgentCreatedTasks=true&limit=1000`);
+    state.createdFromSource = fromSource.length === 1 && fromSource[0]?.id === task.id;
+    const attachments = await ctx.input.api.get<ChatQueryAttachmentReceipt[]>(`/api/issues/${task.id}/attachments`), original = state.briefAttachment as ChatQueryAttachmentReceipt;
+    const documents = await ctx.input.api.get<Entity[]>(`/api/issues/${task.id}/documents`), bodies: Entity[] = [];
+    for (const doc of documents) bodies.push(await ctx.input.api.get<Entity>(`/api/issues/${task.id}/documents/${encodeURIComponent(doc.key!)}`));
+    const context = `${task.description ?? ""}\n${bodies.map(doc => doc.body).join("\n")}`;
+    state.linkedToLaunch = [...context.matchAll(/\[[^\]]*\]\(([^)\s]+)\)/g)].some(match => {
+      try { const url = new URL(match[1]!, ctx.input.api.baseURL), parts = url.pathname.split("/").map(decodeURIComponent), i = parts.indexOf("issues");
+        return url.origin === ctx.input.api.baseURL && i >= 0 && [(state.parent as Entity).id, (state.parent as Entity).identifier].includes(parts[i + 1]); }
+      catch { return false; }
+    });
+    const copied = attachments.find(row => row.sha256 === state.briefHash), path = copied?.contentPath ?? (context.includes(original.contentPath) ? original.contentPath : undefined);
+    state.childHasBrief = !!path;
+    const key = keys.get(ctx); if (!key || key.workerId !== (state.worker as Entity).id) throw new Error("Actual Morgan brief probe identity is missing");
+    if (path) {
+      const target = new URL(path, ctx.input.api.baseURL); if (target.origin !== ctx.input.api.baseURL || !target.pathname.startsWith("/api/attachments/")) throw new Error("Delegated brief escaped the fixture API");
+      const response = await fetch(target, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}` } });
+      state.workerBriefReadStatus = response.status;
+      if (response.ok) { const bytes = new Uint8Array(await response.arrayBuffer()); if (bytes.byteLength > 65_536) throw new Error("Delegated fixture brief size exceeded"); state.workerBriefHash = hash(bytes); }
+      else await response.body?.cancel();
+    }
+    await writeFile((state.gate as { gate: string }).gate, "The human has authorized this delegated page implementation. Read the actual task-bound approved design brief, implement its headline and signup design as website.html, retain its exact reference in the delivery document, and complete this same work task. Do not publish or create further tasks.", "utf8");
+    const delivered = await observeCompletionUpdate({ page: ctx.input.page, api: ctx.input.api, sourceId: String(state.sourceId), workerId: task.id,
+      marker: String(state.marker), fixtureRequest: ctx.scenario.query, allRuns: () => ctx.allRuns(), evidence: ctx.input.evidence, capture: ctx.input.capture });
+    state.delivery = completionDelivery(delivered);
+  },
+  async observe(ctx, state) {
+    const o = await observeRuntime(ctx, state), parent = await ctx.input.api.get<Entity>(`/api/issues/${(state.parent as Entity).id}`);
+    state.parentOwnedByCeo = parent.assigneeAgentId === ctx.input.fixtures.agent.id;
+    const original = state.briefAttachment as ChatQueryAttachmentReceipt, attachment = (await ctx.input.api.get<ChatQueryAttachmentReceipt[]>(`/api/issues/${parent.id}/attachments`)).find(row => row.id === original.id);
+    state.parentContextRetained = !!attachment && attachment.sha256 === original.sha256 && attachment.issueCommentId === state.briefCommentId;
+    const task = state.task as Entity | undefined;
+    if (task) {
+      state.task = await ctx.input.api.get<Entity>(`/api/issues/${task.id}`);
+      const workerRun = o.runs.find(row => row.agentId === (state.worker as Entity).id);
+      if (workerRun) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, task.id, workerRun.id, ["website.html"]);
+        state.actualWebsiteArtifact = artifact?.contentVerified === true && hasLaunchHeadline(artifact.content)
+          && o.revisions.some(row => row.createdByRunId === workerRun.id && row.createdByAgentId === (state.worker as Entity).id && (row.body ?? "").includes(String(state.marker))
+            && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); }
+    }
+    return o;
+  },
+  async verifyResult(ctx, state) {
+    const source = ctx.input.page.url();
+    for (const task of [state.parent as Entity, state.task as Entity]) { await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
+      await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("heading", { name: task.title!, exact: true }).first()).toBeVisible(); }
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+  async cleanup(ctx, state) { try { const key = keys.get(ctx); keys.delete(ctx); if (key) await ctx.input.api.delete(`/api/agents/${key.workerId}/keys/${key.id}`, { allowNotFound: true }); }
+    finally { await releaseRuntimeWorker(state); } },
+})];