test(evals): exercise delegated task creation and lost send recovery

tests/runner-e2e/chat-catalog-domains/internal-projects.ts

@@ -1,3 +1,4 @@
+import { projectWorkCases } from "./internal-task-creation.js";
 import type { ChatCatalogCase } from "../chat-catalog-contract.js";
 import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity, type InternalObservation } from "./internal-support.js";
 import { makeDefinition, retryDefinition, fixtureWorker } from "./internal-definition.js";
@@ -38,6 +39,7 @@ export function grade_projects(value: unknown) {
 
 const define = (id: string, setup: ChatCatalogCase["setup"]) => makeDefinition(id, setup, grade_projects);
 export const projectsCases: readonly ChatCatalogCase[] = [
+  ...projectWorkCases,
   retryDefinition("projects.create-project", grade_projects, initialProjectCreation),
   define("projects.archive-project", async ctx => {
     const p = await project(ctx, "Launch", { status: "completed" });

tests/runner-e2e/chat-catalog-domains/internal-task-creation.ts

@@ -0,0 +1,135 @@
+import { createHash } from "node:crypto";
+import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { ChatRun, ChatIssue } from "../chat-flow.js";
+import { dropChatSendAcknowledgement } from "../lost-send.js";
+import { pollUntil } from "../api.js";
+import { makeDefinition, fixtureWorker } from "./internal-definition.js";
+import { observeInternal, project, task, entity, observation, checks, hasLink, sameTask,
+  type Entity, type InternalObservation } from "./internal-support.js";
+
+interface Attachment extends Entity { contentType?: string; originalFilename?: string; filename?: string; byteSize?: number }
+interface CreationObservation extends InternalObservation {
+  source: ChatIssue | null;
+  replies: Entity[];
+  sourceComments: Array<Entity & { clientRequestId?: string }>;
+  websiteFiles: Array<{ issueId: string; attachmentId: string; body: string; sha256: string; createdAt?: string }>;
+}
+function boundIssue(run: ChatRun) { return (run as ChatRun & { nativeIssueId?: string }).nativeIssueId ?? run.contextSnapshot?.issueId; }
+function newTasks(o: InternalObservation) {
+  const priorIds = Array.isArray(o.state.priorTasks) ? (o.state.priorTasks as Entity[]).map(row => row.id) : [];
+  return o.tasks.filter(row => !priorIds.includes(row.id));
+}
+function creationObservation(value: unknown): CreationObservation | undefined {
+  const o = observation(value) as CreationObservation | undefined;
+  return o && Array.isArray(o.replies) && Array.isArray(o.sourceComments) && Array.isArray(o.websiteFiles) ? o : undefined;
+}
+export function gradeDelegatedCreation(value: unknown) {
+  const c = checks(); const o = creationObservation(value);
+  c.add("complete-creation-evidence", !!o, "Creation needs actual conversation, task, worker-run and delivered-file observations.");
+  if (!o) return c.result();
+  const created = newTasks(o); const saved = created[0]; const worker = entity(o.state, "worker");
+  const launch = o.projects.filter(row => row.name === "Launch"); const p = launch[0];
+  const workRuns = saved ? o.runs.filter(run => boundIssue(run) === saved.id) : [];
+  c.add("one-ordinary-company-task", created.length === 1 && !!saved && saved.companyId === o.companyId
+    && saved.assigneeAgentId === worker.id && saved.parentId == null && saved.projectId === p?.id && saved.id !== o.source?.id,
+    "One ordinary task belongs to the intended Launch project and single eligible Morgan, separately from standing Chat.");
+  c.add("requested-work-captured", !!saved && (o.scenarioId === "projects.project-work"
+    ? /\bmobile\b/i.test(`${saved.title} ${saved.description ?? ""}`) && /\blayout\b/i.test(`${saved.title} ${saved.description ?? ""}`)
+    : /\b(?:website|web site|landing.page)\b/i.test(`${saved.title} ${saved.description ?? ""}`)),
+    "The actual new task captures the requested website or mobile-layout work.");
+  c.add("one-launch-and-attribution", launch.length === 1 && p?.companyId === o.companyId && !!saved
+    && o.activity.filter(row => row.action === "issue.created" && row.entityId === saved.id).length === 1
+    && o.activity.some(row => row.action === "issue.created" && row.entityId === saved.id && row.agentId === o.leadId),
+    "Exactly one intended Launch and one CEO-attributed task creation exist.");
+  c.add("retained-chat-and-links", o.source?.assigneeAgentId === o.leadId && o.source.status === "in_review"
+    && o.source.conversationState === "waiting" && !o.source.scheduledRetry && !o.source.activeRecoveryAction
+    && !!saved && hasLink(o, "issues", saved) && !!p && (o.scenarioId !== "tasks.website-without-project" || hasLink(o, "projects", p)),
+    "The original source remains a settled standing conversation and renders the actual saved result links.");
+  c.add("separate-admitted-worker", o.scenarioId === "projects.project-work" && workRuns.length === 0 || workRuns.length === 1
+    && workRuns[0]?.agentId === worker.id && ["queued", "running", "succeeded"].includes(workRuns[0]?.status ?? ""),
+    "Requested execution is a separate Morgan run bound to the actual new task; project task creation may remain saved.");
+  const readinessClaim = /\b(?:website|site|landing.page)\b(?!\s+task\b)[^.!?\n]{0,60}\b(?:is ready|is live|is built|is complete|has been built|has been deployed)\b/i;
+  c.add("no-premature-website-success", o.replies.every(reply => !readinessClaim.test(reply.body ?? "") || !!saved && saved.status === "done"
+    && workRuns.some(run => run.status === "succeeded") && o.websiteFiles.some(file => file.issueId === saved.id
+      && /<html\b/i.test(file.body) && /<body\b/i.test(file.body) && /<h[1-6]\b/i.test(file.body)
+      && Date.parse(file.createdAt ?? "") <= Date.parse(reply.createdAt ?? ""))),
+    "A claimed finished website requires independently downloaded HTML from the executed task, rather than task creation alone.");
+  const priorTasks = Array.isArray(o.state.priorTasks) ? o.state.priorTasks as Entity[] : [];
+  c.add("existing-work-preserved", priorTasks.every(before => sameTask(before, o.tasks.find(row => row.id === before.id))),
+    "Existing tasks, including similarly named work in other projects, remain unchanged.");
+  if (o.scenarioId === "projects.project-work") {
+    const open = priorTasks.filter(row => row.projectId === p?.id && !["done", "cancelled"].includes(row.status ?? ""));
+    c.add("launch-leftover-summary", open.length > 0 && open.every(row => hasLink(o, "issues", row))
+      && priorTasks.filter(row => row.projectId !== p?.id).every(row => !hasLink(o, "issues", row)),
+      "The leftover-work summary is grounded in Launch's unfinished work and excludes the other project's distractor.");
+  }
+  if (o.scenarioId === "tasks.website-without-project") {
+    const transport = o.state.transport as { commentId?: string; clientRequestId?: string; replayedCommentId?: string; firstTaskId?: string; firstProjectId?: string } | undefined;
+    const requestRuns = transport ? o.runs.filter(run => run.contextSnapshot?.wakeCommentId === transport.commentId
+      || Array.isArray(run.contextSnapshot?.wakeCommentIds) && run.contextSnapshot.wakeCommentIds.includes(transport.commentId)) : [];
+    c.add("committed-lost-ack-recovery", !!transport && typeof transport.commentId === "string" && !!transport.commentId
+      && typeof transport.clientRequestId === "string" && !!transport.clientRequestId && transport.commentId === transport.replayedCommentId
+      && o.sourceComments.filter(row => row.clientRequestId === transport.clientRequestId).map(row => row.id).join() === transport.commentId
+      && saved?.id === transport.firstTaskId && p?.id === transport.firstProjectId && requestRuns.length === 1,
+      "A real lost HTTP acknowledgement and exact client-request replay retain one original comment, project, task and request-bound run.");
+    c.add("new-project-is-agent-created", !!p && o.activity.some(row => row.action === "project.created" && row.entityId === p.id && row.agentId === o.leadId),
+      "The harness did not seed the requested new project.");
+  }
+  return c.result();
+}
+async function observeCreation(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<CreationObservation> {
+  const o = await observeInternal(ctx, state); const source = ctx.issue();
+  const sourceComments = source ? await ctx.input.api.get<CreationObservation["sourceComments"]>(`/api/issues/${source.id}/comments?order=asc`) : [];
+  const replies = sourceComments.filter(row => row.authorAgentId === o.leadId && o.runs.some(run => run.id === row.createdByRunId));
+  const websiteFiles: CreationObservation["websiteFiles"] = [];
+  for (const saved of newTasks(o)) {
+    const attachments = await ctx.input.api.get<Attachment[]>(`/api/issues/${saved.id}/attachments`);
+    for (const file of attachments.filter(row => /\.html?$/i.test(row.originalFilename ?? row.filename ?? "") || row.contentType === "text/html").slice(0, 3)) {
+      if (file.byteSize !== undefined && file.byteSize > 1_048_576) continue;
+      const response = await ctx.input.api.request.get(`/api/attachments/${file.id}/content`);
+      if (!response.ok()) continue;
+      const body = await response.text();
+      if (Buffer.byteLength(body) <= 1_048_576) websiteFiles.push({ issueId: saved.id, attachmentId: file.id, body,
+        sha256: createHash("sha256").update(body).digest("hex"), createdAt: file.createdAt });
+    }
+  }
+  return { ...o, source: source ?? null, replies, sourceComments, websiteFiles };
+}
+function creationDefinition(scenarioId: string): ChatCatalogCase {
+  let interception: Awaited<ReturnType<typeof dropChatSendAcknowledgement>> | undefined;
+  const base = makeDefinition(scenarioId, async ctx => {
+    const worker = await fixtureWorker(ctx); const priorTasks: Entity[] = [];
+    if (scenarioId !== "tasks.website-without-project") {
+      const p = await project(ctx, "Launch", { description: "Acorn Notes is a note-taking product for small research teams. The website should explain it and offer signup." });
+      if (scenarioId === "projects.project-work") {
+        priorTasks.push(await task(ctx, "Signup copy", p.id));
+        priorTasks.push(await task(ctx, "Product brief", p.id, { status: "done" }));
+        const other = await project(ctx, "Experiments"); priorTasks.push(await task(ctx, "Signup copy", other.id));
+      }
+    }
+    if (scenarioId === "tasks.website-without-project") interception = await dropChatSendAcknowledgement(ctx.input.page, ctx.scenario.query);
+    return { worker, priorTasks };
+  }, gradeDelegatedCreation);
+  return { ...base, observe: observeCreation, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-task-creation.ts", "lost-send.ts"],
+    expectedRunCount: 3, minimumRunCount: scenarioId === "projects.project-work" ? 1 : 2,
+    runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 },
+    async act(ctx, state) {
+      if (scenarioId !== "tasks.website-without-project") return;
+      if (!interception) throw new Error("Lost acknowledgement fixture was not armed");
+      const sent = await interception.committed;
+      const first = await pollUntil({ label: "Committed project and task before lost-ack replay", deadlineAt: Date.now() + 150_000,
+        load: () => observeCreation(ctx, state), accept: o => newTasks(o).length === 1 && o.projects.filter(row => row.name === "Launch").length === 1
+          && ctx.issue()?.conversationState === "waiting" });
+      await interception.dispose();
+      const replay = await ctx.input.api.post<{ id: string }>(sent.path, sent.data);
+      state.transport = { commentId: sent.commentId, clientRequestId: String(sent.data.clientRequestId), replayedCommentId: replay.id,
+        firstTaskId: newTasks(first)[0]!.id, firstProjectId: first.projects.find(row => row.name === "Launch")!.id };
+      await ctx.input.evidence("chat-catalog-lost-ack-transport.json", { fault: "server committed; browser acknowledgement reset", ...state.transport as object,
+        originalRequestBody: sent.data.body, retry: "Exact original production clientRequestId and request payload; no new human prompt." });
+      await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
+    },
+    async cleanup() { await interception?.dispose(); interception = undefined; },
+  };
+}
+export const delegatedTaskCases: readonly ChatCatalogCase[] = [creationDefinition("tasks.website-task"), creationDefinition("tasks.website-without-project")];
+export const projectWorkCases: readonly ChatCatalogCase[] = [creationDefinition("projects.project-work")];

tests/runner-e2e/chat-catalog-domains/internal-tasks.ts

@@ -1,3 +1,4 @@
+import { delegatedTaskCases } from "./internal-task-creation.js";
 import type { ChatCatalogCase } from "../chat-catalog-contract.js";
 import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity } from "./internal-support.js";
 import { makeDefinition, retryDefinition, fixtureWorker } from "./internal-definition.js";
@@ -54,6 +55,7 @@ export function grade_tasks(value: unknown) {
 
 const define = (id: string, setup: ChatCatalogCase["setup"]) => makeDefinition(id, setup, grade_tasks);
 export const tasksCases: readonly ChatCatalogCase[] = [
+  ...delegatedTaskCases,
   ambiguousTask(),
   define("tasks.edit-properties", async ctx => {
     const p = await project(ctx); const worker = await fixtureWorker(ctx);

tests/runner-e2e/chat-catalog-domains/internal.test.ts

@@ -19,6 +19,25 @@ function valid(scenario: string): InternalObservation {
       o.projects[0] = { ...o.projects[0]!, archivedAt: "2026-10-26T12:00:00Z" }; o.state.originalBody = "Delivered"; o.state.originalActivityId = "original";
       o.documents = [{ id: "doc", body: "Delivered" }]; o.activity = [{ id: "original", action: "issue.created", entityId: "task" }, event("project.updated", "p")]; break;
     }
+    case "tasks.website-task":
+    case "tasks.website-without-project":
+    case "projects.project-work": {
+      delete o.state.task; o.state.worker = { id: "morgan" }; o.state.priorTasks = [];
+      o.tasks[0] = { ...o.tasks[0]!, title: scenario === "projects.project-work" ? "Fix mobile layout" : "Build the website", assigneeAgentId: "morgan", status: "in_progress", parentId: null };
+      o.activity = [event("issue.created", "task"), event("project.created", "p")];
+      o.runs[0]!.contextSnapshot = { issueId: "chat", wakeCommentId: "request" };
+      o.runs.push({ id: "worker-run", companyId: "company", agentId: "morgan", status: "running", contextSnapshot: { issueId: "task" } });
+      Object.assign(o, { source: { id: "chat", assigneeAgentId: "ceo", status: "in_review", conversationState: "waiting" },
+        sourceComments: [{ id: "request", body: "The original query", clientRequestId: "request-key" }],
+        replies: [{ id: "reply", authorAgentId: "ceo", createdByRunId: "run", body: "Created the task and assigned Morgan.", createdAt: "2026-10-26T12:00:00Z" }], websiteFiles: [] });
+      if (scenario === "tasks.website-without-project") o.state.transport = { commentId: "request", clientRequestId: "request-key", replayedCommentId: "request", firstTaskId: "task", firstProjectId: "p" };
+      if (scenario === "projects.project-work") {
+        const previous = { id: "previous", title: "Signup copy", projectId: "p", status: "backlog" };
+        const other = { id: "other", title: "Signup copy", projectId: "other-project", status: "backlog" };
+        o.state.priorTasks = [previous, other]; o.tasks.push(previous, other); o.links.push("/ACORN/issues/previous");
+      }
+      break;
+    }
     case "tasks.edit-properties": o.state.worker = { id: "morgan" }; o.tasks = [{ ...o.tasks[0]!, priority: "high", assigneeAgentId: "morgan" }]; o.activity = [event("issue.updated", "task")]; break;
     case "tasks.ambiguous-task": {
       const other = { ...o.tasks[0]!, id: "other", projectId: "other-project", title: "Landing page" };
@@ -92,6 +111,9 @@ function valid(scenario: string): InternalObservation {
 const corrupt: Record<string, (o: InternalObservation) => void> = {
   "projects.create-project": o => o.projects.push({ ...o.projects[0]!, id: "duplicate" }),
   "projects.archive-project": o => { o.documents = []; },
+  "tasks.website-task": o => { o.tasks[0]!.assigneeAgentId = "another-worker"; },
+  "tasks.website-without-project": o => { (o.state.transport as { replayedCommentId: string }).replayedCommentId = "duplicate"; },
+  "projects.project-work": o => { o.tasks[0]!.projectId = "other-project"; },
   "tasks.edit-properties": o => { o.tasks[0]!.assigneeAgentId = "someone-else"; },
   "tasks.ambiguous-task": o => { (o.state.clarification as InternalObservation).tasks[0]!.priority = "high"; },
   "tasks.comment-with-reference": o => { o.comments[0]!.authorAgentId = null; o.comments[0]!.authorUserId = "human"; },
@@ -203,3 +225,28 @@ describe("goal oracle phase binding", () => {
     expect(gradeInternal(o).passed).toBe(false);
   });
 });
+
+
+describe("delegated creation calibration", () => {
+  it("rejects invented finished website without delivered bytes", () => {
+    const o = valid("tasks.website-task"); Object.assign(o, { replies: [{ id: "reply", body: "The website is ready.", createdAt: "2026-10-26T12:00:00Z" }] });
+    expect(gradeInternal(o).passed).toBe(false);
+  });
+  it("accepts genuine active work without pretending it has completed", () => {
+    expect(gradeInternal(valid("tasks.website-task")).passed).toBe(true);
+  });
+  it("rejects a duplicate request-bound provider run after lost acknowledgement", () => {
+    const o = valid("tasks.website-without-project"); o.runs.push({ ...o.runs[0]!, id: "duplicate-run" });
+    expect(gradeInternal(o).passed).toBe(false);
+  });
+  it("rejects an ungrounded other-project leftover summary", () => {
+    const o = valid("projects.project-work"); o.links = o.links.filter(href => !href.endsWith("previous")); o.links.push("/ACORN/issues/other");
+    expect(gradeInternal(o).passed).toBe(false);
+  });
+});
+
+
+it("distinguishes a ready saved task from a claimed finished website", () => {
+  const o = valid("tasks.website-task"); Object.assign(o, { replies: [{ id: "reply", body: "The website task is ready for Morgan.", createdAt: "2026-10-26T12:00:00Z" }] });
+  expect(gradeInternal(o).passed).toBe(true);
+});