test(evals): grade first-work's rendered site by visible heading and business name, not hero heading text

The worker put "Acorn Notes" in the brand link and a value proposition in
the h1, so the heading-name locator failed on a correct site. The scripted
human now checks for a visible heading and the setup's business name
anywhere visible. The task title keyword checks are dropped; the delivered
HTML proves the work is the business website.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -3712,7 +3712,7 @@
       "chat-catalog-domains/internal-support.ts",
       "chat-scenarios/onboarding.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.identity.v1",
+    "oracleVersion": "paperclip.chat-catalog.identity.first-work.v2",
     "expectedRunCount": 3,
     "minimumRunCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/identity-first-work.ts

@@ -30,18 +30,17 @@ export function gradeFirstWork(value: unknown) {
   c.add("actual-first-request-context", o.state.beforeSourceHumanComments === 0 && o.state.beforeSourceRuns === 0 && o.humanQueries.length === 1
     && o.humanQueries[0]?.trim() === "Make me a simple website introducing our business.", "The canonical request is the first real human instruction in this ready company's conversation.");
   c.add("separate-requested-ordinary-work", o.tasks.length === 1 && work?.companyId === o.companyId && work.assigneeAgentId === worker.id && work.id !== o.state.sourceId && work.parentId == null
-    && work.status === "done" && hasLink(o, "issues", work) && /website|landing.page|web site/i.test(`${work.title} ${work.description}`)
-    && /Acorn Notes/i.test(`${work.title} ${work.description}`) && o.state.sourceStillConversation === true
-    && o.activity.some(row => row.action === "issue.created" && row.entityId === work.id && row.agentId === o.leadId), "The first chat captures the actual known business in one linked ordinary completed task; its standing conversation stays separate.");
+    && work.status === "done" && hasLink(o, "issues", work) && o.state.sourceStillConversation === true
+    && o.activity.some(row => row.action === "issue.created" && row.entityId === work.id && row.agentId === o.leadId), "The first chat creates one linked ordinary completed task; its standing conversation stays separate. Its delivered HTML (graded below), not its model-written title, proves it is the business website.");
   c.add("actual-run-attributed-website", !!work && !!website && website.attachment.issueId === work.id && website.attachment.companyId === o.companyId
     && website.sha256 === website.attachment.sha256 && website.attachment.createdByAgentId === worker.id
     && o.runs.some(run => run.id === website.attachment.originatingRunId && run.agentId === worker.id && run.status === "succeeded"
       && (run.contextSnapshot?.issueId === work.id || (run as unknown as { nativeIssueId?: string }).nativeIssueId === work.id))
     && /<html\b/i.test(website.body) && /<body\b/i.test(website.body) && /<h[1-6]\b/i.test(website.body)
     && /Acorn Notes/i.test(website.body) && /research|science/i.test(website.body), "Authorized exact HTML bytes match the immutable actual upload hash and originating worker run, introducing the real business rather than an empty onboarding task.");
-  c.add("actual-review-and-usable-rendering", o.state.visibleArtifactName === true && o.state.renderedBusinessHeading === true && o.state.renderedBusinessDescription === true
+  c.add("actual-review-and-usable-rendering", o.state.visibleArtifactName === true && o.state.renderedHeading === true && o.state.renderedBusinessName === true && o.state.renderedBusinessDescription === true
     && o.state.renderedBytesMatch === true && o.state.renderedWithoutScriptExecution === true,
-    "The real task link exposes its delivered file, and the exact downloaded HTML renders a visible business heading/description in an isolated browser with scripts disabled.");
+    "The real task link exposes its delivered file, and the exact downloaded HTML renders a visible heading, the business name and its description in an isolated browser with scripts disabled. The name may sit in a brand link rather than the hero heading.");
   return c.result();
 }
 export const firstWorkIdentityCases: readonly ChatCatalogCase[] = [identityDefinition("onboarding.first-work", async ctx => {
@@ -50,7 +49,7 @@ export const firstWorkIdentityCases: readonly ChatCatalogCase[] = [identityDefin
   const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments`);
   return { sourceId: source.id, worker, project: p, beforeSourceHumanComments: comments.filter(comment => comment.authorUserId).length, beforeSourceRuns: (await ctx.allRuns()).length };
 }, gradeFirstWork, {
-  authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 600_000,
+  oracleVersion: "paperclip.chat-catalog.identity.first-work.v2", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 600_000,
   sourceFiles: ["chat-catalog-domains/identity-first-work.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"],
   async act(ctx, state) {
     const o = await pollUntil({ label: "first real completed website and delivered HTML", deadlineAt: Date.now() + 360_000,
@@ -71,11 +70,14 @@ export const firstWorkIdentityCases: readonly ChatCatalogCase[] = [identityDefin
       // No rewrite, fabricated server, or script execution: inspect the exact
       // authorized delivered artifact bytes through the browser's file loader.
       await isolated.route(/^https?:\/\//, route => route.abort()); const page = await isolated.newPage(); await page.goto(pathToFileURL(filename).href);
-      await expect(page.getByRole("heading", { name: /Acorn Notes/i }).first()).toBeVisible(); state.renderedBusinessHeading = true;
-      await expect(page.getByText(/research|science/i).first()).toBeVisible(); state.renderedBusinessDescription = true; state.renderedWithoutScriptExecution = true;
+      // The business name is the setup's literal; where the page places it
+      // (hero heading, brand link, title banner) is the worker's design choice.
+      await expect(page.getByRole("heading").filter({ visible: true }).first()).toBeVisible(); state.renderedHeading = true;
+      await expect(page.locator("body").getByText(/Acorn Notes/i).filter({ visible: true }).first()).toBeVisible(); state.renderedBusinessName = true;
+      await expect(page.locator("body").getByText(/research|science/i).filter({ visible: true }).first()).toBeVisible(); state.renderedBusinessDescription = true; state.renderedWithoutScriptExecution = true;
     } finally { await isolated.close(); }
     await ctx.input.evidence("identity-real-first-website.json", { taskId: work.id, attachmentId: file.attachment.id, originatingRunId: file.attachment.originatingRunId,
-      sha256: file.sha256, visibleArtifactName: state.visibleArtifactName, renderedBusinessHeading: state.renderedBusinessHeading, renderedBusinessDescription: state.renderedBusinessDescription,
+      sha256: file.sha256, visibleArtifactName: state.visibleArtifactName, renderedHeading: state.renderedHeading, renderedBusinessName: state.renderedBusinessName, renderedBusinessDescription: state.renderedBusinessDescription,
       artifactBytesRewritten: false, scriptsExecuted: false, externallyPublished: false });
     await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
   },

tests/runner-e2e/chat-catalog-domains/identity-onboarding-work.test.ts

@@ -17,7 +17,7 @@ function website() {
   o.runs = [{ id: "worker-run", companyId: "acorn", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "website" } }];
   o.state = { worker: { id: "morgan" }, sourceId: "chat", beforeSourceHumanComments: 0, beforeSourceRuns: 0, sourceStillConversation: true, reviewedAttachmentId: "html",
     websites: [{ attachment: { id: "html", issueId: "website", companyId: "acorn", sha256: "bytes", createdByAgentId: "morgan", originatingRunId: "worker-run" }, sha256: "bytes", body: "<html><body><h1>Acorn Notes</h1><p>Research notes for science teams.</p></body></html>" }],
-    visibleArtifactName: true, renderedBusinessHeading: true, renderedBusinessDescription: true, renderedBytesMatch: true, renderedWithoutScriptExecution: true }; return o;
+    visibleArtifactName: true, renderedHeading: true, renderedBusinessName: true, renderedBusinessDescription: true, renderedBytesMatch: true, renderedWithoutScriptExecution: true }; return o;
 }
 describe("human company creation and genuine first completed website", () => {
   it("validates an actual human-created separate company with a real CEO", () => { expect(gradeNewCompany(company()).passed).toBe(true); expect(gradeNewCompany(undefined).passed).toBe(false); });
@@ -26,5 +26,15 @@ describe("human company creation and genuine first completed website", () => {
   it("validates the actual run-bound delivered and rendered website", () => { expect(gradeFirstWork(website()).passed).toBe(true); expect(gradeFirstWork(undefined).passed).toBe(false); });
   it("rejects empty task creation even with a finished website reply", () => { const o = website(); o.reply = "The website is complete."; o.state.websites = []; expect(gradeFirstWork(o).passed).toBe(false); });
   it("rejects artifact createdByRunId fiction instead of immutable originatingRunId", () => { const o = website(); const file = (o.state.websites as Array<{ attachment: Record<string, unknown> }>)[0]!; file.attachment.createdByRunId = "worker-run"; delete file.attachment.originatingRunId; expect(gradeFirstWork(o).passed).toBe(false); });
+  it("accepts a reworded task title and a business name placed in the brand link rather than the hero heading", () => {
+    const o = website(); Object.assign(o.tasks[0]!, { title: "Build the launch page", description: "One static page from the Launch brief." });
+    const file = (o.state.websites as Array<{ body: string }>)[0]!; file.body = "<html><body><header><a class=\"brand\">Acorn Notes</a></header><h1>Capture every observation</h1><p>A notebook for small science teams.</p></body></html>";
+    expect(gradeFirstWork(o).passed).toBe(true);
+  });
+  it("rejects a rendered page without a visible heading or without the business name", () => {
+    const heading = website(); heading.state.renderedHeading = false; expect(gradeFirstWork(heading).passed).toBe(false);
+    const name = website(); name.state.renderedBusinessName = false; expect(gradeFirstWork(name).passed).toBe(false);
+    const bytes = website(); (bytes.state.websites as Array<{ body: string }>)[0]!.body = "<html><body><h1>Our company</h1><p>Research notes for science teams.</p></body></html>"; expect(gradeFirstWork(bytes).passed).toBe(false);
+  });
   it("rejects rendered fixture bytes that differ from the actual authorized download", () => { const o = website(); o.state.renderedBytesMatch = false; expect(gradeFirstWork(o).passed).toBe(false); });
 });