diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 9d378bab12a..03dd0142eb0 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -3712,7 +3712,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.first-work.v2", "expectedRunCount": 3, "minimumRunCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-first-work.ts b/tests/runner-e2e/chat-catalog-domains/identity-first-work.ts index 7542fcc5adb..f205b1db167 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-first-work.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-first-work.ts @@ -30,18 +30,17 @@ export function gradeFirstWork(value: unknown) { c.add("actual-first-request-context", o.state.beforeSourceHumanComments === 0 && o.state.beforeSourceRuns === 0 && o.humanQueries.length === 1 && o.humanQueries[0]?.trim() === "Make me a simple website introducing our business.", "The canonical request is the first real human instruction in this ready company's conversation."); c.add("separate-requested-ordinary-work", o.tasks.length === 1 && work?.companyId === o.companyId && work.assigneeAgentId === worker.id && work.id !== o.state.sourceId && work.parentId == null - && work.status === "done" && hasLink(o, "issues", work) && /website|landing.page|web site/i.test(`${work.title} ${work.description}`) - && /Acorn Notes/i.test(`${work.title} ${work.description}`) && o.state.sourceStillConversation === true - && o.activity.some(row => row.action === "issue.created" && row.entityId === work.id && row.agentId === o.leadId), "The first chat captures the actual known business in one linked ordinary completed task; its standing conversation stays separate."); + && work.status === "done" && hasLink(o, "issues", work) && o.state.sourceStillConversation === true + && o.activity.some(row => row.action === "issue.created" && row.entityId === work.id && row.agentId === o.leadId), "The first chat creates one linked ordinary completed task; its standing conversation stays separate. Its delivered HTML (graded below), not its model-written title, proves it is the business website."); c.add("actual-run-attributed-website", !!work && !!website && website.attachment.issueId === work.id && website.attachment.companyId === o.companyId && website.sha256 === website.attachment.sha256 && website.attachment.createdByAgentId === worker.id && o.runs.some(run => run.id === website.attachment.originatingRunId && run.agentId === worker.id && run.status === "succeeded" && (run.contextSnapshot?.issueId === work.id || (run as unknown as { nativeIssueId?: string }).nativeIssueId === work.id)) && / { @@ -50,7 +49,7 @@ export const firstWorkIdentityCases: readonly ChatCatalogCase[] = [identityDefin const comments = await ctx.input.api.get(`/api/issues/${source.id}/comments`); return { sourceId: source.id, worker, project: p, beforeSourceHumanComments: comments.filter(comment => comment.authorUserId).length, beforeSourceRuns: (await ctx.allRuns()).length }; }, gradeFirstWork, { - authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 600_000, + oracleVersion: "paperclip.chat-catalog.identity.first-work.v2", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 600_000, sourceFiles: ["chat-catalog-domains/identity-first-work.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"], async act(ctx, state) { const o = await pollUntil({ label: "first real completed website and delivered HTML", deadlineAt: Date.now() + 360_000, @@ -71,11 +70,14 @@ export const firstWorkIdentityCases: readonly ChatCatalogCase[] = [identityDefin // No rewrite, fabricated server, or script execution: inspect the exact // authorized delivered artifact bytes through the browser's file loader. await isolated.route(/^https?:\/\//, route => route.abort()); const page = await isolated.newPage(); await page.goto(pathToFileURL(filename).href); - await expect(page.getByRole("heading", { name: /Acorn Notes/i }).first()).toBeVisible(); state.renderedBusinessHeading = true; - await expect(page.getByText(/research|science/i).first()).toBeVisible(); state.renderedBusinessDescription = true; state.renderedWithoutScriptExecution = true; + // The business name is the setup's literal; where the page places it + // (hero heading, brand link, title banner) is the worker's design choice. + await expect(page.getByRole("heading").filter({ visible: true }).first()).toBeVisible(); state.renderedHeading = true; + await expect(page.locator("body").getByText(/Acorn Notes/i).filter({ visible: true }).first()).toBeVisible(); state.renderedBusinessName = true; + await expect(page.locator("body").getByText(/research|science/i).filter({ visible: true }).first()).toBeVisible(); state.renderedBusinessDescription = true; state.renderedWithoutScriptExecution = true; } finally { await isolated.close(); } await ctx.input.evidence("identity-real-first-website.json", { taskId: work.id, attachmentId: file.attachment.id, originatingRunId: file.attachment.originatingRunId, - sha256: file.sha256, visibleArtifactName: state.visibleArtifactName, renderedBusinessHeading: state.renderedBusinessHeading, renderedBusinessDescription: state.renderedBusinessDescription, + sha256: file.sha256, visibleArtifactName: state.visibleArtifactName, renderedHeading: state.renderedHeading, renderedBusinessName: state.renderedBusinessName, renderedBusinessDescription: state.renderedBusinessDescription, artifactBytesRewritten: false, scriptsExecuted: false, externallyPublished: false }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-onboarding-work.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-onboarding-work.test.ts index 334ed45c7d1..df70e66d483 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-onboarding-work.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-onboarding-work.test.ts @@ -17,7 +17,7 @@ function website() { o.runs = [{ id: "worker-run", companyId: "acorn", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "website" } }]; o.state = { worker: { id: "morgan" }, sourceId: "chat", beforeSourceHumanComments: 0, beforeSourceRuns: 0, sourceStillConversation: true, reviewedAttachmentId: "html", websites: [{ attachment: { id: "html", issueId: "website", companyId: "acorn", sha256: "bytes", createdByAgentId: "morgan", originatingRunId: "worker-run" }, sha256: "bytes", body: "

Acorn Notes

Research notes for science teams.

" }], - visibleArtifactName: true, renderedBusinessHeading: true, renderedBusinessDescription: true, renderedBytesMatch: true, renderedWithoutScriptExecution: true }; return o; + visibleArtifactName: true, renderedHeading: true, renderedBusinessName: true, renderedBusinessDescription: true, renderedBytesMatch: true, renderedWithoutScriptExecution: true }; return o; } describe("human company creation and genuine first completed website", () => { it("validates an actual human-created separate company with a real CEO", () => { expect(gradeNewCompany(company()).passed).toBe(true); expect(gradeNewCompany(undefined).passed).toBe(false); }); @@ -26,5 +26,15 @@ describe("human company creation and genuine first completed website", () => { it("validates the actual run-bound delivered and rendered website", () => { expect(gradeFirstWork(website()).passed).toBe(true); expect(gradeFirstWork(undefined).passed).toBe(false); }); it("rejects empty task creation even with a finished website reply", () => { const o = website(); o.reply = "The website is complete."; o.state.websites = []; expect(gradeFirstWork(o).passed).toBe(false); }); it("rejects artifact createdByRunId fiction instead of immutable originatingRunId", () => { const o = website(); const file = (o.state.websites as Array<{ attachment: Record }>)[0]!; file.attachment.createdByRunId = "worker-run"; delete file.attachment.originatingRunId; expect(gradeFirstWork(o).passed).toBe(false); }); + it("accepts a reworded task title and a business name placed in the brand link rather than the hero heading", () => { + const o = website(); Object.assign(o.tasks[0]!, { title: "Build the launch page", description: "One static page from the Launch brief." }); + const file = (o.state.websites as Array<{ body: string }>)[0]!; file.body = "
Acorn Notes

Capture every observation

A notebook for small science teams.

"; + expect(gradeFirstWork(o).passed).toBe(true); + }); + it("rejects a rendered page without a visible heading or without the business name", () => { + const heading = website(); heading.state.renderedHeading = false; expect(gradeFirstWork(heading).passed).toBe(false); + const name = website(); name.state.renderedBusinessName = false; expect(gradeFirstWork(name).passed).toBe(false); + const bytes = website(); (bytes.state.websites as Array<{ body: string }>)[0]!.body = "

Our company

Research notes for science teams.

"; expect(gradeFirstWork(bytes).passed).toBe(false); + }); it("rejects rendered fixture bytes that differ from the actual authorized download", () => { const o = website(); o.state.renderedBytesMatch = false; expect(gradeFirstWork(o).passed).toBe(false); }); });