diff --git a/tests/runner-e2e/chat-catalog-domains/identity-first-work.ts b/tests/runner-e2e/chat-catalog-domains/identity-first-work.ts new file mode 100644 index 00000000000..7542fcc5adb --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-first-work.ts @@ -0,0 +1,84 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect } from "@playwright/test"; +import type { IssueAttachment } from "../../../packages/shared/src/types/issue.js"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { checks, entity, hasLink, project, type Entity } from "./internal-support.js"; +import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat } from "./identity-support.js"; + +interface Website { attachment: IssueAttachment; body: string; sha256: string } +const folders = new WeakMap(); +async function actualWebsite(ctx: ChatCatalogContext, state: Record) { + const o = await observeIdentity(ctx, state); const files: Website[] = []; + for (const task of o.tasks) { + const attachments = await ctx.input.api.get(`/api/issues/${task.id}/attachments`); + for (const attachment of attachments.filter(row => row.contentType === "text/html" || /\.html?$/i.test(row.originalFilename ?? "")).slice(0, 3)) { + if (attachment.byteSize > 1_048_576) continue; + const bytes = await ctx.input.api.getAttachmentContent(attachment.id); + files.push({ attachment, body: bytes.toString("utf8"), sha256: createHash("sha256").update(bytes).digest("hex") }); + } + } + state.websites = files; return o; +} +export function gradeFirstWork(value: unknown) { + const c = checks(); const o = identityObservation(value); c.add("complete-first-website-work", !!o, "Actual first request, ordinary linked execution, authorized artifact bytes and browser rendering are required."); if (!o) return c.result(); + const work = o.tasks[0]; const worker = entity(o.state, "worker"); const websites = o.state.websites as Website[] | undefined; + const website = websites?.find(file => file.attachment.id === o.state.reviewedAttachmentId); + c.add("actual-first-request-context", o.state.beforeSourceHumanComments === 0 && o.state.beforeSourceRuns === 0 && o.humanQueries.length === 1 + && o.humanQueries[0]?.trim() === "Make me a simple website introducing our business.", "The canonical request is the first real human instruction in this ready company's conversation."); + c.add("separate-requested-ordinary-work", o.tasks.length === 1 && work?.companyId === o.companyId && work.assigneeAgentId === worker.id && work.id !== o.state.sourceId && work.parentId == null + && work.status === "done" && hasLink(o, "issues", work) && /website|landing.page|web site/i.test(`${work.title} ${work.description}`) + && /Acorn Notes/i.test(`${work.title} ${work.description}`) && o.state.sourceStillConversation === true + && o.activity.some(row => row.action === "issue.created" && row.entityId === work.id && row.agentId === o.leadId), "The first chat captures the actual known business in one linked ordinary completed task; its standing conversation stays separate."); + c.add("actual-run-attributed-website", !!work && !!website && website.attachment.issueId === work.id && website.attachment.companyId === o.companyId + && website.sha256 === website.attachment.sha256 && website.attachment.createdByAgentId === worker.id + && o.runs.some(run => run.id === website.attachment.originatingRunId && run.agentId === worker.id && run.status === "succeeded" + && (run.contextSnapshot?.issueId === work.id || (run as unknown as { nativeIssueId?: string }).nativeIssueId === work.id)) + && / { + const source = await openStandingChat(ctx); const worker = await identityWorker(ctx); + const p = await project(ctx, "Launch", { description: "Acorn Notes is a research notebook for small science teams. It helps teams capture observations, organize evidence and share research notes. Our business website should introduce these benefits and offer a contact link. No external publishing is requested." }); + const comments = await ctx.input.api.get(`/api/issues/${source.id}/comments`); + return { sourceId: source.id, worker, project: p, beforeSourceHumanComments: comments.filter(comment => comment.authorUserId).length, beforeSourceRuns: (await ctx.allRuns()).length }; +}, gradeFirstWork, { + authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 600_000, + sourceFiles: ["chat-catalog-domains/identity-first-work.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"], + async act(ctx, state) { + const o = await pollUntil({ label: "first real completed website and delivered HTML", deadlineAt: Date.now() + 360_000, + load: () => actualWebsite(ctx, state), accept: current => current.tasks.length === 1 && current.tasks[0]?.status === "done" && (state.websites as Website[]).some(file => /Acorn Notes/i.test(file.body)) + && current.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + const work = o.tasks[0]!; const file = (state.websites as Website[]).find(file => /Acorn Notes/i.test(file.body))!; state.reviewedAttachmentId = file.attachment.id; + const href = o.links.find(link => hasLink({ ...o, links: [link] }, "issues", work)); if (!href) throw new Error("First website has no actual task review link"); + const source = ctx.input.page.url(); await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + if (!file.attachment.originalFilename) throw new Error("Delivered website has no visible file name"); + await expect(ctx.input.page.getByText(file.attachment.originalFilename, { exact: true }).first()).toBeVisible(); state.visibleArtifactName = true; + const privateDir = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR; if (!privateDir) throw new Error("Attempt-private output directory is required"); + const folder = await mkdtemp(path.join(privateDir, "identity-website-")); folders.set(ctx, folder); + const filename = path.join(folder, "website.html"); const bytes = await ctx.input.api.getAttachmentContent(file.attachment.id); + await writeFile(filename, bytes, { mode: 0o600 }); state.renderedBytesMatch = createHash("sha256").update(bytes).digest("hex") === file.sha256; + const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Actual website rendering requires a browser"); + const isolated = await browser.newContext({ javaScriptEnabled: false }); + try { + // No rewrite, fabricated server, or script execution: inspect the exact + // authorized delivered artifact bytes through the browser's file loader. + await isolated.route(/^https?:\/\//, route => route.abort()); const page = await isolated.newPage(); await page.goto(pathToFileURL(filename).href); + await expect(page.getByRole("heading", { name: /Acorn Notes/i }).first()).toBeVisible(); state.renderedBusinessHeading = true; + await expect(page.getByText(/research|science/i).first()).toBeVisible(); state.renderedBusinessDescription = true; state.renderedWithoutScriptExecution = true; + } finally { await isolated.close(); } + await ctx.input.evidence("identity-real-first-website.json", { taskId: work.id, attachmentId: file.attachment.id, originatingRunId: file.attachment.originatingRunId, + sha256: file.sha256, visibleArtifactName: state.visibleArtifactName, renderedBusinessHeading: state.renderedBusinessHeading, renderedBusinessDescription: state.renderedBusinessDescription, + artifactBytesRewritten: false, scriptsExecuted: false, externallyPublished: false }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { const o = await actualWebsite(ctx, state); state.sourceStillConversation = ctx.issue()?.status === "in_review" && ctx.issue()?.assigneeAgentId === ctx.input.fixtures.agent.id && ctx.issue()?.conversationState === "waiting"; return o; }, + async cleanup(ctx) { const folder = folders.get(ctx); if (folder) await rm(folder, { recursive: true, force: true }); folders.delete(ctx); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-new-company.ts b/tests/runner-e2e/chat-catalog-domains/identity-new-company.ts new file mode 100644 index 00000000000..306f9aa34cb --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-new-company.ts @@ -0,0 +1,64 @@ +import { expect } from "@playwright/test"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { Agent } from "../../../packages/shared/src/types/agent.js"; +import { finishIdentityWizard, type WizardReceipt } from "./identity-wizard.js"; +import { checks, type Entity } from "./internal-support.js"; +import { identityDefinition, identityObservation, observeIdentity, readAgentResource, waitForReply } from "./identity-support.js"; + +const companies = new WeakMap(); +export function gradeNewCompany(value: unknown) { + const c = checks(); const o = identityObservation(value); c.add("complete-human-company-onboarding", !!o, "Actual human wizard, first-account readiness and separate company/agent authority are required."); if (!o) return c.result(); + const wizard = o.state.wizard as WizardReceipt | undefined; const agent = o.state.newAgent as Agent | undefined; + c.add("guidance-before-human-creation", o.state.companyCountAfterReply === 1 && /website|business/i.test(o.reply) && /onboarding|creat.*(?:company|organization)|company.*(?:flow|wizard|switcher)|organization.*(?:wizard|switcher)/i.test(o.reply) + && !/\bI(?:'ve| have)?\s+(?:created|set up)\s+(?:a |the |your )?(?:new )?(?:company|organization)/i.test(o.reply), "The source agent preserves the website-business request and guides the actual person without claiming it created another company."); + c.add("actual-ready-company-and-ceo", !!wizard && wizard.companyId !== o.companyId && agent?.id === wizard.firstAgentId && agent.companyId === wizard.companyId + && agent.role === "ceo" && agent.lifecycleState === "ready" && wizard.firstAgentReady && wizard.hiredThroughWizard && wizard.actualHarness === "codex_local" + && wizard.hello.checks.some(check => check.code === "codex_hello_probe_passed") && wizard.account.ownership === "personal" + && wizard.account.ownerUserId === o.state.humanId && wizard.account.responsibleUserId === o.state.humanId && o.state.humanRoleConfigurationStatus === 200, + "The real company wizard creates an independently tested account/first agent; an explicit authenticated human public configuration establishes the actual CEO role, not a renamed general profile."); + c.add("distinct-human-and-agent-authority", !!wizard && wizard.companyId !== wizard.firstAgentId && wizard.firstAgentId !== o.leadId + && o.state.newCompanyOwnedByHuman === true && o.state.oldAgentForeignReadStatus === 403 && o.state.newAgentVisible === true && o.state.oldAgentStillSourceCompany === true, + "The same real human owns the new company, its CEO is a separate actual agent, and the original agent retains only its source-company access."); + c.add("no-duplicate-or-secret-leak", !!wizard && wizard.beforeCompanyIds.length === 1 && wizard.beforeCompanyIds[0] === o.companyId + && wizard.afterCompanyIds.length === 2 && new Set(wizard.afterCompanyIds).size === 2 && wizard.afterAgentIds.length === 1 + && wizard.draftContainsCredential === false && wizard.publicAgentContainsCredential === false && wizard.heartbeatRunIds.length === 0 + && o.state.sourceMembershipsUnchanged === true, "The human flow creates one company/first agent without duplicating source identities, dispatching nonexistent work, leaking credentials or changing original memberships."); + return c.result(); +} +export const newCompanyIdentityCases: readonly ChatCatalogCase[] = [identityDefinition("onboarding.new-company", async ctx => { + if (!ctx.input.privateInputs?.credentials.OPENAI_API_KEY) throw new Error("Setup blocker: human company onboarding needs declared private OPENAI_API_KEY; source chat subscription cannot substitute"); + const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + return { humanId: session.user.id, sourceMembers: await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/members`) }; +}, gradeNewCompany, { + authentication: "human-session", requiredCredentials: ["OPENAI_API_KEY"], attemptTimeoutMs: 600_000, + sourceFiles: ["chat-catalog-domains/identity-new-company.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"], + async act(ctx, state) { + await waitForReply(ctx); state.companyCountAfterReply = (await ctx.input.api.get("/api/companies")).length; + if (state.companyCountAfterReply !== 1) throw new Error("The source agent changed company identities before the human wizard"); + const credential = ctx.input.privateInputs!.credentials.OPENAI_API_KEY!; ctx.input.privateSecrets!.push(credential); const source = ctx.input.page.url(); + const wizard = await finishIdentityWizard({ page: ctx.input.page, origin: ctx.input.api.baseURL, companyName: `Acorn Website Business ${ctx.input.nonce}`, agentName: "Website business CEO", harness: "codex_local", credential, + deadlineAt: Date.now() + 360_000, entry: "new-company", evidence: ctx.input.evidence, onCompany(company) { companies.set(ctx, company.id); } }); + state.wizard = wizard; + // The current supported wizard intentionally hires a neutral general role. + // Role selection is therefore an explicit authenticated human public API + // configuration step after the real wizard, recorded separately from UI. + const configured = await ctx.input.page.context().request.patch(`${ctx.input.api.baseURL}/api/agents/${wizard.firstAgentId}`, { data: { role: "ceo", title: "Website business CEO" }, headers: { Origin: ctx.input.api.baseURL } }); + state.humanRoleConfigurationStatus = configured.status(); if (!configured.ok()) throw new Error("Human public CEO configuration failed"); + state.newAgent = await ctx.input.api.get(`/api/agents/${wizard.firstAgentId}`); + const membership = await ctx.input.api.get<{ members: Array<{ principalId: string; membershipRole: string }> }>(`/api/companies/${wizard.companyId}/members`); + state.newCompanyOwnedByHuman = membership.members.some(member => member.principalId === state.humanId && member.membershipRole === "owner"); + state.oldAgentForeignReadStatus = (await readAgentResource(ctx, `/api/companies/${wizard.companyId}/agents`)).status; + const company = await ctx.input.api.get(`/api/companies/${wizard.companyId}`); + await ctx.input.page.goto(`/${company.issuePrefix}/agents/${wizard.firstAgentId}`, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByText("Website business CEO", { exact: true }).first()).toBeVisible(); state.newAgentVisible = true; + await ctx.input.evidence("identity-human-new-company-authority.json", { sourceCompanyId: ctx.input.fixtures.company.id, newCompanyId: wizard.companyId, newAgentId: wizard.firstAgentId, + humanId: state.humanId, actualCeoRole: (state.newAgent as Agent).role, roleStep: "authenticated-human-public-configuration", originalAgentForeignReadStatus: state.oldAgentForeignReadStatus }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { + state.oldAgentStillSourceCompany = (await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`)).companyId === ctx.input.fixtures.company.id; + state.sourceMembershipsUnchanged = JSON.stringify(state.sourceMembers) === JSON.stringify(await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/members`)); + return observeIdentity(ctx, state); + }, + async cleanup(ctx) { const id = companies.get(ctx); if (id) await ctx.input.api.delete(`/api/companies/${id}`, { allowNotFound: true }); companies.delete(ctx); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-onboarding-work.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-onboarding-work.test.ts new file mode 100644 index 00000000000..8c41d48ed3f --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-onboarding-work.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { gradeNewCompany } from "./identity-new-company.js"; +import { gradeFirstWork } from "./identity-first-work.js"; +import type { IdentityObservation } from "./identity-support.js"; +function base(scenarioId: string): IdentityObservation { return { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], tasks: [], projects: [], goals: [], interactions: [], documents: [], revisions: [], comments: [], links: [], runs: [], activity: [], reply: "", humanQueries: [], state: {} }; } +function company() { + const o = base("onboarding.new-company"); o.reply = "Use the company switcher to create an organization for your website business; finish the human onboarding wizard."; + o.state = { humanId: "alex", companyCountAfterReply: 1, wizard: { companyId: "new-company", firstAgentId: "new-ceo", firstAgentReady: true, hiredThroughWizard: true, actualHarness: "codex_local", hello: { checks: [{ code: "codex_hello_probe_passed" }] }, + account: { ownership: "personal", ownerUserId: "alex", responsibleUserId: "alex" }, beforeCompanyIds: ["acorn"], afterCompanyIds: ["acorn", "new-company"], afterAgentIds: ["new-ceo"], draftContainsCredential: false, publicAgentContainsCredential: false, heartbeatRunIds: [] }, + newAgent: { id: "new-ceo", companyId: "new-company", role: "ceo", lifecycleState: "ready" }, humanRoleConfigurationStatus: 200, newCompanyOwnedByHuman: true, + oldAgentForeignReadStatus: 403, newAgentVisible: true, oldAgentStillSourceCompany: true, sourceMembershipsUnchanged: true }; return o; +} +function website() { + const o = base("onboarding.first-work"); const work = { id: "website", companyId: "acorn", title: "Acorn Notes website", description: "Website introducing Acorn Notes for science teams", status: "done", assigneeAgentId: "morgan" }; + o.tasks = [work]; o.links = ["/ACO/issues/website"]; o.humanQueries = ["Make me a simple website introducing our business."]; + o.activity = [{ id: "created", action: "issue.created", entityId: "website", agentId: "ceo" }]; + o.runs = [{ id: "worker-run", companyId: "acorn", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "website" } }]; + o.state = { worker: { id: "morgan" }, sourceId: "chat", beforeSourceHumanComments: 0, beforeSourceRuns: 0, sourceStillConversation: true, reviewedAttachmentId: "html", + websites: [{ attachment: { id: "html", issueId: "website", companyId: "acorn", sha256: "bytes", createdByAgentId: "morgan", originatingRunId: "worker-run" }, sha256: "bytes", body: "

Acorn Notes

Research notes for science teams.

" }], + visibleArtifactName: true, renderedBusinessHeading: true, renderedBusinessDescription: true, renderedBytesMatch: true, renderedWithoutScriptExecution: true }; return o; +} +describe("human company creation and genuine first completed website", () => { + it("validates an actual human-created separate company with a real CEO", () => { expect(gradeNewCompany(company()).passed).toBe(true); expect(gradeNewCompany(undefined).passed).toBe(false); }); + it("rejects merely renaming the neutral general agent CEO", () => { const o = company(); (o.state.newAgent as { role: string }).role = "general"; expect(gradeNewCompany(o).passed).toBe(false); }); + it("rejects cross-company agent access or human impersonation", () => { const o = company(); o.state.oldAgentForeignReadStatus = 200; expect(gradeNewCompany(o).passed).toBe(false); const wrong = company(); wrong.state.humanRoleConfigurationStatus = 403; expect(gradeNewCompany(wrong).passed).toBe(false); }); + it("validates the actual run-bound delivered and rendered website", () => { expect(gradeFirstWork(website()).passed).toBe(true); expect(gradeFirstWork(undefined).passed).toBe(false); }); + it("rejects empty task creation even with a finished website reply", () => { const o = website(); o.reply = "The website is complete."; o.state.websites = []; expect(gradeFirstWork(o).passed).toBe(false); }); + it("rejects artifact createdByRunId fiction instead of immutable originatingRunId", () => { const o = website(); const file = (o.state.websites as Array<{ attachment: Record }>)[0]!; file.attachment.createdByRunId = "worker-run"; delete file.attachment.originatingRunId; expect(gradeFirstWork(o).passed).toBe(false); }); + it("rejects rendered fixture bytes that differ from the actual authorized download", () => { const o = website(); o.state.renderedBytesMatch = false; expect(gradeFirstWork(o).passed).toBe(false); }); +});