From 62c6745930a45ed19008b3fba960217d151b17b0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 19:28:56 -0500 Subject: [PATCH] test(e2e): author real identity onboarding boundaries --- .../identity-onboarding.ts | 44 +++-- .../identity-wizard-boundaries.test.ts | 44 +++++ .../identity-wizard-boundaries.ts | 100 ++++++++++++ .../chat-catalog-domains/identity-wizard.ts | 150 ++++++++++++++++++ 4 files changed, 323 insertions(+), 15 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-wizard.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-onboarding.ts b/tests/runner-e2e/chat-catalog-domains/identity-onboarding.ts index 25662b562b3..7a80f9e067c 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-onboarding.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-onboarding.ts @@ -4,10 +4,12 @@ import { createServer } from "node:net"; import { mkdtemp, mkdir, readFile, readdir, rm, open } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import { expect, type Browser, type Page } from "@playwright/test"; -import { installedReleaseLaunch } from "../installed-release.js"; +import { fileURLToPath } from "node:url"; +import { expect, type Browser, type BrowserContext, type Page } from "@playwright/test"; +import { installedReleaseEnvironment, installedReleaseLaunch } from "../installed-release.js"; import { pollUntil } from "../api.js"; import { checks } from "./internal-support.js"; +import { ONBOARDING_FIRST_TASK_ORIGIN_KIND } from "../../../packages/shared/src/constants.js"; export interface EmptyInstallationReceipt { journeyKind: "operator-before-chat"; query: "Help me set up Paperclip on this machine."; querySent: false; @@ -25,39 +27,43 @@ export async function runEmptyInstallationBoundary(input: { /** Attempt-owned ignored/private result path; raw operator logs can contain bootstrap tokens. */ privateLogPath: string; deadlineAt: number; evidence(name: string, value: unknown): Promise; - finishWizard(target: { origin: string; page: Page; deadlineAt: number }): Promise<{ companyId: string; firstAgentId: string }>; + finishWizard(target: { origin: string; page: Page; deadlineAt: number }): Promise<{ companyId: string; firstAgentId: string; conversationPath: string }>; }): Promise { const launch = installedReleaseLaunch(input.installedCli); const cliSha256 = createHash("sha256").update(await readFile(launch.cli)).digest("hex"); if (cliSha256 !== input.expectedCliSha256 || launch.version !== input.expectedCliVersion) throw new Error("Empty installation candidate differs from its declared immutable CLI pin"); + if (!path.isAbsolute(input.privateLogPath)) throw new Error("Operator logs require an absolute attempt-owned private result path"); const directory = await mkdtemp(path.join(tmpdir(), "paperclip-chat-onboarding-")); - const home = path.join(directory, "instance"); await mkdir(home, { mode: 0o700 }); + const home = path.join(directory, "instance"); await mkdir(home, { mode: 0o700 }).catch(async error => { await rm(directory, { recursive: true, force: true }); throw error; }); const initialHomeEntries = (await readdir(home)).length; if (initialHomeEntries !== 0) throw new Error("Operator installation did not start from an empty isolated home"); - const config = path.join(home, "config.json"); const port = await unusedLoopbackPort(); + const config = path.join(home, "config.json"); const port = await unusedLoopbackPort().catch(async error => { await rm(directory, { recursive: true, force: true }); throw error; }); const origin = `http://127.0.0.1:${port}`; - if (!path.isAbsolute(input.privateLogPath) || path.relative(directory, input.privateLogPath).startsWith("..") === false) throw new Error("Operator logs require an external attempt-owned private result path"); - const log = await open(input.privateLogPath, "wx", 0o600); + if (path.relative(directory, input.privateLogPath).startsWith("..") === false) { await rm(directory, { recursive: true, force: true }); throw new Error("Operator logs require an external attempt-owned private result path"); } + const log = await open(input.privateLogPath, "wx", 0o600).catch(async error => { await rm(directory, { recursive: true, force: true }); throw error; }); // Closed environment prevents the existing instance, DB, provider credentials // or eval qualification flags from becoming a prerequisite of this install. - const environment: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: process.env.HOME, USER: process.env.USER, + const environment = installedReleaseEnvironment({ PATH: process.env.PATH, HOME: home, USER: process.env.USER, TMPDIR: process.env.TMPDIR, SHELL: process.env.SHELL, LANG: process.env.LANG, PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: config, PAPERCLIP_INSTANCE_ID: `onboarding-${randomUUID()}`, PORT: String(port), PAPERCLIP_OPEN_ON_LISTEN: "false", PAPERCLIP_DEPLOYMENT_MODE: "local_trusted", - PAPERCLIP_BIND: "loopback", PAPERCLIP_BIND_HOST: "127.0.0.1", SERVE_UI: "true", DO_NOT_TRACK: "1" }; + PAPERCLIP_BIND: "loopback", PAPERCLIP_BIND_HOST: "127.0.0.1", SERVE_UI: "true", DO_NOT_TRACK: "1" }, + fileURLToPath(new URL("../../../", import.meta.url)), path.join(process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? directory, "provider-bin")); const processHandle = spawn(process.execPath, [...launch.args, "--config", config], { cwd: launch.cwd, env: environment, detached: true, stdio: ["ignore", log.fd, log.fd], }); let spawnFailed = false; processHandle.once("error", () => { spawnFailed = true; }); const exited = new Promise(resolve => processHandle.once("exit", () => resolve())); - const context = await input.browser.newContext({ baseURL: origin }); const page = await context.newPage(); + let context: BrowserContext | undefined; let receipt: EmptyInstallationReceipt | undefined; let failed = false; async function get(endpoint: string): Promise { + if (!context) throw new Error("Owned operator browser is not available"); const response = await context.request.get(origin + endpoint, { timeout: 5_000 }); if (!response.ok()) throw new Error(`Owned installation read returned HTTP ${response.status()}`); return response.json() as Promise; } try { + context = await input.browser.newContext({ baseURL: origin }); const page = await context.newPage(); const health = await pollUntil({ label: "operator-created installation readiness", deadlineAt: Math.min(input.deadlineAt, Date.now() + 180_000), intervalMs: 1_000, load: () => get<{ status: string; deploymentMode: string }>("/api/health"), accept: result => result.status === "ok", reject: () => spawnFailed || processHandle.exitCode !== null ? "Public onboarding CLI exited before readiness" : undefined }); @@ -74,8 +80,14 @@ export async function runEmptyInstallationBoundary(input: { const agent = await get<{ id: string; companyId: string; lifecycleState: string; status: string }>(`/api/agents/${chosen.firstAgentId}`); expect(company.id).toBe(chosen.companyId); expect(agent.companyId).toBe(company.id); expect(agent.lifecycleState).toBe("ready"); expect(["idle", "active"]).toContain(agent.status); - await page.goto(`/${company.issuePrefix}/chats/${agent.id}`, { waitUntil: "domcontentloaded" }); - await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); + const destination = new URL(chosen.conversationPath, origin); const parts = destination.pathname.split("/"); + if (destination.origin !== origin || parts[1] !== company.issuePrefix || !["issues", "chats"].includes(parts[2]!)) throw new Error("Later conversation path does not belong to the actual new company"); + if (parts[2] === "issues") { + const conversation = await get<{ companyId: string; assigneeAgentId: string; originKind: string }>(`/api/issues/${parts[3]}`); + expect(conversation.companyId).toBe(company.id); expect(conversation.assigneeAgentId).toBe(agent.id); expect(conversation.originKind).toBe(ONBOARDING_FIRST_TASK_ORIGIN_KIND); + } else expect(parts[3]).toBe(agent.id); + await page.goto(destination.href, { waitUntil: "domcontentloaded" }); + await expect(page.getByTestId("task-chat-composer-input").or(page.getByTestId("task-chat-composer-takeover")).first()).toBeVisible(); const runs = await get(`/api/companies/${company.id}/heartbeat-runs?limit=100`); expect(runs).toHaveLength(0); receipt = { journeyKind: "operator-before-chat", query: "Help me set up Paperclip on this machine.", querySent: false, @@ -85,7 +97,8 @@ export async function runEmptyInstallationBoundary(input: { return receipt; } catch (error) { failed = true; throw error; } finally { - await context.close(); + let browserCloseFailed = false; + try { await context?.close(); } catch { browserCloseFailed = true; } // This child owns its new process group; signals never target the harness // controller, an attached instance, or an unverified database process. if (processHandle.pid && processHandle.exitCode === null) { @@ -95,8 +108,9 @@ export async function runEmptyInstallationBoundary(input: { if (processHandle.exitCode === null) { try { process.kill(-processHandle.pid, "SIGKILL"); } catch { /* Already exited. */ } await exited; } } await log.close(); await rm(directory, { recursive: true, force: true }); - if (receipt) { receipt.cleanup = "complete"; await input.evidence("identity-empty-installation-result.json", receipt); } - else await input.evidence("identity-empty-installation-result.json", { journeyKind: "operator-before-chat", querySent: false, failed, cleanup: "complete" }); + if (receipt) { receipt.cleanup = browserCloseFailed ? "failed" : "complete"; await input.evidence("identity-empty-installation-result.json", receipt); } + else await input.evidence("identity-empty-installation-result.json", { journeyKind: "operator-before-chat", querySent: false, failed, cleanup: browserCloseFailed ? "failed" : "complete" }); + if (browserCloseFailed && !failed) throw new Error("Owned installation browser cleanup failed after the target process and private home were removed"); } } export function gradeEmptyInstallation(value: unknown) { diff --git a/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.test.ts new file mode 100644 index 00000000000..4768447eabb --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { gradeWizardBoundary, wizardIdentityBoundaryCases } from "./identity-wizard-boundaries.js"; +import type { WizardReceipt } from "./identity-wizard.js"; +function valid(id: string) { + const claude = id === "onboarding.choose-first-runtime"; const owner = id === "onboarding.new-instance" ? "local-board" : "alex"; + const wizard: WizardReceipt = { companyId: "acorn", firstAgentId: "morgan", companyName: "Acorn", agentName: "Morgan", requestedHarness: claude ? "claude_local" : "codex_local", + actualHarness: claude ? "claude_local" : "codex_local", environmentId: "local-target", environmentDriver: "local", actualProbeEnvironmentId: "local-target", + beforeCompanyIds: [], afterCompanyIds: ["acorn"], beforeReadyAgents: 0, afterAgentIds: ["morgan"], + hello: { httpStatus: 200, routeHarness: claude ? "claude_local" : "codex_local", adapterType: "paperclip_runner", status: "pass", checks: [{ code: claude ? "acpx_hello_probe_passed" : "codex_hello_probe_passed", level: "info" }] }, + hiredThroughWizard: true, firstAgentReady: true, chatComposerVisible: true, heartbeatRunIds: [], readyConversationPath: "/ACO/issues/ACO-1", + draftCompanyId: "acorn", draftAgentId: "morgan", restoredCompanyId: "acorn", restoredAgentId: "morgan", secureFieldRequestedAgain: true, draftRetainedHarness: true, + draftContainsCredential: false, publicAgentContainsCredential: false, verificationKind: "public-native-hello-probe", firstWorkExecuted: false, probeUsage: "unavailable-outside-heartbeat-accounting", + account: { id: "account", grantId: "grant", provider: claude ? "anthropic" : "openai", method: "api_key", status: "connected", ownership: "personal", ownerUserId: owner, + currentUserId: owner, responsibleUserId: owner, bindingMode: "responsible_user" } }; + return { wizard, personId: "alex", returnedPersonId: "alex", ...(id === "onboarding.new-instance" ? { installation: { journeyKind: "operator-before-chat", query: "Help me set up Paperclip on this machine.", querySent: false, + cliVersion: "0.3.1", cliSha256: "a".repeat(64), initialHomeEntries: 0, companiesBeforeWizard: 0, deploymentMode: "local_trusted", configCreatedByOperatorCLI: true, + selectedBind: "loopback", companyId: "acorn", firstAgentId: "morgan", firstAgentReady: true, chatComposerVisible: true, cleanup: "complete" } } : {}) }; +} +describe("full real pre-chat wizard acceptance", () => { + for (const definition of wizardIdentityBoundaryCases) it(`${definition.scenarioId}: complete, wrong provider and missing phases`, () => { + const o = valid(definition.scenarioId); expect(definition.grade(o), JSON.stringify(definition.grade(o).checks)).toMatchObject({ passed: true }); + const wrong = structuredClone(o); wrong.wizard.actualHarness = wrong.wizard.actualHarness === "claude_local" ? "codex_local" : "claude_local"; + expect(definition.grade(wrong).passed).toBe(false); expect(definition.grade(undefined).passed).toBe(false); + expect(definition.grade({ wizard: { firstAgentReady: true } }).passed).toBe(false); + }); + it.each([{ hello: { httpStatus: 200, routeHarness: "claude_local", adapterType: "paperclip_runner", status: "pass", checks: [] } }, + { firstAgentReady: false }, { actualProbeEnvironmentId: "some-other-target" }, { draftContainsCredential: true }, { heartbeatRunIds: ["accidental-query"] }])("rejects unproven or wrong wizard result %j", change => { + const o = valid("onboarding.choose-first-runtime"); Object.assign(o.wizard, change); expect(gradeWizardBoundary(o, "onboarding.choose-first-runtime").passed).toBe(false); + }); + it("rejects an unavailable account despite a misleading successful hello", () => { + const o = valid("onboarding.choose-first-runtime"); o.wizard.hello.checks.push({ code: "adapter_auth_missing", level: "warn" }); + expect(gradeWizardBoundary(o, "onboarding.choose-first-runtime").passed).toBe(false); + }); + it("requires the same draft identities, proper secure reentry and account owner on reload", () => { + for (const change of [{ restoredAgentId: "duplicate-agent" }, { restoredCompanyId: "new-company" }, { secureFieldRequestedAgain: false }]) { + const o = valid("onboarding.resume-wizard"); Object.assign(o.wizard, change); expect(gradeWizardBoundary(o, "onboarding.resume-wizard").passed).toBe(false); + } + const wrong = valid("onboarding.resume-wizard"); wrong.wizard.account.ownerUserId = "riley"; expect(gradeWizardBoundary(wrong, "onboarding.resume-wizard").passed).toBe(false); + }); + it("requires actual empty installation identity and synchronous owned cleanup", () => { + const o = valid("onboarding.new-instance"); o.installation!.cleanup = "failed"; expect(gradeWizardBoundary(o, "onboarding.new-instance").passed).toBe(false); + const missing = valid("onboarding.new-instance"); delete missing.installation; expect(gradeWizardBoundary(missing, "onboarding.new-instance").passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.ts b/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.ts new file mode 100644 index 00000000000..e82c13cf2bc --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.ts @@ -0,0 +1,100 @@ +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import { RunnerApi } from "../api.js"; +import { installedReleaseLaunch } from "../installed-release.js"; +import type { IdentityBoundaryCase, IdentityBoundaryContext } from "./identity-boundary-contract.js"; +import { createAuthenticatedPerson } from "./identity-auth.js"; +import { checks } from "./internal-support.js"; +import { finishIdentityWizard, type WizardReceipt } from "./identity-wizard.js"; +import { gradeEmptyInstallation, runEmptyInstallationBoundary, type EmptyInstallationReceipt } from "./identity-onboarding.js"; + +interface BoundaryReceipt { wizard: WizardReceipt; installation?: EmptyInstallationReceipt; personId?: string; returnedPersonId?: string } +type Person = Awaited>; +const owned = new WeakMap(); +export function gradeWizardBoundary(value: unknown, scenarioId: string) { + const c = checks(); const o = value as BoundaryReceipt | undefined; const w = o?.wizard; + const expected = scenarioId === "onboarding.choose-first-runtime" ? "claude_local" : "codex_local"; + c.add("complete-public-wizard", !!w && typeof w.companyId === "string" && typeof w.firstAgentId === "string" && w.companyId !== w.firstAgentId + && w.hiredThroughWizard && w.firstAgentReady && w.chatComposerVisible && w.beforeReadyAgents === 0, + "The normal company/account/first-agent flow completes; a saved provisional agent or setup screen cannot qualify."); + c.add("requested-harness-and-real-target", !!w && w.requestedHarness === expected && w.actualHarness === expected && w.environmentDriver === "local" + && !!w.environmentId && w.actualProbeEnvironmentId === w.environmentId, + "The requested Codex or Claude Code harness is retained on the actual local target; no silent provider/runtime substitution is accepted."); + const helloCodes = expected === "claude_local" ? ["claude_hello_probe_passed", "acpx_hello_probe_passed"] : ["codex_hello_probe_passed"]; + c.add("actual-model-readiness-test", !!w && w.verificationKind === "public-native-hello-probe" && w.hello?.httpStatus === 200 && w.hello.routeHarness === expected + && ["pass", "warn"].includes(w.hello.status) && Array.isArray(w.hello.checks) && w.hello.checks.some(row => helloCodes.includes(row.code)) + && !w.hello.checks.some(row => row.level === "error" || row.code === "adapter_auth_missing" || /_auth_required$/.test(row.code)) && w.firstWorkExecuted === false && w.probeUsage === "unavailable-outside-heartbeat-accounting", + "The actual wizard's selected harness/account test returns a real successful hello; its usage is explicitly unknown and it is distinguished from later task execution."); + const account = w?.account; const provider = expected === "claude_local" ? "anthropic" : "openai"; + c.add("selected-account-belongs-to-person", !!account?.id && !!account.grantId && account.provider === provider && account.method === "api_key" && account.status === "connected" + && account.ownership === "personal" && typeof account.currentUserId === "string" && account.ownerUserId === account.currentUserId + && account.responsibleUserId === account.currentUserId && account.bindingMode === "responsible_user", + "Independent public account and agent reads retain the real selected person's eligible provider account and responsible-user binding."); + c.add("no-duplicate-company-or-agent", !!w && Array.isArray(w.beforeCompanyIds) && Array.isArray(w.afterCompanyIds) && w.beforeCompanyIds.length === 0 + && w.afterCompanyIds.length === 1 && w.afterCompanyIds[0] === w.companyId && Array.isArray(w.afterAgentIds) && w.afterAgentIds.length === 1 && w.afterAgentIds[0] === w.firstAgentId, + "A fresh target gains precisely one actual company and one first agent; the saved provisional identity is reused."); + c.add("secret-free-draft-and-zero-chat-dispatch", !!w && w.draftContainsCredential === false && w.publicAgentContainsCredential === false + && Array.isArray(w.heartbeatRunIds) && w.heartbeatRunIds.length === 0 && typeof w.readyConversationPath === "string" && w.readyConversationPath.startsWith("/"), + "The real draft/public configuration contains no raw key and no pre-chat request is credited with agent work; later usable conversation input is independently visible."); + if (scenarioId === "onboarding.new-instance") { + const installation = gradeEmptyInstallation(o?.installation); + for (const row of installation.checks) c.add(`installation-${row.id}`, row.passed, row.detail); + c.add("installation-and-wizard-same-target", !!w && !!o?.installation && o.installation.companyId === w.companyId && o.installation.firstAgentId === w.firstAgentId, + "The genuinely empty pinned installation and later ready wizard identities belong to the same independently started target."); + } else { + c.add("real-authorized-human", typeof o?.personId === "string" && o.personId !== "local-board" && o.returnedPersonId === o.personId + && account?.ownerUserId === o.personId, "An actual authenticated person completes their own saved wizard and retains account ownership."); + if (scenarioId === "onboarding.resume-wizard") c.add("restored-nonsecret-draft", !!w && w.draftCompanyId === w.companyId && w.restoredCompanyId === w.companyId + && w.draftAgentId === w.firstAgentId && w.restoredAgentId === w.firstAgentId && w.draftRetainedHarness === true && w.secureFieldRequestedAgain === true, + "A real reload restores existing company/agent IDs and the chosen harness while requesting the missing key through its secure field again."); + } + return c.result(); +} +const sources = ["chat-catalog-domains/identity-wizard-boundaries.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-onboarding.ts", + "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "installed-release.ts", "chat-scenarios/onboarding.yaml"]; +export const wizardIdentityBoundaryCases: readonly IdentityBoundaryCase[] = ["onboarding.new-instance", "onboarding.choose-first-runtime", "onboarding.resume-wizard"].map(scenarioId => ({ + scenarioId, journeyKind: "browser-boundary", boundaryTarget: scenarioId === "onboarding.new-instance" ? "empty-installed-target" : "authenticated-harness", + ...(scenarioId === "onboarding.new-instance" ? {} : { authentication: "human-session" as const }), + requiredCredentials: [scenarioId === "onboarding.choose-first-runtime" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"], + sourceFiles: sources, oracleVersion: "paperclip.chat-catalog.identity.v1", expectedRunCount: 0, expectedHumanMessageCount: 0, turnTimeoutMs: 180_000, attemptTimeoutMs: 600_000, + grade: value => gradeWizardBoundary(value, scenarioId), + async run(ctx) { + const credentialName = scenarioId === "onboarding.choose-first-runtime" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"; + const credential = ctx.credentials[credentialName]; + if (!credential) { + await ctx.evidence("identity-wizard-setup-blocker.json", { scenarioId, missingCredentialName: credentialName, originalQuerySent: false, nativeHeartbeatRuns: 0 }); + throw new Error(`Setup blocker: selected wizard requires the private ${credentialName} input; no provider substitution is allowed`); + } + ctx.privateSecrets.push(credential); const browser = ctx.page.context().browser(); if (!browser) throw new Error("Public wizard requires a real browser"); + if (scenarioId === "onboarding.new-instance") { + const cli = process.env.PAPERCLIP_CHAT_BOUNDARY_CLI; const digest = process.env.PAPERCLIP_CHAT_BOUNDARY_CLI_SHA256; + if (!cli || !digest) throw new Error("Setup blocker: empty target requires the actual installed CLI and declared immutable identity pins"); + await mkdir(ctx.privateDir, { recursive: true, mode: 0o700 }); let wizard: WizardReceipt | undefined; + const installation = await runEmptyInstallationBoundary({ browser, installedCli: cli, expectedCliSha256: digest, expectedCliVersion: installedReleaseLaunch(cli).version, + privateLogPath: path.join(ctx.privateDir, `operator-${ctx.nonce}.private.log`), deadlineAt: ctx.deadlineAt, evidence: ctx.evidence, + async finishWizard(target) { + wizard = await finishIdentityWizard({ ...target, credential, harness: "codex_local", companyName: `Acorn installation ${ctx.nonce}`, agentName: "Morgan", evidence: ctx.evidence, + onCompany(company) { ctx.observeResources({ companyId: company.id, runIds: [] }); } }); + ctx.observeResources({ companyId: wizard.companyId, agentId: wizard.firstAgentId, runIds: wizard.heartbeatRunIds }); + return { companyId: wizard.companyId, firstAgentId: wizard.firstAgentId, conversationPath: wizard.readyConversationPath }; + } }); + return { installation, wizard } satisfies Partial; + } + const person = await createAuthenticatedPerson({ api: ctx.api, browser, secrets: ctx.privateSecrets, name: "Alex" }); + const api = new RunnerApi(person.context.request); person.bindApiSession(api); const resource = { person, api, companyId: undefined as string | undefined }; owned.set(ctx, resource); + await api.post("/api/bootstrap/claim"); const page = await person.context.newPage(); + const wizard = await finishIdentityWizard({ page, origin: api.baseURL, credential, deadlineAt: ctx.deadlineAt, harness: scenarioId === "onboarding.choose-first-runtime" ? "claude_local" : "codex_local", + resume: scenarioId === "onboarding.resume-wizard", companyName: `Acorn wizard ${ctx.nonce}`, agentName: "Morgan", evidence: ctx.evidence, + onCompany(company) { resource.companyId = company.id; ctx.observeResources({ companyId: company.id, runIds: [] }); } }); + ctx.observeResources({ companyId: wizard.companyId, agentId: wizard.firstAgentId, runIds: wizard.heartbeatRunIds }); + const session = await api.get<{ user: { id: string } }>("/api/auth/get-session"); + return { wizard, personId: person.id, returnedPersonId: session.user.id } satisfies BoundaryReceipt; + }, + async cleanup(ctx) { + // Empty-installation helper synchronously owns and removes its separate + // process group/home/browser before returning, including on partial failure. + const resource = owned.get(ctx); if (!resource) return; + try { if (resource.companyId) await resource.api.delete(`/api/companies/${resource.companyId}`, { allowNotFound: true }); } + finally { await resource.person.dispose(); owned.delete(ctx); } + }, +})); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-wizard.ts b/tests/runner-e2e/chat-catalog-domains/identity-wizard.ts new file mode 100644 index 00000000000..2ec63c0e7df --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-wizard.ts @@ -0,0 +1,150 @@ +import { expect, type Page, type Response } from "@playwright/test"; +import { pollUntil } from "../api.js"; +import type { AiConnectionBinding, AiConnectionList } from "../../../packages/shared/src/ai-connections.js"; + +export type WizardHarness = "codex_local" | "claude_local"; +interface Company { id: string; name: string; issuePrefix: string; defaultResponsibleUserId?: string | null } +interface Agent { id: string; companyId: string; name: string; adapterType: string; adapterConfig: { provider?: string; acpxAgent?: string }; runtimeConfig: { aiConnection?: AiConnectionBinding }; lifecycleState: string; status: string; pauseReason?: string | null } +export interface WizardReceipt { + companyId: string; firstAgentId: string; companyName: string; agentName: string; requestedHarness: WizardHarness; + actualHarness: WizardHarness; environmentId: string; environmentDriver: string; actualProbeEnvironmentId: string; + beforeCompanyIds: string[]; afterCompanyIds: string[]; beforeReadyAgents: number; afterAgentIds: string[]; + hello: { httpStatus: number; routeHarness: string; adapterType: string; status: string; checks: Array<{ code: string; level: string }> }; + hiredThroughWizard: boolean; firstAgentReady: boolean; chatComposerVisible: boolean; heartbeatRunIds: string[]; + readyConversationPath: string; + account: { id: string; grantId: string; provider: string; method: string; status: string; ownership: string; ownerUserId?: string; currentUserId: string; responsibleUserId?: string | null; bindingMode: string }; + draftCompanyId?: string; draftAgentId?: string; restoredCompanyId?: string; restoredAgentId?: string; secureFieldRequestedAgain?: boolean; + draftRetainedHarness?: boolean; draftContainsCredential: boolean; publicAgentContainsCredential: boolean; + verificationKind: "public-native-hello-probe"; firstWorkExecuted: false; probeUsage: "unavailable-outside-heartbeat-accounting"; +} +/** The actual public wizard owns all company/account/first-agent mutations. + * APIs only read its resulting identities; secrets enter the normal masked + * human field and are never included in receipts or browser draft snapshots. */ +export async function finishIdentityWizard(input: { + page: Page; origin: string; companyName: string; agentName: string; harness: WizardHarness; credential: string; + deadlineAt: number; resume?: boolean; entry?: "empty" | "new-company"; + onCompany(company: Company): void; evidence(name: string, value: unknown): Promise; +}): Promise { + const { page, origin } = input; const remaining = () => Math.max(1, Math.min(180_000, input.deadlineAt - Date.now())); + const get = async (endpoint: string): Promise => { + const response = await page.context().request.get(origin + endpoint, { timeout: Math.min(30_000, remaining()) }); + if (!response.ok()) throw new Error(`Public wizard read returned HTTP ${response.status()}`); return response.json() as Promise; + }; + const before = await get("/api/companies"); + if (input.entry === "new-company") { + await page.getByRole("button", { name: /^Open .+ (?:organization|workspace|company) switcher$/i }).click(); + await page.getByRole("menuitem", { name: "Create organization", exact: true }).click(); + } else await page.goto(`${origin}/onboarding`, { waitUntil: "domcontentloaded" }); + await expect(page.getByRole("heading", { name: "What is the name of your organization?", exact: true })).toBeVisible(); + await page.locator("#onboarding-company-name").fill(input.companyName); + await page.getByRole("button", { name: "Continue", exact: true }).click(); + await expect(page.locator("#onboarding-agent-name")).toBeVisible(); + const company = await pollUntil({ label: "wizard-created exact company", deadlineAt: input.deadlineAt, + load: () => get("/api/companies"), accept: rows => rows.filter(row => !before.some(old => old.id === row.id) && row.name === input.companyName).length === 1 }).then(rows => rows.find(row => !before.some(old => old.id === row.id) && row.name === input.companyName)!); + input.onCompany(company); + await page.locator("#onboarding-agent-name").fill(input.agentName); await page.getByRole("button", { name: "Next", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Connect a model", exact: true })).toBeVisible(); + const setupAgents = await pollUntil({ label: "public first-agent setup record", deadlineAt: input.deadlineAt, + load: () => get(`/api/companies/${company.id}/agents`), accept: rows => rows.length === 1 && rows[0]?.name === input.agentName }); + const provisional = setupAgents[0]!; + const beforeReadyAgents = setupAgents.filter(row => row.lifecycleState === "ready" && ["idle", "active"].includes(row.status)).length; + if (beforeReadyAgents) throw new Error("Wizard prerequisite already had a usable first agent before account setup"); + const environments = await get>(`/api/companies/${company.id}/environments?driver=local`); + if (environments.length !== 1 || environments[0]?.driver !== "local") throw new Error("Wizard requires one real available local target; no silent target substitution is allowed"); + const environment = environments[0]!; + const choose = async () => { + await page.getByRole("radio", { name: input.harness === "claude_local" ? /^Claude/ : /^OpenAI/ }).click(); + const apiMode = page.getByRole("button", { name: "Use API key instead", exact: true }); + const key = page.getByPlaceholder("Enter API key here", { exact: true }); + await expect(apiMode.or(key).first()).toBeVisible(); + if (await apiMode.isVisible()) await apiMode.click(); + await expect(key).toBeVisible(); return key; + }; + let key = await choose(); + let draftCompanyId: string | undefined, draftAgentId: string | undefined, restoredCompanyId: string | undefined, restoredAgentId: string | undefined; + let secureFieldRequestedAgain: boolean | undefined, draftRetainedHarness: boolean | undefined; + await key.fill(input.credential); + // Retain only booleans/identity fields from browser-local storage. Never emit + // its raw JSON, which is exactly the privacy surface under test. + const draft = async () => page.evaluate(secret => { + const raw = localStorage.getItem("paperclip-onboarding-state") ?? ""; let state: Record = {}; + try { state = JSON.parse(raw) as Record; } catch { /* Readiness below stays false. */ } + return { companyId: typeof state.createdCompanyId === "string" ? state.createdCompanyId : null, + agentId: typeof state.setupAgentId === "string" ? state.setupAgentId : null, harness: typeof state.adapterType === "string" ? state.adapterType : null, + containsCredential: Object.values(localStorage).some(value => String(value).includes(secret)) }; + }, input.credential); + const saved = await pollUntil({ label: "nonsecret wizard draft persisted", deadlineAt: input.deadlineAt, load: draft, + accept: value => value.companyId === company.id && value.agentId === provisional.id && value.harness === input.harness }); + if (saved.containsCredential) throw new Error("The actual wizard draft leaked its secure input; contents withheld"); + if (input.resume) { + draftCompanyId = saved.companyId!; draftAgentId = saved.agentId!; + await page.reload({ waitUntil: "domcontentloaded" }); + await expect(page.getByRole("heading", { name: "Connect a model", exact: true })).toBeVisible(); + const restored = await draft(); restoredCompanyId = restored.companyId!; restoredAgentId = restored.agentId!; draftRetainedHarness = restored.harness === input.harness; + key = page.getByPlaceholder("Enter API key here", { exact: true }); await expect(key).toBeVisible(); + secureFieldRequestedAgain = await key.inputValue() === ""; + if (!secureFieldRequestedAgain || restored.containsCredential) throw new Error("Reload did not request the secure input again; values withheld"); + const existing = await get(`/api/companies/${company.id}/agents`); + if (existing.length !== 1 || existing[0]?.id !== provisional.id || existing[0]?.lifecycleState === "ready") throw new Error("Wizard reload duplicated or prematurely completed the first agent"); + await key.fill(input.credential); + } + let hello: WizardReceipt["hello"] | undefined; let actualProbeEnvironmentId = ""; let hiredThroughWizard = false; + const reads = new Set>(); + const onResponse = (response: Response) => { + const url = new URL(response.url()); if (url.origin !== origin || response.request().method() !== "POST") return; + const test = url.pathname === `/api/companies/${company.id}/adapters/${input.harness}/test-environment`; + const hire = url.pathname === `/api/companies/${company.id}/agent-hires`; + if (!test && !hire) return; + const pending = (async () => { + if (hire) { hiredThroughWizard = response.ok(); return; } + const request = response.request().postDataJSON() as { environmentId?: unknown }; + actualProbeEnvironmentId = typeof request.environmentId === "string" ? request.environmentId : ""; + const body = await response.json() as { adapterType?: string; status?: string; checks?: Array<{ code?: string; level?: string }> }; + hello = { httpStatus: response.status(), routeHarness: input.harness, adapterType: body.adapterType ?? "", status: body.status ?? "", + checks: (body.checks ?? []).map(row => ({ code: row.code ?? "", level: row.level ?? "" })) }; + })(); reads.add(pending); void pending.finally(() => reads.delete(pending)).catch(() => {}); + }; + page.on("response", onResponse); + try { + await page.getByRole("button", { name: /^(?:Connect|Next)$/, exact: true }).last().click(); + await expect(page.getByRole("button", { name: "Get started", exact: true })).toBeVisible({ timeout: remaining() }); + await Promise.all([...reads]); + } finally { page.off("response", onResponse); } + const expectedHello = input.harness === "claude_local" ? ["claude_hello_probe_passed", "acpx_hello_probe_passed"] : ["codex_hello_probe_passed"]; + if (!hello || hello.httpStatus !== 200 || !["pass", "warn"].includes(hello.status) || hello.checks.some(row => row.level === "error" || row.code === "adapter_auth_missing" || /_auth_required$/.test(row.code)) || !hello.checks.some(row => expectedHello.includes(row.code)) + || actualProbeEnvironmentId !== environment.id || !hiredThroughWizard) throw new Error("Actual wizard account/harness/target verification is incomplete; a saved profile alone is not readiness"); + const agents = await get(`/api/companies/${company.id}/agents`); + if (agents.length !== 1 || agents[0]?.id !== provisional.id) throw new Error("Wizard completion duplicated or replaced the saved first agent"); + const agent = await pollUntil({ label: "wizard first agent independently ready", deadlineAt: input.deadlineAt, + load: () => get(`/api/agents/${provisional.id}`), accept: row => row.lifecycleState === "ready" && ["idle", "active"].includes(row.status) }); + const actualHarness = agent.adapterType === "claude_local" || agent.adapterType === "codex_local" ? agent.adapterType + : agent.adapterType === "paperclip_runner" && agent.adapterConfig.provider === "acpx" && agent.adapterConfig.acpxAgent === "claude" ? "claude_local" + : agent.adapterType === "paperclip_runner" && agent.adapterConfig.provider === "codex" ? "codex_local" : undefined; + if (actualHarness !== input.harness || agent.companyId !== company.id || JSON.stringify(agent).includes(input.credential)) throw new Error("Public ready agent differs from the requested harness/company or exposes its credential; contents withheld"); + const accounts = await get(`/api/companies/${company.id}/ai-connections`); + const responsibleCompany = await get(`/api/companies/${company.id}`); + const provider = input.harness === "claude_local" ? "anthropic" : "openai"; + const account = accounts.connections.find(row => row.companyId === company.id && row.provider === provider && row.method === "api_key" && row.ownership === "personal" + && row.ownerUserId === accounts.currentUserId && row.isDefault && row.status === "connected"); + if (!account || responsibleCompany.defaultResponsibleUserId !== accounts.currentUserId || agent.runtimeConfig.aiConnection?.provider !== provider + || agent.runtimeConfig.aiConnection.method !== "api_key" || agent.runtimeConfig.aiConnection.mode !== "responsible_user") throw new Error("Ready wizard agent does not retain the actual responsible person's selected provider account"); + await page.getByRole("button", { name: "Get started", exact: true }).click(); + await expect(page.getByTestId("task-chat-composer-input").or(page.getByTestId("task-chat-composer-takeover")).first()).toBeVisible(); + const settings = await get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental"); + if (settings.enableAgentChat) { + await page.goto(`${origin}/${company.issuePrefix}/chats/${agent.id}`, { waitUntil: "domcontentloaded" }); + await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); + } + const runIds = (await get>(`/api/companies/${company.id}/heartbeat-runs?limit=100`)).map(row => row.id); + const afterCompanies = await get("/api/companies"); + const finalDraft = await draft(); + const receipt: WizardReceipt = { companyId: company.id, firstAgentId: agent.id, companyName: company.name, agentName: agent.name, requestedHarness: input.harness, actualHarness, + environmentId: environment.id, environmentDriver: environment.driver, actualProbeEnvironmentId, beforeCompanyIds: before.map(row => row.id), afterCompanyIds: afterCompanies.map(row => row.id), + beforeReadyAgents, afterAgentIds: agents.map(row => row.id), hello, hiredThroughWizard, firstAgentReady: true, chatComposerVisible: true, heartbeatRunIds: runIds, readyConversationPath: new URL(page.url()).pathname, + account: { id: account.id, grantId: account.grantId, provider: account.provider, method: account.method, status: account.status, ownership: account.ownership, + ownerUserId: account.ownerUserId, currentUserId: accounts.currentUserId, responsibleUserId: responsibleCompany.defaultResponsibleUserId, bindingMode: agent.runtimeConfig.aiConnection.mode }, + ...(input.resume ? { draftCompanyId, draftAgentId, restoredCompanyId, restoredAgentId, secureFieldRequestedAgain, draftRetainedHarness } : {}), + draftContainsCredential: saved.containsCredential || finalDraft.containsCredential, publicAgentContainsCredential: false, + verificationKind: "public-native-hello-probe", firstWorkExecuted: false, probeUsage: "unavailable-outside-heartbeat-accounting" }; + await input.evidence("identity-public-wizard-result.json", receipt); return receipt; +}