From ad4b0162ae83059235b591fb476f2d6926f618a6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 21:12:12 -0500 Subject: [PATCH] test(e2e): isolate identity entry and verify post-retry chat result --- .../identity-access-grants.ts | 2 +- .../identity-accounting.test.ts | 15 ++++-- .../identity-accounting.ts | 52 ++++++++++++++++--- .../chat-catalog-domains/identity-auth.ts | 40 +++++++++----- .../identity-navigation-boundaries.ts | 2 +- .../identity-navigation-preferences.ts | 2 +- .../identity-new-company.ts | 2 +- .../identity-observability.ts | 9 ++-- .../identity-private-browser-privacy.spec.ts | 29 +++++++++++ .../identity-private-browser-worker.ts | 51 ++++++++++++++++++ .../identity-private-browser.ts | 39 ++++++++++++++ .../chat-catalog-domains/identity-profile.ts | 2 +- .../identity-questions-and-approvals.ts | 2 +- .../identity-sidebar-order.ts | 2 +- .../chat-catalog-domains/identity-sign-in.ts | 2 +- .../identity-wizard-boundaries.ts | 2 +- .../chat-catalog-domains/identity-wizard.ts | 20 +++++-- 17 files changed, 231 insertions(+), 42 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-private-browser-privacy.spec.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-private-browser-worker.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-private-browser.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts index 17873cb3935..f42935cde30 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts @@ -33,7 +33,7 @@ export function gradeAccessGrant(value: unknown) { } else c.add("known-grant-case", false, "No permission oracle exists for this case."); return c.result(); } -const sources = ["chat-catalog-domains/identity-access-grants.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/access.yaml", +const sources = ["chat-catalog-domains/identity-access-grants.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/access.yaml", "../../maps/agent-access-map/access.md", "../../server/src/routes/access.ts", "../../server/src/routes/agents.ts", "../../packages/shared/src/validators/agent.ts"]; export const accessGrantIdentityCases: readonly ChatCatalogCase[] = ["access.invite-teammate", "access.manage-membership"].map(scenarioId => identityDefinition(scenarioId, async ctx => { const permissionKey = scenarioId.endsWith("invite-teammate") ? "users:invite" : "users:manage_permissions"; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts index 12ae92ac805..c258b03de9a 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts @@ -3,8 +3,8 @@ import { accountingIdentityCases } from "./identity-accounting.js"; import type { IdentityObservation } from "./identity-support.js"; function valid(scenarioId: string): IdentityObservation { const repair = scenarioId.endsWith("repair-cost-recording"); - return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", company: { id: "company" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], interactions: [], humanQueries: [], - runs: [{ id: "original", companyId: "company", agentId: "morgan", status: "succeeded", runtimeMode: "native" }], + return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", company: { id: "company" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], interactions: [], humanQueries: repair ? ["Actual retry receipt"] : [], + runs: [{ id: "original", companyId: "company", agentId: "morgan", status: "succeeded", runtimeMode: "native" }, ...(repair ? [{ id: "final", companyId: "company", agentId: "ceo", status: "succeeded", runtimeMode: "native" }] : [])], links: ["/ACO/issues/morgan-receipt", "/ACO/issues/riley-receipt"], reply: repair ? "Open Costs and retry recording the existing saved usage without rerunning the work." : "Launch this week: Morgan $1.37; Riley $2.63; recorded metered API total $4.00. Accounting is incomplete: unresolved usage is pending. Subscription fees stay separate.", activity: repair ? [{ id: "retry", action: "accounting.retry_requested", entityId: "original", actorType: "user" }] : [], @@ -12,8 +12,8 @@ function valid(scenarioId: string): IdentityObservation { originalRun: { id: "original", status: "succeeded", usageJson: { inputTokens: 10 } }, afterOriginalRun: { id: "original", status: "succeeded", usageJson: { inputTokens: 10 } }, originalUsagePreserved: true, originalLogPreserved: true, originalTaskPreserved: true, unresolvedBefore: { canRetry: true, runs: [{ runId: "original", reason: "retryable" }], events: [] }, receiptOrigin: "public-imported-synthetic-fixture", weekFrom: "2026-10-26T00:00:00Z", weekTo: "2026-11-02T00:00:00Z", - issueCosts: [{ costCents: 137 }, { costCents: 263 }], projectCosts: [{ projectId: "launch", costCents: 400 }], agentRetryStatus: 403, humanRetryHttpStatus: 200, retryAccounted: false, retryReceiptVisible: true, - confirmedPricedRepair: false, afterStillUnresolved: true } }; + issueCosts: [{ costCents: 137 }, { costCents: 263 }], projectCosts: [{ projectId: "launch", costCents: 400 }], agentRetryStatus: 403, humanRetryHttpStatus: 200, retryAccounted: false, retryReceiptVisible: true, retryQuality: { runs: [{ runId: "original", reason: "retryable" }], events: [] }, retryLedger: [], + initialReply: "Open Costs and retry recording the existing saved usage without rerunning the work.", finalReply: "The saved cost still remains unresolved; it is not a confirmed charge.", finalReplyRunId: "final", retryHumanFollowup: "Actual retry receipt", confirmedPricedRepair: false, afterStillUnresolved: true } }; } describe("actual saved native accounting contexts", () => { for (const d of accountingIdentityCases) { @@ -25,6 +25,11 @@ describe("actual saved native accounting contexts", () => { it("does not silently include prior-week/other-project receipts", () => { const o = valid("budgets-costs.spend-breakdown"); o.state.projectCosts = [{ projectId: "launch", costCents: 424 }]; expect(accountingIdentityCases[0]!.grade(o).passed).toBe(false); }); it("does not count unresolved native usage as free", () => { const o = valid("budgets-costs.spend-breakdown"); o.reply += " The pending usage is free."; expect(accountingIdentityCases[0]!.grade(o).passed).toBe(false); }); it("a HTTP200 retry alone does not establish repaired accounting", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.afterStillUnresolved = false; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); - it("a genuine confirmed priced repair is distinct from still-unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(true); }); + it("a genuine confirmed priced repair is distinct from still-unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; o.state.retryQuality = { runs: [], events: [] }; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 0 }]; o.state.finalReply = "The saved usage cost is now recorded. The original work was not rerun."; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(true); }); + it("post-retry false repair claim fails while unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.finalReply = "The missing cost is now fully repaired."; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); + it("post-retry false unresolved claim fails after confirmed recording", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; o.state.retryQuality = { runs: [], events: [] }; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 0 }]; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); + it("an earlier reply cannot substitute for the distinct post-action native reply", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.finalReplyRunId = "original"; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); + it("a price-estimate ledger cannot substitute for confirmed recording", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 1 }]; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); + it("no hidden human receipt or missing phase may pass", () => { const o = valid("budgets-costs.repair-cost-recording"); o.humanQueries = []; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); it("an actual agent retry bypass fails", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.agentRetryStatus = 200; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts index 54a005b9aa9..fd151f316ae 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts @@ -1,8 +1,10 @@ import { createHash } from "node:crypto"; import { expect } from "@playwright/test"; +import type { IssueComment } from "../../../packages/shared/src/types/issue.js"; import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; -import type { CostDataQualityReport, CostByProject, CostEvent, IssueCostSummary } from "../../../packages/shared/src/types/cost.js"; +import type { CostDataQualityReport, CostByProject, CostEvent, IssueCostSummary, CostByAgentModel } from "../../../packages/shared/src/types/cost.js"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import { sendChatMessage } from "../chat-flow.js"; import { pollUntil } from "../api.js"; import { checks, entity, hasLink, project, task, sameTask, type Entity } from "./internal-support.js"; import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, probeAgentPermission } from "./identity-support.js"; @@ -55,10 +57,27 @@ export function gradeAccounting(value: unknown) { && o.state.humanRetryHttpStatus === 200 && typeof o.state.retryAccounted === "boolean" && o.state.retryReceiptVisible === true && o.activity.filter(row => row.action === "accounting.retry_requested" && row.entityId === before?.id && row.actorType === "user").length === 1, "The authorized person clicks the actual saved-cost retry control for exactly that genuine retryable run; the agent cannot impersonate board accounting authority."); - c.add("honest-repair-or-unresolved-result", o.state.confirmedPricedRepair === true || o.state.afterStillUnresolved === true, - "Independent accounting readback distinguishes an actually repaired priced charge from a still-unresolved receipt/estimate; a successful HTTP request alone cannot claim repair."); - c.add("guides-accounting-without-rerun", /cost|accounting|usage/i.test(o.reply) && /retry|record/i.test(o.reply) && /saved|existing|without.*run|not.*rerun|no.*rerun/i.test(o.reply) - && !/\bI(?:'ve| have)?\s+(?:fixed|repaired|recorded|rerun|restarted)\b/i.test(o.reply), "The canonical response guides the real saved-usage control without claiming an unauthorized repair or rerunning the work."); + const ledger = o.state.retryLedger as CostByAgentModel[] | undefined; + const afterQuality = o.state.retryQuality as CostDataQualityReport | undefined; + const actualUnresolved = !!afterQuality && ([...afterQuality.runs, ...afterQuality.events].some(row => row.runId === before?.id) || ledger?.some(row => row.estimatedEventCount > 0) === true); + const actualRecorded = !!afterQuality && !actualUnresolved && o.state.retryAccounted === true && !!ledger?.length && ledger.every(row => row.agentId === entity(o.state, "worker").id && row.eventCount > 0 && row.estimatedEventCount === 0); + c.add("independent-post-retry-ledger-classification", !!afterQuality && Array.isArray(ledger) && o.state.afterStillUnresolved === actualUnresolved && o.state.confirmedPricedRepair === actualRecorded && (actualUnresolved || actualRecorded), + "Actual saved quality records and the sole original worker's ledger, not HTTP success or fixture booleans alone, classify the human retry result."); + const finalReply = typeof o.state.finalReply === "string" ? o.state.finalReply : ""; + const initialReply = typeof o.state.initialReply === "string" ? o.state.initialReply : ""; + const finalRun = o.runs.find(run => run.id === o.state.finalReplyRunId); + c.add("actual-post-retry-human-and-native-reply", typeof o.state.retryHumanFollowup === "string" && o.humanQueries.includes(o.state.retryHumanFollowup) + && !!finalRun && finalRun.agentId === o.leadId && finalRun.status === "succeeded" && finalRun.runtimeMode === "native" && !!finalReply.trim(), + "The actual human reports the real retry receipt and independently read accounting outcome, followed by a distinct persisted native chat reply."); + const recordedClaim = /(?:cost|charge|usage|accounting)[^.\n]{0,65}(?:recorded|repaired|resolved|recovered)|(?:recorded|repaired|resolved|recovered)[^.\n]{0,65}(?:cost|charge|usage|accounting)/i; + const unresolvedClaim = /(?:still|remains?|not yet|not fully)[^.\n]{0,65}(?:unresolved|pending|estimated|incomplete|unconfirmed)|(?:unresolved|pending|estimated|incomplete|unconfirmed)[^.\n]{0,65}(?:still|remains?)/i; + c.add("honest-post-retry-accounting-result", o.state.confirmedPricedRepair === true + ? recordedClaim.test(finalReply) && !unresolvedClaim.test(finalReply) && !/\b(?:not|never|wasn.t|isn.t)\s+(?:actually\s+)?(?:recorded|repaired|resolved)/i.test(finalReply) + : o.state.afterStillUnresolved === true && unresolvedClaim.test(finalReply) && !/\b(?:now|fully|successfully)\s+(?:recorded|repaired|resolved|recovered)|\b(?:fixed|repaired|resolved)\s+(?:the|that|its|your)\s+(?:missing\s+)?(?:cost|charge|usage|accounting)/i.test(finalReply), + "The post-action reply distinguishes independently confirmed recorded charges from still-unresolved/estimated accounting; the initial guidance cannot predict the human result."); + c.add("guides-accounting-without-rerun", /cost|accounting|usage/i.test(initialReply) && /retry|record/i.test(initialReply) && /saved|existing|without.*run|not.*rerun|no.*rerun/i.test(initialReply) + && !/\bI(?:'ve| have)?\s+(?:fixed|repaired|recorded|rerun|restarted)\b/i.test(initialReply) + && !/\bI(?:'ve| have)?\s+(?:rerun|restarted)\b/i.test(finalReply), "Initial guidance uses the saved-usage control; neither phase claims a new execution of the original work."); } else c.add("known-accounting-case", false, "No accounting oracle exists for this case."); return c.result(); } @@ -78,6 +97,10 @@ async function observeAccounting(ctx: ChatCatalogContext, state: Record ctx.input.api.get(`/api/issues/${entity(state, key).id}/cost-summary`))); state.projectCosts = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/costs/by-project?from=${encodeURIComponent(String(state.weekFrom))}&to=${encodeURIComponent(String(state.weekTo))}`); } + if (ctx.scenario.id.endsWith("repair-cost-recording") && state.finalReplyRunId) { + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + state.finalReply = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === state.finalReplyRunId).map(row => row.body ?? "").join("\n"); + } return observeIdentity(ctx, state); } async function verifyAccountingLink(ctx: ChatCatalogContext, state: Record, value: unknown) { @@ -117,11 +140,14 @@ export const accountingIdentityCases: readonly ChatCatalogCase[] = [ await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The completed native run requiring saved-cost recovery is ${state.originalRunId}, on task ${state.preflightTask.identifier}. Its public accounting quality record is actually retryable and retains saved usage. Only the person may use cost-recording retry; do not start or rerun the task.` }); return { ...state }; }, gradeAccounting, { - authentication: "human-session", sourceFiles: sources, expectedRunCount: 2, attemptTimeoutMs: 600_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, + authentication: "human-session", sourceFiles: sources, expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 780_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, setupPolicy: { funding: "standard", readiness: "all-ready", invariantCheckIds: ["genuine-unresolved-saved-native-usage"] }, async act(ctx, state) { await pollUntil({ label: "accounting guidance source settled", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, accept: rows => rows.length === 2 && rows.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + const initialComments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + state.initialReply = initialComments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body ?? "").join("\n"); + const previousRunIds = new Set((await ctx.allRuns()).map(run => run.id)); const root = `/api/companies/${ctx.input.fixtures.company.id}`; state.agentRetryStatus = await probeAgentPermission(ctx, `${root}/costs/retry-accounting`, "POST", { runId: state.originalRunId }); const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/activity/costs?period=all`, { waitUntil: "domcontentloaded" }); await ctx.input.page.getByRole("button", { name: /run costs? unavailable|billing types? unknown/i }).first().click(); @@ -130,11 +156,21 @@ export const accountingIdentityCases: readonly ChatCatalogCase[] = [ await row.getByRole("button", { name: "Retry cost recording", exact: true }).click(); const receipt = await response; state.humanRetryHttpStatus = receipt.status(); const result = await receipt.json() as { accounted: boolean }; state.retryAccounted = result.accounted; await expect(ctx.input.page.getByText(result.accounted ? "Costs recorded." : "No new cost data is available.", { exact: true })).toBeVisible(); state.retryReceiptVisible = true; await expect(ctx.input.page.getByText("The agent was not rerun.", { exact: true })).toBeVisible(); - const after = await ctx.input.api.get(`${root}/costs/data-quality?period=all`); state.afterStillUnresolved = [...after.runs, ...after.events].some(row => row.runId === state.originalRunId); - state.confirmedPricedRepair = result.accounted === true && state.afterStillUnresolved === false; + const after = await ctx.input.api.get(`${root}/costs/data-quality?period=all`); state.retryQuality = after; state.afterStillUnresolved = [...after.runs, ...after.events].some(row => row.runId === state.originalRunId); + const ledger = (await ctx.input.api.get(`${root}/costs/by-agent-model?period=all`)).filter(row => row.agentId === entity(state, "worker").id); + state.retryLedger = ledger; + state.confirmedPricedRepair = result.accounted === true && state.afterStillUnresolved === false && ledger.length > 0 && ledger.every(row => row.eventCount > 0 && row.estimatedEventCount === 0); + state.afterStillUnresolved = state.afterStillUnresolved === true || ledger.some(row => row.estimatedEventCount > 0); + if (!state.confirmedPricedRepair && !state.afterStillUnresolved) throw new Error("Retry returned no independently classifiable recorded or unresolved accounting outcome"); await ctx.input.evidence("identity-saved-usage-retry.json", { runId: state.originalRunId, actualHumanHttpStatus: receipt.status(), accounted: result.accounted, confirmedPricedRepair: state.confirmedPricedRepair, afterStillUnresolved: state.afterStillUnresolved, originalRunNeverRerun: (await ctx.allRuns()).filter(run => run.agentId === entity(state, "worker").id).length === 1 }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + const followup = `I clicked Retry cost recording for run ${state.originalRunId}. The actual HTTP ${receipt.status()} receipt says accounted=${result.accounted}. Fresh saved accounting shows ${state.afterStillUnresolved ? "remaining unresolved or estimated accounting" : "no remaining unresolved accounting"}; recorded ledger: ${JSON.stringify(ledger.map(row => ({ billingType: row.billingType, costCents: row.costCents, eventCount: row.eventCount, estimatedEventCount: row.estimatedEventCount })))}. What does this actual result mean for the missing cost? Do not run the original task again.`; + state.retryHumanFollowup = followup; await sendChatMessage(ctx.input.page, followup); + const finalRuns = await pollUntil({ label: "actual post-retry native accounting explanation", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, + accept: rows => rows.length === 3 && rows.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + const finalRun = finalRuns.find(run => !previousRunIds.has(run.id) && run.agentId === ctx.input.fixtures.agent.id && run.runtimeMode === "native"); + if (!finalRun) throw new Error("No distinct native response followed the factual human retry receipt"); state.finalReplyRunId = finalRun.id; }, }), ]; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-auth.ts b/tests/runner-e2e/chat-catalog-domains/identity-auth.ts index a288aef0f99..0595e64474c 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-auth.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-auth.ts @@ -1,9 +1,25 @@ +import { mkdtemp, writeFile, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { privateIdentityBrowser } from "./identity-private-browser.js"; import { randomBytes, randomUUID } from "node:crypto"; import { expect, type Browser, type BrowserContext, type Page } from "@playwright/test"; import type { RunnerApi } from "../api.js"; +/** Finite session transfer. Only the private file path reaches Playwright Test. */ +export async function installIdentitySession(target: BrowserContext, origin: string, cookie: string) { + const directory = await mkdtemp(path.join(tmpdir(), "identity-human-session-")); + try { + const file = path.join(directory, "storage.json"); + await writeFile(file, JSON.stringify({ cookies: cookie.split("; ").filter(Boolean).map(value => ({ name: value.slice(0, value.indexOf("=")), value: value.slice(value.indexOf("=") + 1), + domain: new URL(origin).hostname, path: "/", expires: -1, httpOnly: true, secure: false, sameSite: "Lax" as const })), origins: [] }), { mode: 0o600 }); + await target.setStorageState(file); + } finally { await rm(directory, { recursive: true, force: true }); } +} + /** Credentials remain in this closure and the attempt redactor, never case state/evidence. - * Call before enabling tracing on the new context, as in the public MCP suite. */ + * Browser installation takes only a private state filename; password entry runs + * in a standalone child with no Playwright Test API-step recorder. */ export async function createAuthenticatedPerson(input: { api: RunnerApi; browser: Browser; secrets: string[]; name: string; email?: string; }) { @@ -26,11 +42,7 @@ export async function createAuthenticatedPerson(input: { const user = (await response.json() as { user: { id: string; name: string } }).user; if (!user?.id || !cookie) throw new Error("Signup did not establish a real user session"); const context = await input.browser.newContext({ baseURL: input.api.baseURL }); - async function installSession(target: BrowserContext) { - await target.clearCookies(); - await target.addCookies(cookie.split("; ").map(value => ({ name: value.slice(0, value.indexOf("=")), value: value.slice(value.indexOf("=") + 1), - url: input.api.baseURL, httpOnly: true, sameSite: "Lax" as const }))); - } + async function installSession(target: BrowserContext) { await installIdentitySession(target, input.api.baseURL, cookie); } await installSession(context); async function request(path: string, method = "GET", data?: unknown) { const url = new URL(path, input.api.baseURL); @@ -39,6 +51,7 @@ export async function createAuthenticatedPerson(input: { ...(data === undefined ? {} : { "content-type": "application/json" }) }, ...(data === undefined ? {} : { body: JSON.stringify(data) }) }); } return { id: user.id, name: user.name, context, request, + async installSessionOn(target: BrowserContext) { await installSession(target); }, bindApiSession(api: RunnerApi) { if (api.baseURL !== input.api.baseURL || !cookie) throw new Error("Public human API binding requires the same authenticated instance"); api.setBrowserSession(cookie); @@ -48,13 +61,16 @@ export async function createAuthenticatedPerson(input: { async expireSession() { await auth("sign-out", {}); cookie = ""; }, async signInThroughBrowser(page: Page, returnPath: string) { if (page.context() !== context || !returnPath.startsWith("/") || returnPath.startsWith("//")) throw new Error("Human sign-in target must be the person's same-origin browser"); - await page.getByLabel("Email", { exact: true }).fill(email); - await page.getByLabel("Password", { exact: true }).fill(password); - await page.getByRole("button", { name: "Sign In", exact: true }).click(); + await privateIdentityBrowser({ kind: "sign-in", origin: input.api.baseURL, startPath: page.url(), returnPath, email, password, cookies: [] }, { + async storageState(file) { + const state = JSON.parse(await readFile(file, "utf8")) as { cookies: Array<{ name: string; value: string }> }; + cookie = state.cookies.map(value => `${value.name}=${value.value}`).join("; "); + for (const value of state.cookies) input.secrets.push(value.value); + await context.setStorageState(file); + }, + }); + await page.goto(new URL(returnPath, input.api.baseURL).href, { waitUntil: "domcontentloaded" }); await expect(page).toHaveURL(new URL(returnPath, input.api.baseURL).href); - const cookies = await context.cookies(input.api.baseURL); - cookie = cookies.map(value => `${value.name}=${value.value}`).join("; "); - for (const value of cookies) input.secrets.push(value.value); if (!cookie) throw new Error("Human browser sign-in returned no session"); }, async signOut() { await auth("sign-out", {}); cookie = ""; await context.clearCookies(); }, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-navigation-boundaries.ts b/tests/runner-e2e/chat-catalog-domains/identity-navigation-boundaries.ts index 97f2b99522b..f2b1ace0d05 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-navigation-boundaries.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-navigation-boundaries.ts @@ -44,7 +44,7 @@ export function gradeNavigationBoundary(value: unknown) { } else c.add("known-navigation-boundary", false, "No oracle exists for this case."); return c.result(); } -const sources = ["chat-catalog-domains/identity-navigation-boundaries.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"]; +const sources = ["chat-catalog-domains/identity-navigation-boundaries.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"]; export const navigationBoundaryIdentityCases: readonly ChatCatalogCase[] = [ identityDefinition("navigation-preferences.broken-link", async ctx => { const source = await openStandingChat(ctx); const p = await project(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts b/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts index 5a0caf10fa4..234334f73ce 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-navigation-preferences.ts @@ -70,7 +70,7 @@ export const navigationIdentityCases = [ } catch (error) { await observer.dispose(); searchObservers.delete(ctx); throw error; } }, gradeNavigation, { authentication: "human-session", - sourceFiles: ["chat-catalog-domains/identity-navigation-preferences.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"], + sourceFiles: ["chat-catalog-domains/identity-navigation-preferences.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"], async act(ctx, state) { await waitForReply(ctx); const source = ctx.input.page.url(); await ctx.input.page.keyboard.press("ControlOrMeta+k"); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-new-company.ts b/tests/runner-e2e/chat-catalog-domains/identity-new-company.ts index 306f9aa34cb..e227798d868 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-new-company.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-new-company.ts @@ -31,7 +31,7 @@ export const newCompanyIdentityCases: readonly ChatCatalogCase[] = [identityDefi return { humanId: session.user.id, sourceMembers: await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/members`) }; }, gradeNewCompany, { authentication: "human-session", requiredCredentials: ["OPENAI_API_KEY"], attemptTimeoutMs: 600_000, - sourceFiles: ["chat-catalog-domains/identity-new-company.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"], + sourceFiles: ["chat-catalog-domains/identity-new-company.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"], async act(ctx, state) { await waitForReply(ctx); state.companyCountAfterReply = (await ctx.input.api.get("/api/companies")).length; if (state.companyCountAfterReply !== 1) throw new Error("The source agent changed company identities before the human wizard"); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-observability.ts b/tests/runner-e2e/chat-catalog-domains/identity-observability.ts index c0453e46e9e..4b499f94e7c 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-observability.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-observability.ts @@ -1,3 +1,4 @@ +import { privateIdentityBrowser } from "./identity-private-browser.js"; import { createServer, type Server } from "node:http"; import { randomBytes, createHash } from "node:crypto"; import { gunzipSync } from "node:zlib"; @@ -56,7 +57,7 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD return { beforeSettings: before.settings, collectorUrl: target.url, syntheticCollector: true }; }, gradeObservability, { authentication: "human-session", expectedRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 600_000, - sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"], + sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"], async act(ctx, state) { await waitForReply(ctx); const target = collectors.get(ctx); if (!target) throw new Error("Owned synthetic collector is missing"); const root = `/api/companies/${ctx.input.fixtures.company.id}/observability`; const source = ctx.input.page.url(); @@ -64,10 +65,8 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD state.agentWriteStatus = await probeAgentPermission(ctx, root, "PUT", { expectedRevision: null, enabled: true, destination: { provider: "otlp", endpointBaseUrl: target.url }, credentials: { mode: "replace", headers: [] }, capture: { userPrompts: false, assistantResponses: false, toolDetails: false, toolContent: false } }); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded" }); const section = ctx.input.page.locator('section[aria-labelledby="agent-observability-heading"]'); - await section.getByLabel("Destination", { exact: true }).selectOption("otlp"); await section.getByLabel("OTLP base URL", { exact: true }).fill(target.url); - await section.getByRole("button", { name: "Add header", exact: true }).click(); await section.getByRole("textbox", { name: "Header name", exact: true }).fill("Authorization"); - await section.getByLabel("Value for Authorization", { exact: true }).fill(`Bearer ${target.key}`); - await section.getByRole("switch", { name: "Enable export", exact: true }).setChecked(true); await section.getByRole("button", { name: "Save changes", exact: true }).click(); + await privateIdentityBrowser({ kind: "observability", origin: ctx.input.api.baseURL, startPath: ctx.input.page.url(), cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL), endpoint: target.url, credential: `Bearer ${target.key}` }); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(section.getByRole("button", { name: "Test saved settings", exact: true })).toBeEnabled(); const saved = await ctx.input.api.get(root); state.saved = saved; state.publicSettingsContainsKey = JSON.stringify(saved).includes(target.key); const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/test`); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-private-browser-privacy.spec.ts b/tests/runner-e2e/chat-catalog-domains/identity-private-browser-privacy.spec.ts new file mode 100644 index 00000000000..01fbb19e851 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-private-browser-privacy.spec.ts @@ -0,0 +1,29 @@ +import { randomUUID } from "node:crypto"; +import { appendFile } from "node:fs/promises"; +import { test, expect } from "@playwright/test"; +import { installIdentitySession } from "./identity-auth.js"; +import { privateIdentityBrowser } from "./identity-private-browser.js"; +// Offline opt-in calibration only: actual human-session recording policy, +// actual failed fill in standalone child, no product/provider/auth server. +test("private identity failed input is absent from human-session reports", async () => { + test.skip(!process.env.PAPERCLIP_IDENTITY_CANARY_PRIVATE_FILE, "Explicit offline privacy calibration only"); + const credential = `identity-private-${randomUUID()}`; + await appendFile(process.env.PAPERCLIP_IDENTITY_CANARY_PRIVATE_FILE!, credential + "\n", { mode: 0o600 }); + const receipt = await privateIdentityBrowser({ kind: "canary-failure", origin: "http://127.0.0.1:1", startPath: "/", cookies: [], credential }); + expect(receipt.failedApiCallObserved).toBe(true); + // A deliberate parent failure makes JUnit/blob/HTML/error-context observable. + // Its message and recorded call arguments have never received the canary. + test.fail(); throw new Error("Deliberate safe parent failure after private failed-input canary"); +}); + +for (const failure of [false, true]) test(`private session file transfer ${failure ? "controlled failure" : "success"} retains no cookie`, async ({ browser }) => { + test.skip(!process.env.PAPERCLIP_IDENTITY_CANARY_PRIVATE_FILE, "Explicit offline privacy calibration only"); + const credential = `identity-cookie-${randomUUID()}`; + await appendFile(process.env.PAPERCLIP_IDENTITY_CANARY_PRIVATE_FILE!, credential + "\n", { mode: 0o600 }); + const context = await browser.newContext(); + try { + await installIdentitySession(context, "http://127.0.0.1:1", `identity.synthetic=${credential}`); + expect((await context.cookies()).some(cookie => cookie.name === "identity.synthetic" && cookie.value === credential)).toBe(true); + if (failure) { test.fail(); throw new Error("Deliberate safe failure after private session transfer"); } + } finally { await context.close(); } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-private-browser-worker.ts b/tests/runner-e2e/chat-catalog-domains/identity-private-browser-worker.ts new file mode 100644 index 00000000000..fe231f28529 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-private-browser-worker.ts @@ -0,0 +1,51 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { chromium, test, expect } from "@playwright/test"; +import type { PrivateIdentityOperation, PrivateIdentityReceipt } from "./identity-private-browser.js"; +import { runIdentityWizardInPrivateBrowser } from "./identity-wizard.js"; +let privateValues: string[] = []; +async function main(): Promise { + try { test.info(); throw new Error("Attached Test worker refused"); } catch (error) { if (error instanceof Error && error.message === "Attached Test worker refused") throw error; } + const request = process.argv[2]!; const directory = path.dirname(request); + const input = JSON.parse(await readFile(request, "utf8")) as PrivateIdentityOperation & { executablePath?: string; channel?: string }; + privateValues = [...input.cookies.map(cookie => cookie.value), ...(input.kind === "sign-in" ? [input.password] : [input.credential])].filter(Boolean); + const origin = new URL(input.origin); + if (origin.hostname !== "127.0.0.1" || origin.protocol !== "http:" || new URL(input.startPath, origin).origin !== origin.origin) throw new Error("Private identity target must be a disposable loopback instance"); + if (!["sign-in", "wizard", "observability", "canary-failure"].includes(input.kind)) throw new Error("Unknown finite identity operation"); + const browser = await chromium.launch({ headless: true, ...(input.executablePath ? { executablePath: input.executablePath } : {}), ...(input.channel ? { channel: input.channel } : {}) }); + try { + const context = await browser.newContext({ baseURL: origin.origin }); await context.addCookies(input.cookies); const page = await context.newPage(); page.setDefaultTimeout(30_000); + const receipt: PrivateIdentityReceipt = { operation: input.kind, completed: true, processIsolation: true, testRunnerAttached: false, apiStepRecorder: false, tracingEnabled: false, videoEnabled: false }; + if (input.kind === "canary-failure") { + await page.setContent(''); + await page.getByLabel("Private credential", { exact: true }).fill(input.credential); + if (await page.getByLabel("Private credential", { exact: true }).inputValue() !== input.credential) throw new Error("Private canary entry failed"); + // Deliberate failed API call reproduces the proven leak trigger, only here + // outside the Test recorder. The parent receives a constant failure. + let observed = false; + try { await page.getByLabel("Deliberately absent credential field").fill(input.credential, { timeout: 25 }); } + catch (error) { observed = error instanceof Error && error.name === "TimeoutError"; } + if (!observed) throw new Error("Expected private failed-call canary was not exercised"); + receipt.failedApiCallObserved = true; return receipt; + } + await page.goto(new URL(input.startPath, origin).href, { waitUntil: "domcontentloaded" }); + if (input.kind === "sign-in") { + if (!input.returnPath.startsWith("/") || input.returnPath.startsWith("//")) throw new Error("Invalid human return path"); + await page.getByLabel("Email", { exact: true }).fill(input.email); await page.getByLabel("Password", { exact: true }).fill(input.password); + await page.getByRole("button", { name: "Sign In", exact: true }).click(); await expect(page).toHaveURL(new URL(input.returnPath, origin).href); + await writeFile(path.join(directory, "storage.json"), JSON.stringify(await context.storageState()), { mode: 0o600 }); + } else if (input.kind === "wizard") { + receipt.wizard = await runIdentityWizardInPrivateBrowser({ ...input, page, onCompany: company => { void company; }, + async registerCompany(company) { await writeFile(path.join(directory, "company.json"), JSON.stringify({ id: company.id }), { mode: 0o600 }); }, evidence: async () => {} }); + } else { + const section = page.locator('section[aria-labelledby="agent-observability-heading"]'); + await section.getByLabel("Destination", { exact: true }).selectOption("otlp"); await section.getByLabel("OTLP base URL", { exact: true }).fill(input.endpoint); + await section.getByRole("button", { name: "Add header", exact: true }).click(); await section.getByRole("textbox", { name: "Header name", exact: true }).fill("Authorization"); + await section.getByLabel("Value for Authorization", { exact: true }).fill(input.credential); + await section.getByRole("switch", { name: "Enable export", exact: true }).setChecked(true); await section.getByRole("button", { name: "Save changes", exact: true }).click(); + await expect(section.getByRole("button", { name: "Test saved settings", exact: true })).toBeEnabled(); + } + return receipt; + } finally { await browser.close(); } +} +main().then(receipt => { const output = JSON.stringify(receipt); if (privateValues.some(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64")].some(encoded => output.includes(encoded)))) throw new Error("Private identity receipt contains a credential"); process.stdout.write(output); }).catch(() => { process.stderr.write("Private identity browser operation failed; details withheld"); process.exitCode = 1; }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-private-browser.ts b/tests/runner-e2e/chat-catalog-domains/identity-private-browser.ts new file mode 100644 index 00000000000..9c1320c005d --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-private-browser.ts @@ -0,0 +1,39 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtemp, readFile, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { chromium, type Cookie } from "@playwright/test"; +import type { WizardReceipt, WizardHarness } from "./identity-wizard.js"; +export const identityPrivateBrowserSources = ["chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts"]; +export type PrivateIdentityOperation = + | { kind: "sign-in"; origin: string; startPath: string; returnPath: string; email: string; password: string; cookies: Cookie[] } + | { kind: "wizard"; origin: string; startPath: string; cookies: Cookie[]; credential: string; companyName: string; agentName: string; harness: WizardHarness; resume?: boolean; entry?: "empty" | "new-company"; deadlineAt: number } + | { kind: "observability"; origin: string; startPath: string; cookies: Cookie[]; endpoint: string; credential: string } + | { kind: "canary-failure"; origin: string; startPath: string; cookies: Cookie[]; credential: string }; +export interface PrivateIdentityReceipt { processIsolation: true; testRunnerAttached: false; apiStepRecorder: false; tracingEnabled: false; videoEnabled: false; operation: PrivateIdentityOperation["kind"]; completed: true; failedApiCallObserved?: boolean; wizard?: WizardReceipt } +/** Request/state files are outside artifacts, mode0600, removed even on failure. + * No private value enters Test APIs, argv, environment, stdout or error text. */ +export async function privateIdentityBrowser(input: PrivateIdentityOperation, callbacks: { storageState?(file: string): Promise; company?(id: string): void } = {}): Promise { + const directory = await mkdtemp(path.join(tmpdir(), "paperclip-identity-private-")); + try { + const request = path.join(directory, "request.json"); const channel = process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL; + await writeFile(request, JSON.stringify({ ...input, executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ?? (channel ? undefined : chromium.executablePath()), channel }), { mode: 0o600 }); + const worker = fileURLToPath(new URL("./identity-private-browser-worker.ts", import.meta.url)); + const loader = fileURLToPath(new URL("../../../server/node_modules/tsx/dist/loader.mjs", import.meta.url)); + let output: string; + try { output = (await promisify(execFile)(process.execPath, ["--import", loader, worker, request], { timeout: input.kind === "wizard" ? Math.max(1, Math.min(540_000, input.deadlineAt - Date.now())) : 120_000, + maxBuffer: 32_768, env: { PATH: process.env.PATH ?? "", NODE_ENV: "test", DO_NOT_TRACK: "1", HOME: directory, TMPDIR: directory } })).stdout; } + catch { throw new Error("Private identity browser operation failed; child diagnostics withheld"); } + let receipt: PrivateIdentityReceipt; + try { receipt = JSON.parse(output) as PrivateIdentityReceipt; } catch { throw new Error("Invalid closed private identity receipt"); } + if (receipt.operation !== input.kind || receipt.completed !== true || receipt.processIsolation !== true || receipt.testRunnerAttached !== false || receipt.apiStepRecorder !== false || receipt.tracingEnabled !== false || receipt.videoEnabled !== false + || Object.keys(receipt).some(key => !["operation", "completed", "processIsolation", "testRunnerAttached", "apiStepRecorder", "tracingEnabled", "videoEnabled", "failedApiCallObserved", "wizard"].includes(key))) throw new Error("Invalid closed private identity receipt"); + if (input.kind === "sign-in") await callbacks.storageState?.(path.join(directory, "storage.json")); + return receipt; + } finally { + try { const company = JSON.parse(await readFile(path.join(directory, "company.json"), "utf8")) as { id: string }; if (/^[a-f0-9-]{36}$/i.test(company.id)) callbacks.company?.(company.id); } catch { /* no company created */ } + await rm(directory, { recursive: true, force: true }); + } +} diff --git a/tests/runner-e2e/chat-catalog-domains/identity-profile.ts b/tests/runner-e2e/chat-catalog-domains/identity-profile.ts index 90344ccd967..d1e2c9521fc 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-profile.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-profile.ts @@ -37,7 +37,7 @@ export const profileIdentityCases: readonly ChatCatalogCase[] = [identityDefinit } catch (error) { await observer.dispose(); observers.delete(ctx); throw error; } }, gradeProfile, { authentication: "human-session", - sourceFiles: ["chat-catalog-domains/identity-profile.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"], + sourceFiles: ["chat-catalog-domains/identity-profile.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"], async act(ctx, state) { await waitForReply(ctx); state.beforeHumanName = (await ctx.input.api.get("/api/auth/get-session")).user.name; state.agentWriteStatus = await probeAgentPermission(ctx, "/api/auth/profile", "PATCH", { name: "Alex Chen" }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts index df916cb60cb..649cebb46c2 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts @@ -155,7 +155,7 @@ export const questionIdentityCases = [ return { original, originalHumanName: session.user.name, alexId: session.user.id, rileyId: riley.id, realHumanIdentities: true, visibleToAlex: true }; }, gradeQuestions, { authentication: "human-session", - sourceFiles: ["chat-catalog-domains/identity-questions-and-approvals.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"], + sourceFiles: ["chat-catalog-domains/identity-questions-and-approvals.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"], async act(ctx, state) { await waitForReply(ctx); const response = await ctx.input.page.context().request.post(`${ctx.input.api.baseURL}/api/issues/${ctx.issue()!.id}/interactions/${entity(state, "original").id}/accept`, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-sidebar-order.ts b/tests/runner-e2e/chat-catalog-domains/identity-sidebar-order.ts index b4c5166df1b..420bd78cbb5 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-sidebar-order.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-sidebar-order.ts @@ -51,7 +51,7 @@ export const sidebarOrderIdentityCases: readonly ChatCatalogCase[] = [identityDe } catch (error) { await observers.get(ctx)?.dispose(); observers.delete(ctx); await ctx.input.api.patch("/api/instance/settings/experimental", { enableStreamlinedUi: settings.enableStreamlinedUi }); throw error; } }, gradeSidebarOrder, { authentication: "human-session", - sourceFiles: ["chat-catalog-domains/identity-sidebar-order.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"], + sourceFiles: ["chat-catalog-domains/identity-sidebar-order.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"], async act(ctx, state) { await waitForReply(ctx); const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/sidebar-preferences/me`; state.orderAfterReply = await ctx.input.api.get(endpoint); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-sign-in.ts b/tests/runner-e2e/chat-catalog-domains/identity-sign-in.ts index 3558d396cbc..86b2e93581f 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-sign-in.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-sign-in.ts @@ -37,7 +37,7 @@ export function gradeSignInBoundary(value: unknown) { export const signInIdentityBoundaryCases: readonly IdentityBoundaryCase[] = [{ scenarioId: "access.sign-in", journeyKind: "browser-boundary", boundaryTarget: "authenticated-harness", authentication: "human-session", oracleVersion: "paperclip.chat-catalog.identity.v1", expectedRunCount: 0, expectedHumanMessageCount: 0, turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000, - sourceFiles: ["chat-catalog-domains/identity-sign-in.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "live-fixtures.ts", "chat-scenarios/access.yaml"], + sourceFiles: ["chat-catalog-domains/identity-sign-in.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "live-fixtures.ts", "chat-scenarios/access.yaml"], grade: gradeSignInBoundary, async run(ctx) { const browser = ctx.page.context().browser(); if (!browser) throw new Error("Real human boundary needs a browser"); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.ts b/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.ts index e82c13cf2bc..1fe9920cf5a 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-wizard-boundaries.ts @@ -51,7 +51,7 @@ export function gradeWizardBoundary(value: unknown, scenarioId: string) { return c.result(); } const sources = ["chat-catalog-domains/identity-wizard-boundaries.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-onboarding.ts", - "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "installed-release.ts", "chat-scenarios/onboarding.yaml"]; + "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "installed-release.ts", "chat-scenarios/onboarding.yaml"]; export const wizardIdentityBoundaryCases: readonly IdentityBoundaryCase[] = ["onboarding.new-instance", "onboarding.choose-first-runtime", "onboarding.resume-wizard"].map(scenarioId => ({ scenarioId, journeyKind: "browser-boundary", boundaryTarget: scenarioId === "onboarding.new-instance" ? "empty-installed-target" : "authenticated-harness", ...(scenarioId === "onboarding.new-instance" ? {} : { authentication: "human-session" as const }), diff --git a/tests/runner-e2e/chat-catalog-domains/identity-wizard.ts b/tests/runner-e2e/chat-catalog-domains/identity-wizard.ts index 2ec63c0e7df..7a41703d671 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-wizard.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-wizard.ts @@ -1,4 +1,5 @@ -import { expect, type Page, type Response } from "@playwright/test"; +import { expect, test, type Page, type Response } from "@playwright/test"; +import { privateIdentityBrowser } from "./identity-private-browser.js"; import { pollUntil } from "../api.js"; import type { AiConnectionBinding, AiConnectionList } from "../../../packages/shared/src/ai-connections.js"; @@ -20,11 +21,23 @@ export interface WizardReceipt { /** The actual public wizard owns all company/account/first-agent mutations. * APIs only read its resulting identities; secrets enter the normal masked * human field and are never included in receipts or browser draft snapshots. */ -export async function finishIdentityWizard(input: { +interface WizardInput { page: Page; origin: string; companyName: string; agentName: string; harness: WizardHarness; credential: string; deadlineAt: number; resume?: boolean; entry?: "empty" | "new-company"; onCompany(company: Company): void; evidence(name: string, value: unknown): Promise; -}): Promise { +} +export async function finishIdentityWizard(input: WizardInput): Promise { + const receipt = await privateIdentityBrowser({ kind: "wizard", origin: input.origin, startPath: input.page.url() === "about:blank" ? "/" : input.page.url(), cookies: await input.page.context().cookies(input.origin), + credential: input.credential, companyName: input.companyName, agentName: input.agentName, harness: input.harness, deadlineAt: input.deadlineAt, resume: input.resume, entry: input.entry }, + { company(id) { input.onCompany({ id, name: input.companyName, issuePrefix: "" }); } }); + if (!receipt.wizard) throw new Error("Private wizard returned no bounded result"); + await input.page.goto(new URL(receipt.wizard.readyConversationPath, input.origin).href, { waitUntil: "domcontentloaded" }); + await input.evidence("identity-public-wizard-result.json", receipt.wizard); return receipt.wizard; +} +/** Called only by the standalone worker, never in the Playwright Test process. */ +export async function runIdentityWizardInPrivateBrowser(input: WizardInput & { registerCompany(company: Company): Promise }): Promise { + let attached = false; try { test.info(); attached = true; } catch { /* outside Test */ } + if (attached) throw new Error("Credential-bearing wizard must run only in the private child"); const { page, origin } = input; const remaining = () => Math.max(1, Math.min(180_000, input.deadlineAt - Date.now())); const get = async (endpoint: string): Promise => { const response = await page.context().request.get(origin + endpoint, { timeout: Math.min(30_000, remaining()) }); @@ -41,6 +54,7 @@ export async function finishIdentityWizard(input: { await expect(page.locator("#onboarding-agent-name")).toBeVisible(); const company = await pollUntil({ label: "wizard-created exact company", deadlineAt: input.deadlineAt, load: () => get("/api/companies"), accept: rows => rows.filter(row => !before.some(old => old.id === row.id) && row.name === input.companyName).length === 1 }).then(rows => rows.find(row => !before.some(old => old.id === row.id) && row.name === input.companyName)!); + await input.registerCompany(company); input.onCompany(company); await page.locator("#onboarding-agent-name").fill(input.agentName); await page.getByRole("button", { name: "Next", exact: true }).click(); await expect(page.getByRole("heading", { name: "Connect a model", exact: true })).toBeVisible();