test(e2e): isolate identity entry and verify post-retry chat result

This commit is contained in:
Dotta committed 2026-10-26 21:12:12 -05:00
1 parent b72f0f7cbb
commit ad4b0162ae
17 files changed
+231 -42

No files matched your search

@@ -33,7 +33,7 @@ export function gradeAccessGrant(value: unknown) {
} else c.add("known-grant-case", false, "No permission oracle exists for this case.");
return c.result();
}
const sources = ["chat-catalog-domains/identity-access-grants.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/access.yaml",
const sources = ["chat-catalog-domains/identity-access-grants.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/access.yaml",
"../../maps/agent-access-map/access.md", "../../server/src/routes/access.ts", "../../server/src/routes/agents.ts", "../../packages/shared/src/validators/agent.ts"];
export const accessGrantIdentityCases: readonly ChatCatalogCase[] = ["access.invite-teammate", "access.manage-membership"].map(scenarioId => identityDefinition(scenarioId, async ctx => {
const permissionKey = scenarioId.endsWith("invite-teammate") ? "users:invite" : "users:manage_permissions";
@@ -3,8 +3,8 @@ import { accountingIdentityCases } from "./identity-accounting.js";
import type { IdentityObservation } from "./identity-support.js";
function valid(scenarioId: string): IdentityObservation {
const repair = scenarioId.endsWith("repair-cost-recording");
return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", company: { id: "company" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], interactions: [], humanQueries: [],
runs: [{ id: "original", companyId: "company", agentId: "morgan", status: "succeeded", runtimeMode: "native" }],
return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", company: { id: "company" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], interactions: [], humanQueries: repair ? ["Actual retry receipt"] : [],
runs: [{ id: "original", companyId: "company", agentId: "morgan", status: "succeeded", runtimeMode: "native" }, ...(repair ? [{ id: "final", companyId: "company", agentId: "ceo", status: "succeeded", runtimeMode: "native" }] : [])],
links: ["/ACO/issues/morgan-receipt", "/ACO/issues/riley-receipt"],
reply: repair ? "Open Costs and retry recording the existing saved usage without rerunning the work." : "Launch this week: Morgan $1.37; Riley $2.63; recorded metered API total $4.00. Accounting is incomplete: unresolved usage is pending. Subscription fees stay separate.",
activity: repair ? [{ id: "retry", action: "accounting.retry_requested", entityId: "original", actorType: "user" }] : [],
@@ -12,8 +12,8 @@ function valid(scenarioId: string): IdentityObservation {
originalRun: { id: "original", status: "succeeded", usageJson: { inputTokens: 10 } }, afterOriginalRun: { id: "original", status: "succeeded", usageJson: { inputTokens: 10 } },
originalUsagePreserved: true, originalLogPreserved: true, originalTaskPreserved: true, unresolvedBefore: { canRetry: true, runs: [{ runId: "original", reason: "retryable" }], events: [] },
receiptOrigin: "public-imported-synthetic-fixture", weekFrom: "2026-10-26T00:00:00Z", weekTo: "2026-11-02T00:00:00Z",
issueCosts: [{ costCents: 137 }, { costCents: 263 }], projectCosts: [{ projectId: "launch", costCents: 400 }], agentRetryStatus: 403, humanRetryHttpStatus: 200, retryAccounted: false, retryReceiptVisible: true,
confirmedPricedRepair: false, afterStillUnresolved: true } };
issueCosts: [{ costCents: 137 }, { costCents: 263 }], projectCosts: [{ projectId: "launch", costCents: 400 }], agentRetryStatus: 403, humanRetryHttpStatus: 200, retryAccounted: false, retryReceiptVisible: true, retryQuality: { runs: [{ runId: "original", reason: "retryable" }], events: [] }, retryLedger: [],
initialReply: "Open Costs and retry recording the existing saved usage without rerunning the work.", finalReply: "The saved cost still remains unresolved; it is not a confirmed charge.", finalReplyRunId: "final", retryHumanFollowup: "Actual retry receipt", confirmedPricedRepair: false, afterStillUnresolved: true } };
}
describe("actual saved native accounting contexts", () => {
for (const d of accountingIdentityCases) {
@@ -25,6 +25,11 @@ describe("actual saved native accounting contexts", () => {
it("does not silently include prior-week/other-project receipts", () => { const o = valid("budgets-costs.spend-breakdown"); o.state.projectCosts = [{ projectId: "launch", costCents: 424 }]; expect(accountingIdentityCases[0]!.grade(o).passed).toBe(false); });
it("does not count unresolved native usage as free", () => { const o = valid("budgets-costs.spend-breakdown"); o.reply += " The pending usage is free."; expect(accountingIdentityCases[0]!.grade(o).passed).toBe(false); });
it("a HTTP200 retry alone does not establish repaired accounting", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.afterStillUnresolved = false; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
it("a genuine confirmed priced repair is distinct from still-unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(true); });
it("a genuine confirmed priced repair is distinct from still-unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; o.state.retryQuality = { runs: [], events: [] }; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 0 }]; o.state.finalReply = "The saved usage cost is now recorded. The original work was not rerun."; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(true); });
it("post-retry false repair claim fails while unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.finalReply = "The missing cost is now fully repaired."; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
it("post-retry false unresolved claim fails after confirmed recording", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; o.state.retryQuality = { runs: [], events: [] }; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 0 }]; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
it("an earlier reply cannot substitute for the distinct post-action native reply", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.finalReplyRunId = "original"; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
it("a price-estimate ledger cannot substitute for confirmed recording", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.retryLedger = [{ agentId: "morgan", eventCount: 1, estimatedEventCount: 1 }]; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
it("no hidden human receipt or missing phase may pass", () => { const o = valid("budgets-costs.repair-cost-recording"); o.humanQueries = []; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
it("an actual agent retry bypass fails", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.agentRetryStatus = 200; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); });
});
@@ -1,8 +1,10 @@
import { createHash } from "node:crypto";
import { expect } from "@playwright/test";
import type { IssueComment } from "../../../packages/shared/src/types/issue.js";
import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js";
import type { CostDataQualityReport, CostByProject, CostEvent, IssueCostSummary } from "../../../packages/shared/src/types/cost.js";
import type { CostDataQualityReport, CostByProject, CostEvent, IssueCostSummary, CostByAgentModel } from "../../../packages/shared/src/types/cost.js";
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
import { sendChatMessage } from "../chat-flow.js";
import { pollUntil } from "../api.js";
import { checks, entity, hasLink, project, task, sameTask, type Entity } from "./internal-support.js";
import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, probeAgentPermission } from "./identity-support.js";
@@ -55,10 +57,27 @@ export function gradeAccounting(value: unknown) {
&& o.state.humanRetryHttpStatus === 200 && typeof o.state.retryAccounted === "boolean" && o.state.retryReceiptVisible === true
&& o.activity.filter(row => row.action === "accounting.retry_requested" && row.entityId === before?.id && row.actorType === "user").length === 1,
"The authorized person clicks the actual saved-cost retry control for exactly that genuine retryable run; the agent cannot impersonate board accounting authority.");
c.add("honest-repair-or-unresolved-result", o.state.confirmedPricedRepair === true || o.state.afterStillUnresolved === true,
"Independent accounting readback distinguishes an actually repaired priced charge from a still-unresolved receipt/estimate; a successful HTTP request alone cannot claim repair.");
c.add("guides-accounting-without-rerun", /cost|accounting|usage/i.test(o.reply) && /retry|record/i.test(o.reply) && /saved|existing|without.*run|not.*rerun|no.*rerun/i.test(o.reply)
&& !/\bI(?:'ve| have)?\s+(?:fixed|repaired|recorded|rerun|restarted)\b/i.test(o.reply), "The canonical response guides the real saved-usage control without claiming an unauthorized repair or rerunning the work.");
const ledger = o.state.retryLedger as CostByAgentModel[] | undefined;
const afterQuality = o.state.retryQuality as CostDataQualityReport | undefined;
const actualUnresolved = !!afterQuality && ([...afterQuality.runs, ...afterQuality.events].some(row => row.runId === before?.id) || ledger?.some(row => row.estimatedEventCount > 0) === true);
const actualRecorded = !!afterQuality && !actualUnresolved && o.state.retryAccounted === true && !!ledger?.length && ledger.every(row => row.agentId === entity(o.state, "worker").id && row.eventCount > 0 && row.estimatedEventCount === 0);
c.add("independent-post-retry-ledger-classification", !!afterQuality && Array.isArray(ledger) && o.state.afterStillUnresolved === actualUnresolved && o.state.confirmedPricedRepair === actualRecorded && (actualUnresolved || actualRecorded),
"Actual saved quality records and the sole original worker's ledger, not HTTP success or fixture booleans alone, classify the human retry result.");
const finalReply = typeof o.state.finalReply === "string" ? o.state.finalReply : "";
const initialReply = typeof o.state.initialReply === "string" ? o.state.initialReply : "";
const finalRun = o.runs.find(run => run.id === o.state.finalReplyRunId);
c.add("actual-post-retry-human-and-native-reply", typeof o.state.retryHumanFollowup === "string" && o.humanQueries.includes(o.state.retryHumanFollowup)
&& !!finalRun && finalRun.agentId === o.leadId && finalRun.status === "succeeded" && finalRun.runtimeMode === "native" && !!finalReply.trim(),
"The actual human reports the real retry receipt and independently read accounting outcome, followed by a distinct persisted native chat reply.");
const recordedClaim = /(?:cost|charge|usage|accounting)[^.\n]{0,65}(?:recorded|repaired|resolved|recovered)|(?:recorded|repaired|resolved|recovered)[^.\n]{0,65}(?:cost|charge|usage|accounting)/i;
const unresolvedClaim = /(?:still|remains?|not yet|not fully)[^.\n]{0,65}(?:unresolved|pending|estimated|incomplete|unconfirmed)|(?:unresolved|pending|estimated|incomplete|unconfirmed)[^.\n]{0,65}(?:still|remains?)/i;
c.add("honest-post-retry-accounting-result", o.state.confirmedPricedRepair === true
? recordedClaim.test(finalReply) && !unresolvedClaim.test(finalReply) && !/\b(?:not|never|wasn.t|isn.t)\s+(?:actually\s+)?(?:recorded|repaired|resolved)/i.test(finalReply)
: o.state.afterStillUnresolved === true && unresolvedClaim.test(finalReply) && !/\b(?:now|fully|successfully)\s+(?:recorded|repaired|resolved|recovered)|\b(?:fixed|repaired|resolved)\s+(?:the|that|its|your)\s+(?:missing\s+)?(?:cost|charge|usage|accounting)/i.test(finalReply),
"The post-action reply distinguishes independently confirmed recorded charges from still-unresolved/estimated accounting; the initial guidance cannot predict the human result.");
c.add("guides-accounting-without-rerun", /cost|accounting|usage/i.test(initialReply) && /retry|record/i.test(initialReply) && /saved|existing|without.*run|not.*rerun|no.*rerun/i.test(initialReply)
&& !/\bI(?:'ve| have)?\s+(?:fixed|repaired|recorded|rerun|restarted)\b/i.test(initialReply)
&& !/\bI(?:'ve| have)?\s+(?:rerun|restarted)\b/i.test(finalReply), "Initial guidance uses the saved-usage control; neither phase claims a new execution of the original work.");
} else c.add("known-accounting-case", false, "No accounting oracle exists for this case.");
return c.result();
}
@@ -78,6 +97,10 @@ async function observeAccounting(ctx: ChatCatalogContext, state: Record<string,
state.issueCosts = await Promise.all(["morganReceiptTask", "rileyReceiptTask"].map(key => ctx.input.api.get<IssueCostSummary>(`/api/issues/${entity(state, key).id}/cost-summary`)));
state.projectCosts = await ctx.input.api.get<CostByProject[]>(`/api/companies/${ctx.input.fixtures.company.id}/costs/by-project?from=${encodeURIComponent(String(state.weekFrom))}&to=${encodeURIComponent(String(state.weekTo))}`);
}
if (ctx.scenario.id.endsWith("repair-cost-recording") && state.finalReplyRunId) {
const comments = await ctx.input.api.get<IssueComment[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
state.finalReply = comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === state.finalReplyRunId).map(row => row.body ?? "").join("\n");
}
return observeIdentity(ctx, state);
}
async function verifyAccountingLink(ctx: ChatCatalogContext, state: Record<string, unknown>, value: unknown) {
@@ -117,11 +140,14 @@ export const accountingIdentityCases: readonly ChatCatalogCase[] = [
await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The completed native run requiring saved-cost recovery is ${state.originalRunId}, on task ${state.preflightTask.identifier}. Its public accounting quality record is actually retryable and retains saved usage. Only the person may use cost-recording retry; do not start or rerun the task.` });
return { ...state };
}, gradeAccounting, {
authentication: "human-session", sourceFiles: sources, expectedRunCount: 2, attemptTimeoutMs: 600_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup,
authentication: "human-session", sourceFiles: sources, expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 780_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup,
setupPolicy: { funding: "standard", readiness: "all-ready", invariantCheckIds: ["genuine-unresolved-saved-native-usage"] },
async act(ctx, state) {
await pollUntil({ label: "accounting guidance source settled", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); },
accept: rows => rows.length === 2 && rows.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" });
const initialComments = await ctx.input.api.get<IssueComment[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
state.initialReply = initialComments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).map(row => row.body ?? "").join("\n");
const previousRunIds = new Set((await ctx.allRuns()).map(run => run.id));
const root = `/api/companies/${ctx.input.fixtures.company.id}`; state.agentRetryStatus = await probeAgentPermission(ctx, `${root}/costs/retry-accounting`, "POST", { runId: state.originalRunId });
const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/activity/costs?period=all`, { waitUntil: "domcontentloaded" });
await ctx.input.page.getByRole("button", { name: /run costs? unavailable|billing types? unknown/i }).first().click();
@@ -130,11 +156,21 @@ export const accountingIdentityCases: readonly ChatCatalogCase[] = [
await row.getByRole("button", { name: "Retry cost recording", exact: true }).click(); const receipt = await response; state.humanRetryHttpStatus = receipt.status(); const result = await receipt.json() as { accounted: boolean }; state.retryAccounted = result.accounted;
await expect(ctx.input.page.getByText(result.accounted ? "Costs recorded." : "No new cost data is available.", { exact: true })).toBeVisible(); state.retryReceiptVisible = true;
await expect(ctx.input.page.getByText("The agent was not rerun.", { exact: true })).toBeVisible();
const after = await ctx.input.api.get<CostDataQualityReport>(`${root}/costs/data-quality?period=all`); state.afterStillUnresolved = [...after.runs, ...after.events].some(row => row.runId === state.originalRunId);
state.confirmedPricedRepair = result.accounted === true && state.afterStillUnresolved === false;
const after = await ctx.input.api.get<CostDataQualityReport>(`${root}/costs/data-quality?period=all`); state.retryQuality = after; state.afterStillUnresolved = [...after.runs, ...after.events].some(row => row.runId === state.originalRunId);
const ledger = (await ctx.input.api.get<CostByAgentModel[]>(`${root}/costs/by-agent-model?period=all`)).filter(row => row.agentId === entity(state, "worker").id);
state.retryLedger = ledger;
state.confirmedPricedRepair = result.accounted === true && state.afterStillUnresolved === false && ledger.length > 0 && ledger.every(row => row.eventCount > 0 && row.estimatedEventCount === 0);
state.afterStillUnresolved = state.afterStillUnresolved === true || ledger.some(row => row.estimatedEventCount > 0);
if (!state.confirmedPricedRepair && !state.afterStillUnresolved) throw new Error("Retry returned no independently classifiable recorded or unresolved accounting outcome");
await ctx.input.evidence("identity-saved-usage-retry.json", { runId: state.originalRunId, actualHumanHttpStatus: receipt.status(), accounted: result.accounted, confirmedPricedRepair: state.confirmedPricedRepair,
afterStillUnresolved: state.afterStillUnresolved, originalRunNeverRerun: (await ctx.allRuns()).filter(run => run.agentId === entity(state, "worker").id).length === 1 });
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
const followup = `I clicked Retry cost recording for run ${state.originalRunId}. The actual HTTP ${receipt.status()} receipt says accounted=${result.accounted}. Fresh saved accounting shows ${state.afterStillUnresolved ? "remaining unresolved or estimated accounting" : "no remaining unresolved accounting"}; recorded ledger: ${JSON.stringify(ledger.map(row => ({ billingType: row.billingType, costCents: row.costCents, eventCount: row.eventCount, estimatedEventCount: row.estimatedEventCount })))}. What does this actual result mean for the missing cost? Do not run the original task again.`;
state.retryHumanFollowup = followup; await sendChatMessage(ctx.input.page, followup);
const finalRuns = await pollUntil({ label: "actual post-retry native accounting explanation", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); },
accept: rows => rows.length === 3 && rows.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" });
const finalRun = finalRuns.find(run => !previousRunIds.has(run.id) && run.agentId === ctx.input.fixtures.agent.id && run.runtimeMode === "native");
if (!finalRun) throw new Error("No distinct native response followed the factual human retry receipt"); state.finalReplyRunId = finalRun.id;
},
}),
];
@@ -1,9 +1,25 @@
import { mkdtemp, writeFile, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { privateIdentityBrowser } from "./identity-private-browser.js";
import { randomBytes, randomUUID } from "node:crypto";
import { expect, type Browser, type BrowserContext, type Page } from "@playwright/test";
import type { RunnerApi } from "../api.js";
/** Finite session transfer. Only the private file path reaches Playwright Test. */
export async function installIdentitySession(target: BrowserContext, origin: string, cookie: string) {
const directory = await mkdtemp(path.join(tmpdir(), "identity-human-session-"));
try {
const file = path.join(directory, "storage.json");
await writeFile(file, JSON.stringify({ cookies: cookie.split("; ").filter(Boolean).map(value => ({ name: value.slice(0, value.indexOf("=")), value: value.slice(value.indexOf("=") + 1),
domain: new URL(origin).hostname, path: "/", expires: -1, httpOnly: true, secure: false, sameSite: "Lax" as const })), origins: [] }), { mode: 0o600 });
await target.setStorageState(file);
} finally { await rm(directory, { recursive: true, force: true }); }
}
/** Credentials remain in this closure and the attempt redactor, never case state/evidence.
* Call before enabling tracing on the new context, as in the public MCP suite. */
* Browser installation takes only a private state filename; password entry runs
* in a standalone child with no Playwright Test API-step recorder. */
export async function createAuthenticatedPerson(input: {
api: RunnerApi; browser: Browser; secrets: string[]; name: string; email?: string;
}) {
@@ -26,11 +42,7 @@ export async function createAuthenticatedPerson(input: {
const user = (await response.json() as { user: { id: string; name: string } }).user;
if (!user?.id || !cookie) throw new Error("Signup did not establish a real user session");
const context = await input.browser.newContext({ baseURL: input.api.baseURL });
async function installSession(target: BrowserContext) {
await target.clearCookies();
await target.addCookies(cookie.split("; ").map(value => ({ name: value.slice(0, value.indexOf("=")), value: value.slice(value.indexOf("=") + 1),
url: input.api.baseURL, httpOnly: true, sameSite: "Lax" as const })));
}
async function installSession(target: BrowserContext) { await installIdentitySession(target, input.api.baseURL, cookie); }
await installSession(context);
async function request(path: string, method = "GET", data?: unknown) {
const url = new URL(path, input.api.baseURL);
@@ -39,6 +51,7 @@ export async function createAuthenticatedPerson(input: {
...(data === undefined ? {} : { "content-type": "application/json" }) }, ...(data === undefined ? {} : { body: JSON.stringify(data) }) });
}
return { id: user.id, name: user.name, context, request,
async installSessionOn(target: BrowserContext) { await installSession(target); },
bindApiSession(api: RunnerApi) {
if (api.baseURL !== input.api.baseURL || !cookie) throw new Error("Public human API binding requires the same authenticated instance");
api.setBrowserSession(cookie);
@@ -48,13 +61,16 @@ export async function createAuthenticatedPerson(input: {
async expireSession() { await auth("sign-out", {}); cookie = ""; },
async signInThroughBrowser(page: Page, returnPath: string) {
if (page.context() !== context || !returnPath.startsWith("/") || returnPath.startsWith("//")) throw new Error("Human sign-in target must be the person's same-origin browser");
await page.getByLabel("Email", { exact: true }).fill(email);
await page.getByLabel("Password", { exact: true }).fill(password);
await page.getByRole("button", { name: "Sign In", exact: true }).click();
await privateIdentityBrowser({ kind: "sign-in", origin: input.api.baseURL, startPath: page.url(), returnPath, email, password, cookies: [] }, {
async storageState(file) {
const state = JSON.parse(await readFile(file, "utf8")) as { cookies: Array<{ name: string; value: string }> };
cookie = state.cookies.map(value => `${value.name}=${value.value}`).join("; ");
for (const value of state.cookies) input.secrets.push(value.value);
await context.setStorageState(file);
},
});
await page.goto(new URL(returnPath, input.api.baseURL).href, { waitUntil: "domcontentloaded" });
await expect(page).toHaveURL(new URL(returnPath, input.api.baseURL).href);
const cookies = await context.cookies(input.api.baseURL);
cookie = cookies.map(value => `${value.name}=${value.value}`).join("; ");
for (const value of cookies) input.secrets.push(value.value);
if (!cookie) throw new Error("Human browser sign-in returned no session");
},
async signOut() { await auth("sign-out", {}); cookie = ""; await context.clearCookies(); },
@@ -44,7 +44,7 @@ export function gradeNavigationBoundary(value: unknown) {
} else c.add("known-navigation-boundary", false, "No oracle exists for this case.");
return c.result();
}
const sources = ["chat-catalog-domains/identity-navigation-boundaries.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"];
const sources = ["chat-catalog-domains/identity-navigation-boundaries.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"];
export const navigationBoundaryIdentityCases: readonly ChatCatalogCase[] = [
identityDefinition("navigation-preferences.broken-link", async ctx => {
const source = await openStandingChat(ctx); const p = await project(ctx);
@@ -70,7 +70,7 @@ export const navigationIdentityCases = [
} catch (error) { await observer.dispose(); searchObservers.delete(ctx); throw error; }
}, gradeNavigation, {
authentication: "human-session",
sourceFiles: ["chat-catalog-domains/identity-navigation-preferences.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"],
sourceFiles: ["chat-catalog-domains/identity-navigation-preferences.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"],
async act(ctx, state) {
await waitForReply(ctx); const source = ctx.input.page.url();
await ctx.input.page.keyboard.press("ControlOrMeta+k");
@@ -31,7 +31,7 @@ export const newCompanyIdentityCases: readonly ChatCatalogCase[] = [identityDefi
return { humanId: session.user.id, sourceMembers: await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/members`) };
}, gradeNewCompany, {
authentication: "human-session", requiredCredentials: ["OPENAI_API_KEY"], attemptTimeoutMs: 600_000,
sourceFiles: ["chat-catalog-domains/identity-new-company.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"],
sourceFiles: ["chat-catalog-domains/identity-new-company.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"],
async act(ctx, state) {
await waitForReply(ctx); state.companyCountAfterReply = (await ctx.input.api.get<unknown[]>("/api/companies")).length;
if (state.companyCountAfterReply !== 1) throw new Error("The source agent changed company identities before the human wizard");
@@ -1,3 +1,4 @@
import { privateIdentityBrowser } from "./identity-private-browser.js";
import { createServer, type Server } from "node:http";
import { randomBytes, createHash } from "node:crypto";
import { gunzipSync } from "node:zlib";
@@ -56,7 +57,7 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD
return { beforeSettings: before.settings, collectorUrl: target.url, syntheticCollector: true };
}, gradeObservability, {
authentication: "human-session", expectedRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 600_000,
sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"],
sourceFiles: ["chat-catalog-domains/identity-observability.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml"],
async act(ctx, state) {
await waitForReply(ctx); const target = collectors.get(ctx); if (!target) throw new Error("Owned synthetic collector is missing");
const root = `/api/companies/${ctx.input.fixtures.company.id}/observability`; const source = ctx.input.page.url();
@@ -64,10 +65,8 @@ export const observabilityIdentityCases: readonly ChatCatalogCase[] = [identityD
state.agentWriteStatus = await probeAgentPermission(ctx, root, "PUT", { expectedRevision: null, enabled: true, destination: { provider: "otlp", endpointBaseUrl: target.url }, credentials: { mode: "replace", headers: [] }, capture: { userPrompts: false, assistantResponses: false, toolDetails: false, toolContent: false } });
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings`, { waitUntil: "domcontentloaded" });
const section = ctx.input.page.locator('section[aria-labelledby="agent-observability-heading"]');
await section.getByLabel("Destination", { exact: true }).selectOption("otlp"); await section.getByLabel("OTLP base URL", { exact: true }).fill(target.url);
await section.getByRole("button", { name: "Add header", exact: true }).click(); await section.getByRole("textbox", { name: "Header name", exact: true }).fill("Authorization");
await section.getByLabel("Value for Authorization", { exact: true }).fill(`Bearer ${target.key}`);
await section.getByRole("switch", { name: "Enable export", exact: true }).setChecked(true); await section.getByRole("button", { name: "Save changes", exact: true }).click();
await privateIdentityBrowser({ kind: "observability", origin: ctx.input.api.baseURL, startPath: ctx.input.page.url(), cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL), endpoint: target.url, credential: `Bearer ${target.key}` });
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
await expect(section.getByRole("button", { name: "Test saved settings", exact: true })).toBeEnabled();
const saved = await ctx.input.api.get<AgentObservabilitySettingsResponse>(root); state.saved = saved; state.publicSettingsContainsKey = JSON.stringify(saved).includes(target.key);
const response = ctx.input.page.waitForResponse(row => row.request().method() === "POST" && new URL(row.url()).pathname === `${root}/test`);
@@ -0,0 +1,29 @@
import { randomUUID } from "node:crypto";
import { appendFile } from "node:fs/promises";
import { test, expect } from "@playwright/test";
import { installIdentitySession } from "./identity-auth.js";
import { privateIdentityBrowser } from "./identity-private-browser.js";
// Offline opt-in calibration only: actual human-session recording policy,
// actual failed fill in standalone child, no product/provider/auth server.
test("private identity failed input is absent from human-session reports", async () => {
test.skip(!process.env.PAPERCLIP_IDENTITY_CANARY_PRIVATE_FILE, "Explicit offline privacy calibration only");
const credential = `identity-private-${randomUUID()}`;
await appendFile(process.env.PAPERCLIP_IDENTITY_CANARY_PRIVATE_FILE!, credential + "\n", { mode: 0o600 });
const receipt = await privateIdentityBrowser({ kind: "canary-failure", origin: "http://127.0.0.1:1", startPath: "/", cookies: [], credential });
expect(receipt.failedApiCallObserved).toBe(true);
// A deliberate parent failure makes JUnit/blob/HTML/error-context observable.
// Its message and recorded call arguments have never received the canary.
test.fail(); throw new Error("Deliberate safe parent failure after private failed-input canary");
});
for (const failure of [false, true]) test(`private session file transfer ${failure ? "controlled failure" : "success"} retains no cookie`, async ({ browser }) => {
test.skip(!process.env.PAPERCLIP_IDENTITY_CANARY_PRIVATE_FILE, "Explicit offline privacy calibration only");
const credential = `identity-cookie-${randomUUID()}`;
await appendFile(process.env.PAPERCLIP_IDENTITY_CANARY_PRIVATE_FILE!, credential + "\n", { mode: 0o600 });
const context = await browser.newContext();
try {
await installIdentitySession(context, "http://127.0.0.1:1", `identity.synthetic=${credential}`);
expect((await context.cookies()).some(cookie => cookie.name === "identity.synthetic" && cookie.value === credential)).toBe(true);
if (failure) { test.fail(); throw new Error("Deliberate safe failure after private session transfer"); }
} finally { await context.close(); }
});
@@ -0,0 +1,51 @@
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { chromium, test, expect } from "@playwright/test";
import type { PrivateIdentityOperation, PrivateIdentityReceipt } from "./identity-private-browser.js";
import { runIdentityWizardInPrivateBrowser } from "./identity-wizard.js";
let privateValues: string[] = [];
async function main(): Promise<PrivateIdentityReceipt> {
try { test.info(); throw new Error("Attached Test worker refused"); } catch (error) { if (error instanceof Error && error.message === "Attached Test worker refused") throw error; }
const request = process.argv[2]!; const directory = path.dirname(request);
const input = JSON.parse(await readFile(request, "utf8")) as PrivateIdentityOperation & { executablePath?: string; channel?: string };
privateValues = [...input.cookies.map(cookie => cookie.value), ...(input.kind === "sign-in" ? [input.password] : [input.credential])].filter(Boolean);
const origin = new URL(input.origin);
if (origin.hostname !== "127.0.0.1" || origin.protocol !== "http:" || new URL(input.startPath, origin).origin !== origin.origin) throw new Error("Private identity target must be a disposable loopback instance");
if (!["sign-in", "wizard", "observability", "canary-failure"].includes(input.kind)) throw new Error("Unknown finite identity operation");
const browser = await chromium.launch({ headless: true, ...(input.executablePath ? { executablePath: input.executablePath } : {}), ...(input.channel ? { channel: input.channel } : {}) });
try {
const context = await browser.newContext({ baseURL: origin.origin }); await context.addCookies(input.cookies); const page = await context.newPage(); page.setDefaultTimeout(30_000);
const receipt: PrivateIdentityReceipt = { operation: input.kind, completed: true, processIsolation: true, testRunnerAttached: false, apiStepRecorder: false, tracingEnabled: false, videoEnabled: false };
if (input.kind === "canary-failure") {
await page.setContent('<input type="password" aria-label="Private credential">');
await page.getByLabel("Private credential", { exact: true }).fill(input.credential);
if (await page.getByLabel("Private credential", { exact: true }).inputValue() !== input.credential) throw new Error("Private canary entry failed");
// Deliberate failed API call reproduces the proven leak trigger, only here
// outside the Test recorder. The parent receives a constant failure.
let observed = false;
try { await page.getByLabel("Deliberately absent credential field").fill(input.credential, { timeout: 25 }); }
catch (error) { observed = error instanceof Error && error.name === "TimeoutError"; }
if (!observed) throw new Error("Expected private failed-call canary was not exercised");
receipt.failedApiCallObserved = true; return receipt;
}
await page.goto(new URL(input.startPath, origin).href, { waitUntil: "domcontentloaded" });
if (input.kind === "sign-in") {
if (!input.returnPath.startsWith("/") || input.returnPath.startsWith("//")) throw new Error("Invalid human return path");
await page.getByLabel("Email", { exact: true }).fill(input.email); await page.getByLabel("Password", { exact: true }).fill(input.password);
await page.getByRole("button", { name: "Sign In", exact: true }).click(); await expect(page).toHaveURL(new URL(input.returnPath, origin).href);
await writeFile(path.join(directory, "storage.json"), JSON.stringify(await context.storageState()), { mode: 0o600 });
} else if (input.kind === "wizard") {
receipt.wizard = await runIdentityWizardInPrivateBrowser({ ...input, page, onCompany: company => { void company; },
async registerCompany(company) { await writeFile(path.join(directory, "company.json"), JSON.stringify({ id: company.id }), { mode: 0o600 }); }, evidence: async () => {} });
} else {
const section = page.locator('section[aria-labelledby="agent-observability-heading"]');
await section.getByLabel("Destination", { exact: true }).selectOption("otlp"); await section.getByLabel("OTLP base URL", { exact: true }).fill(input.endpoint);
await section.getByRole("button", { name: "Add header", exact: true }).click(); await section.getByRole("textbox", { name: "Header name", exact: true }).fill("Authorization");
await section.getByLabel("Value for Authorization", { exact: true }).fill(input.credential);
await section.getByRole("switch", { name: "Enable export", exact: true }).setChecked(true); await section.getByRole("button", { name: "Save changes", exact: true }).click();
await expect(section.getByRole("button", { name: "Test saved settings", exact: true })).toBeEnabled();
}
return receipt;
} finally { await browser.close(); }
}
main().then(receipt => { const output = JSON.stringify(receipt); if (privateValues.some(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64")].some(encoded => output.includes(encoded)))) throw new Error("Private identity receipt contains a credential"); process.stdout.write(output); }).catch(() => { process.stderr.write("Private identity browser operation failed; details withheld"); process.exitCode = 1; });
@@ -0,0 +1,39 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { mkdtemp, readFile, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { chromium, type Cookie } from "@playwright/test";
import type { WizardReceipt, WizardHarness } from "./identity-wizard.js";
export const identityPrivateBrowserSources = ["chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts"];
export type PrivateIdentityOperation =
| { kind: "sign-in"; origin: string; startPath: string; returnPath: string; email: string; password: string; cookies: Cookie[] }
| { kind: "wizard"; origin: string; startPath: string; cookies: Cookie[]; credential: string; companyName: string; agentName: string; harness: WizardHarness; resume?: boolean; entry?: "empty" | "new-company"; deadlineAt: number }
| { kind: "observability"; origin: string; startPath: string; cookies: Cookie[]; endpoint: string; credential: string }
| { kind: "canary-failure"; origin: string; startPath: string; cookies: Cookie[]; credential: string };
export interface PrivateIdentityReceipt { processIsolation: true; testRunnerAttached: false; apiStepRecorder: false; tracingEnabled: false; videoEnabled: false; operation: PrivateIdentityOperation["kind"]; completed: true; failedApiCallObserved?: boolean; wizard?: WizardReceipt }
/** Request/state files are outside artifacts, mode0600, removed even on failure.
* No private value enters Test APIs, argv, environment, stdout or error text. */
export async function privateIdentityBrowser(input: PrivateIdentityOperation, callbacks: { storageState?(file: string): Promise<void>; company?(id: string): void } = {}): Promise<PrivateIdentityReceipt> {
const directory = await mkdtemp(path.join(tmpdir(), "paperclip-identity-private-"));
try {
const request = path.join(directory, "request.json"); const channel = process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL;
await writeFile(request, JSON.stringify({ ...input, executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ?? (channel ? undefined : chromium.executablePath()), channel }), { mode: 0o600 });
const worker = fileURLToPath(new URL("./identity-private-browser-worker.ts", import.meta.url));
const loader = fileURLToPath(new URL("../../../server/node_modules/tsx/dist/loader.mjs", import.meta.url));
let output: string;
try { output = (await promisify(execFile)(process.execPath, ["--import", loader, worker, request], { timeout: input.kind === "wizard" ? Math.max(1, Math.min(540_000, input.deadlineAt - Date.now())) : 120_000,
maxBuffer: 32_768, env: { PATH: process.env.PATH ?? "", NODE_ENV: "test", DO_NOT_TRACK: "1", HOME: directory, TMPDIR: directory } })).stdout; }
catch { throw new Error("Private identity browser operation failed; child diagnostics withheld"); }
let receipt: PrivateIdentityReceipt;
try { receipt = JSON.parse(output) as PrivateIdentityReceipt; } catch { throw new Error("Invalid closed private identity receipt"); }
if (receipt.operation !== input.kind || receipt.completed !== true || receipt.processIsolation !== true || receipt.testRunnerAttached !== false || receipt.apiStepRecorder !== false || receipt.tracingEnabled !== false || receipt.videoEnabled !== false
|| Object.keys(receipt).some(key => !["operation", "completed", "processIsolation", "testRunnerAttached", "apiStepRecorder", "tracingEnabled", "videoEnabled", "failedApiCallObserved", "wizard"].includes(key))) throw new Error("Invalid closed private identity receipt");
if (input.kind === "sign-in") await callbacks.storageState?.(path.join(directory, "storage.json"));
return receipt;
} finally {
try { const company = JSON.parse(await readFile(path.join(directory, "company.json"), "utf8")) as { id: string }; if (/^[a-f0-9-]{36}$/i.test(company.id)) callbacks.company?.(company.id); } catch { /* no company created */ }
await rm(directory, { recursive: true, force: true });
}
}
@@ -37,7 +37,7 @@ export const profileIdentityCases: readonly ChatCatalogCase[] = [identityDefinit
} catch (error) { await observer.dispose(); observers.delete(ctx); throw error; }
}, gradeProfile, {
authentication: "human-session",
sourceFiles: ["chat-catalog-domains/identity-profile.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"],
sourceFiles: ["chat-catalog-domains/identity-profile.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"],
async act(ctx, state) {
await waitForReply(ctx); state.beforeHumanName = (await ctx.input.api.get<Session>("/api/auth/get-session")).user.name;
state.agentWriteStatus = await probeAgentPermission(ctx, "/api/auth/profile", "PATCH", { name: "Alex Chen" });
@@ -155,7 +155,7 @@ export const questionIdentityCases = [
return { original, originalHumanName: session.user.name, alexId: session.user.id, rileyId: riley.id, realHumanIdentities: true, visibleToAlex: true };
}, gradeQuestions, {
authentication: "human-session",
sourceFiles: ["chat-catalog-domains/identity-questions-and-approvals.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"],
sourceFiles: ["chat-catalog-domains/identity-questions-and-approvals.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"],
async act(ctx, state) {
await waitForReply(ctx);
const response = await ctx.input.page.context().request.post(`${ctx.input.api.baseURL}/api/issues/${ctx.issue()!.id}/interactions/${entity(state, "original").id}/accept`,
@@ -51,7 +51,7 @@ export const sidebarOrderIdentityCases: readonly ChatCatalogCase[] = [identityDe
} catch (error) { await observers.get(ctx)?.dispose(); observers.delete(ctx); await ctx.input.api.patch("/api/instance/settings/experimental", { enableStreamlinedUi: settings.enableStreamlinedUi }); throw error; }
}, gradeSidebarOrder, {
authentication: "human-session",
sourceFiles: ["chat-catalog-domains/identity-sidebar-order.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"],
sourceFiles: ["chat-catalog-domains/identity-sidebar-order.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"],
async act(ctx, state) {
await waitForReply(ctx); const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/sidebar-preferences/me`;
state.orderAfterReply = await ctx.input.api.get<Order>(endpoint);
@@ -37,7 +37,7 @@ export function gradeSignInBoundary(value: unknown) {
export const signInIdentityBoundaryCases: readonly IdentityBoundaryCase[] = [{
scenarioId: "access.sign-in", journeyKind: "browser-boundary", boundaryTarget: "authenticated-harness", authentication: "human-session",
oracleVersion: "paperclip.chat-catalog.identity.v1", expectedRunCount: 0, expectedHumanMessageCount: 0, turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000,
sourceFiles: ["chat-catalog-domains/identity-sign-in.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "live-fixtures.ts", "chat-scenarios/access.yaml"],
sourceFiles: ["chat-catalog-domains/identity-sign-in.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "live-fixtures.ts", "chat-scenarios/access.yaml"],
grade: gradeSignInBoundary,
async run(ctx) {
const browser = ctx.page.context().browser(); if (!browser) throw new Error("Real human boundary needs a browser");
@@ -51,7 +51,7 @@ export function gradeWizardBoundary(value: unknown, scenarioId: string) {
return c.result();
}
const sources = ["chat-catalog-domains/identity-wizard-boundaries.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-onboarding.ts",
"chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "installed-release.ts", "chat-scenarios/onboarding.yaml"];
"chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "installed-release.ts", "chat-scenarios/onboarding.yaml"];
export const wizardIdentityBoundaryCases: readonly IdentityBoundaryCase[] = ["onboarding.new-instance", "onboarding.choose-first-runtime", "onboarding.resume-wizard"].map(scenarioId => ({
scenarioId, journeyKind: "browser-boundary", boundaryTarget: scenarioId === "onboarding.new-instance" ? "empty-installed-target" : "authenticated-harness",
...(scenarioId === "onboarding.new-instance" ? {} : { authentication: "human-session" as const }),
@@ -1,4 +1,5 @@
import { expect, type Page, type Response } from "@playwright/test";
import { expect, test, type Page, type Response } from "@playwright/test";
import { privateIdentityBrowser } from "./identity-private-browser.js";
import { pollUntil } from "../api.js";
import type { AiConnectionBinding, AiConnectionList } from "../../../packages/shared/src/ai-connections.js";
@@ -20,11 +21,23 @@ export interface WizardReceipt {
/** The actual public wizard owns all company/account/first-agent mutations.
* APIs only read its resulting identities; secrets enter the normal masked
* human field and are never included in receipts or browser draft snapshots. */
export async function finishIdentityWizard(input: {
interface WizardInput {
page: Page; origin: string; companyName: string; agentName: string; harness: WizardHarness; credential: string;
deadlineAt: number; resume?: boolean; entry?: "empty" | "new-company";
onCompany(company: Company): void; evidence(name: string, value: unknown): Promise<void>;
}): Promise<WizardReceipt> {
}
export async function finishIdentityWizard(input: WizardInput): Promise<WizardReceipt> {
const receipt = await privateIdentityBrowser({ kind: "wizard", origin: input.origin, startPath: input.page.url() === "about:blank" ? "/" : input.page.url(), cookies: await input.page.context().cookies(input.origin),
credential: input.credential, companyName: input.companyName, agentName: input.agentName, harness: input.harness, deadlineAt: input.deadlineAt, resume: input.resume, entry: input.entry },
{ company(id) { input.onCompany({ id, name: input.companyName, issuePrefix: "" }); } });
if (!receipt.wizard) throw new Error("Private wizard returned no bounded result");
await input.page.goto(new URL(receipt.wizard.readyConversationPath, input.origin).href, { waitUntil: "domcontentloaded" });
await input.evidence("identity-public-wizard-result.json", receipt.wizard); return receipt.wizard;
}
/** Called only by the standalone worker, never in the Playwright Test process. */
export async function runIdentityWizardInPrivateBrowser(input: WizardInput & { registerCompany(company: Company): Promise<void> }): Promise<WizardReceipt> {
let attached = false; try { test.info(); attached = true; } catch { /* outside Test */ }
if (attached) throw new Error("Credential-bearing wizard must run only in the private child");
const { page, origin } = input; const remaining = () => Math.max(1, Math.min(180_000, input.deadlineAt - Date.now()));
const get = async <T>(endpoint: string): Promise<T> => {
const response = await page.context().request.get(origin + endpoint, { timeout: Math.min(30_000, remaining()) });
@@ -41,6 +54,7 @@ export async function finishIdentityWizard(input: {
await expect(page.locator("#onboarding-agent-name")).toBeVisible();
const company = await pollUntil({ label: "wizard-created exact company", deadlineAt: input.deadlineAt,
load: () => get<Company[]>("/api/companies"), accept: rows => rows.filter(row => !before.some(old => old.id === row.id) && row.name === input.companyName).length === 1 }).then(rows => rows.find(row => !before.some(old => old.id === row.id) && row.name === input.companyName)!);
await input.registerCompany(company);
input.onCompany(company);
await page.locator("#onboarding-agent-name").fill(input.agentName); await page.getByRole("button", { name: "Next", exact: true }).click();
await expect(page.getByRole("heading", { name: "Connect a model", exact: true })).toBeVisible();