test(evals): add finite private public-origin sign-in operation

tests/runner-e2e/chat-catalog-domains/external-public-signin-contract.test.ts

@@ -0,0 +1,74 @@
+import { expect, it } from "vitest";
+import {
+  validPublicSignInInput,
+  publicSignInStorageMatches,
+} from "./external-public-signin-contract.js";
+const input = () => ({
+  receiptPath: "/private/tmp/attempt/public-origin.json",
+  executionId: "execution",
+  attemptId: "attempt",
+  backendOrigin: "http://127.0.0.1:3399",
+  publicOrigin: "https://owned.example.test",
+  userId: "generated-person",
+  email: "mcp-12345678-1234-1234-1234-123456789012@example.test",
+  password: "synthetic-generated-private-password",
+});
+const storage = () => ({
+  cookies: [
+    {
+      name: "session",
+      value: "synthetic-private-cookie",
+      domain: "owned.example.test",
+      path: "/",
+      secure: true,
+    },
+  ],
+  origins: [
+    {
+      origin: "https://owned.example.test",
+      localStorage: [{ name: "preference", value: "private-storage-value" }],
+    },
+  ],
+});
+it("accepts only generated fixture-person HTTPS sign-in with exact loopback backend and explicit private receipt", () => {
+  expect(validPublicSignInInput(input())).toBe(true);
+  for (const change of [
+    { publicOrigin: "http://owned.example.test" },
+    { publicOrigin: "https://user:secret@owned.example.test" },
+    { publicOrigin: "https://owned.example.test/auth" },
+    { backendOrigin: "http://production.example.test" },
+    { email: "someone@real.example" },
+    { password: "short" },
+    { receiptPath: "relative.json" },
+  ])
+    expect(validPublicSignInInput({ ...input(), ...change })).toBe(false);
+});
+it("rejects loopback cookies, broader-domain cookies and another origin's storage", () => {
+  expect(publicSignInStorageMatches(storage(), input().publicOrigin)).toBe(
+    true,
+  );
+  for (const change of [
+    (s: ReturnType<typeof storage>) => {
+      s.cookies[0]!.domain = "127.0.0.1";
+    },
+    (s: ReturnType<typeof storage>) => {
+      s.cookies[0]!.domain = ".example.test";
+    },
+    (s: ReturnType<typeof storage>) => {
+      s.cookies[0]!.secure = false;
+    },
+    (s: ReturnType<typeof storage>) => {
+      s.origins[0]!.origin = "http://127.0.0.1:3399";
+    },
+    (s: ReturnType<typeof storage>) => {
+      s.cookies = [];
+    },
+  ]) {
+    const s = storage();
+    change(s);
+    expect(publicSignInStorageMatches(s, input().publicOrigin)).toBe(false);
+  }
+  expect(publicSignInStorageMatches(undefined, input().publicOrigin)).toBe(
+    false,
+  );
+});

tests/runner-e2e/chat-catalog-domains/external-public-signin-contract.ts

@@ -0,0 +1,87 @@
+export interface PublicSignInInput {
+  receiptPath: string;
+  executionId: string;
+  attemptId: string;
+  backendOrigin: string;
+  publicOrigin: string;
+  userId: string;
+  email: string;
+  password: string;
+}
+export interface PublicSignInStorage {
+  cookies: Array<{
+    name: string;
+    value: string;
+    domain: string;
+    path: string;
+    secure: boolean;
+  }>;
+  origins: Array<{
+    origin: string;
+    localStorage: Array<{ name: string; value: string }>;
+  }>;
+}
+export function validPublicSignInInput(value: PublicSignInInput) {
+  try {
+    const origin = new URL(value.publicOrigin),
+      backend = new URL(value.backendOrigin);
+    return (
+      value.publicOrigin === origin.origin &&
+      origin.protocol === "https:" &&
+      !origin.username &&
+      !origin.password &&
+      value.backendOrigin === backend.origin &&
+      backend.hostname === "127.0.0.1" &&
+      backend.protocol === "http:" &&
+      value.receiptPath.startsWith("/") &&
+      !/[\r\n\0]/.test(value.receiptPath) &&
+      [value.executionId, value.attemptId, value.userId].every(
+        (id) => typeof id === "string" && /^[a-zA-Z0-9_.:-]{1,200}$/.test(id),
+      ) &&
+      /^mcp-[a-f0-9-]{36}@example\.test$/.test(value.email) &&
+      typeof value.password === "string" &&
+      value.password.length >= 16 &&
+      value.password.length <= 8192
+    );
+  } catch {
+    return false;
+  }
+}
+export function publicSignInStorageMatches(
+  value: unknown,
+  publicOrigin: string,
+): value is PublicSignInStorage {
+  const storage = value as PublicSignInStorage | undefined;
+  try {
+    const origin = new URL(publicOrigin);
+    return (
+      !!storage &&
+      Array.isArray(storage.cookies) &&
+      storage.cookies.length > 0 &&
+      storage.cookies.length <= 30 &&
+      storage.cookies.every(
+        (cookie) =>
+          cookie.domain === origin.hostname &&
+          cookie.secure === true &&
+          cookie.path === "/" &&
+          typeof cookie.name === "string" &&
+          typeof cookie.value === "string" &&
+          !!cookie.value,
+      ) &&
+      Array.isArray(storage.origins) &&
+      storage.origins.length <= 1 &&
+      storage.origins.every(
+        (entry) =>
+          entry.origin === publicOrigin &&
+          Array.isArray(entry.localStorage) &&
+          entry.localStorage.length <= 200 &&
+          entry.localStorage.every(
+            (item) =>
+              typeof item.name === "string" && typeof item.value === "string",
+          ),
+      )
+    );
+  } catch {
+    return false;
+  }
+}

tests/runner-e2e/chat-catalog-domains/external-public-signin-worker.ts

@@ -0,0 +1,139 @@
+import { readFile, writeFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { chromium, test } from "@playwright/test";
+import { readValidatedPublicOriginReceipt } from "../chat-public-origin.js";
+import {
+  validPublicSignInInput,
+  publicSignInStorageMatches,
+  type PublicSignInInput,
+} from "./external-public-signin-contract.js";
+
+async function main() {
+  let attached = false;
+  try {
+    test.info();
+    attached = true;
+  } catch {
+    /* standalone process */
+  }
+  if (attached)
+    throw new Error("Private public sign-in cannot run inside Test");
+  const file = process.argv[2]!;
+  const metadata = await stat(file);
+  if (
+    !metadata.isFile() ||
+    metadata.mode & 0o077 ||
+    metadata.size > 32768 ||
+    metadata.uid !== process.getuid?.()
+  )
+    throw new Error("Invalid private public sign-in input file");
+  const input = JSON.parse(
+    await readFile(file, "utf8"),
+  ) as PublicSignInInput & { executablePath?: string; channel?: string };
+  if (!validPublicSignInInput(input))
+    throw new Error("Invalid finite public sign-in input");
+  const origin = await readValidatedPublicOriginReceipt({
+    receiptPath: input.receiptPath,
+    executionId: input.executionId,
+    attemptId: input.attemptId,
+    backendOrigin: input.backendOrigin,
+  });
+  if (origin.publicOrigin !== input.publicOrigin)
+    throw new Error(
+      "Public sign-in origin does not match this validated owned attempt",
+    );
+  const browser = await chromium.launch({
+    headless: true,
+    ...(input.executablePath ? { executablePath: input.executablePath } : {}),
+    ...(input.channel ? { channel: input.channel } : {}),
+  });
+  try {
+    const context = await browser.newContext({ baseURL: origin.publicOrigin });
+    let requests = 0;
+    await context.route("**/*", async (route) => {
+      if (
+        ++requests > 200 ||
+        new URL(route.request().url()).origin !== origin.publicOrigin
+      )
+        await route.abort();
+      else await route.continue();
+    });
+    const page = await context.newPage();
+    page.setDefaultTimeout(30000);
+    await page.goto(`${origin.publicOrigin}/auth?next=%2F`, {
+      waitUntil: "domcontentloaded",
+    });
+    await page
+      .getByRole("heading", { name: "Sign in to Paperclip", exact: true })
+      .waitFor();
+    const email = page.getByLabel("Email", { exact: true }),
+      password = page.getByLabel("Password", { exact: true });
+    if ((await password.getAttribute("type")) !== "password")
+      throw new Error("Public sign-in password field is not protected");
+    await email.fill(input.email);
+    await password.fill(input.password);
+    const signedIn = page.waitForResponse(
+      (response) =>
+        response.url() === `${origin.publicOrigin}/api/auth/sign-in/email` &&
+        response.request().method() === "POST" &&
+        response.ok(),
+    );
+    await page.getByRole("button", { name: "Sign In", exact: true }).click();
+    const response = await signedIn;
+    const submitted = response.request().postDataJSON() as {
+      email?: string;
+      password?: string;
+    };
+    const result = (await response.json()) as { user?: { id?: string } };
+    if (
+      submitted.email !== input.email ||
+      submitted.password !== input.password ||
+      result.user?.id !== input.userId
+    )
+      throw new Error(
+        "Public sign-in form did not authenticate the generated fixture person",
+      );
+    const sessionResponse = await context.request.get(
+      `${origin.publicOrigin}/api/auth/get-session`,
+      { maxRedirects: 0 },
+    );
+    if (
+      !sessionResponse.ok() ||
+      (await sessionResponse.json()).user?.id !== input.userId
+    )
+      throw new Error(
+        "Public session identity does not match the generated fixture person",
+      );
+    await password.waitFor({ state: "detached" });
+    if (new URL(page.url()).origin !== origin.publicOrigin)
+      throw new Error(
+        "Public sign-in navigated outside its exact owned origin",
+      );
+    const storage = await context.storageState();
+    if (!publicSignInStorageMatches(storage, origin.publicOrigin))
+      throw new Error("Public sign-in produced an unexpected storage domain");
+    await writeFile(
+      path.join(path.dirname(file), "storage.json"),
+      JSON.stringify(storage),
+      { mode: 0o600 },
+    );
+    return {
+      processIsolation: true,
+      testRunnerAttached: false,
+      tracingEnabled: false,
+      videoEnabled: false,
+      apiStepRecorder: false,
+      userId: input.userId,
+      signedIn: true,
+      publicDomainOnly: true,
+    };
+  } finally {
+    await browser.close();
+  }
+}
+main()
+  .then((receipt) => process.stdout.write(JSON.stringify(receipt)))
+  .catch(() => {
+    process.stderr.write("Private public sign-in failed; diagnostics withheld");
+    process.exitCode = 1;
+  });

tests/runner-e2e/chat-catalog-domains/external-public-signin.ts

@@ -0,0 +1,130 @@
+import { mkdtemp, writeFile, readFile, rm, chmod } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { chromium } from "@playwright/test";
+import {
+  validPublicSignInInput,
+  publicSignInStorageMatches,
+  type PublicSignInInput,
+} from "./external-public-signin-contract.js";
+
+export interface PublicSignInReceipt {
+  processIsolation: true;
+  testRunnerAttached: false;
+  tracingEnabled: false;
+  videoEnabled: false;
+  apiStepRecorder: false;
+  userId: string;
+  signedIn: true;
+  publicDomainOnly: true;
+}
+/** One generated fixture person signs in through the real supplied HTTPS front
+ * door. Neither a loopback cookie nor a general navigation command is accepted. */
+export async function privatePublicSignIn(
+  input: PublicSignInInput,
+  privateSecrets: string[],
+  applyStorage: (privateFile: string) => Promise<void>,
+): Promise<PublicSignInReceipt> {
+  privateSecrets.push(input.password);
+  if (!validPublicSignInInput(input))
+    throw new Error("Invalid finite generated-person public sign-in request");
+  const directory = await mkdtemp(
+    path.join(tmpdir(), "paperclip-private-public-signin-"),
+  );
+  await chmod(directory, 0o700);
+  try {
+    const request = path.join(directory, "request.json"),
+      storagePath = path.join(directory, "storage.json");
+    const channel = process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL;
+    await writeFile(
+      request,
+      JSON.stringify({
+        ...input,
+        executablePath:
+          process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ??
+          (channel ? undefined : chromium.executablePath()),
+        channel,
+      }),
+      { mode: 0o600 },
+    );
+    const worker = fileURLToPath(
+      new URL("./external-public-signin-worker.ts", import.meta.url),
+    );
+    const loader = fileURLToPath(
+      new URL(
+        "../../../server/node_modules/tsx/dist/loader.mjs",
+        import.meta.url,
+      ),
+    );
+    let stdout: string;
+    try {
+      const result = await promisify(execFile)(
+        process.execPath,
+        ["--import", loader, worker, request],
+        {
+          timeout: 120000,
+          maxBuffer: 16384,
+          env: {
+            PATH: process.env.PATH ?? "",
+            NODE_ENV: "test",
+            DO_NOT_TRACK: "1",
+            HOME: directory,
+            TMPDIR: directory,
+          },
+        },
+      );
+      stdout = result.stdout;
+    } catch {
+      throw new Error(
+        "Private public sign-in failed; child output and private handoff withheld",
+      );
+    }
+    let storage: unknown;
+    try {
+      storage = JSON.parse(await readFile(storagePath, "utf8"));
+    } catch {
+      throw new Error("Private public sign-in storage handoff is unavailable");
+    }
+    // Register actual values before any parent validation or instrumented import.
+    const raw = storage as {
+      cookies?: Array<{ value?: unknown }>;
+      origins?: Array<{ localStorage?: Array<{ value?: unknown }> }>;
+    };
+    for (const cookie of raw?.cookies ?? [])
+      if (typeof cookie.value === "string") privateSecrets.push(cookie.value);
+    for (const origin of raw?.origins ?? [])
+      for (const item of origin.localStorage ?? [])
+        if (typeof item.value === "string" && item.value)
+          privateSecrets.push(item.value);
+    if (!publicSignInStorageMatches(storage, input.publicOrigin))
+      throw new Error("Public sign-in storage escaped its exact public domain");
+    let receipt: PublicSignInReceipt;
+    try {
+      receipt = JSON.parse(stdout) as PublicSignInReceipt;
+    } catch {
+      throw new Error("Invalid closed public sign-in receipt");
+    }
+    if (
+      Object.keys(receipt).sort().join(",") !==
+        "apiStepRecorder,processIsolation,publicDomainOnly,signedIn,testRunnerAttached,tracingEnabled,userId,videoEnabled" ||
+      receipt.processIsolation !== true ||
+      receipt.testRunnerAttached !== false ||
+      receipt.tracingEnabled !== false ||
+      receipt.videoEnabled !== false ||
+      receipt.apiStepRecorder !== false ||
+      receipt.userId !== input.userId ||
+      receipt.signedIn !== true ||
+      receipt.publicDomainOnly !== true
+    )
+      throw new Error(
+        "Public sign-in receipt failed its closed identity/isolation contract",
+      );
+    await applyStorage(storagePath);
+    return receipt;
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}