PaperClipAI/tests/runner-e2e/chat-catalog-domains/external-secure-key.ts

import { expect } from "@playwright/test";
import { uploadFactualQueryAttachment, verifyChatQueryAttachments, type ChatQueryAttachmentReceipt, type ChatQueryHumanComment } from "../chat-query-attachments.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import type { ToolConnection } from "../../../packages/shared/src/types/tool-access.js";
import { collectRunEvents } from "../run-observations.js";
import { privateScanJson, bytesContainSecret, scanReachableSecretOutputs } from "./external-secret-scan.js";
import { startNotesProvider, type NotesProvider } from "./external-provider.js";
import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
import { waitForReply } from "./identity-support.js";
import { privateBrowserEntry } from "./external-private-browser.js";
const fixtures = new WeakMap<ChatCatalogContext, NotesProvider>();
/** Nonsecret account selection supplied by the real fixture human. No route,
 * credential, tool invocation or expected setup result is provided to the model. */
export function secureProviderFacts(providerURL: string) {
  const provider = new URL(providerURL);
  if (provider.protocol !== "http:" || provider.hostname !== "127.0.0.1" || !provider.port || provider.pathname !== "/" || provider.search || provider.hash || provider.username || provider.password)
    throw new Error("Account context must identify only the actual owned provider origin");
  return `# Selected account\n\nAccount: Synthetic notes service (controlled test provider).\nAuthorized server origin: ${provider.origin}\n`;
}
export function gradeSecureKey(value: unknown) {
  const c = checks(), o = externalObservation(value);
  c.add("complete-secure-key-evidence", !!o, "Actual secure form submission, saved secret references, provider health and private output/draft scans are required."); if (!o) return c.result();
  c.add("actual-human-account-context", o.state.factualContextVerified === true,
    "The selected nonsecret account is delivered as an actual human-uploaded canonical attachment with independently verified source and bytes.");
  c.add("actual-secure-credential-control", o.state.challengeVisible === true && o.state.passwordInput === true && o.state.autocompleteOff === true && o.state.humanSubmittedSecurely === true
    && o.state.secureBrowserUntraced === true && o.state.visibleReady === true, "A separate untraced human browser enters the credential only into the product's password control and sees actual completed setup.");
  const connection = o.state.connectionAfter as { companyId: string; createdByUserId: string | null; providerURL: unknown; id: string; status: string; enabled: boolean; healthStatus: string; secretIds: string[]; serializedValueLeaked: boolean } | undefined;
  c.add("stored-reference-and-verified-connectivity", !!connection && connection.companyId === o.companyId && connection.createdByUserId === o.state.personId && connection.providerURL === o.state.serverUrl && connection.id === o.state.connectionId && connection.status === "active" && connection.enabled === true
    && connection.healthStatus === "ok" && connection.secretIds.length >= 1 && connection.serializedValueLeaked === false && Number(o.state.providerAuthorizedCatalogReads) > 0
    && o.connections.length === 1 && o.state.secureSecretsExist === true,
    "Exactly one active connection stores real credential references and a successful authenticated provider catalog check; no value is copied into public connection configuration.");
  c.add("no-key-in-history-or-drafts", o.state.storageClean === true && o.state.secretScanComplete === true && o.state.secretLeaked === false,
    "Private scans check actual browser storage, persisted comments, documents/revisions, attachments/artifacts, run events and run results for the actual credential bytes.");
  c.add("supported-secure-entry-handoff", o.links.some(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === o.state.setupPath; } catch { return false; } }),
    "The model actually links the supported secure setup entry rather than merely mentioning that credentials exist.");
  // Reply wording is not graded: the required secure-entry link is the handoff, and the private scans
  // prove the credential never reached chat, documents, attachments or run output.
  return c.result();
}
export const secureKeyExternalCases = [definition("connection-setup.secure-api-key", async ctx => {
  if (!ctx.input.privateSecrets) throw new Error("Secure setup requires private credential redaction ownership");
  const provider = await startNotesProvider("Synthetic key-authenticated fixture"); fixtures.set(ctx, provider); ctx.input.privateSecrets.push(provider.token);
  try {
    await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
    const factualContextAttachment = await uploadFactualQueryAttachment(ctx, {
      filename: "provider-account.md", body: secureProviderFacts(provider.url),
    });
    return { serverUrl: provider.url, personId: factualContextAttachment.createdByUserId, factualContextAttachment, setupPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/byo` };
  } catch (error) { fixtures.delete(ctx); await provider.close(); throw error; }
}, gradeSecureKey, {
  oracleVersion: "paperclip.chat-catalog.external.v9", authentication: "human-session",
  queryAttachments: [{ stateKey: "factualContextAttachment", filename: "provider-account.md" }],
  attemptTimeoutMs: 420_000,
  sourceFiles: ["chat-catalog-domains/external-secure-key.ts", "chat-query-attachments.ts", "../../ui/src/App.tsx", "../../ui/src/pages/apps/app-connect-policy.ts", "../../ui/src/features/connections/ConnectionSetupFlow.tsx", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-secret-scan.ts", "run-observations.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml"],
  async act(ctx, state) {
    await waitForReply(ctx); const provider = fixtures.get(ctx); if (!provider) throw new Error("Secure provider fixture missing");
    // The saved reply can arrive before it renders; wait for the rendered link instead of reading anchors once.
    const renderedTarget = async () => (await ctx.input.page.locator(`[id^="comment-"] a`).evaluateAll(anchors => anchors.map(anchor => (anchor as HTMLAnchorElement).href)))
      .find(href => { try { const url = new URL(href); return url.origin === new URL(ctx.input.api.baseURL).origin && url.pathname === state.setupPath; } catch { return false; } });
    let target: string | undefined;
    await expect.poll(async () => (target = await renderedTarget()) !== undefined, { timeout: 30_000 }).toBe(true).catch(() => undefined);
    if (!target) throw new Error("The model did not link the intended secure setup entry");
    const receipt = await privateBrowserEntry({ mode: "secure-key", baseURL: ctx.input.api.baseURL, setupURL: target, providerURL: provider.url,
      credential: provider.token, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) });
    Object.assign(state, receipt, { secureBrowserUntraced: receipt.processIsolation && !receipt.testRunnerAttached && !receipt.tracingEnabled && !receipt.videoEnabled && !receipt.apiStepRecorder });
    await ctx.input.evidence("chat-catalog-secure-entry-boundary.json", { processIsolation: receipt.processIsolation, testRunnerAttached: receipt.testRunnerAttached,
      tracingEnabled: receipt.tracingEnabled, videoEnabled: receipt.videoEnabled, apiStepRecorder: receipt.apiStepRecorder, connectionId: receipt.connectionId });
  },
  async observe(ctx, state) {
    const provider = fixtures.get(ctx); if (!provider) throw new Error("Secure fixture missing");
    const connection = await ctx.input.api.get<ToolConnection>(`/api/tool-connections/${state.connectionId}`);
    const sourceIssue = ctx.issue();
    if (!sourceIssue) throw new Error("Secure account context has no canonical source");
    const comments = await ctx.input.api.get<ChatQueryHumanComment[]>(`/api/issues/${sourceIssue.id}/comments?order=asc`);
    const human = comments.find(row => row.authorUserId && !row.authorAgentId);
    state.factualContextVerified = !!human && await verifyChatQueryAttachments(ctx, [state.factualContextAttachment as ChatQueryAttachmentReceipt], human);
    const secretIds = [...new Set(connection.credentialSecretRefs.map(ref => ref.secretId))];
    state.connectionAfter = { companyId: connection.companyId, createdByUserId: connection.createdByUserId, providerURL: connection.config?.url, id: connection.id, status: connection.status, enabled: connection.enabled, healthStatus: connection.healthStatus, secretIds, serializedValueLeaked: JSON.stringify(connection).includes(provider.token) };
    state.secretIds = secretIds; const o = await observeExternal(ctx, state); state.secureSecretsExist = secretIds.length > 0 && secretIds.every(id => o.secrets.some(secret => secret.id === id));
    state.providerAuthorizedCatalogReads = provider.calls.filter(call => call.authorized && call.method === "tools/list").length;
    const events: unknown[] = []; for (const run of o.runs) events.push(...await collectRunEvents(afterSeq => privateScanJson(ctx, `/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)));
    const source = ctx.issue(); if (!source) throw new Error("Secure output scan needs the actual source task"); const reachable = new Set([source.id]);
    for (;;) { const count = reachable.size; for (const task of o.tasks) if (task.parentId && reachable.has(task.parentId)) reachable.add(task.id); if (count === reachable.size) break; }
    const scan = await scanReachableSecretOutputs(ctx, [...reachable], provider.token);
    state.secretScanComplete = scan.complete; state.secretLeaked = scan.leaked || bytesContainSecret(Buffer.from(JSON.stringify([events, o.runs])), provider.token);
    return { ...o, state };
  },
  async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); },
  async cleanup(ctx, state) {
    const provider = fixtures.get(ctx); fixtures.delete(ctx);
    try { if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true });
      for (const id of Array.isArray(state.secretIds) ? state.secretIds : []) if (typeof id === "string") await ctx.input.api.delete(`/api/secrets/${id}`, { allowNotFound: true });
    } finally { await provider?.close(); }
  },
})];