test(evals): isolate declared case credentials and private provider inputs

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-boundary-fingerprint.ts

@@ -1,3 +1,4 @@
+import { chatPrivateInputRequirements, chatPrivateInputNameIdentity } from "./chat-private-inputs.js";
 import { createHash } from "node:crypto";
 import { readFileSync, realpathSync } from "node:fs";
 import path from "node:path";
@@ -32,7 +33,7 @@ export function chatBoundaryFingerprint(descriptor: ChatCatalogDescriptor, execu
   return fingerprintChatDependencyClosure({ root,
     entries: [`tests/runner-e2e/${descriptor.modulePath.slice(2)}`, "tests/runner-e2e/chat-boundary-flow.ts", "tests/runner-e2e/chat-boundary-fingerprint.ts",
       "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts", ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)],
-    selected: { descriptor, journeyKind: "browser-boundary", executionId: execution.id, environment: execution.environment.id, installedIdentity },
+    selected: { descriptor, journeyKind: "browser-boundary", executionId: execution.id, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity: chatPrivateInputNameIdentity(execution.id, env), requiredCredentialNames: execution.requiredCredentials, environment: execution.environment.id, installedIdentity },
     selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts",
     normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor },
   });

tests/runner-e2e/chat-catalog-contract.ts

@@ -1,4 +1,6 @@
 import type { ChatFlowInput, ChatIssue, ChatRun } from "./chat-flow.js";
+import type { CredentialName } from "./types.js";
+import type { ChatExternalInputRequirement } from "./chat-private-inputs.js";
 import type { ChatScenario } from "./chat-feature-cases.js";
 
 export interface ChatCatalogCheck { id: string; passed: boolean; detail: string }
@@ -32,6 +34,8 @@ export interface ChatCatalogQueryAttachment {
 /** One concrete scenario, with production API setup and a case-specific independent oracle. */
 export interface ChatCatalogCase {
   scenarioId: string;
+  requiredCredentials?: readonly CredentialName[];
+  externalInput?: ChatExternalInputRequirement;
   journeyKind?: "agent-chat";
   authentication?: "local-trusted" | "human-session";
   setupPolicy?: ChatCatalogSetupPolicy;

tests/runner-e2e/chat-catalog-fingerprint.ts

@@ -1,3 +1,4 @@
+import { chatPrivateInputRequirements, chatPrivateInputNameIdentity } from "./chat-private-inputs.js";
 import { createHash } from "node:crypto";
 import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
 import path from "node:path";
@@ -100,7 +101,7 @@ export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, se
     entries: [selectedModule, "tests/runner-e2e/chat-catalog-flow.ts", "tests/runner-e2e/chat-catalog-fingerprint.ts", "tests/runner-e2e/chat-catalog-dynamic-resolutions.json",
       "tests/runner-e2e/chat-feature-flow.ts", "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts",
       "tests/runner-e2e/chat-scenarios/defaults.yaml", ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)],
-    selected: { descriptor, selectedExecution, lane, attachmentFormatter, scenario: { id, capabilities, context, query, expected_outcome } },
+    selected: { descriptor, selectedExecution, lane, attachmentFormatter, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity: chatPrivateInputNameIdentity((selectedExecution as { executionId: string }).executionId), scenario: { id, capabilities, context, query, expected_outcome } },
     reviewedDynamicResolutions,
     selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts",
     normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor,
@@ -112,7 +113,7 @@ export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, se
 export function normalizedChatCatalogExecution(execution: MatrixExecution) {
   return { executionId: execution.id, suiteId: execution.suite.id, flow: execution.task.flow, journeyKind: "agent-chat",
     profile: { id: execution.profile.id, provider: execution.profile.provider, model: execution.profile.model, credential: execution.profile.credential },
-    environment: execution.environment.id, task: { expectedRunCount: execution.task.expectedRunCount,
+    environment: execution.environment.id, requiredCredentialNames: execution.requiredCredentials, task: { expectedRunCount: execution.task.expectedRunCount,
       minimumExpectedRunCount: execution.task.minimumExpectedRunCount, turnTimeoutMs: execution.task.turnTimeoutMs,
       attemptTimeoutMs: execution.task.attemptTimeoutMs } };
 }

tests/runner-e2e/chat-catalog-registry.ts

@@ -45,7 +45,7 @@ export async function loadChatCatalogImplementation(
   const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId);
   if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`);
   const implementation = selected[0] as ChatCatalogCase;
-  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments"] as const) {
+  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput"] as const) {
     if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`);
   }
   if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario")
@@ -72,7 +72,7 @@ export async function loadIdentityBoundaryImplementation(descriptor: ChatCatalog
   const selected = Array.isArray(rows) ? rows.filter(row => row.scenarioId === descriptor.scenarioId) : [];
   if (selected.length !== 1) throw new Error("Missing or duplicate browser boundary implementation");
   const implementation = selected[0] as IdentityBoundaryCase;
-  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "boundaryTarget", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments"] as const) {
+  for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "boundaryTarget", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput"] as const) {
     if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime boundary descriptor drift: ${descriptor.scenarioId}.${key}`);
   }
   if ([implementation.run, implementation.grade, implementation.cleanup].some(value => typeof value !== "function") || descriptor.queryBindings?.length || descriptor.queryAttachments?.length

tests/runner-e2e/chat-feature-cases.ts

@@ -85,6 +85,7 @@ export const chatFeatureTasks: readonly RunnerTaskFixture[] = CHAT_FEATURE_CASES
 export const chatCatalogTasks: readonly RunnerTaskFixture[] = chatCatalogCases.map(implementation => {
   const scenario = chatCatalogScenarios.find(row => row.id === implementation.scenarioId)!;
   return {
+    requiredCredentials: implementation.requiredCredentials,
     id: scenario.id, label: scenario.id, groups: ["chat"], flow: "agent_chat", workMode: "standard",
     expectedRunCount: implementation.expectedRunCount, minimumExpectedRunCount: implementation.minimumRunCount,
     automaticRetryPolicy: "single_attempt", turnTimeoutMs: implementation.turnTimeoutMs,

tests/runner-e2e/chat-flow.ts

@@ -1,3 +1,4 @@
+import type { ChatPrivateInputs } from "./chat-private-inputs.js";
 import { chatCatalogAllowedRunStatuses } from "./chat-catalog-run-policy.js";
 import { runHiringTemplateFlow } from "./hiring-template-flow.js";
 import { gradeHiringTemplateTurns } from "./hiring-template-turn-accounting.js";
@@ -365,6 +366,7 @@ export interface ChatFlowInput {
   workspacePath: string;
   /** Attempt-private redaction sink for synthetic secondary-user credentials. Never evidence. */
   privateSecrets?: string[];
+  privateInputs?: ChatPrivateInputs;
   restart: () => Promise<void>;
   observe: (issue: ChatIssue, runs: ChatRun[]) => void;
   capture: (id: string, label: string, file: string) => Promise<void>;

tests/runner-e2e/chat-private-inputs.test.ts

@@ -0,0 +1,44 @@
+import { mkdtempSync, writeFileSync, statSync, chmodSync, rmSync } from "node:fs";
+import path from "node:path";
+import { tmpdir } from "node:os";
+import { afterEach, expect, it } from "vitest";
+import { stageChatPrivateInputs, readChatPrivateInputs, chatPrivateInputNameIdentity, scrubChatPrivateEnvironment, CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js";
+import type { MatrixExecution } from "./types.js";
+import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js";
+const roots: string[] = [];
+afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
+function fixture() { const root = mkdtempSync(path.join(tmpdir(), "chat-private-")); roots.push(root); return root; }
+const execution = { id: "selected", profile: { credential: "CODEX_AUTH_JSON" }, environment: {} } as MatrixExecution;
+it("stages only declared credentials privately and removes them from unrelated process environments", () => {
+  const root = fixture(), secrets: string[] = [];
+  const descriptor = { requiredCredentials: ["OPENAI_API_KEY"] } satisfies Pick<ChatCatalogDescriptor, "requiredCredentials" | "externalInput">;
+  const env = { CODEX_AUTH_JSON: "profile-login", OPENAI_API_KEY: "wizard-value", ANTHROPIC_API_KEY: "other-value", PAPERCLIP_CHAT_EVAL_UNUSED: "unrelated-value" };
+  const staged = stageChatPrivateInputs({ execution, descriptor, privateDir: root, env, secrets });
+  expect(statSync(staged.filename).mode & 0o077).toBe(0);
+  const privateEnv = { [CHAT_PRIVATE_INPUTS_ENV]: staged.filename, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root };
+  const inputs = readChatPrivateInputs("selected", secrets, privateEnv);
+  expect(inputs.credentials.OPENAI_API_KEY).toBe("wizard-value"); expect(inputs.read("ANTHROPIC_API_KEY")).toBeUndefined();
+  expect(secrets).toContain("wizard-value");
+  expect(() => readChatPrivateInputs("other", [], privateEnv)).toThrow("another execution");
+  const child = scrubChatPrivateEnvironment({ ...env, ...privateEnv }, [execution]);
+  expect(child.CODEX_AUTH_JSON).toBe("profile-login"); expect(child.OPENAI_API_KEY).toBeUndefined(); expect(child.PAPERCLIP_CHAT_EVAL_UNUSED).toBeUndefined();
+  expect(scrubChatPrivateEnvironment(child, [], true)[CHAT_PRIVATE_INPUTS_ENV]).toBeUndefined();
+  expect(JSON.stringify(chatPrivateInputNameIdentity("selected", privateEnv))).not.toContain("wizard-value");
+  chmodSync(staged.filename, 0o644); expect(() => readChatPrivateInputs("selected", [], privateEnv)).toThrow("private0600");
+});
+it("resolves only the selected provider references and fingerprints names rather than credential values", () => {
+  const root = fixture(), configFile = path.join(root, "external.json"), secrets: string[] = [];
+  writeFileSync(configFile, JSON.stringify({ schemaVersion: 1, providers: {
+    notion: { disposable: true, resourceIds: ["test-page"], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_AUTH"], prerequisites: { workspaceId: "test-space", tokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_AUTH", unrelated: "not-selected" } },
+    slack: { disposable: true, resourceIds: ["unrelated"], credentialEnv: ["PAPERCLIP_CHAT_EVAL_SLACK_AUTH"], prerequisites: {} },
+  } }), { mode: 0o600 });
+  const descriptor = { externalInput: { provider: "notion", fields: ["workspaceId", "tokenEnv"] } } satisfies Pick<ChatCatalogDescriptor, "requiredCredentials" | "externalInput">;
+  const staged = stageChatPrivateInputs({ execution, descriptor, privateDir: root, secrets, env: { PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG: configFile, PAPERCLIP_CHAT_EVAL_NOTION_AUTH: "notion-private", PAPERCLIP_CHAT_EVAL_SLACK_AUTH: "slack-private" } });
+  const env = { [CHAT_PRIVATE_INPUTS_ENV]: staged.filename, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root };
+  const inputs = readChatPrivateInputs("selected", secrets, env);
+  expect(inputs.read("PAPERCLIP_CHAT_EVAL_NOTION_AUTH")).toBe("notion-private"); expect(inputs.read("PAPERCLIP_CHAT_EVAL_SLACK_AUTH")).toBeUndefined();
+  expect(JSON.stringify(inputs.configuration("external-provider"))).not.toContain("slack");
+  expect(JSON.stringify(inputs.configuration("external-provider"))).not.toContain("not-selected");
+  const identity = JSON.stringify(chatPrivateInputNameIdentity("selected", env));
+  expect(identity).toContain("PAPERCLIP_CHAT_EVAL_NOTION_AUTH"); expect(identity).not.toContain("notion-private"); expect(identity).not.toContain("test-space");
+});

tests/runner-e2e/chat-private-inputs.ts

@@ -0,0 +1,92 @@
+import { createHash } from "node:crypto";
+import { lstatSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { CREDENTIAL_NAMES, type CredentialName, type MatrixExecution } from "./types.js";
+import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js";
+
+export const CHAT_PRIVATE_INPUTS_ENV = "PAPERCLIP_CHAT_PRIVATE_INPUTS";
+const externalConfigEnv = "PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG";
+export interface ChatExternalInputRequirement { provider: "notion" | "slack" | "agentmail" | "vault" | "github"; fields: readonly string[] }
+export interface ChatPrivateInputs {
+  readonly credentials: Readonly<Partial<Record<CredentialName, string>>>;
+  read(name: string): string | undefined;
+  configuration(name: "external-provider"): unknown;
+}
+interface PrivateEnvelope { version: 1; executionId: string; credentials: Partial<Record<CredentialName, string>>; references: Record<string, string>; external?: unknown }
+function readPrivateJson(filename: string, ownedRoot?: string): unknown {
+  const info = lstatSync(filename);
+  if (!info.isFile() || info.isSymbolicLink() || info.mode & 0o077 || info.size > 262144
+    || (ownedRoot && !realpathSync(filename).startsWith(realpathSync(ownedRoot) + path.sep)))
+    throw new Error("Selected-case inputs require a bounded private0600 owned file; contents withheld");
+  try { return JSON.parse(readFileSync(filename, "utf8")); } catch { throw new Error("Selected-case private inputs are invalid; contents withheld"); }
+}
+export function chatPrivateInputRequirements(descriptor?: Pick<ChatCatalogDescriptor, "requiredCredentials" | "externalInput">) {
+  const credentialNames = [...new Set(descriptor?.requiredCredentials ?? [])].sort();
+  if (credentialNames.some(name => !CREDENTIAL_NAMES.includes(name))) throw new Error("Unknown selected-case credential name");
+  const external = descriptor?.externalInput;
+  if (external && (!["notion", "slack", "agentmail", "vault", "github"].includes(external.provider)
+    || !external.fields.length || external.fields.some(field => !/^[a-zA-Z][a-zA-Z0-9]*$/.test(field)))) throw new Error("Invalid selected external configuration requirements");
+  const requirements = { credentialNames, external: external ? { provider: external.provider, fields: [...external.fields].sort() } : null };
+  return { ...requirements, digest: createHash("sha256").update(JSON.stringify(requirements)).digest("hex") };
+}
+export function stageChatPrivateInputs(input: { execution: MatrixExecution; descriptor?: Pick<ChatCatalogDescriptor, "requiredCredentials" | "externalInput">; privateDir: string; env: NodeJS.ProcessEnv; secrets: string[] }) {
+  const requirements = chatPrivateInputRequirements(input.descriptor);
+  const envelope: PrivateEnvelope = { version: 1, executionId: input.execution.id, credentials: {}, references: {} };
+  for (const name of requirements.credentialNames) {
+    const value = input.env[name]?.trim(); if (!value) throw new Error(`Missing selected-case credential ${name}`);
+    envelope.credentials[name] = value; input.secrets.push(value);
+  }
+  if (requirements.external) {
+    const ref = input.env[externalConfigEnv]; if (!ref) throw new Error("Missing selected-case private external configuration");
+    const config = readPrivateJson(ref) as { schemaVersion?: number; providers?: Record<string, { disposable?: boolean; resourceIds?: string[]; credentialEnv?: string[]; prerequisites?: Record<string, string> }> };
+    const binding = config.schemaVersion === 1 ? config.providers?.[requirements.external.provider] : undefined;
+    if (!binding || binding.disposable !== true || !Array.isArray(binding.resourceIds) || !binding.resourceIds.length
+      || !Array.isArray(binding.credentialEnv) || !binding.credentialEnv.length || !binding.prerequisites
+      || requirements.external.fields.some(field => !binding.prerequisites?.[field])) throw new Error("Selected disposable external configuration lacks declared prerequisites; contents withheld");
+    const selected = { disposable: true, resourceIds: binding.resourceIds, credentialEnv: binding.credentialEnv,
+      prerequisites: Object.fromEntries(requirements.external.fields.map(field => [field, binding.prerequisites![field]])) };
+    const names = [...binding.credentialEnv, ...requirements.external.fields.filter(field => field.endsWith("Env")).map(field => binding.prerequisites![field]!)];
+    for (const name of new Set(names)) {
+      if (!/^PAPERCLIP_CHAT_EVAL_[A-Z0-9_]+$/.test(name)) throw new Error("External credential reference is not a declared private environment name");
+      const value = input.env[name]?.trim(); if (!value) throw new Error(`Missing selected external environment reference ${name}`);
+      envelope.references[name] = value; input.secrets.push(value);
+    }
+    envelope.external = { schemaVersion: 1, providers: { [requirements.external.provider]: selected } };
+  }
+  const filename = path.join(input.privateDir, "chat-selected-inputs.json");
+  writeFileSync(filename, JSON.stringify(envelope), { mode: 0o600, flag: "wx" });
+  return { filename, requirements };
+}
+export function readChatPrivateInputs(executionId: string, secrets: string[], env: NodeJS.ProcessEnv = process.env): ChatPrivateInputs {
+  const reference = env[CHAT_PRIVATE_INPUTS_ENV], root = env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT;
+  if (!reference) return { credentials: Object.freeze({}), read: () => undefined, configuration: () => undefined };
+  if (!root) throw new Error("Selected private input ownership root is missing");
+  const envelope = readPrivateJson(reference, root) as PrivateEnvelope;
+  if (envelope.version !== 1 || envelope.executionId !== executionId || !envelope.credentials || !envelope.references)
+    throw new Error("Private inputs belong to another execution; contents withheld");
+  for (const value of [...Object.values(envelope.credentials), ...Object.values(envelope.references)]) {
+    if (typeof value !== "string" || !value.trim()) throw new Error("Invalid private credential value; contents withheld");
+    secrets.push(value);
+  }
+  return { credentials: Object.freeze(envelope.credentials), read: name => envelope.references[name] ?? envelope.credentials[name as CredentialName],
+    configuration: name => name === "external-provider" ? envelope.external : undefined };
+}
+export function scrubChatPrivateEnvironment(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[], removeReference = false) {
+  for (const key of Object.keys(env)) if (key.startsWith("PAPERCLIP_CHAT_EVAL_") || key === externalConfigEnv || removeReference && key === CHAT_PRIVATE_INPUTS_ENV) delete env[key];
+  if (executions.length) {
+    const profileNames = new Set(executions.flatMap(execution => [execution.profile.credential, execution.environment.credential].filter(Boolean)));
+    for (const name of CREDENTIAL_NAMES) if (!profileNames.has(name)) delete env[name];
+  }
+  return env;
+}
+/** Public fingerprint projection contains reference names and field names only. */
+export function chatPrivateInputNameIdentity(executionId: string, env: NodeJS.ProcessEnv = process.env) {
+  const reference = env[CHAT_PRIVATE_INPUTS_ENV], root = env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT;
+  if (!reference) return null;
+  if (!root) throw new Error("Selected private input ownership root is missing");
+  const envelope = readPrivateJson(reference, root) as PrivateEnvelope;
+  if (envelope.executionId !== executionId) throw new Error("Private input identity belongs to another execution");
+  const external = envelope.external as { providers?: Record<string, { prerequisites?: Record<string, unknown> }> } | undefined;
+  return { credentialNames: Object.keys(envelope.credentials).sort(), referenceNames: Object.keys(envelope.references).sort(),
+    externalRequirements: Object.entries(external?.providers ?? {}).map(([provider, binding]) => ({ provider, fields: Object.keys(binding.prerequisites ?? {}).sort() })) };
+}

tests/runner-e2e/harness-env.ts

@@ -1,3 +1,4 @@
+import { scrubChatPrivateEnvironment } from "./chat-private-inputs.js";
 import { installedDaytonaPluginKeys } from "./installed-daytona-plugin.js";
 import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
 import { assertInstalledCliSelection, installedCliKeys, usesInstalledCli } from "./installed-cli.js";
@@ -131,6 +132,8 @@ export function buildRunnerE2EProcessEnvironment(
   const installed = usesInstalledCli(source);
   if (installed) assertInstalledCliSelection(source, executions);
   const result = { ...source };
+  if (executions.some(execution => execution.suite.id === "chat-features")) scrubChatPrivateEnvironment(result, executions);
+  else scrubChatPrivateEnvironment(result, []);
   delete result.PAPERCLIP_E2E_GATEWAY_PHASE_PATH;
   delete result.PAPERCLIP_E2E_OTLP_RELAY_URL;
   // Announcements are unrelated to the scenarios and obscure screenshot evidence.
@@ -194,7 +197,7 @@ export function buildPaperclipServerEnvironment(
   source: NodeJS.ProcessEnv,
   overrides: NodeJS.ProcessEnv = {},
 ): NodeJS.ProcessEnv {
-  const result = { ...source };
+  const result = scrubChatPrivateEnvironment({ ...source }, [], true);
   for (const key of Object.keys(result)) {
     if (PROVIDER_SECRET_KEY.test(key)) delete result[key];
   }

tests/runner-e2e/launch.ts

@@ -1,3 +1,4 @@
+import { stageChatPrivateInputs, CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js";
 import { findChatCatalogCase } from "./chat-catalog-registry.js";
 import { recoverWorkerTiming } from "./worker-timing.js";
 import { openEvalOperations, type EvalOperations } from "./operations.js";
@@ -549,6 +550,10 @@ async function runAttempt(input: {
     };
     const credentials = normalizedSecrets(Object.values(credentialSources));
     attemptSecrets = credentials;
+    const selectedDescriptor = execution.suite.id === "chat-features" ? findChatCatalogCase(execution.task.id) : undefined;
+    const needsPrivateInputs = selectedDescriptor && ((selectedDescriptor.requiredCredentials?.length ?? 0) > 0 || selectedDescriptor.externalInput);
+    if (needsPrivateInputs && executions.length !== 1) throw new Error("Private case inputs require one explicitly selected execution");
+    const privateInputs = needsPrivateInputs ? stageChatPrivateInputs({ execution, descriptor: selectedDescriptor, privateDir, env: process.env, secrets: credentials }) : undefined;
     if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 });
     if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 });
     const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness(
@@ -557,6 +562,7 @@ async function runAttempt(input: {
     );
     const childEnv: NodeJS.ProcessEnv = {
       ...buildRunnerE2EProcessEnvironment(process.env, executions),
+      [CHAT_PRIVATE_INPUTS_ENV]: privateInputs?.filename,
       PATH: providerPath,
       PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify(
         executions.map((candidate) => candidate.id),

tests/runner-e2e/live-fixtures.ts

@@ -191,7 +191,9 @@ export async function setupLiveFixtures(input: {
     async setup(resolved) {
       const company = value<CompanyRecord>(resolved, "company");
       const refs: SecretReferenceMap = {};
-      for (const credentialName of execution.requiredCredentials) {
+      for (const credentialName of execution.suite.id === "chat-features"
+        ? [...new Set([execution.profile.credential, execution.environment.credential].filter((name): name is CredentialName => !!name))]
+        : execution.requiredCredentials) {
         if (["GROK_AUTH_JSON", "CODEX_AUTH_JSON"].includes(credentialName)) continue;
         const rawValue = input.credentials[credentialName];
         if (!rawValue) throw new Error(`Missing credential ${credentialName}`);

tests/runner-e2e/runner.spec.ts

@@ -1,3 +1,4 @@
+import { readChatPrivateInputs } from "./chat-private-inputs.js";
 import { findChatCatalogCase } from "./chat-catalog-registry.js";
 import { runChatBoundaryFlow } from "./chat-boundary-flow.js";
 import { chatCatalogScenarios } from "./chat-catalog-inventory.js";
@@ -624,6 +625,7 @@ for (const execution of executions) {
     }
     const credentials = credentialValues();
     const secrets = normalizedSecrets([...Object.values(credentials), ...EVAL_OBSERVABILITY_SECRET_NAMES.map(name => process.env[name]), process.env.PAPERCLIP_E2E_GATEWAY_TOKEN]);
+    const privateInputs = execution.suite.id === "chat-features" ? readChatPrivateInputs(execution.id, secrets) : undefined;
     const traceConfig = evalBraintrustConfig(process.env);
     const trace = traceConfig ? new EvalAttemptTrace(traceConfig, {
       campaignId: process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? "",
@@ -927,7 +929,7 @@ for (const execution of executions) {
         if (!scenario || !catalogDescriptor.boundaryTarget) throw new Error("Boundary scenario or target identity missing");
         await workerTiming.enter("execution"); trace?.phase("execution");
         const checks = await runChatBoundaryFlow({ journeyKind: "browser-boundary", boundaryTarget: catalogDescriptor.boundaryTarget,
-          scenario, page, api, execution, nonce, credentials, privateSecrets: secrets, privateDir, deadlineAt: startedAtMs + deadlineMs,
+          scenario, page, api, execution, nonce, credentials: { ...credentials, ...privateInputs?.credentials }, privateSecrets: secrets, privateDir, deadlineAt: startedAtMs + deadlineMs,
           capture: captureScreenshot,
           evidence: async (name, value) => {
             await writeSanitizedJson(snapshotsDir, name, value, secrets);
@@ -1296,7 +1298,7 @@ for (const execution of executions) {
         selectedRuns = contextIntegrity.runs as unknown as RunRecord[];
       } else if (execution.task.flow === "agent_chat") {
         const chat = await runChatFlow({
-          privateSecrets: secrets,
+          privateSecrets: secrets, privateInputs,
           page, api, fixtures, execution, nonce, workspacePath,
           restart: () => restartChatServer(page, () => restartIsolatedPaperclipServer({ api, requestId: `chat-${nonce}`, deadlineAt: startedAtMs + deadlineMs })),
           observe: (chatIssue, chatRuns) => { issue = chatIssue; selectedRuns = chatRuns; },