diff --git a/tests/runner-e2e/chat-boundary-fingerprint.ts b/tests/runner-e2e/chat-boundary-fingerprint.ts index 79afb7cd6ad..6130626a6c2 100644 --- a/tests/runner-e2e/chat-boundary-fingerprint.ts +++ b/tests/runner-e2e/chat-boundary-fingerprint.ts @@ -1,3 +1,4 @@ +import { chatPrivateInputRequirements, chatPrivateInputNameIdentity } from "./chat-private-inputs.js"; import { createHash } from "node:crypto"; import { readFileSync, realpathSync } from "node:fs"; import path from "node:path"; @@ -32,7 +33,7 @@ export function chatBoundaryFingerprint(descriptor: ChatCatalogDescriptor, execu return fingerprintChatDependencyClosure({ root, entries: [`tests/runner-e2e/${descriptor.modulePath.slice(2)}`, "tests/runner-e2e/chat-boundary-flow.ts", "tests/runner-e2e/chat-boundary-fingerprint.ts", "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts", ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)], - selected: { descriptor, journeyKind: "browser-boundary", executionId: execution.id, environment: execution.environment.id, installedIdentity }, + selected: { descriptor, journeyKind: "browser-boundary", executionId: execution.id, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity: chatPrivateInputNameIdentity(execution.id, env), requiredCredentialNames: execution.requiredCredentials, environment: execution.environment.id, installedIdentity }, selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts", normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor }, }); diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 6c773c9d536..58887ee31fb 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -1,4 +1,6 @@ import type { ChatFlowInput, ChatIssue, ChatRun } from "./chat-flow.js"; +import type { CredentialName } from "./types.js"; +import type { ChatExternalInputRequirement } from "./chat-private-inputs.js"; import type { ChatScenario } from "./chat-feature-cases.js"; export interface ChatCatalogCheck { id: string; passed: boolean; detail: string } @@ -32,6 +34,8 @@ export interface ChatCatalogQueryAttachment { /** One concrete scenario, with production API setup and a case-specific independent oracle. */ export interface ChatCatalogCase { scenarioId: string; + requiredCredentials?: readonly CredentialName[]; + externalInput?: ChatExternalInputRequirement; journeyKind?: "agent-chat"; authentication?: "local-trusted" | "human-session"; setupPolicy?: ChatCatalogSetupPolicy; diff --git a/tests/runner-e2e/chat-catalog-fingerprint.ts b/tests/runner-e2e/chat-catalog-fingerprint.ts index 92f2602bfe5..e0439381026 100644 --- a/tests/runner-e2e/chat-catalog-fingerprint.ts +++ b/tests/runner-e2e/chat-catalog-fingerprint.ts @@ -1,3 +1,4 @@ +import { chatPrivateInputRequirements, chatPrivateInputNameIdentity } from "./chat-private-inputs.js"; import { createHash } from "node:crypto"; import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; @@ -100,7 +101,7 @@ export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, se entries: [selectedModule, "tests/runner-e2e/chat-catalog-flow.ts", "tests/runner-e2e/chat-catalog-fingerprint.ts", "tests/runner-e2e/chat-catalog-dynamic-resolutions.json", "tests/runner-e2e/chat-feature-flow.ts", "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts", "tests/runner-e2e/chat-scenarios/defaults.yaml", ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)], - selected: { descriptor, selectedExecution, lane, attachmentFormatter, scenario: { id, capabilities, context, query, expected_outcome } }, + selected: { descriptor, selectedExecution, lane, attachmentFormatter, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity: chatPrivateInputNameIdentity((selectedExecution as { executionId: string }).executionId), scenario: { id, capabilities, context, query, expected_outcome } }, reviewedDynamicResolutions, selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts", normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor, @@ -112,7 +113,7 @@ export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, se export function normalizedChatCatalogExecution(execution: MatrixExecution) { return { executionId: execution.id, suiteId: execution.suite.id, flow: execution.task.flow, journeyKind: "agent-chat", profile: { id: execution.profile.id, provider: execution.profile.provider, model: execution.profile.model, credential: execution.profile.credential }, - environment: execution.environment.id, task: { expectedRunCount: execution.task.expectedRunCount, + environment: execution.environment.id, requiredCredentialNames: execution.requiredCredentials, task: { expectedRunCount: execution.task.expectedRunCount, minimumExpectedRunCount: execution.task.minimumExpectedRunCount, turnTimeoutMs: execution.task.turnTimeoutMs, attemptTimeoutMs: execution.task.attemptTimeoutMs } }; } diff --git a/tests/runner-e2e/chat-catalog-registry.ts b/tests/runner-e2e/chat-catalog-registry.ts index e38e2cc0ac4..e9a123252b3 100644 --- a/tests/runner-e2e/chat-catalog-registry.ts +++ b/tests/runner-e2e/chat-catalog-registry.ts @@ -45,7 +45,7 @@ export async function loadChatCatalogImplementation( const selected = implementations.filter((row: ChatCatalogCase) => row.scenarioId === descriptor.scenarioId); if (selected.length !== 1) throw new Error(`Missing or duplicate runtime case: ${descriptor.scenarioId}`); const implementation = selected[0] as ChatCatalogCase; - for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments"] as const) { + for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput"] as const) { if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime descriptor drift: ${descriptor.scenarioId}.${key}`); } if ((descriptor.setupPolicy?.funding === "preserve-scenario" || descriptor.setupPolicy?.readiness === "preserve-scenario") @@ -72,7 +72,7 @@ export async function loadIdentityBoundaryImplementation(descriptor: ChatCatalog const selected = Array.isArray(rows) ? rows.filter(row => row.scenarioId === descriptor.scenarioId) : []; if (selected.length !== 1) throw new Error("Missing or duplicate browser boundary implementation"); const implementation = selected[0] as IdentityBoundaryCase; - for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "boundaryTarget", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments"] as const) { + for (const key of ["sourceFiles", "oracleVersion", "expectedRunCount", "minimumRunCount", "expectedHumanMessageCount", "setupHumanMessageCount", "turnTimeoutMs", "attemptTimeoutMs", "coverageGaps", "journeyKind", "boundaryTarget", "authentication", "setupPolicy", "runPolicy", "queryOutcome", "queryBindings", "queryAttachments", "requiredCredentials", "externalInput"] as const) { if (JSON.stringify(implementation[key]) !== JSON.stringify(descriptor[key])) throw new Error(`Runtime boundary descriptor drift: ${descriptor.scenarioId}.${key}`); } if ([implementation.run, implementation.grade, implementation.cleanup].some(value => typeof value !== "function") || descriptor.queryBindings?.length || descriptor.queryAttachments?.length diff --git a/tests/runner-e2e/chat-feature-cases.ts b/tests/runner-e2e/chat-feature-cases.ts index 16cdfd5ae3a..42328efba05 100644 --- a/tests/runner-e2e/chat-feature-cases.ts +++ b/tests/runner-e2e/chat-feature-cases.ts @@ -85,6 +85,7 @@ export const chatFeatureTasks: readonly RunnerTaskFixture[] = CHAT_FEATURE_CASES export const chatCatalogTasks: readonly RunnerTaskFixture[] = chatCatalogCases.map(implementation => { const scenario = chatCatalogScenarios.find(row => row.id === implementation.scenarioId)!; return { + requiredCredentials: implementation.requiredCredentials, id: scenario.id, label: scenario.id, groups: ["chat"], flow: "agent_chat", workMode: "standard", expectedRunCount: implementation.expectedRunCount, minimumExpectedRunCount: implementation.minimumRunCount, automaticRetryPolicy: "single_attempt", turnTimeoutMs: implementation.turnTimeoutMs, diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index 9caa4bbd4d5..9c50be4e6dd 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -1,3 +1,4 @@ +import type { ChatPrivateInputs } from "./chat-private-inputs.js"; import { chatCatalogAllowedRunStatuses } from "./chat-catalog-run-policy.js"; import { runHiringTemplateFlow } from "./hiring-template-flow.js"; import { gradeHiringTemplateTurns } from "./hiring-template-turn-accounting.js"; @@ -365,6 +366,7 @@ export interface ChatFlowInput { workspacePath: string; /** Attempt-private redaction sink for synthetic secondary-user credentials. Never evidence. */ privateSecrets?: string[]; + privateInputs?: ChatPrivateInputs; restart: () => Promise; observe: (issue: ChatIssue, runs: ChatRun[]) => void; capture: (id: string, label: string, file: string) => Promise; diff --git a/tests/runner-e2e/chat-private-inputs.test.ts b/tests/runner-e2e/chat-private-inputs.test.ts new file mode 100644 index 00000000000..d1a8b3be47c --- /dev/null +++ b/tests/runner-e2e/chat-private-inputs.test.ts @@ -0,0 +1,44 @@ +import { mkdtempSync, writeFileSync, statSync, chmodSync, rmSync } from "node:fs"; +import path from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, expect, it } from "vitest"; +import { stageChatPrivateInputs, readChatPrivateInputs, chatPrivateInputNameIdentity, scrubChatPrivateEnvironment, CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js"; +import type { MatrixExecution } from "./types.js"; +import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js"; +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +function fixture() { const root = mkdtempSync(path.join(tmpdir(), "chat-private-")); roots.push(root); return root; } +const execution = { id: "selected", profile: { credential: "CODEX_AUTH_JSON" }, environment: {} } as MatrixExecution; +it("stages only declared credentials privately and removes them from unrelated process environments", () => { + const root = fixture(), secrets: string[] = []; + const descriptor = { requiredCredentials: ["OPENAI_API_KEY"] } satisfies Pick; + const env = { CODEX_AUTH_JSON: "profile-login", OPENAI_API_KEY: "wizard-value", ANTHROPIC_API_KEY: "other-value", PAPERCLIP_CHAT_EVAL_UNUSED: "unrelated-value" }; + const staged = stageChatPrivateInputs({ execution, descriptor, privateDir: root, env, secrets }); + expect(statSync(staged.filename).mode & 0o077).toBe(0); + const privateEnv = { [CHAT_PRIVATE_INPUTS_ENV]: staged.filename, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root }; + const inputs = readChatPrivateInputs("selected", secrets, privateEnv); + expect(inputs.credentials.OPENAI_API_KEY).toBe("wizard-value"); expect(inputs.read("ANTHROPIC_API_KEY")).toBeUndefined(); + expect(secrets).toContain("wizard-value"); + expect(() => readChatPrivateInputs("other", [], privateEnv)).toThrow("another execution"); + const child = scrubChatPrivateEnvironment({ ...env, ...privateEnv }, [execution]); + expect(child.CODEX_AUTH_JSON).toBe("profile-login"); expect(child.OPENAI_API_KEY).toBeUndefined(); expect(child.PAPERCLIP_CHAT_EVAL_UNUSED).toBeUndefined(); + expect(scrubChatPrivateEnvironment(child, [], true)[CHAT_PRIVATE_INPUTS_ENV]).toBeUndefined(); + expect(JSON.stringify(chatPrivateInputNameIdentity("selected", privateEnv))).not.toContain("wizard-value"); + chmodSync(staged.filename, 0o644); expect(() => readChatPrivateInputs("selected", [], privateEnv)).toThrow("private0600"); +}); +it("resolves only the selected provider references and fingerprints names rather than credential values", () => { + const root = fixture(), configFile = path.join(root, "external.json"), secrets: string[] = []; + writeFileSync(configFile, JSON.stringify({ schemaVersion: 1, providers: { + notion: { disposable: true, resourceIds: ["test-page"], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_AUTH"], prerequisites: { workspaceId: "test-space", tokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_AUTH", unrelated: "not-selected" } }, + slack: { disposable: true, resourceIds: ["unrelated"], credentialEnv: ["PAPERCLIP_CHAT_EVAL_SLACK_AUTH"], prerequisites: {} }, + } }), { mode: 0o600 }); + const descriptor = { externalInput: { provider: "notion", fields: ["workspaceId", "tokenEnv"] } } satisfies Pick; + const staged = stageChatPrivateInputs({ execution, descriptor, privateDir: root, secrets, env: { PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG: configFile, PAPERCLIP_CHAT_EVAL_NOTION_AUTH: "notion-private", PAPERCLIP_CHAT_EVAL_SLACK_AUTH: "slack-private" } }); + const env = { [CHAT_PRIVATE_INPUTS_ENV]: staged.filename, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root }; + const inputs = readChatPrivateInputs("selected", secrets, env); + expect(inputs.read("PAPERCLIP_CHAT_EVAL_NOTION_AUTH")).toBe("notion-private"); expect(inputs.read("PAPERCLIP_CHAT_EVAL_SLACK_AUTH")).toBeUndefined(); + expect(JSON.stringify(inputs.configuration("external-provider"))).not.toContain("slack"); + expect(JSON.stringify(inputs.configuration("external-provider"))).not.toContain("not-selected"); + const identity = JSON.stringify(chatPrivateInputNameIdentity("selected", env)); + expect(identity).toContain("PAPERCLIP_CHAT_EVAL_NOTION_AUTH"); expect(identity).not.toContain("notion-private"); expect(identity).not.toContain("test-space"); +}); diff --git a/tests/runner-e2e/chat-private-inputs.ts b/tests/runner-e2e/chat-private-inputs.ts new file mode 100644 index 00000000000..9b6842eebc3 --- /dev/null +++ b/tests/runner-e2e/chat-private-inputs.ts @@ -0,0 +1,92 @@ +import { createHash } from "node:crypto"; +import { lstatSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { CREDENTIAL_NAMES, type CredentialName, type MatrixExecution } from "./types.js"; +import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js"; + +export const CHAT_PRIVATE_INPUTS_ENV = "PAPERCLIP_CHAT_PRIVATE_INPUTS"; +const externalConfigEnv = "PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG"; +export interface ChatExternalInputRequirement { provider: "notion" | "slack" | "agentmail" | "vault" | "github"; fields: readonly string[] } +export interface ChatPrivateInputs { + readonly credentials: Readonly>>; + read(name: string): string | undefined; + configuration(name: "external-provider"): unknown; +} +interface PrivateEnvelope { version: 1; executionId: string; credentials: Partial>; references: Record; external?: unknown } +function readPrivateJson(filename: string, ownedRoot?: string): unknown { + const info = lstatSync(filename); + if (!info.isFile() || info.isSymbolicLink() || info.mode & 0o077 || info.size > 262144 + || (ownedRoot && !realpathSync(filename).startsWith(realpathSync(ownedRoot) + path.sep))) + throw new Error("Selected-case inputs require a bounded private0600 owned file; contents withheld"); + try { return JSON.parse(readFileSync(filename, "utf8")); } catch { throw new Error("Selected-case private inputs are invalid; contents withheld"); } +} +export function chatPrivateInputRequirements(descriptor?: Pick) { + const credentialNames = [...new Set(descriptor?.requiredCredentials ?? [])].sort(); + if (credentialNames.some(name => !CREDENTIAL_NAMES.includes(name))) throw new Error("Unknown selected-case credential name"); + const external = descriptor?.externalInput; + if (external && (!["notion", "slack", "agentmail", "vault", "github"].includes(external.provider) + || !external.fields.length || external.fields.some(field => !/^[a-zA-Z][a-zA-Z0-9]*$/.test(field)))) throw new Error("Invalid selected external configuration requirements"); + const requirements = { credentialNames, external: external ? { provider: external.provider, fields: [...external.fields].sort() } : null }; + return { ...requirements, digest: createHash("sha256").update(JSON.stringify(requirements)).digest("hex") }; +} +export function stageChatPrivateInputs(input: { execution: MatrixExecution; descriptor?: Pick; privateDir: string; env: NodeJS.ProcessEnv; secrets: string[] }) { + const requirements = chatPrivateInputRequirements(input.descriptor); + const envelope: PrivateEnvelope = { version: 1, executionId: input.execution.id, credentials: {}, references: {} }; + for (const name of requirements.credentialNames) { + const value = input.env[name]?.trim(); if (!value) throw new Error(`Missing selected-case credential ${name}`); + envelope.credentials[name] = value; input.secrets.push(value); + } + if (requirements.external) { + const ref = input.env[externalConfigEnv]; if (!ref) throw new Error("Missing selected-case private external configuration"); + const config = readPrivateJson(ref) as { schemaVersion?: number; providers?: Record }> }; + const binding = config.schemaVersion === 1 ? config.providers?.[requirements.external.provider] : undefined; + if (!binding || binding.disposable !== true || !Array.isArray(binding.resourceIds) || !binding.resourceIds.length + || !Array.isArray(binding.credentialEnv) || !binding.credentialEnv.length || !binding.prerequisites + || requirements.external.fields.some(field => !binding.prerequisites?.[field])) throw new Error("Selected disposable external configuration lacks declared prerequisites; contents withheld"); + const selected = { disposable: true, resourceIds: binding.resourceIds, credentialEnv: binding.credentialEnv, + prerequisites: Object.fromEntries(requirements.external.fields.map(field => [field, binding.prerequisites![field]])) }; + const names = [...binding.credentialEnv, ...requirements.external.fields.filter(field => field.endsWith("Env")).map(field => binding.prerequisites![field]!)]; + for (const name of new Set(names)) { + if (!/^PAPERCLIP_CHAT_EVAL_[A-Z0-9_]+$/.test(name)) throw new Error("External credential reference is not a declared private environment name"); + const value = input.env[name]?.trim(); if (!value) throw new Error(`Missing selected external environment reference ${name}`); + envelope.references[name] = value; input.secrets.push(value); + } + envelope.external = { schemaVersion: 1, providers: { [requirements.external.provider]: selected } }; + } + const filename = path.join(input.privateDir, "chat-selected-inputs.json"); + writeFileSync(filename, JSON.stringify(envelope), { mode: 0o600, flag: "wx" }); + return { filename, requirements }; +} +export function readChatPrivateInputs(executionId: string, secrets: string[], env: NodeJS.ProcessEnv = process.env): ChatPrivateInputs { + const reference = env[CHAT_PRIVATE_INPUTS_ENV], root = env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; + if (!reference) return { credentials: Object.freeze({}), read: () => undefined, configuration: () => undefined }; + if (!root) throw new Error("Selected private input ownership root is missing"); + const envelope = readPrivateJson(reference, root) as PrivateEnvelope; + if (envelope.version !== 1 || envelope.executionId !== executionId || !envelope.credentials || !envelope.references) + throw new Error("Private inputs belong to another execution; contents withheld"); + for (const value of [...Object.values(envelope.credentials), ...Object.values(envelope.references)]) { + if (typeof value !== "string" || !value.trim()) throw new Error("Invalid private credential value; contents withheld"); + secrets.push(value); + } + return { credentials: Object.freeze(envelope.credentials), read: name => envelope.references[name] ?? envelope.credentials[name as CredentialName], + configuration: name => name === "external-provider" ? envelope.external : undefined }; +} +export function scrubChatPrivateEnvironment(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[], removeReference = false) { + for (const key of Object.keys(env)) if (key.startsWith("PAPERCLIP_CHAT_EVAL_") || key === externalConfigEnv || removeReference && key === CHAT_PRIVATE_INPUTS_ENV) delete env[key]; + if (executions.length) { + const profileNames = new Set(executions.flatMap(execution => [execution.profile.credential, execution.environment.credential].filter(Boolean))); + for (const name of CREDENTIAL_NAMES) if (!profileNames.has(name)) delete env[name]; + } + return env; +} +/** Public fingerprint projection contains reference names and field names only. */ +export function chatPrivateInputNameIdentity(executionId: string, env: NodeJS.ProcessEnv = process.env) { + const reference = env[CHAT_PRIVATE_INPUTS_ENV], root = env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; + if (!reference) return null; + if (!root) throw new Error("Selected private input ownership root is missing"); + const envelope = readPrivateJson(reference, root) as PrivateEnvelope; + if (envelope.executionId !== executionId) throw new Error("Private input identity belongs to another execution"); + const external = envelope.external as { providers?: Record }> } | undefined; + return { credentialNames: Object.keys(envelope.credentials).sort(), referenceNames: Object.keys(envelope.references).sort(), + externalRequirements: Object.entries(external?.providers ?? {}).map(([provider, binding]) => ({ provider, fields: Object.keys(binding.prerequisites ?? {}).sort() })) }; +} diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index 2ea3cd91a40..8a8db216d3d 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -1,3 +1,4 @@ +import { scrubChatPrivateEnvironment } from "./chat-private-inputs.js"; import { installedDaytonaPluginKeys } from "./installed-daytona-plugin.js"; import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; import { assertInstalledCliSelection, installedCliKeys, usesInstalledCli } from "./installed-cli.js"; @@ -131,6 +132,8 @@ export function buildRunnerE2EProcessEnvironment( const installed = usesInstalledCli(source); if (installed) assertInstalledCliSelection(source, executions); const result = { ...source }; + if (executions.some(execution => execution.suite.id === "chat-features")) scrubChatPrivateEnvironment(result, executions); + else scrubChatPrivateEnvironment(result, []); delete result.PAPERCLIP_E2E_GATEWAY_PHASE_PATH; delete result.PAPERCLIP_E2E_OTLP_RELAY_URL; // Announcements are unrelated to the scenarios and obscure screenshot evidence. @@ -194,7 +197,7 @@ export function buildPaperclipServerEnvironment( source: NodeJS.ProcessEnv, overrides: NodeJS.ProcessEnv = {}, ): NodeJS.ProcessEnv { - const result = { ...source }; + const result = scrubChatPrivateEnvironment({ ...source }, [], true); for (const key of Object.keys(result)) { if (PROVIDER_SECRET_KEY.test(key)) delete result[key]; } diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 75175521f99..7b0feffbc25 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,3 +1,4 @@ +import { stageChatPrivateInputs, CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js"; import { findChatCatalogCase } from "./chat-catalog-registry.js"; import { recoverWorkerTiming } from "./worker-timing.js"; import { openEvalOperations, type EvalOperations } from "./operations.js"; @@ -549,6 +550,10 @@ async function runAttempt(input: { }; const credentials = normalizedSecrets(Object.values(credentialSources)); attemptSecrets = credentials; + const selectedDescriptor = execution.suite.id === "chat-features" ? findChatCatalogCase(execution.task.id) : undefined; + const needsPrivateInputs = selectedDescriptor && ((selectedDescriptor.requiredCredentials?.length ?? 0) > 0 || selectedDescriptor.externalInput); + if (needsPrivateInputs && executions.length !== 1) throw new Error("Private case inputs require one explicitly selected execution"); + const privateInputs = needsPrivateInputs ? stageChatPrivateInputs({ execution, descriptor: selectedDescriptor, privateDir, env: process.env, secrets: credentials }) : undefined; if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 }); if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 }); const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness( @@ -557,6 +562,7 @@ async function runAttempt(input: { ); const childEnv: NodeJS.ProcessEnv = { ...buildRunnerE2EProcessEnvironment(process.env, executions), + [CHAT_PRIVATE_INPUTS_ENV]: privateInputs?.filename, PATH: providerPath, PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify( executions.map((candidate) => candidate.id), diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 348db2d08af..7f7c04838a5 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -191,7 +191,9 @@ export async function setupLiveFixtures(input: { async setup(resolved) { const company = value(resolved, "company"); const refs: SecretReferenceMap = {}; - for (const credentialName of execution.requiredCredentials) { + for (const credentialName of execution.suite.id === "chat-features" + ? [...new Set([execution.profile.credential, execution.environment.credential].filter((name): name is CredentialName => !!name))] + : execution.requiredCredentials) { if (["GROK_AUTH_JSON", "CODEX_AUTH_JSON"].includes(credentialName)) continue; const rawValue = input.credentials[credentialName]; if (!rawValue) throw new Error(`Missing credential ${credentialName}`); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 43442c5737f..90c1a2ee45a 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,4 @@ +import { readChatPrivateInputs } from "./chat-private-inputs.js"; import { findChatCatalogCase } from "./chat-catalog-registry.js"; import { runChatBoundaryFlow } from "./chat-boundary-flow.js"; import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; @@ -624,6 +625,7 @@ for (const execution of executions) { } const credentials = credentialValues(); const secrets = normalizedSecrets([...Object.values(credentials), ...EVAL_OBSERVABILITY_SECRET_NAMES.map(name => process.env[name]), process.env.PAPERCLIP_E2E_GATEWAY_TOKEN]); + const privateInputs = execution.suite.id === "chat-features" ? readChatPrivateInputs(execution.id, secrets) : undefined; const traceConfig = evalBraintrustConfig(process.env); const trace = traceConfig ? new EvalAttemptTrace(traceConfig, { campaignId: process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? "", @@ -927,7 +929,7 @@ for (const execution of executions) { if (!scenario || !catalogDescriptor.boundaryTarget) throw new Error("Boundary scenario or target identity missing"); await workerTiming.enter("execution"); trace?.phase("execution"); const checks = await runChatBoundaryFlow({ journeyKind: "browser-boundary", boundaryTarget: catalogDescriptor.boundaryTarget, - scenario, page, api, execution, nonce, credentials, privateSecrets: secrets, privateDir, deadlineAt: startedAtMs + deadlineMs, + scenario, page, api, execution, nonce, credentials: { ...credentials, ...privateInputs?.credentials }, privateSecrets: secrets, privateDir, deadlineAt: startedAtMs + deadlineMs, capture: captureScreenshot, evidence: async (name, value) => { await writeSanitizedJson(snapshotsDir, name, value, secrets); @@ -1296,7 +1298,7 @@ for (const execution of executions) { selectedRuns = contextIntegrity.runs as unknown as RunRecord[]; } else if (execution.task.flow === "agent_chat") { const chat = await runChatFlow({ - privateSecrets: secrets, + privateSecrets: secrets, privateInputs, page, api, fixtures, execution, nonce, workspacePath, restart: () => restartChatServer(page, () => restartIsolatedPaperclipServer({ api, requestId: `chat-${nonce}`, deadlineAt: startedAtMs + deadlineMs })), observe: (chatIssue, chatRuns) => { issue = chatIssue; selectedRuns = chatRuns; },