From 5f8a3c8eec4e65d8aac2ab868c1f8efda35a13fe Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 19:41:13 -0500 Subject: [PATCH] test(evals): erase selected secrets outside retained diagnostics Co-Authored-By: Paperclip --- tests/runner-e2e/chat-boundary-fingerprint.ts | 8 +++++--- tests/runner-e2e/chat-catalog-fingerprint.test.ts | 14 ++++++++++++-- tests/runner-e2e/chat-catalog-fingerprint.ts | 7 ++++--- tests/runner-e2e/chat-private-inputs.ts | 2 +- tests/runner-e2e/launch.ts | 7 ++++++- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/tests/runner-e2e/chat-boundary-fingerprint.ts b/tests/runner-e2e/chat-boundary-fingerprint.ts index 6130626a6c2..2e20cb5a8e8 100644 --- a/tests/runner-e2e/chat-boundary-fingerprint.ts +++ b/tests/runner-e2e/chat-boundary-fingerprint.ts @@ -9,9 +9,10 @@ import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js"; import type { MatrixExecution } from "./types.js"; /** Independent boundary lane: never relax assertChatCatalogFingerprintLane. */ -export function chatBoundaryFingerprint(descriptor: ChatCatalogDescriptor, execution: MatrixExecution, env: NodeJS.ProcessEnv = process.env) { +export function chatBoundaryFingerprint(descriptor: ChatCatalogDescriptor, execution: MatrixExecution, env: NodeJS.ProcessEnv = process.env, retainedPrivateInputNames?: ReturnType) { if (descriptor.journeyKind !== "browser-boundary" || execution.environment.id !== "local" || execution.suite.id !== "chat-features") throw new Error("Boundary fingerprint requires an explicitly selected local browser boundary"); + const privateInputNameIdentity = retainedPrivateInputNames === undefined ? chatPrivateInputNameIdentity(execution.id, env) : retainedPrivateInputNames; let installedIdentity: unknown = null; if (descriptor.boundaryTarget === "empty-installed-target") { const cli = env.PAPERCLIP_CHAT_BOUNDARY_CLI; const expectedCli = env.PAPERCLIP_CHAT_BOUNDARY_CLI_SHA256; @@ -30,11 +31,12 @@ export function chatBoundaryFingerprint(descriptor: ChatCatalogDescriptor, execu installedIdentity = { version: cliManifest.version, cliSha256, serverSha256 }; } else if (descriptor.boundaryTarget !== "authenticated-harness") throw new Error("Boundary target identity is undeclared"); const root = fileURLToPath(new URL("../../", import.meta.url)); - return fingerprintChatDependencyClosure({ root, + const result = fingerprintChatDependencyClosure({ root, entries: [`tests/runner-e2e/${descriptor.modulePath.slice(2)}`, "tests/runner-e2e/chat-boundary-flow.ts", "tests/runner-e2e/chat-boundary-fingerprint.ts", "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts", ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)], - selected: { descriptor, journeyKind: "browser-boundary", executionId: execution.id, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity: chatPrivateInputNameIdentity(execution.id, env), requiredCredentialNames: execution.requiredCredentials, environment: execution.environment.id, installedIdentity }, + selected: { descriptor, journeyKind: "browser-boundary", executionId: execution.id, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity, requiredCredentialNames: execution.requiredCredentials, environment: execution.environment.id, installedIdentity }, selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts", normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor }, }); + return { ...result, privateInputNameIdentity, privateInputRequirements: chatPrivateInputRequirements(descriptor) }; } diff --git a/tests/runner-e2e/chat-catalog-fingerprint.test.ts b/tests/runner-e2e/chat-catalog-fingerprint.test.ts index cce867c4640..1f41a22fb17 100644 --- a/tests/runner-e2e/chat-catalog-fingerprint.test.ts +++ b/tests/runner-e2e/chat-catalog-fingerprint.test.ts @@ -2,11 +2,11 @@ import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { afterEach, expect, it } from "vitest"; +import { afterEach, expect, it, vi } from "vitest"; import { assertChatCatalogFingerprintLane, chatCatalogCaseFingerprint, fingerprintChatDependencyClosure } from "./chat-catalog-fingerprint.js"; import { findChatCatalogCase, loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js"; const directories: string[] = []; -afterEach(() => { for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); }); +afterEach(() => { vi.unstubAllEnvs(); for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); }); function fixture() { const root = mkdtempSync(path.join(tmpdir(), "chat-fingerprint-")); directories.push(root); const write = (file: string, body: string) => writeFileSync(path.join(root, file), body); @@ -79,3 +79,13 @@ it("fails closed when preserved prerequisites or a rejected query lack their ind const rejected = { ...base, queryOutcome: "rejected" as const }; await expect(loadChatCatalogImplementation(rejected, async () => ({ cases: [{ ...rejected, ...callbacks }] }))).rejects.toThrow("independent rejection evidence"); }); + +it("replays the retained public name-only identity after raw selected inputs have been removed", () => { + const descriptor = findChatCatalogCase("projects.create-project")!; + const execution = { executionId: "offline-selected", suiteId: "chat-features", flow: "agent_chat", environment: "local" }; + const names = { credentialNames: ["OPENAI_API_KEY"], referenceNames: [], externalRequirements: [] }; + const before = chatCatalogCaseFingerprint(descriptor, execution, names).digest; + vi.stubEnv("PAPERCLIP_CHAT_PRIVATE_INPUTS", "/removed/owned-secret-file.json"); + expect(chatCatalogCaseFingerprint(descriptor, execution, names).digest).toBe(before); + expect(() => chatCatalogCaseFingerprint(descriptor, execution)).toThrow(); +}); diff --git a/tests/runner-e2e/chat-catalog-fingerprint.ts b/tests/runner-e2e/chat-catalog-fingerprint.ts index e0439381026..bfeb0c75582 100644 --- a/tests/runner-e2e/chat-catalog-fingerprint.ts +++ b/tests/runner-e2e/chat-catalog-fingerprint.ts @@ -85,8 +85,9 @@ export function assertChatCatalogFingerprintLane(selected: unknown, env: NodeJS. installedLane: false, externalJourneyDispatch: false }; } -export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, selectedExecution: unknown) { +export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, selectedExecution: unknown, retainedPrivateInputNames?: ReturnType) { const root = fileURLToPath(new URL("../../", import.meta.url)); + const privateInputNameIdentity = retainedPrivateInputNames === undefined ? chatPrivateInputNameIdentity((selectedExecution as { executionId: string }).executionId) : retainedPrivateInputNames; const lane = assertChatCatalogFingerprintLane(selectedExecution); const reviewedDynamicResolutions = JSON.parse(readFileSync(new URL("./chat-catalog-dynamic-resolutions.json", import.meta.url), "utf8")); const selectedModule = `tests/runner-e2e/${descriptor.modulePath.slice(2)}`; @@ -101,13 +102,13 @@ export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, se entries: [selectedModule, "tests/runner-e2e/chat-catalog-flow.ts", "tests/runner-e2e/chat-catalog-fingerprint.ts", "tests/runner-e2e/chat-catalog-dynamic-resolutions.json", "tests/runner-e2e/chat-feature-flow.ts", "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts", "tests/runner-e2e/chat-scenarios/defaults.yaml", ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)], - selected: { descriptor, selectedExecution, lane, attachmentFormatter, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity: chatPrivateInputNameIdentity((selectedExecution as { executionId: string }).executionId), scenario: { id, capabilities, context, query, expected_outcome } }, + selected: { descriptor, selectedExecution, lane, attachmentFormatter, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity, scenario: { id, capabilities, context, query, expected_outcome } }, reviewedDynamicResolutions, selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts", normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor, "tests/runner-e2e/chat-campaign-manifest.json": { id, capabilities, context, query, expected_outcome } }, }); - return { ...result, attachmentFormatter }; + return { ...result, attachmentFormatter, privateInputNameIdentity, privateInputRequirements: chatPrivateInputRequirements(descriptor) }; } export function normalizedChatCatalogExecution(execution: MatrixExecution) { diff --git a/tests/runner-e2e/chat-private-inputs.ts b/tests/runner-e2e/chat-private-inputs.ts index 9b6842eebc3..800f413fa9f 100644 --- a/tests/runner-e2e/chat-private-inputs.ts +++ b/tests/runner-e2e/chat-private-inputs.ts @@ -72,7 +72,7 @@ export function readChatPrivateInputs(executionId: string, secrets: string[], en configuration: name => name === "external-provider" ? envelope.external : undefined }; } export function scrubChatPrivateEnvironment(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[], removeReference = false) { - for (const key of Object.keys(env)) if (key.startsWith("PAPERCLIP_CHAT_EVAL_") || key === externalConfigEnv || removeReference && key === CHAT_PRIVATE_INPUTS_ENV) delete env[key]; + for (const key of Object.keys(env)) if (key.startsWith("PAPERCLIP_CHAT_EVAL_") || key === externalConfigEnv || removeReference && [CHAT_PRIVATE_INPUTS_ENV, "PAPERCLIP_CHAT_INSTALLED_PLAN", "PAPERCLIP_CHAT_OPERATOR_INSTALL_RECEIPT", "PAPERCLIP_CHAT_INSTALLED_SOURCE_CONFIG"].includes(key)) delete env[key]; if (executions.length) { const profileNames = new Set(executions.flatMap(execution => [execution.profile.credential, execution.environment.credential].filter(Boolean))); for (const name of CREDENTIAL_NAMES) if (!profileNames.has(name)) delete env[name]; diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 7b0feffbc25..3a9985af4f8 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -503,6 +503,7 @@ async function runAttempt(input: { const sharedMemoryBaseline = snapshotDarwinSharedMemory(); const temporaryParent = await realpath(os.tmpdir()); const temporaryRoot = await createRunnerE2ETemporaryRoot(temporaryParent); + const selectedPrivateInputsDirectory = path.join(temporaryRoot, "selected-private-inputs"); const publishedResults: RunnerE2EResult[] = []; const publishedResultPaths = new Map(); let attemptSecrets: string[] = []; @@ -553,7 +554,8 @@ async function runAttempt(input: { const selectedDescriptor = execution.suite.id === "chat-features" ? findChatCatalogCase(execution.task.id) : undefined; const needsPrivateInputs = selectedDescriptor && ((selectedDescriptor.requiredCredentials?.length ?? 0) > 0 || selectedDescriptor.externalInput); if (needsPrivateInputs && executions.length !== 1) throw new Error("Private case inputs require one explicitly selected execution"); - const privateInputs = needsPrivateInputs ? stageChatPrivateInputs({ execution, descriptor: selectedDescriptor, privateDir, env: process.env, secrets: credentials }) : undefined; + if (needsPrivateInputs) await mkdir(selectedPrivateInputsDirectory, { mode: 0o700 }); + const privateInputs = needsPrivateInputs ? stageChatPrivateInputs({ execution, descriptor: selectedDescriptor, privateDir: selectedPrivateInputsDirectory, env: process.env, secrets: credentials }) : undefined; if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 }); if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 }); const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness( @@ -882,6 +884,9 @@ async function runAttempt(input: { try { await gateway?.close(); } catch (error) { gatewayCleanupError = error; } if (!processCleanupFailed) reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline); let cleanupError: unknown; + // Raw selected keys are never diagnostic/archive artifacts, even when recovery state is retained. + try { await rm(selectedPrivateInputsDirectory, { recursive: true, force: true }); } + catch { cleanupError = new Error("Selected private credential cleanup failed; contents withheld"); } const resourceAdmissionStarted = await access(path.join(temporaryRoot, "artifacts-private", "resource-admission-started")) .then(() => true).catch((error: NodeJS.ErrnoException) => error.code !== "ENOENT"); if (mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults })) {