Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-05 03:51:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): erase selected secrets outside retained diagnostics

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-26 19:41:13 -05:00
1 parent 99513b6c48
commit 5f8a3c8eec
5 files changed
+28 -10
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-boundary-fingerprint.ts
+5 -3
View File
@@ -9,9 +9,10 @@ import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js";
import type { MatrixExecution } from "./types.js";
/** Independent boundary lane: never relax assertChatCatalogFingerprintLane. */
export function chatBoundaryFingerprint(descriptor: ChatCatalogDescriptor, execution: MatrixExecution, env: NodeJS.ProcessEnv = process.env) {
export function chatBoundaryFingerprint(descriptor: ChatCatalogDescriptor, execution: MatrixExecution, env: NodeJS.ProcessEnv = process.env, retainedPrivateInputNames?: ReturnType<typeof chatPrivateInputNameIdentity>) {
if (descriptor.journeyKind !== "browser-boundary" || execution.environment.id !== "local" || execution.suite.id !== "chat-features")
throw new Error("Boundary fingerprint requires an explicitly selected local browser boundary");
const privateInputNameIdentity = retainedPrivateInputNames === undefined ? chatPrivateInputNameIdentity(execution.id, env) : retainedPrivateInputNames;
let installedIdentity: unknown = null;
if (descriptor.boundaryTarget === "empty-installed-target") {
const cli = env.PAPERCLIP_CHAT_BOUNDARY_CLI; const expectedCli = env.PAPERCLIP_CHAT_BOUNDARY_CLI_SHA256;
@@ -30,11 +31,12 @@ export function chatBoundaryFingerprint(descriptor: ChatCatalogDescriptor, execu
installedIdentity = { version: cliManifest.version, cliSha256, serverSha256 };
} else if (descriptor.boundaryTarget !== "authenticated-harness") throw new Error("Boundary target identity is undeclared");
const root = fileURLToPath(new URL("../../", import.meta.url));
return fingerprintChatDependencyClosure({ root,
const result = fingerprintChatDependencyClosure({ root,
entries: [`tests/runner-e2e/${descriptor.modulePath.slice(2)}`, "tests/runner-e2e/chat-boundary-flow.ts", "tests/runner-e2e/chat-boundary-fingerprint.ts",
"tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts", ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)],
selected: { descriptor, journeyKind: "browser-boundary", executionId: execution.id, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity: chatPrivateInputNameIdentity(execution.id, env), requiredCredentialNames: execution.requiredCredentials, environment: execution.environment.id, installedIdentity },
selected: { descriptor, journeyKind: "browser-boundary", executionId: execution.id, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity, requiredCredentialNames: execution.requiredCredentials, environment: execution.environment.id, installedIdentity },
selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts",
normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor },
});
return { ...result, privateInputNameIdentity, privateInputRequirements: chatPrivateInputRequirements(descriptor) };
}
tests/runner-e2e/chat-catalog-fingerprint.test.ts
+12 -2
View File
@@ -2,11 +2,11 @@ import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { afterEach, expect, it } from "vitest";
import { afterEach, expect, it, vi } from "vitest";
import { assertChatCatalogFingerprintLane, chatCatalogCaseFingerprint, fingerprintChatDependencyClosure } from "./chat-catalog-fingerprint.js";
import { findChatCatalogCase, loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js";
const directories: string[] = [];
afterEach(() => { for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); });
afterEach(() => { vi.unstubAllEnvs(); for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); });
function fixture() {
const root = mkdtempSync(path.join(tmpdir(), "chat-fingerprint-")); directories.push(root);
const write = (file: string, body: string) => writeFileSync(path.join(root, file), body);
@@ -79,3 +79,13 @@ it("fails closed when preserved prerequisites or a rejected query lack their ind
const rejected = { ...base, queryOutcome: "rejected" as const };
await expect(loadChatCatalogImplementation(rejected, async () => ({ cases: [{ ...rejected, ...callbacks }] }))).rejects.toThrow("independent rejection evidence");
});
it("replays the retained public name-only identity after raw selected inputs have been removed", () => {
const descriptor = findChatCatalogCase("projects.create-project")!;
const execution = { executionId: "offline-selected", suiteId: "chat-features", flow: "agent_chat", environment: "local" };
const names = { credentialNames: ["OPENAI_API_KEY"], referenceNames: [], externalRequirements: [] };
const before = chatCatalogCaseFingerprint(descriptor, execution, names).digest;
vi.stubEnv("PAPERCLIP_CHAT_PRIVATE_INPUTS", "/removed/owned-secret-file.json");
expect(chatCatalogCaseFingerprint(descriptor, execution, names).digest).toBe(before);
expect(() => chatCatalogCaseFingerprint(descriptor, execution)).toThrow();
});
tests/runner-e2e/chat-catalog-fingerprint.ts
+4 -3
View File
@@ -85,8 +85,9 @@ export function assertChatCatalogFingerprintLane(selected: unknown, env: NodeJS.
installedLane: false, externalJourneyDispatch: false };
}
export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, selectedExecution: unknown) {
export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, selectedExecution: unknown, retainedPrivateInputNames?: ReturnType<typeof chatPrivateInputNameIdentity>) {
const root = fileURLToPath(new URL("../../", import.meta.url));
const privateInputNameIdentity = retainedPrivateInputNames === undefined ? chatPrivateInputNameIdentity((selectedExecution as { executionId: string }).executionId) : retainedPrivateInputNames;
const lane = assertChatCatalogFingerprintLane(selectedExecution);
const reviewedDynamicResolutions = JSON.parse(readFileSync(new URL("./chat-catalog-dynamic-resolutions.json", import.meta.url), "utf8"));
const selectedModule = `tests/runner-e2e/${descriptor.modulePath.slice(2)}`;
@@ -101,13 +102,13 @@ export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, se
entries: [selectedModule, "tests/runner-e2e/chat-catalog-flow.ts", "tests/runner-e2e/chat-catalog-fingerprint.ts", "tests/runner-e2e/chat-catalog-dynamic-resolutions.json",
"tests/runner-e2e/chat-feature-flow.ts", "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts",
"tests/runner-e2e/chat-scenarios/defaults.yaml", ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)],
selected: { descriptor, selectedExecution, lane, attachmentFormatter, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity: chatPrivateInputNameIdentity((selectedExecution as { executionId: string }).executionId), scenario: { id, capabilities, context, query, expected_outcome } },
selected: { descriptor, selectedExecution, lane, attachmentFormatter, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity, scenario: { id, capabilities, context, query, expected_outcome } },
reviewedDynamicResolutions,
selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts",
normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor,
"tests/runner-e2e/chat-campaign-manifest.json": { id, capabilities, context, query, expected_outcome } },
});
return { ...result, attachmentFormatter };
return { ...result, attachmentFormatter, privateInputNameIdentity, privateInputRequirements: chatPrivateInputRequirements(descriptor) };
}
export function normalizedChatCatalogExecution(execution: MatrixExecution) {
tests/runner-e2e/chat-private-inputs.ts
+1 -1
View File
@@ -72,7 +72,7 @@ export function readChatPrivateInputs(executionId: string, secrets: string[], en
configuration: name => name === "external-provider" ? envelope.external : undefined };
}
export function scrubChatPrivateEnvironment(env: NodeJS.ProcessEnv, executions: readonly MatrixExecution[], removeReference = false) {
for (const key of Object.keys(env)) if (key.startsWith("PAPERCLIP_CHAT_EVAL_") || key === externalConfigEnv || removeReference && key === CHAT_PRIVATE_INPUTS_ENV) delete env[key];
for (const key of Object.keys(env)) if (key.startsWith("PAPERCLIP_CHAT_EVAL_") || key === externalConfigEnv || removeReference && [CHAT_PRIVATE_INPUTS_ENV, "PAPERCLIP_CHAT_INSTALLED_PLAN", "PAPERCLIP_CHAT_OPERATOR_INSTALL_RECEIPT", "PAPERCLIP_CHAT_INSTALLED_SOURCE_CONFIG"].includes(key)) delete env[key];
if (executions.length) {
const profileNames = new Set(executions.flatMap(execution => [execution.profile.credential, execution.environment.credential].filter(Boolean)));
for (const name of CREDENTIAL_NAMES) if (!profileNames.has(name)) delete env[name];
tests/runner-e2e/launch.ts
+6 -1
View File
@@ -503,6 +503,7 @@ async function runAttempt(input: {
const sharedMemoryBaseline = snapshotDarwinSharedMemory();
const temporaryParent = await realpath(os.tmpdir());
const temporaryRoot = await createRunnerE2ETemporaryRoot(temporaryParent);
const selectedPrivateInputsDirectory = path.join(temporaryRoot, "selected-private-inputs");
const publishedResults: RunnerE2EResult[] = [];
const publishedResultPaths = new Map<string, string>();
let attemptSecrets: string[] = [];
@@ -553,7 +554,8 @@ async function runAttempt(input: {
const selectedDescriptor = execution.suite.id === "chat-features" ? findChatCatalogCase(execution.task.id) : undefined;
const needsPrivateInputs = selectedDescriptor && ((selectedDescriptor.requiredCredentials?.length ?? 0) > 0 || selectedDescriptor.externalInput);
if (needsPrivateInputs && executions.length !== 1) throw new Error("Private case inputs require one explicitly selected execution");
const privateInputs = needsPrivateInputs ? stageChatPrivateInputs({ execution, descriptor: selectedDescriptor, privateDir, env: process.env, secrets: credentials }) : undefined;
if (needsPrivateInputs) await mkdir(selectedPrivateInputsDirectory, { mode: 0o700 });
const privateInputs = needsPrivateInputs ? stageChatPrivateInputs({ execution, descriptor: selectedDescriptor, privateDir: selectedPrivateInputsDirectory, env: process.env, secrets: credentials }) : undefined;
if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 });
if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 });
const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness(
@@ -882,6 +884,9 @@ async function runAttempt(input: {
try { await gateway?.close(); } catch (error) { gatewayCleanupError = error; }
if (!processCleanupFailed) reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline);
let cleanupError: unknown;
// Raw selected keys are never diagnostic/archive artifacts, even when recovery state is retained.
try { await rm(selectedPrivateInputsDirectory, { recursive: true, force: true }); }
catch { cleanupError = new Error("Selected private credential cleanup failed; contents withheld"); }
const resourceAdmissionStarted = await access(path.join(temporaryRoot, "artifacts-private", "resource-admission-started"))
.then(() => true).catch((error: NodeJS.ErrnoException) => error.code !== "ENOENT");
if (mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults })) {