From a0c5ad98050331b709fb79be5f8683869a230a13 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 01:31:00 -0500 Subject: [PATCH] test(evals): scope vault AWS credentials to selected server --- .../chat-vault-private-server.test.ts | 218 +++++++++++++++ tests/runner-e2e/chat-vault-private-server.ts | 252 ++++++++++++++++++ tests/runner-e2e/harness-env.ts | 12 + 3 files changed, 482 insertions(+) create mode 100644 tests/runner-e2e/chat-vault-private-server.test.ts create mode 100644 tests/runner-e2e/chat-vault-private-server.ts diff --git a/tests/runner-e2e/chat-vault-private-server.test.ts b/tests/runner-e2e/chat-vault-private-server.test.ts new file mode 100644 index 00000000000..ff654c6bdfb --- /dev/null +++ b/tests/runner-e2e/chat-vault-private-server.test.ts @@ -0,0 +1,218 @@ +import { afterEach, expect, it } from "vitest"; +import { mkdtemp, mkdir, writeFile, rm, chmod } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js"; +import { + buildPaperclipServerEnvironment, + buildRunnerE2EProcessEnvironment, +} from "./harness-env.js"; +import { + readSelectedVaultServerEnvironment, + privateVaultMetadataMatches, + validVaultBinding, + vaultPrivateFields, + vaultExecutionId, +} from "./chat-vault-private-server.js"; +import type { MatrixExecution } from "./types.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const directory of roots.splice(0)) + await rm(directory, { recursive: true, force: true }); +}); +function binding() { + return { + disposable: true, + resourceIds: [ + "arn:aws:secretsmanager:us-east-1:123456789012:secret:disposable/deployment-AbCdEf", + "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012", + ], + credentialEnv: [ + "PAPERCLIP_CHAT_EVAL_AWS_ACCESS", + "PAPERCLIP_CHAT_EVAL_AWS_SECRET", + "PAPERCLIP_CHAT_EVAL_AWS_SESSION", + "PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED", + ], + prerequisites: { + provider: "aws_secrets_manager", + region: "us-east-1", + deploymentId: "disposable-catalog", + kmsKeyId: + "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012", + secretReference: + "arn:aws:secretsmanager:us-east-1:123456789012:secret:disposable/deployment-AbCdEf", + accessKeyIdEnv: "PAPERCLIP_CHAT_EVAL_AWS_ACCESS", + secretAccessKeyEnv: "PAPERCLIP_CHAT_EVAL_AWS_SECRET", + sessionTokenEnv: "PAPERCLIP_CHAT_EVAL_AWS_SESSION", + expectedValueEnv: "PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED", + }, + }; +} +async function fixture() { + const root = await mkdtemp( + path.join(tmpdir(), "paperclip-runner-e2e-vault-control-"), + ); + roots.push(root); + const directory = path.join(root, "selected-private-inputs"); + await mkdir(directory, { mode: 0o700 }); + const filename = path.join(directory, "chat-selected-inputs.json"), + b = binding(); + const envelope = { + version: 1, + executionId: vaultExecutionId, + references: { + PAPERCLIP_CHAT_EVAL_AWS_ACCESS: "ASIA1234567890123456", + PAPERCLIP_CHAT_EVAL_AWS_SECRET: "synthetic-selected-secret-key", + PAPERCLIP_CHAT_EVAL_AWS_SESSION: "synthetic-selected-session-token", + PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED: "synthetic-vault-oracle-only", + }, + external: { schemaVersion: 1, providers: { vault: b } }, + declaredReferenceNames: b.credentialEnv, + declaredExternalRequirements: [ + { provider: "vault", fields: [...vaultPrivateFields] }, + ], + setupBlockers: [] as string[], + }; + const save = () => + writeFile(filename, JSON.stringify(envelope), { mode: 0o600 }); + await save(); + return { + envelope, + save, + filename, + input: { + executionIds: [vaultExecutionId], + temporaryRoot: root, + backendOrigin: "http://127.0.0.1:3399", + env: { + [CHAT_PRIVATE_INPUTS_ENV]: filename, + AWS_ACCESS_KEY_ID: "AMBIENT-DO-NOT-USE", + AWS_PROFILE: "unrelated-personal-profile", + PAPERCLIP_SECRETS_AWS_ENDPOINT: "http://unrelated.test", + }, + }, + }; +} +it("supplies only explicitly selected server credentials/config and excludes oracle values/default profiles", async () => { + const f = await fixture(), + env = await readSelectedVaultServerEnvironment(f.input); + expect(env.AWS_ACCESS_KEY_ID).toBe( + f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_ACCESS, + ); + expect(env.AWS_SECRET_ACCESS_KEY).toBe( + f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_SECRET, + ); + expect(env.AWS_SESSION_TOKEN).toBe( + f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_SESSION, + ); + expect(env.AWS_EC2_METADATA_DISABLED).toBe("true"); + expect(env.AWS_PROFILE).toBeUndefined(); + expect(env.PAPERCLIP_SECRETS_AWS_ENDPOINT).toBeUndefined(); + expect(JSON.stringify(env)).not.toContain( + f.envelope.references.PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED, + ); + expect( + Object.keys(env).some((name) => name.startsWith("PAPERCLIP_CHAT_EVAL_")), + ).toBe(false); + expect(env.AWS_CONFIG_FILE).toBe(env.AWS_SHARED_CREDENTIALS_FILE); +}); +it("keeps missing selected inputs as reserved setup prerequisites without using ambient AWS", async () => { + const f = await fixture(); + expect( + await readSelectedVaultServerEnvironment({ + ...f.input, + env: { AWS_ACCESS_KEY_ID: "ambient" }, + }), + ).toEqual({}); + f.envelope.setupBlockers = [ + "missing-external-reference:PAPERCLIP_CHAT_EVAL_AWS_ACCESS", + ]; + await f.save(); + expect(await readSelectedVaultServerEnvironment(f.input)).toEqual({}); + expect( + await readSelectedVaultServerEnvironment({ + ...f.input, + executionIds: ["unrelated"], + }), + ).toEqual({}); +}); +it("rejects another execution, unsafe private files, undeclared references and cross-account resources", async () => { + const f = await fixture(); + await expect( + readSelectedVaultServerEnvironment({ + ...f.input, + executionIds: [vaultExecutionId, "unrelated"], + }), + ).rejects.toThrow(/single selected/); + f.envelope.executionId = "another-execution"; + await f.save(); + await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow( + /another execution/, + ); + f.envelope.executionId = vaultExecutionId; + f.envelope.declaredReferenceNames = []; + await f.save(); + await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow( + /selected reference/, + ); + f.envelope.declaredReferenceNames = binding().credentialEnv; + await f.save(); + await chmod(f.filename, 0o644); + await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow( + /private and owned/, + ); + expect(privateVaultMetadataMatches({ uid: 101, mode: 0o600 }, 101)).toBe( + true, + ); + expect(privateVaultMetadataMatches({ uid: 102, mode: 0o600 }, 101)).toBe( + false, + ); + const b = binding(); + expect(validVaultBinding(b)).toBe(true); + b.prerequisites.kmsKeyId = b.prerequisites.kmsKeyId.replace( + "123456789012", + "987654321012", + ); + expect(validVaultBinding(b)).toBe(false); + const other = binding(); + other.resourceIds = []; + expect(validVaultBinding(other)).toBe(false); +}); +it("scrubs ambient AWS and vault SDK config while preserving explicit unrelated suite mechanisms", () => { + const ambient = { + AWS_ACCESS_KEY_ID: "ambient", + AWS_SECRET_ACCESS_KEY: "ambient-secret", + AWS_PROFILE: "personal", + AWS_CONTAINER_CREDENTIALS_FULL_URI: "https://unrelated.test", + AWS_WEB_IDENTITY_TOKEN_FILE: "/private/personal-token", + PAPERCLIP_SECRETS_AWS_ENDPOINT: "https://unrelated.test", + KEEP: "yes", + }; + expect(buildPaperclipServerEnvironment(ambient)).toEqual({ KEEP: "yes" }); + expect( + buildPaperclipServerEnvironment(ambient, { + AWS_ACCESS_KEY_ID: "explicit-selected", + }).AWS_ACCESS_KEY_ID, + ).toBe("explicit-selected"); + const execution = { + suite: { id: "chat-features" }, + task: { id: "secrets.vault-reference" }, + profile: { generation: "native", provider: "codex" }, + environment: { id: "local" }, + } as MatrixExecution; + const chat = buildRunnerE2EProcessEnvironment(ambient, [execution]); + expect( + Object.keys(chat).some( + (name) => + name.startsWith("AWS_") || name.startsWith("PAPERCLIP_SECRETS_AWS_"), + ), + ).toBe(false); + const other = buildRunnerE2EProcessEnvironment(ambient, [ + { + ...execution, + suite: { ...execution.suite, id: "explicit-unrelated-suite" }, + }, + ]); + expect(other.AWS_ACCESS_KEY_ID).toBe("ambient"); +}); diff --git a/tests/runner-e2e/chat-vault-private-server.ts b/tests/runner-e2e/chat-vault-private-server.ts new file mode 100644 index 00000000000..d0554e7b4cd --- /dev/null +++ b/tests/runner-e2e/chat-vault-private-server.ts @@ -0,0 +1,252 @@ +import { constants } from "node:fs"; +import { lstat, realpath, open, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js"; + +export const vaultScenarioId = "secrets.vault-reference"; +export const vaultExecutionId = `chat-features.runner-codex-subscription.local.${vaultScenarioId}`; +export const vaultPrivateFields = [ + "provider", + "region", + "deploymentId", + "kmsKeyId", + "secretReference", + "accessKeyIdEnv", + "secretAccessKeyEnv", + "sessionTokenEnv", + "expectedValueEnv", +] as const; +type Binding = { + disposable: boolean; + resourceIds: string[]; + credentialEnv: string[]; + prerequisites: Record; +}; +interface Envelope { + version: number; + executionId: string; + references: Record; + external?: { schemaVersion?: number; providers?: { vault?: Binding } }; + declaredReferenceNames?: string[]; + declaredExternalRequirements?: Array<{ provider: string; fields: string[] }>; + setupBlockers?: string[]; +} +export function privateVaultMetadataMatches( + info: { uid: number; mode: number }, + expectedUid: number, +) { + return info.uid === expectedUid && !(info.mode & 0o077); +} +export function validVaultBinding( + binding: Binding | undefined, +): binding is Binding { + if ( + !binding || + binding.disposable !== true || + !Array.isArray(binding.resourceIds) || + !Array.isArray(binding.credentialEnv) || + !binding.prerequisites + ) + return false; + const p = binding.prerequisites; + if ( + vaultPrivateFields.some( + (field) => typeof p[field] !== "string" || !p[field].trim(), + ) || + p.provider !== "aws_secrets_manager" || + !/^[a-z]{2}(?:-[a-z]+)+-\d$/.test(p.region!) || + !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,80}$/.test(p.deploymentId!) + ) + return false; + const secret = + /^arn:(aws(?:-us-gov|-cn)?):secretsmanager:([^:]+):(\d{12}):secret:([^\r\n]+)$/.exec( + p.secretReference!, + ); + const kms = + /^arn:(aws(?:-us-gov|-cn)?):kms:([^:]+):(\d{12}):key\/([a-f0-9-]{36})$/.exec( + p.kmsKeyId!, + ); + if ( + !secret || + !kms || + secret[1] !== kms[1] || + secret[2] !== p.region || + kms[2] !== p.region || + secret[3] !== kms[3] || + ![p.secretReference, p.kmsKeyId].every((ref) => + binding.resourceIds.includes(ref!), + ) + ) + return false; + const names = [ + p.accessKeyIdEnv!, + p.secretAccessKeyEnv!, + p.sessionTokenEnv!, + p.expectedValueEnv!, + ]; + return ( + new Set(names).size === names.length && + names.every( + (name) => + /^PAPERCLIP_CHAT_EVAL_[A-Z0-9_]+$/.test(name) && + binding.credentialEnv.includes(name), + ) && + binding.credentialEnv.every((name) => names.includes(name)) + ); +} +/** Only the selected product server receives these credentials. Oracle values, + * config pointers, profiles and SDK fallback sources never cross this boundary. */ +export async function readSelectedVaultServerEnvironment(input: { + executionIds: readonly string[]; + temporaryRoot: string; + backendOrigin: string; + env: NodeJS.ProcessEnv; +}): Promise { + if (!input.executionIds.includes(vaultExecutionId)) return {}; + if (input.executionIds.length !== 1) + throw new Error( + "Vault runtime inputs require the exact single selected source execution", + ); + const backend = new URL(input.backendOrigin); + if ( + backend.protocol !== "http:" || + backend.hostname !== "127.0.0.1" || + backend.origin !== input.backendOrigin + ) + throw new Error("Vault runtime inputs require the owned loopback backend"); + const filename = input.env[CHAT_PRIVATE_INPUTS_ENV]; + if (!filename) return {}; + const directory = path.join(input.temporaryRoot, "selected-private-inputs"); + if (filename !== path.join(directory, "chat-selected-inputs.json")) + throw new Error( + "Vault runtime input path escaped its owned attempt directory", + ); + const uid = process.getuid?.(); + if (uid === undefined) + throw new Error("Vault runtime input ownership cannot be verified"); + for (const target of [input.temporaryRoot, directory]) { + const info = await lstat(target); + if ( + !info.isDirectory() || + info.isSymbolicLink() || + !privateVaultMetadataMatches(info, uid) + ) + throw new Error("Vault runtime input directory is not private and owned"); + } + const handle = await open( + filename, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + let envelope: Envelope; + try { + const before = await handle.stat(); + if ( + !before.isFile() || + !privateVaultMetadataMatches(before, uid) || + before.size > 262144 || + !(await realpath(filename)).startsWith( + (await realpath(input.temporaryRoot)) + path.sep, + ) + ) + throw new Error("Vault runtime input file is not private and owned"); + const bytes = await handle.readFile(), + after = await handle.stat(), + named = await lstat(filename); + if ( + before.ino !== after.ino || + before.dev !== after.dev || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + named.ino !== before.ino || + named.dev !== before.dev + ) + throw new Error("Vault private input changed during read"); + try { + envelope = JSON.parse(bytes.toString("utf8")) as Envelope; + } catch { + throw new Error("Vault runtime input JSON is invalid; contents withheld"); + } + } finally { + await handle.close(); + } + if (envelope.version !== 1 || envelope.executionId !== vaultExecutionId) + throw new Error("Vault runtime input belongs to another execution"); + // Ordinary reserved setup handles missing selected provider inputs. It never + // receives an ambient fallback merely because the private binding is absent. + if (!envelope.external?.providers?.vault || envelope.setupBlockers?.length) + return {}; + const binding = envelope.external.providers.vault; + if (envelope.external.schemaVersion !== 1 || !validVaultBinding(binding)) + throw new Error( + "Vault runtime binding does not designate the exact allowed AWS resources", + ); + const declaration = envelope.declaredExternalRequirements; + if ( + declaration?.length !== 1 || + declaration[0]?.provider !== "vault" || + [...declaration[0].fields].sort().join() !== + [...vaultPrivateFields].sort().join() + ) + throw new Error( + "Vault runtime fields differ from the selected fingerprinted declaration", + ); + const p = binding.prerequisites; + const read = ( + field: "accessKeyIdEnv" | "secretAccessKeyEnv" | "sessionTokenEnv", + ) => { + const name = p[field]!; + if (!envelope.declaredReferenceNames?.includes(name)) + throw new Error( + "Vault runtime credential is not an explicitly selected reference", + ); + const value = envelope.references?.[name]; + if ( + typeof value !== "string" || + !value || + /[\r\n\0]/.test(value) || + value.length > 16384 + ) + throw new Error( + "Selected AWS runtime credential is missing or invalid; value withheld", + ); + return value; + }; + const accessKey = read("accessKeyIdEnv"), + secretKey = read("secretAccessKeyEnv"), + sessionToken = read("sessionTokenEnv"); + if ( + !/^ASIA[A-Z0-9]{12,124}$/.test(accessKey) || + secretKey.length < 16 || + sessionToken.length < 16 + ) + throw new Error( + "Vault runtime requires explicitly selected temporary AWS credentials", + ); + const awsConfig = path.join(directory, "aws-vault-empty-config"); + try { + await writeFile(awsConfig, "", { mode: 0o600, flag: "wx" }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const info = await lstat(awsConfig); + if ( + !info.isFile() || + info.isSymbolicLink() || + info.size !== 0 || + !privateVaultMetadataMatches(info, uid) + ) + throw new Error("Selected empty AWS config file is unsafe"); + } + return { + AWS_ACCESS_KEY_ID: accessKey, + AWS_SECRET_ACCESS_KEY: secretKey, + AWS_SESSION_TOKEN: sessionToken, + AWS_REGION: p.region, + AWS_DEFAULT_REGION: p.region, + AWS_EC2_METADATA_DISABLED: "true", + AWS_CONFIG_FILE: awsConfig, + AWS_SHARED_CREDENTIALS_FILE: awsConfig, + PAPERCLIP_SECRETS_AWS_REGION: p.region, + PAPERCLIP_SECRETS_AWS_DEPLOYMENT_ID: p.deploymentId, + PAPERCLIP_SECRETS_AWS_KMS_KEY_ID: p.kmsKeyId, + }; +} diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index eeb800dda71..af403e887ab 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -1,3 +1,4 @@ +import { PUBLIC_ORIGIN_CONTROL_ENV, PUBLIC_ORIGIN_RECEIPT_ENV } from "./chat-public-origin.js"; import { scrubChatPrivateEnvironment } from "./chat-private-inputs.js"; import { PRIVATE_DIAGNOSTICS_ENV } from "./private-diagnostics.js"; import { installedDaytonaPluginKeys } from "./installed-daytona-plugin.js"; @@ -35,6 +36,13 @@ const AMBIENT_EXTERNAL_STATE_KEYS = [ ] as const; const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|BRAINTRUST|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/; +/** Vault SDK authority is never ambient chat input. An explicitly selected + * server-only vault handoff supplies its own finite overrides after scrubbing. */ +export function scrubAmbientVaultEnvironment(env: NodeJS.ProcessEnv) { + for (const name of Object.keys(env)) if (name.startsWith("AWS_") || name.startsWith("PAPERCLIP_SECRETS_AWS_")) delete env[name]; + return env; +} + export function applyEvalGatewayEnvironment(env: NodeJS.ProcessEnv, credential: string, gateway: { token: string; baseUrl: string; phasePath: string; observability?: { token: string; baseUrl: string }; }) { @@ -133,6 +141,8 @@ export function buildRunnerE2EProcessEnvironment( const installed = usesInstalledCli(source); if (installed) assertInstalledCliSelection(source, executions); const result = { ...source }; + if (executions.some(execution => execution.suite.id === "chat-features")) scrubAmbientVaultEnvironment(result); + delete result[PUBLIC_ORIGIN_CONTROL_ENV]; delete result[PUBLIC_ORIGIN_RECEIPT_ENV]; if (executions.some(execution => execution.suite.id === "chat-features")) scrubChatPrivateEnvironment(result, executions); else scrubChatPrivateEnvironment(result, []); delete result.PAPERCLIP_E2E_GATEWAY_PHASE_PATH; @@ -199,6 +209,8 @@ export function buildPaperclipServerEnvironment( overrides: NodeJS.ProcessEnv = {}, ): NodeJS.ProcessEnv { const result = scrubChatPrivateEnvironment({ ...source }, [], true); + scrubAmbientVaultEnvironment(result); + delete result[PUBLIC_ORIGIN_CONTROL_ENV]; delete result[PUBLIC_ORIGIN_RECEIPT_ENV]; delete result[PRIVATE_DIAGNOSTICS_ENV]; for (const key of Object.keys(result)) { if (PROVIDER_SECRET_KEY.test(key)) delete result[key];