test(evals): scope vault AWS credentials to selected server

tests/runner-e2e/chat-vault-private-server.test.ts

@@ -0,0 +1,218 @@
+import { afterEach, expect, it } from "vitest";
+import { mkdtemp, mkdir, writeFile, rm, chmod } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js";
+import {
+  buildPaperclipServerEnvironment,
+  buildRunnerE2EProcessEnvironment,
+} from "./harness-env.js";
+import {
+  readSelectedVaultServerEnvironment,
+  privateVaultMetadataMatches,
+  validVaultBinding,
+  vaultPrivateFields,
+  vaultExecutionId,
+} from "./chat-vault-private-server.js";
+import type { MatrixExecution } from "./types.js";
+
+const roots: string[] = [];
+afterEach(async () => {
+  for (const directory of roots.splice(0))
+    await rm(directory, { recursive: true, force: true });
+});
+function binding() {
+  return {
+    disposable: true,
+    resourceIds: [
+      "arn:aws:secretsmanager:us-east-1:123456789012:secret:disposable/deployment-AbCdEf",
+      "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012",
+    ],
+    credentialEnv: [
+      "PAPERCLIP_CHAT_EVAL_AWS_ACCESS",
+      "PAPERCLIP_CHAT_EVAL_AWS_SECRET",
+      "PAPERCLIP_CHAT_EVAL_AWS_SESSION",
+      "PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED",
+    ],
+    prerequisites: {
+      provider: "aws_secrets_manager",
+      region: "us-east-1",
+      deploymentId: "disposable-catalog",
+      kmsKeyId:
+        "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012",
+      secretReference:
+        "arn:aws:secretsmanager:us-east-1:123456789012:secret:disposable/deployment-AbCdEf",
+      accessKeyIdEnv: "PAPERCLIP_CHAT_EVAL_AWS_ACCESS",
+      secretAccessKeyEnv: "PAPERCLIP_CHAT_EVAL_AWS_SECRET",
+      sessionTokenEnv: "PAPERCLIP_CHAT_EVAL_AWS_SESSION",
+      expectedValueEnv: "PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED",
+    },
+  };
+}
+async function fixture() {
+  const root = await mkdtemp(
+    path.join(tmpdir(), "paperclip-runner-e2e-vault-control-"),
+  );
+  roots.push(root);
+  const directory = path.join(root, "selected-private-inputs");
+  await mkdir(directory, { mode: 0o700 });
+  const filename = path.join(directory, "chat-selected-inputs.json"),
+    b = binding();
+  const envelope = {
+    version: 1,
+    executionId: vaultExecutionId,
+    references: {
+      PAPERCLIP_CHAT_EVAL_AWS_ACCESS: "ASIA1234567890123456",
+      PAPERCLIP_CHAT_EVAL_AWS_SECRET: "synthetic-selected-secret-key",
+      PAPERCLIP_CHAT_EVAL_AWS_SESSION: "synthetic-selected-session-token",
+      PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED: "synthetic-vault-oracle-only",
+    },
+    external: { schemaVersion: 1, providers: { vault: b } },
+    declaredReferenceNames: b.credentialEnv,
+    declaredExternalRequirements: [
+      { provider: "vault", fields: [...vaultPrivateFields] },
+    ],
+    setupBlockers: [] as string[],
+  };
+  const save = () =>
+    writeFile(filename, JSON.stringify(envelope), { mode: 0o600 });
+  await save();
+  return {
+    envelope,
+    save,
+    filename,
+    input: {
+      executionIds: [vaultExecutionId],
+      temporaryRoot: root,
+      backendOrigin: "http://127.0.0.1:3399",
+      env: {
+        [CHAT_PRIVATE_INPUTS_ENV]: filename,
+        AWS_ACCESS_KEY_ID: "AMBIENT-DO-NOT-USE",
+        AWS_PROFILE: "unrelated-personal-profile",
+        PAPERCLIP_SECRETS_AWS_ENDPOINT: "http://unrelated.test",
+      },
+    },
+  };
+}
+it("supplies only explicitly selected server credentials/config and excludes oracle values/default profiles", async () => {
+  const f = await fixture(),
+    env = await readSelectedVaultServerEnvironment(f.input);
+  expect(env.AWS_ACCESS_KEY_ID).toBe(
+    f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_ACCESS,
+  );
+  expect(env.AWS_SECRET_ACCESS_KEY).toBe(
+    f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_SECRET,
+  );
+  expect(env.AWS_SESSION_TOKEN).toBe(
+    f.envelope.references.PAPERCLIP_CHAT_EVAL_AWS_SESSION,
+  );
+  expect(env.AWS_EC2_METADATA_DISABLED).toBe("true");
+  expect(env.AWS_PROFILE).toBeUndefined();
+  expect(env.PAPERCLIP_SECRETS_AWS_ENDPOINT).toBeUndefined();
+  expect(JSON.stringify(env)).not.toContain(
+    f.envelope.references.PAPERCLIP_CHAT_EVAL_VAULT_EXPECTED,
+  );
+  expect(
+    Object.keys(env).some((name) => name.startsWith("PAPERCLIP_CHAT_EVAL_")),
+  ).toBe(false);
+  expect(env.AWS_CONFIG_FILE).toBe(env.AWS_SHARED_CREDENTIALS_FILE);
+});
+it("keeps missing selected inputs as reserved setup prerequisites without using ambient AWS", async () => {
+  const f = await fixture();
+  expect(
+    await readSelectedVaultServerEnvironment({
+      ...f.input,
+      env: { AWS_ACCESS_KEY_ID: "ambient" },
+    }),
+  ).toEqual({});
+  f.envelope.setupBlockers = [
+    "missing-external-reference:PAPERCLIP_CHAT_EVAL_AWS_ACCESS",
+  ];
+  await f.save();
+  expect(await readSelectedVaultServerEnvironment(f.input)).toEqual({});
+  expect(
+    await readSelectedVaultServerEnvironment({
+      ...f.input,
+      executionIds: ["unrelated"],
+    }),
+  ).toEqual({});
+});
+it("rejects another execution, unsafe private files, undeclared references and cross-account resources", async () => {
+  const f = await fixture();
+  await expect(
+    readSelectedVaultServerEnvironment({
+      ...f.input,
+      executionIds: [vaultExecutionId, "unrelated"],
+    }),
+  ).rejects.toThrow(/single selected/);
+  f.envelope.executionId = "another-execution";
+  await f.save();
+  await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow(
+    /another execution/,
+  );
+  f.envelope.executionId = vaultExecutionId;
+  f.envelope.declaredReferenceNames = [];
+  await f.save();
+  await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow(
+    /selected reference/,
+  );
+  f.envelope.declaredReferenceNames = binding().credentialEnv;
+  await f.save();
+  await chmod(f.filename, 0o644);
+  await expect(readSelectedVaultServerEnvironment(f.input)).rejects.toThrow(
+    /private and owned/,
+  );
+  expect(privateVaultMetadataMatches({ uid: 101, mode: 0o600 }, 101)).toBe(
+    true,
+  );
+  expect(privateVaultMetadataMatches({ uid: 102, mode: 0o600 }, 101)).toBe(
+    false,
+  );
+  const b = binding();
+  expect(validVaultBinding(b)).toBe(true);
+  b.prerequisites.kmsKeyId = b.prerequisites.kmsKeyId.replace(
+    "123456789012",
+    "987654321012",
+  );
+  expect(validVaultBinding(b)).toBe(false);
+  const other = binding();
+  other.resourceIds = [];
+  expect(validVaultBinding(other)).toBe(false);
+});
+it("scrubs ambient AWS and vault SDK config while preserving explicit unrelated suite mechanisms", () => {
+  const ambient = {
+    AWS_ACCESS_KEY_ID: "ambient",
+    AWS_SECRET_ACCESS_KEY: "ambient-secret",
+    AWS_PROFILE: "personal",
+    AWS_CONTAINER_CREDENTIALS_FULL_URI: "https://unrelated.test",
+    AWS_WEB_IDENTITY_TOKEN_FILE: "/private/personal-token",
+    PAPERCLIP_SECRETS_AWS_ENDPOINT: "https://unrelated.test",
+    KEEP: "yes",
+  };
+  expect(buildPaperclipServerEnvironment(ambient)).toEqual({ KEEP: "yes" });
+  expect(
+    buildPaperclipServerEnvironment(ambient, {
+      AWS_ACCESS_KEY_ID: "explicit-selected",
+    }).AWS_ACCESS_KEY_ID,
+  ).toBe("explicit-selected");
+  const execution = {
+    suite: { id: "chat-features" },
+    task: { id: "secrets.vault-reference" },
+    profile: { generation: "native", provider: "codex" },
+    environment: { id: "local" },
+  } as MatrixExecution;
+  const chat = buildRunnerE2EProcessEnvironment(ambient, [execution]);
+  expect(
+    Object.keys(chat).some(
+      (name) =>
+        name.startsWith("AWS_") || name.startsWith("PAPERCLIP_SECRETS_AWS_"),
+    ),
+  ).toBe(false);
+  const other = buildRunnerE2EProcessEnvironment(ambient, [
+    {
+      ...execution,
+      suite: { ...execution.suite, id: "explicit-unrelated-suite" },
+    },
+  ]);
+  expect(other.AWS_ACCESS_KEY_ID).toBe("ambient");
+});

tests/runner-e2e/chat-vault-private-server.ts

@@ -0,0 +1,252 @@
+import { constants } from "node:fs";
+import { lstat, realpath, open, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js";
+
+export const vaultScenarioId = "secrets.vault-reference";
+export const vaultExecutionId = `chat-features.runner-codex-subscription.local.${vaultScenarioId}`;
+export const vaultPrivateFields = [
+  "provider",
+  "region",
+  "deploymentId",
+  "kmsKeyId",
+  "secretReference",
+  "accessKeyIdEnv",
+  "secretAccessKeyEnv",
+  "sessionTokenEnv",
+  "expectedValueEnv",
+] as const;
+type Binding = {
+  disposable: boolean;
+  resourceIds: string[];
+  credentialEnv: string[];
+  prerequisites: Record<string, string>;
+};
+interface Envelope {
+  version: number;
+  executionId: string;
+  references: Record<string, string>;
+  external?: { schemaVersion?: number; providers?: { vault?: Binding } };
+  declaredReferenceNames?: string[];
+  declaredExternalRequirements?: Array<{ provider: string; fields: string[] }>;
+  setupBlockers?: string[];
+}
+export function privateVaultMetadataMatches(
+  info: { uid: number; mode: number },
+  expectedUid: number,
+) {
+  return info.uid === expectedUid && !(info.mode & 0o077);
+}
+export function validVaultBinding(
+  binding: Binding | undefined,
+): binding is Binding {
+  if (
+    !binding ||
+    binding.disposable !== true ||
+    !Array.isArray(binding.resourceIds) ||
+    !Array.isArray(binding.credentialEnv) ||
+    !binding.prerequisites
+  )
+    return false;
+  const p = binding.prerequisites;
+  if (
+    vaultPrivateFields.some(
+      (field) => typeof p[field] !== "string" || !p[field].trim(),
+    ) ||
+    p.provider !== "aws_secrets_manager" ||
+    !/^[a-z]{2}(?:-[a-z]+)+-\d$/.test(p.region!) ||
+    !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,80}$/.test(p.deploymentId!)
+  )
+    return false;
+  const secret =
+    /^arn:(aws(?:-us-gov|-cn)?):secretsmanager:([^:]+):(\d{12}):secret:([^\r\n]+)$/.exec(
+      p.secretReference!,
+    );
+  const kms =
+    /^arn:(aws(?:-us-gov|-cn)?):kms:([^:]+):(\d{12}):key\/([a-f0-9-]{36})$/.exec(
+      p.kmsKeyId!,
+    );
+  if (
+    !secret ||
+    !kms ||
+    secret[1] !== kms[1] ||
+    secret[2] !== p.region ||
+    kms[2] !== p.region ||
+    secret[3] !== kms[3] ||
+    ![p.secretReference, p.kmsKeyId].every((ref) =>
+      binding.resourceIds.includes(ref!),
+    )
+  )
+    return false;
+  const names = [
+    p.accessKeyIdEnv!,
+    p.secretAccessKeyEnv!,
+    p.sessionTokenEnv!,
+    p.expectedValueEnv!,
+  ];
+  return (
+    new Set(names).size === names.length &&
+    names.every(
+      (name) =>
+        /^PAPERCLIP_CHAT_EVAL_[A-Z0-9_]+$/.test(name) &&
+        binding.credentialEnv.includes(name),
+    ) &&
+    binding.credentialEnv.every((name) => names.includes(name))
+  );
+}
+/** Only the selected product server receives these credentials. Oracle values,
+ * config pointers, profiles and SDK fallback sources never cross this boundary. */
+export async function readSelectedVaultServerEnvironment(input: {
+  executionIds: readonly string[];
+  temporaryRoot: string;
+  backendOrigin: string;
+  env: NodeJS.ProcessEnv;
+}): Promise<NodeJS.ProcessEnv> {
+  if (!input.executionIds.includes(vaultExecutionId)) return {};
+  if (input.executionIds.length !== 1)
+    throw new Error(
+      "Vault runtime inputs require the exact single selected source execution",
+    );
+  const backend = new URL(input.backendOrigin);
+  if (
+    backend.protocol !== "http:" ||
+    backend.hostname !== "127.0.0.1" ||
+    backend.origin !== input.backendOrigin
+  )
+    throw new Error("Vault runtime inputs require the owned loopback backend");
+  const filename = input.env[CHAT_PRIVATE_INPUTS_ENV];
+  if (!filename) return {};
+  const directory = path.join(input.temporaryRoot, "selected-private-inputs");
+  if (filename !== path.join(directory, "chat-selected-inputs.json"))
+    throw new Error(
+      "Vault runtime input path escaped its owned attempt directory",
+    );
+  const uid = process.getuid?.();
+  if (uid === undefined)
+    throw new Error("Vault runtime input ownership cannot be verified");
+  for (const target of [input.temporaryRoot, directory]) {
+    const info = await lstat(target);
+    if (
+      !info.isDirectory() ||
+      info.isSymbolicLink() ||
+      !privateVaultMetadataMatches(info, uid)
+    )
+      throw new Error("Vault runtime input directory is not private and owned");
+  }
+  const handle = await open(
+    filename,
+    constants.O_RDONLY | constants.O_NOFOLLOW,
+  );
+  let envelope: Envelope;
+  try {
+    const before = await handle.stat();
+    if (
+      !before.isFile() ||
+      !privateVaultMetadataMatches(before, uid) ||
+      before.size > 262144 ||
+      !(await realpath(filename)).startsWith(
+        (await realpath(input.temporaryRoot)) + path.sep,
+      )
+    )
+      throw new Error("Vault runtime input file is not private and owned");
+    const bytes = await handle.readFile(),
+      after = await handle.stat(),
+      named = await lstat(filename);
+    if (
+      before.ino !== after.ino ||
+      before.dev !== after.dev ||
+      before.size !== after.size ||
+      before.mtimeMs !== after.mtimeMs ||
+      named.ino !== before.ino ||
+      named.dev !== before.dev
+    )
+      throw new Error("Vault private input changed during read");
+    try {
+      envelope = JSON.parse(bytes.toString("utf8")) as Envelope;
+    } catch {
+      throw new Error("Vault runtime input JSON is invalid; contents withheld");
+    }
+  } finally {
+    await handle.close();
+  }
+  if (envelope.version !== 1 || envelope.executionId !== vaultExecutionId)
+    throw new Error("Vault runtime input belongs to another execution");
+  // Ordinary reserved setup handles missing selected provider inputs. It never
+  // receives an ambient fallback merely because the private binding is absent.
+  if (!envelope.external?.providers?.vault || envelope.setupBlockers?.length)
+    return {};
+  const binding = envelope.external.providers.vault;
+  if (envelope.external.schemaVersion !== 1 || !validVaultBinding(binding))
+    throw new Error(
+      "Vault runtime binding does not designate the exact allowed AWS resources",
+    );
+  const declaration = envelope.declaredExternalRequirements;
+  if (
+    declaration?.length !== 1 ||
+    declaration[0]?.provider !== "vault" ||
+    [...declaration[0].fields].sort().join() !==
+      [...vaultPrivateFields].sort().join()
+  )
+    throw new Error(
+      "Vault runtime fields differ from the selected fingerprinted declaration",
+    );
+  const p = binding.prerequisites;
+  const read = (
+    field: "accessKeyIdEnv" | "secretAccessKeyEnv" | "sessionTokenEnv",
+  ) => {
+    const name = p[field]!;
+    if (!envelope.declaredReferenceNames?.includes(name))
+      throw new Error(
+        "Vault runtime credential is not an explicitly selected reference",
+      );
+    const value = envelope.references?.[name];
+    if (
+      typeof value !== "string" ||
+      !value ||
+      /[\r\n\0]/.test(value) ||
+      value.length > 16384
+    )
+      throw new Error(
+        "Selected AWS runtime credential is missing or invalid; value withheld",
+      );
+    return value;
+  };
+  const accessKey = read("accessKeyIdEnv"),
+    secretKey = read("secretAccessKeyEnv"),
+    sessionToken = read("sessionTokenEnv");
+  if (
+    !/^ASIA[A-Z0-9]{12,124}$/.test(accessKey) ||
+    secretKey.length < 16 ||
+    sessionToken.length < 16
+  )
+    throw new Error(
+      "Vault runtime requires explicitly selected temporary AWS credentials",
+    );
+  const awsConfig = path.join(directory, "aws-vault-empty-config");
+  try {
+    await writeFile(awsConfig, "", { mode: 0o600, flag: "wx" });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    const info = await lstat(awsConfig);
+    if (
+      !info.isFile() ||
+      info.isSymbolicLink() ||
+      info.size !== 0 ||
+      !privateVaultMetadataMatches(info, uid)
+    )
+      throw new Error("Selected empty AWS config file is unsafe");
+  }
+  return {
+    AWS_ACCESS_KEY_ID: accessKey,
+    AWS_SECRET_ACCESS_KEY: secretKey,
+    AWS_SESSION_TOKEN: sessionToken,
+    AWS_REGION: p.region,
+    AWS_DEFAULT_REGION: p.region,
+    AWS_EC2_METADATA_DISABLED: "true",
+    AWS_CONFIG_FILE: awsConfig,
+    AWS_SHARED_CREDENTIALS_FILE: awsConfig,
+    PAPERCLIP_SECRETS_AWS_REGION: p.region,
+    PAPERCLIP_SECRETS_AWS_DEPLOYMENT_ID: p.deploymentId,
+    PAPERCLIP_SECRETS_AWS_KMS_KEY_ID: p.kmsKeyId,
+  };
+}

tests/runner-e2e/harness-env.ts

@@ -1,3 +1,4 @@
+import { PUBLIC_ORIGIN_CONTROL_ENV, PUBLIC_ORIGIN_RECEIPT_ENV } from "./chat-public-origin.js";
 import { scrubChatPrivateEnvironment } from "./chat-private-inputs.js";
 import { PRIVATE_DIAGNOSTICS_ENV } from "./private-diagnostics.js";
 import { installedDaytonaPluginKeys } from "./installed-daytona-plugin.js";
@@ -35,6 +36,13 @@ const AMBIENT_EXTERNAL_STATE_KEYS = [
 ] as const;
 const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|BRAINTRUST|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/;
 
+/** Vault SDK authority is never ambient chat input. An explicitly selected
+ * server-only vault handoff supplies its own finite overrides after scrubbing. */
+export function scrubAmbientVaultEnvironment(env: NodeJS.ProcessEnv) {
+  for (const name of Object.keys(env)) if (name.startsWith("AWS_") || name.startsWith("PAPERCLIP_SECRETS_AWS_")) delete env[name];
+  return env;
+}
+
 export function applyEvalGatewayEnvironment(env: NodeJS.ProcessEnv, credential: string, gateway: {
   token: string; baseUrl: string; phasePath: string; observability?: { token: string; baseUrl: string };
 }) {
@@ -133,6 +141,8 @@ export function buildRunnerE2EProcessEnvironment(
   const installed = usesInstalledCli(source);
   if (installed) assertInstalledCliSelection(source, executions);
   const result = { ...source };
+  if (executions.some(execution => execution.suite.id === "chat-features")) scrubAmbientVaultEnvironment(result);
+  delete result[PUBLIC_ORIGIN_CONTROL_ENV]; delete result[PUBLIC_ORIGIN_RECEIPT_ENV];
   if (executions.some(execution => execution.suite.id === "chat-features")) scrubChatPrivateEnvironment(result, executions);
   else scrubChatPrivateEnvironment(result, []);
   delete result.PAPERCLIP_E2E_GATEWAY_PHASE_PATH;
@@ -199,6 +209,8 @@ export function buildPaperclipServerEnvironment(
   overrides: NodeJS.ProcessEnv = {},
 ): NodeJS.ProcessEnv {
   const result = scrubChatPrivateEnvironment({ ...source }, [], true);
+  scrubAmbientVaultEnvironment(result);
+  delete result[PUBLIC_ORIGIN_CONTROL_ENV]; delete result[PUBLIC_ORIGIN_RECEIPT_ENV];
   delete result[PRIVATE_DIAGNOSTICS_ENV];
   for (const key of Object.keys(result)) {
     if (PROVIDER_SECRET_KEY.test(key)) delete result[key];