test(evals): scope private runtime metadata to declared cases

tests/runner-e2e/chat-private-inputs.test.ts

@@ -5,6 +5,7 @@ import { afterEach, expect, it } from "vitest";
 import { stageChatPrivateInputs, readChatPrivateInputs, chatPrivateInputNameIdentity, scrubChatPrivateEnvironment, CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js";
 import type { MatrixExecution } from "./types.js";
 import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js";
+import { runtimePrivateRequirements } from "./chat-runtime-private-requirements.js";
 const roots: string[] = [];
 afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
 function fixture() { const root = mkdtempSync(path.join(tmpdir(), "chat-private-")); roots.push(root); return root; }
@@ -42,3 +43,39 @@ it("resolves only the selected provider references and fingerprints names rather
   const identity = JSON.stringify(chatPrivateInputNameIdentity("selected", env));
   expect(identity).toContain("PAPERCLIP_CHAT_EVAL_NOTION_AUTH"); expect(identity).not.toContain("notion-private"); expect(identity).not.toContain("test-space");
 });
+it("stages only an exact declared runtime metadata contract and standard key without dummy provider refs", () => {
+  const root = fixture(), configFile = path.join(root, "runtime.json");
+  const scenarioId = "execution-environments.prepare-image"; const spec = runtimePrivateRequirements[scenarioId];
+  const descriptor = { scenarioId, externalInput: { provider: spec.provider, fields: spec.fields }, requiredCredentials: spec.credentials };
+  const runtimeExecution = { ...execution, task: { id: scenarioId } } as MatrixExecution;
+  const prerequisites = { imageDigest: "sha256:synthetic-image-pin", imageRef: "owned-image", region: "test-region", disposableNamespace: "owned-test-namespace", toolsManifest: "private-tools-reference", unrelated: "not-selected" };
+  writeFileSync(configFile, JSON.stringify({ schemaVersion: 1, providers: { daytona: { disposable: true, resourceIds: ["owned-test-namespace"], credentialEnv: [], prerequisites } } }), { mode: 0o600 });
+  const staged = stageChatPrivateInputs({ execution: runtimeExecution, descriptor, privateDir: root, secrets: [], env: { PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG: configFile, DAYTONA_API_KEY: "selected-daytona-key", ANTHROPIC_API_KEY: "unrelated-key" } });
+  const env = { [CHAT_PRIVATE_INPUTS_ENV]: staged.filename, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root };
+  const inputs = readChatPrivateInputs("selected", [], env);
+  expect(inputs.credentials.DAYTONA_API_KEY).toBe("selected-daytona-key"); expect(inputs.credentials.ANTHROPIC_API_KEY).toBeUndefined();
+  expect(JSON.stringify(inputs.configuration("external-provider"))).not.toContain("not-selected");
+  const publicIdentity = JSON.stringify(chatPrivateInputNameIdentity("selected", env));
+  expect(publicIdentity).toContain("imageDigest"); expect(publicIdentity).not.toContain("synthetic-image-pin"); expect(publicIdentity).not.toContain("selected-daytona-key");
+  expect(() => stageChatPrivateInputs({ execution, descriptor, privateDir: root, secrets: [], env: {} })).toThrow("another scenario");
+});
+it("rejects undeclared runtime scenarios and fields before resolving or writing private values", () => {
+  const root = fixture(); const spec = runtimePrivateRequirements["execution-environments.prepare-image"];
+  for (const descriptor of [
+    { scenarioId: "tasks.create-backlog", externalInput: { provider: spec.provider, fields: spec.fields }, requiredCredentials: spec.credentials },
+    { scenarioId: "execution-environments.prepare-image", externalInput: { provider: spec.provider, fields: [...spec.fields, "arbitraryCommand"] }, requiredCredentials: spec.credentials },
+    { scenarioId: "execution-environments.prepare-image", externalInput: { provider: spec.provider, fields: spec.fields }, requiredCredentials: [] },
+  ]) expect(() => stageChatPrivateInputs({ execution: { ...execution, task: { id: descriptor.scenarioId } } as MatrixExecution, descriptor, privateDir: root, secrets: [], env: {} })).toThrow("exact selected canonical case");
+});
+it("resolves only the finite SSH private key reference and refuses an alternate ambient name", () => {
+  const root = fixture(), configFile = path.join(root, "ssh.json"); const scenarioId = "execution-environments.test-computer"; const spec = runtimePrivateRequirements[scenarioId];
+  const descriptor = { scenarioId, externalInput: { provider: spec.provider, fields: spec.fields }, requiredCredentials: spec.credentials };
+  const prerequisites = { hostname: "synthetic.test", port: "22", login: "eval", hostKeySha256: "SHA256:synthetic", allowedRemoteRoot: "/owned/disposable", privateKeyEnv: "PAPERCLIP_CHAT_EVAL_SSH_PRIVATE_KEY", qualifiedHarness: "native-codex" };
+  const config = { schemaVersion: 1, providers: { ssh: { disposable: true, resourceIds: ["owned-ssh"], credentialEnv: [], prerequisites } } };
+  writeFileSync(configFile, JSON.stringify(config), { mode: 0o600 });
+  const staged = stageChatPrivateInputs({ execution: { ...execution, task: { id: scenarioId } } as MatrixExecution, descriptor, privateDir: root, secrets: [], env: { PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG: configFile, PAPERCLIP_CHAT_EVAL_SSH_PRIVATE_KEY: "selected-ssh-key" } });
+  const inputs = readChatPrivateInputs("selected", [], { [CHAT_PRIVATE_INPUTS_ENV]: staged.filename, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root });
+  expect(inputs.read("PAPERCLIP_CHAT_EVAL_SSH_PRIVATE_KEY")).toBe("selected-ssh-key"); expect(inputs.read("PAPERCLIP_CHAT_EVAL_OTHER_KEY")).toBeUndefined();
+  prerequisites.privateKeyEnv = "PAPERCLIP_CHAT_EVAL_OTHER_KEY"; writeFileSync(configFile, JSON.stringify(config), { mode: 0o600 });
+  expect(() => stageChatPrivateInputs({ execution: { ...execution, task: { id: scenarioId } } as MatrixExecution, descriptor, privateDir: root, secrets: [], env: { PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG: configFile, PAPERCLIP_CHAT_EVAL_OTHER_KEY: "wrong-key" } })).toThrow("finite contract");
+});

tests/runner-e2e/chat-private-inputs.ts

@@ -3,10 +3,12 @@ import { lstatSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { CREDENTIAL_NAMES, type CredentialName, type MatrixExecution } from "./types.js";
 import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js";
+import { runtimePrivateProviders, runtimePrivateReferenceNames, validateRuntimePrivateRequirement, type RuntimePrivateProvider } from "./chat-runtime-private-requirements.js";
 
 export const CHAT_PRIVATE_INPUTS_ENV = "PAPERCLIP_CHAT_PRIVATE_INPUTS";
 const externalConfigEnv = "PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG";
-export interface ChatExternalInputRequirement { provider: "notion" | "slack" | "agentmail" | "vault" | "github"; fields: readonly string[] }
+export interface ChatExternalInputRequirement { provider: "notion" | "slack" | "agentmail" | "vault" | "github" | RuntimePrivateProvider; fields: readonly string[] }
+type PrivateInputDescriptor = Pick<ChatCatalogDescriptor, "requiredCredentials" | "externalInput"> & Partial<Pick<ChatCatalogDescriptor, "scenarioId">>;
 export interface ChatPrivateInputs {
   readonly credentials: Readonly<Partial<Record<CredentialName, string>>>;
   read(name: string): string | undefined;
@@ -20,16 +22,18 @@ function readPrivateJson(filename: string, ownedRoot?: string): unknown {
     throw new Error("Selected-case inputs require a bounded private0600 owned file; contents withheld");
   try { return JSON.parse(readFileSync(filename, "utf8")); } catch { throw new Error("Selected-case private inputs are invalid; contents withheld"); }
 }
-export function chatPrivateInputRequirements(descriptor?: Pick<ChatCatalogDescriptor, "requiredCredentials" | "externalInput">) {
+export function chatPrivateInputRequirements(descriptor?: PrivateInputDescriptor) {
   const credentialNames = [...new Set(descriptor?.requiredCredentials ?? [])].sort();
   if (credentialNames.some(name => !CREDENTIAL_NAMES.includes(name))) throw new Error("Unknown selected-case credential name");
   const external = descriptor?.externalInput;
-  if (external && (!["notion", "slack", "agentmail", "vault", "github"].includes(external.provider)
+  if (external && (!["notion", "slack", "agentmail", "vault", "github", ...runtimePrivateProviders].includes(external.provider)
     || !external.fields.length || external.fields.some(field => !/^[a-zA-Z][a-zA-Z0-9]*$/.test(field)))) throw new Error("Invalid selected external configuration requirements");
+  if (external) validateRuntimePrivateRequirement(descriptor?.scenarioId, external.provider, external.fields, credentialNames);
   const requirements = { credentialNames, external: external ? { provider: external.provider, fields: [...external.fields].sort() } : null };
   return { ...requirements, digest: createHash("sha256").update(JSON.stringify(requirements)).digest("hex") };
 }
-export function stageChatPrivateInputs(input: { execution: MatrixExecution; descriptor?: Pick<ChatCatalogDescriptor, "requiredCredentials" | "externalInput">; privateDir: string; env: NodeJS.ProcessEnv; secrets: string[] }) {
+export function stageChatPrivateInputs(input: { execution: MatrixExecution; descriptor?: PrivateInputDescriptor; privateDir: string; env: NodeJS.ProcessEnv; secrets: string[] }) {
+  if (input.descriptor?.scenarioId && input.execution.task?.id !== input.descriptor.scenarioId) throw new Error("Selected private input declaration belongs to another scenario");
   const requirements = chatPrivateInputRequirements(input.descriptor);
   const envelope: PrivateEnvelope = { version: 1, executionId: input.execution.id, credentials: {}, references: {} };
   for (const name of requirements.credentialNames) {
@@ -41,11 +45,15 @@ export function stageChatPrivateInputs(input: { execution: MatrixExecution; desc
     const config = readPrivateJson(ref) as { schemaVersion?: number; providers?: Record<string, { disposable?: boolean; resourceIds?: string[]; credentialEnv?: string[]; prerequisites?: Record<string, string> }> };
     const binding = config.schemaVersion === 1 ? config.providers?.[requirements.external.provider] : undefined;
     if (!binding || binding.disposable !== true || !Array.isArray(binding.resourceIds) || !binding.resourceIds.length
-      || !Array.isArray(binding.credentialEnv) || !binding.credentialEnv.length || !binding.prerequisites
-      || requirements.external.fields.some(field => !binding.prerequisites?.[field])) throw new Error("Selected disposable external configuration lacks declared prerequisites; contents withheld");
+      || !Array.isArray(binding.credentialEnv) || (!binding.credentialEnv.length && !runtimePrivateProviders.includes(requirements.external.provider)) || !binding.prerequisites
+      || requirements.external.fields.some(field => typeof binding.prerequisites?.[field] !== "string" || !binding.prerequisites[field]?.trim())) throw new Error("Selected disposable external configuration lacks declared prerequisites; contents withheld");
     const selected = { disposable: true, resourceIds: binding.resourceIds, credentialEnv: binding.credentialEnv,
       prerequisites: Object.fromEntries(requirements.external.fields.map(field => [field, binding.prerequisites![field]])) };
-    const names = [...binding.credentialEnv, ...requirements.external.fields.filter(field => field.endsWith("Env")).map(field => binding.prerequisites![field]!)];
+    const names = [...binding.credentialEnv, ...requirements.external.fields.filter(field => /(?:Env|EnvName)$/.test(field)).map(field => binding.prerequisites![field]!)];
+    if (runtimePrivateProviders.includes(requirements.external.provider)) {
+      const expected = runtimePrivateReferenceNames(requirements.external.provider);
+      if (expected.some(name => !names.includes(name)) || names.some(name => !expected.includes(name))) throw new Error("Runtime private reference names do not match the selected provider's finite contract");
+    }
     for (const name of new Set(names)) {
       if (!/^PAPERCLIP_CHAT_EVAL_[A-Z0-9_]+$/.test(name)) throw new Error("External credential reference is not a declared private environment name");
       const value = input.env[name]?.trim(); if (!value) throw new Error(`Missing selected external environment reference ${name}`);

tests/runner-e2e/chat-runtime-private-requirements.ts

@@ -0,0 +1,29 @@
+/** Finite authoring contract for the actual remote/runtime scenario consumers.
+ * Metadata is private prerequisite context, never proof of a qualified target. */
+export const runtimePrivateRequirements = {
+  "execution-environments.prepare-image": { provider: "daytona", fields: ["imageDigest", "imageRef", "region", "disposableNamespace", "toolsManifest"], credentials: ["DAYTONA_API_KEY"] },
+  "execution-environments.assign-computer": { provider: "daytona", fields: ["imageDigest", "imageRef", "region", "disposableNamespace", "qualifiedHarness"], credentials: ["DAYTONA_API_KEY"] },
+  "agents.ambiguous-cloud-code": { provider: "daytona", fields: ["imageDigest", "imageRef", "region", "disposableNamespace", "qualifiedHarness"], credentials: ["ANTHROPIC_API_KEY", "DAYTONA_API_KEY"] },
+  "execution-environments.add-computer": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
+  "execution-environments.test-computer": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
+  "execution-environments.choose-capable-target": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
+  "agents.existing-grok-bot": { provider: "grok", fields: ["botId", "endpoint", "challengeMethod", "credentialEnvName"], credentials: [] },
+  "agents.managed-profile": { provider: "anthropicManaged", fields: ["lockedEnvironmentId", "operatorProfileId", "retentionPolicyVersion", "retentionAckId", "providerAccountRef"], credentials: ["ANTHROPIC_API_KEY"] },
+  "agents.hire-claude-code": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] },
+  "agents.hire-needs-account": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] },
+  "runs-adapters.validate-hire": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] },
+  "runs-adapters.upgrade-runner": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef", "legacySettingsRef"], credentials: ["ANTHROPIC_API_KEY"] },
+} as const;
+export type RuntimePrivateProvider = typeof runtimePrivateRequirements[keyof typeof runtimePrivateRequirements]["provider"];
+export const runtimePrivateProviders: readonly string[] = ["daytona", "ssh", "grok", "anthropicManaged", "claude"];
+export function validateRuntimePrivateRequirement(scenarioId: string | undefined, provider: string, fields: readonly string[], credentials: readonly string[]) {
+  if (!runtimePrivateProviders.includes(provider)) return false;
+  const spec = runtimePrivateRequirements[scenarioId as keyof typeof runtimePrivateRequirements];
+  const same = (a: readonly string[], b: readonly string[]) => JSON.stringify([...a].sort()) === JSON.stringify([...b].sort());
+  if (!spec || spec.provider !== provider || !same(fields, spec.fields) || !same(credentials, spec.credentials))
+    throw new Error("Runtime private inputs require the exact selected canonical case, fields and credential names");
+  return true;
+}
+export function runtimePrivateReferenceNames(provider: string): readonly string[] {
+  return provider === "ssh" ? ["PAPERCLIP_CHAT_EVAL_SSH_PRIVATE_KEY"] : provider === "grok" ? ["PAPERCLIP_CHAT_EVAL_GROK_BOT_TOKEN"] : [];
+}