diff --git a/tests/runner-e2e/chat-private-inputs.test.ts b/tests/runner-e2e/chat-private-inputs.test.ts index d1a8b3be47c..fc118eab2d6 100644 --- a/tests/runner-e2e/chat-private-inputs.test.ts +++ b/tests/runner-e2e/chat-private-inputs.test.ts @@ -5,6 +5,7 @@ import { afterEach, expect, it } from "vitest"; import { stageChatPrivateInputs, readChatPrivateInputs, chatPrivateInputNameIdentity, scrubChatPrivateEnvironment, CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js"; import type { MatrixExecution } from "./types.js"; import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js"; +import { runtimePrivateRequirements } from "./chat-runtime-private-requirements.js"; const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); function fixture() { const root = mkdtempSync(path.join(tmpdir(), "chat-private-")); roots.push(root); return root; } @@ -42,3 +43,39 @@ it("resolves only the selected provider references and fingerprints names rather const identity = JSON.stringify(chatPrivateInputNameIdentity("selected", env)); expect(identity).toContain("PAPERCLIP_CHAT_EVAL_NOTION_AUTH"); expect(identity).not.toContain("notion-private"); expect(identity).not.toContain("test-space"); }); +it("stages only an exact declared runtime metadata contract and standard key without dummy provider refs", () => { + const root = fixture(), configFile = path.join(root, "runtime.json"); + const scenarioId = "execution-environments.prepare-image"; const spec = runtimePrivateRequirements[scenarioId]; + const descriptor = { scenarioId, externalInput: { provider: spec.provider, fields: spec.fields }, requiredCredentials: spec.credentials }; + const runtimeExecution = { ...execution, task: { id: scenarioId } } as MatrixExecution; + const prerequisites = { imageDigest: "sha256:synthetic-image-pin", imageRef: "owned-image", region: "test-region", disposableNamespace: "owned-test-namespace", toolsManifest: "private-tools-reference", unrelated: "not-selected" }; + writeFileSync(configFile, JSON.stringify({ schemaVersion: 1, providers: { daytona: { disposable: true, resourceIds: ["owned-test-namespace"], credentialEnv: [], prerequisites } } }), { mode: 0o600 }); + const staged = stageChatPrivateInputs({ execution: runtimeExecution, descriptor, privateDir: root, secrets: [], env: { PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG: configFile, DAYTONA_API_KEY: "selected-daytona-key", ANTHROPIC_API_KEY: "unrelated-key" } }); + const env = { [CHAT_PRIVATE_INPUTS_ENV]: staged.filename, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root }; + const inputs = readChatPrivateInputs("selected", [], env); + expect(inputs.credentials.DAYTONA_API_KEY).toBe("selected-daytona-key"); expect(inputs.credentials.ANTHROPIC_API_KEY).toBeUndefined(); + expect(JSON.stringify(inputs.configuration("external-provider"))).not.toContain("not-selected"); + const publicIdentity = JSON.stringify(chatPrivateInputNameIdentity("selected", env)); + expect(publicIdentity).toContain("imageDigest"); expect(publicIdentity).not.toContain("synthetic-image-pin"); expect(publicIdentity).not.toContain("selected-daytona-key"); + expect(() => stageChatPrivateInputs({ execution, descriptor, privateDir: root, secrets: [], env: {} })).toThrow("another scenario"); +}); +it("rejects undeclared runtime scenarios and fields before resolving or writing private values", () => { + const root = fixture(); const spec = runtimePrivateRequirements["execution-environments.prepare-image"]; + for (const descriptor of [ + { scenarioId: "tasks.create-backlog", externalInput: { provider: spec.provider, fields: spec.fields }, requiredCredentials: spec.credentials }, + { scenarioId: "execution-environments.prepare-image", externalInput: { provider: spec.provider, fields: [...spec.fields, "arbitraryCommand"] }, requiredCredentials: spec.credentials }, + { scenarioId: "execution-environments.prepare-image", externalInput: { provider: spec.provider, fields: spec.fields }, requiredCredentials: [] }, + ]) expect(() => stageChatPrivateInputs({ execution: { ...execution, task: { id: descriptor.scenarioId } } as MatrixExecution, descriptor, privateDir: root, secrets: [], env: {} })).toThrow("exact selected canonical case"); +}); +it("resolves only the finite SSH private key reference and refuses an alternate ambient name", () => { + const root = fixture(), configFile = path.join(root, "ssh.json"); const scenarioId = "execution-environments.test-computer"; const spec = runtimePrivateRequirements[scenarioId]; + const descriptor = { scenarioId, externalInput: { provider: spec.provider, fields: spec.fields }, requiredCredentials: spec.credentials }; + const prerequisites = { hostname: "synthetic.test", port: "22", login: "eval", hostKeySha256: "SHA256:synthetic", allowedRemoteRoot: "/owned/disposable", privateKeyEnv: "PAPERCLIP_CHAT_EVAL_SSH_PRIVATE_KEY", qualifiedHarness: "native-codex" }; + const config = { schemaVersion: 1, providers: { ssh: { disposable: true, resourceIds: ["owned-ssh"], credentialEnv: [], prerequisites } } }; + writeFileSync(configFile, JSON.stringify(config), { mode: 0o600 }); + const staged = stageChatPrivateInputs({ execution: { ...execution, task: { id: scenarioId } } as MatrixExecution, descriptor, privateDir: root, secrets: [], env: { PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG: configFile, PAPERCLIP_CHAT_EVAL_SSH_PRIVATE_KEY: "selected-ssh-key" } }); + const inputs = readChatPrivateInputs("selected", [], { [CHAT_PRIVATE_INPUTS_ENV]: staged.filename, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root }); + expect(inputs.read("PAPERCLIP_CHAT_EVAL_SSH_PRIVATE_KEY")).toBe("selected-ssh-key"); expect(inputs.read("PAPERCLIP_CHAT_EVAL_OTHER_KEY")).toBeUndefined(); + prerequisites.privateKeyEnv = "PAPERCLIP_CHAT_EVAL_OTHER_KEY"; writeFileSync(configFile, JSON.stringify(config), { mode: 0o600 }); + expect(() => stageChatPrivateInputs({ execution: { ...execution, task: { id: scenarioId } } as MatrixExecution, descriptor, privateDir: root, secrets: [], env: { PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG: configFile, PAPERCLIP_CHAT_EVAL_OTHER_KEY: "wrong-key" } })).toThrow("finite contract"); +}); diff --git a/tests/runner-e2e/chat-private-inputs.ts b/tests/runner-e2e/chat-private-inputs.ts index 800f413fa9f..0c101c128cb 100644 --- a/tests/runner-e2e/chat-private-inputs.ts +++ b/tests/runner-e2e/chat-private-inputs.ts @@ -3,10 +3,12 @@ import { lstatSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; import path from "node:path"; import { CREDENTIAL_NAMES, type CredentialName, type MatrixExecution } from "./types.js"; import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js"; +import { runtimePrivateProviders, runtimePrivateReferenceNames, validateRuntimePrivateRequirement, type RuntimePrivateProvider } from "./chat-runtime-private-requirements.js"; export const CHAT_PRIVATE_INPUTS_ENV = "PAPERCLIP_CHAT_PRIVATE_INPUTS"; const externalConfigEnv = "PAPERCLIP_CHAT_CATALOG_EXTERNAL_CONFIG"; -export interface ChatExternalInputRequirement { provider: "notion" | "slack" | "agentmail" | "vault" | "github"; fields: readonly string[] } +export interface ChatExternalInputRequirement { provider: "notion" | "slack" | "agentmail" | "vault" | "github" | RuntimePrivateProvider; fields: readonly string[] } +type PrivateInputDescriptor = Pick & Partial>; export interface ChatPrivateInputs { readonly credentials: Readonly>>; read(name: string): string | undefined; @@ -20,16 +22,18 @@ function readPrivateJson(filename: string, ownedRoot?: string): unknown { throw new Error("Selected-case inputs require a bounded private0600 owned file; contents withheld"); try { return JSON.parse(readFileSync(filename, "utf8")); } catch { throw new Error("Selected-case private inputs are invalid; contents withheld"); } } -export function chatPrivateInputRequirements(descriptor?: Pick) { +export function chatPrivateInputRequirements(descriptor?: PrivateInputDescriptor) { const credentialNames = [...new Set(descriptor?.requiredCredentials ?? [])].sort(); if (credentialNames.some(name => !CREDENTIAL_NAMES.includes(name))) throw new Error("Unknown selected-case credential name"); const external = descriptor?.externalInput; - if (external && (!["notion", "slack", "agentmail", "vault", "github"].includes(external.provider) + if (external && (!["notion", "slack", "agentmail", "vault", "github", ...runtimePrivateProviders].includes(external.provider) || !external.fields.length || external.fields.some(field => !/^[a-zA-Z][a-zA-Z0-9]*$/.test(field)))) throw new Error("Invalid selected external configuration requirements"); + if (external) validateRuntimePrivateRequirement(descriptor?.scenarioId, external.provider, external.fields, credentialNames); const requirements = { credentialNames, external: external ? { provider: external.provider, fields: [...external.fields].sort() } : null }; return { ...requirements, digest: createHash("sha256").update(JSON.stringify(requirements)).digest("hex") }; } -export function stageChatPrivateInputs(input: { execution: MatrixExecution; descriptor?: Pick; privateDir: string; env: NodeJS.ProcessEnv; secrets: string[] }) { +export function stageChatPrivateInputs(input: { execution: MatrixExecution; descriptor?: PrivateInputDescriptor; privateDir: string; env: NodeJS.ProcessEnv; secrets: string[] }) { + if (input.descriptor?.scenarioId && input.execution.task?.id !== input.descriptor.scenarioId) throw new Error("Selected private input declaration belongs to another scenario"); const requirements = chatPrivateInputRequirements(input.descriptor); const envelope: PrivateEnvelope = { version: 1, executionId: input.execution.id, credentials: {}, references: {} }; for (const name of requirements.credentialNames) { @@ -41,11 +45,15 @@ export function stageChatPrivateInputs(input: { execution: MatrixExecution; desc const config = readPrivateJson(ref) as { schemaVersion?: number; providers?: Record }> }; const binding = config.schemaVersion === 1 ? config.providers?.[requirements.external.provider] : undefined; if (!binding || binding.disposable !== true || !Array.isArray(binding.resourceIds) || !binding.resourceIds.length - || !Array.isArray(binding.credentialEnv) || !binding.credentialEnv.length || !binding.prerequisites - || requirements.external.fields.some(field => !binding.prerequisites?.[field])) throw new Error("Selected disposable external configuration lacks declared prerequisites; contents withheld"); + || !Array.isArray(binding.credentialEnv) || (!binding.credentialEnv.length && !runtimePrivateProviders.includes(requirements.external.provider)) || !binding.prerequisites + || requirements.external.fields.some(field => typeof binding.prerequisites?.[field] !== "string" || !binding.prerequisites[field]?.trim())) throw new Error("Selected disposable external configuration lacks declared prerequisites; contents withheld"); const selected = { disposable: true, resourceIds: binding.resourceIds, credentialEnv: binding.credentialEnv, prerequisites: Object.fromEntries(requirements.external.fields.map(field => [field, binding.prerequisites![field]])) }; - const names = [...binding.credentialEnv, ...requirements.external.fields.filter(field => field.endsWith("Env")).map(field => binding.prerequisites![field]!)]; + const names = [...binding.credentialEnv, ...requirements.external.fields.filter(field => /(?:Env|EnvName)$/.test(field)).map(field => binding.prerequisites![field]!)]; + if (runtimePrivateProviders.includes(requirements.external.provider)) { + const expected = runtimePrivateReferenceNames(requirements.external.provider); + if (expected.some(name => !names.includes(name)) || names.some(name => !expected.includes(name))) throw new Error("Runtime private reference names do not match the selected provider's finite contract"); + } for (const name of new Set(names)) { if (!/^PAPERCLIP_CHAT_EVAL_[A-Z0-9_]+$/.test(name)) throw new Error("External credential reference is not a declared private environment name"); const value = input.env[name]?.trim(); if (!value) throw new Error(`Missing selected external environment reference ${name}`); diff --git a/tests/runner-e2e/chat-runtime-private-requirements.ts b/tests/runner-e2e/chat-runtime-private-requirements.ts new file mode 100644 index 00000000000..5c03da442f5 --- /dev/null +++ b/tests/runner-e2e/chat-runtime-private-requirements.ts @@ -0,0 +1,29 @@ +/** Finite authoring contract for the actual remote/runtime scenario consumers. + * Metadata is private prerequisite context, never proof of a qualified target. */ +export const runtimePrivateRequirements = { + "execution-environments.prepare-image": { provider: "daytona", fields: ["imageDigest", "imageRef", "region", "disposableNamespace", "toolsManifest"], credentials: ["DAYTONA_API_KEY"] }, + "execution-environments.assign-computer": { provider: "daytona", fields: ["imageDigest", "imageRef", "region", "disposableNamespace", "qualifiedHarness"], credentials: ["DAYTONA_API_KEY"] }, + "agents.ambiguous-cloud-code": { provider: "daytona", fields: ["imageDigest", "imageRef", "region", "disposableNamespace", "qualifiedHarness"], credentials: ["ANTHROPIC_API_KEY", "DAYTONA_API_KEY"] }, + "execution-environments.add-computer": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] }, + "execution-environments.test-computer": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] }, + "execution-environments.choose-capable-target": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] }, + "agents.existing-grok-bot": { provider: "grok", fields: ["botId", "endpoint", "challengeMethod", "credentialEnvName"], credentials: [] }, + "agents.managed-profile": { provider: "anthropicManaged", fields: ["lockedEnvironmentId", "operatorProfileId", "retentionPolicyVersion", "retentionAckId", "providerAccountRef"], credentials: ["ANTHROPIC_API_KEY"] }, + "agents.hire-claude-code": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] }, + "agents.hire-needs-account": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] }, + "runs-adapters.validate-hire": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] }, + "runs-adapters.upgrade-runner": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef", "legacySettingsRef"], credentials: ["ANTHROPIC_API_KEY"] }, +} as const; +export type RuntimePrivateProvider = typeof runtimePrivateRequirements[keyof typeof runtimePrivateRequirements]["provider"]; +export const runtimePrivateProviders: readonly string[] = ["daytona", "ssh", "grok", "anthropicManaged", "claude"]; +export function validateRuntimePrivateRequirement(scenarioId: string | undefined, provider: string, fields: readonly string[], credentials: readonly string[]) { + if (!runtimePrivateProviders.includes(provider)) return false; + const spec = runtimePrivateRequirements[scenarioId as keyof typeof runtimePrivateRequirements]; + const same = (a: readonly string[], b: readonly string[]) => JSON.stringify([...a].sort()) === JSON.stringify([...b].sort()); + if (!spec || spec.provider !== provider || !same(fields, spec.fields) || !same(credentials, spec.credentials)) + throw new Error("Runtime private inputs require the exact selected canonical case, fields and credential names"); + return true; +} +export function runtimePrivateReferenceNames(provider: string): readonly string[] { + return provider === "ssh" ? ["PAPERCLIP_CHAT_EVAL_SSH_PRIVATE_KEY"] : provider === "grok" ? ["PAPERCLIP_CHAT_EVAL_GROK_BOT_TOKEN"] : []; +}