From 5196e94ad2d5d08836a97ae290692700ccbee46a Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 20:15:45 -0500 Subject: [PATCH] test(evals): wire repository fixtures to selected private inputs --- .../internal-repositories-private.test.ts | 28 +++++++++++++++++++ .../internal-repositories.ts | 14 ++++------ 2 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-repositories-private.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/internal-repositories-private.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-repositories-private.test.ts new file mode 100644 index 00000000000..e348836fa37 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-repositories-private.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { repositoryProjectCases } from "./internal-repositories.js"; +import { externalRequirements } from "./external-fixtures.js"; +describe("repository selected-input integration", () => { + for (const definition of repositoryProjectCases) it(`${definition.scenarioId} declares its selected GitHub inputs and never falls back to ambient configuration`, async () => { + expect(definition.externalInput).toEqual(externalRequirements[definition.scenarioId]); + const postSensitive = vi.fn(); + const context = { scenario: { id: definition.scenarioId }, input: { api: { postSensitive }, privateSecrets: [], evidence: async () => {} } } as unknown as ChatCatalogContext; + await expect(definition.setup(context)).rejects.toThrow("selected-case private inputs are unavailable"); + expect(postSensitive).not.toHaveBeenCalled(); + }); + it("uses only the selected reference while retaining actual repository discovery and cleanup guards", async () => { + const declaration = repositoryProjectCases.find(row => row.scenarioId === "projects.add-repository")!; + const binding = { disposable: true, resourceIds: ["42"], credentialEnv: ["PAPERCLIP_CHAT_EVAL_GITHUB_TOKEN"], prerequisites: { siteRepositoryId: "42", siteRepositoryFullName: "owned/site", tokenEnv: "PAPERCLIP_CHAT_EVAL_GITHUB_TOKEN" } }; + const postSensitive = vi.fn(async (path: string, _body?: unknown) => ({ id: path.endsWith("/secrets") ? "secret" : "connection" })); + const remove = vi.fn(async (_path: string, _options?: unknown) => {}); const read = vi.fn((name: string) => name === "PAPERCLIP_CHAT_EVAL_GITHUB_TOKEN" ? "selected-synthetic-token" : undefined); + const secrets: string[] = []; + const context = { scenario: { id: declaration.scenarioId }, input: { nonce: "synthetic", fixtures: { company: { id: "company" } }, privateSecrets: secrets, + privateInputs: { credentials: {}, read, configuration: () => ({ schemaVersion: 1, providers: { github: binding } }) }, evidence: async () => {}, + api: { postSensitive, get: async () => ({ repositories: [{ id: "999", fullName: "owned/site", url: "https://github.com/owned/site", connectionIds: ["connection"] }], failedConnectionCount: 0 }), delete: remove } } } as unknown as ChatCatalogContext; + await expect(declaration.setup(context)).rejects.toThrow("independently expose each explicit disposable repository name and ID"); + expect(postSensitive.mock.calls[0]?.[1]).toMatchObject({ value: "selected-synthetic-token" }); + expect(secrets).toContain("selected-synthetic-token"); + expect(read.mock.calls.every(([name]) => name === "PAPERCLIP_CHAT_EVAL_GITHUB_TOKEN")).toBe(true); + expect(remove.mock.calls.map(call => call[0])).toEqual(["/api/tool-connections/connection", "/api/secrets/secret"]); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-repositories.ts b/tests/runner-e2e/chat-catalog-domains/internal-repositories.ts index aab356a4bef..c140e544b2e 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-repositories.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-repositories.ts @@ -2,7 +2,7 @@ import { expect } from "@playwright/test"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; import { makeDefinition } from "./internal-definition.js"; -import { externalPrerequisiteReceipt, readExternalConfig, requireExternalPrerequisites } from "./external-fixtures.js"; +import { externalRequirements, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js"; import { checks, entity, hasLink, observation, observeInternal, project, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js"; interface Repository { id: string; fullName: string; url: string; connectionIds?: string[] } @@ -14,15 +14,13 @@ interface RepositoryObservation extends InternalObservation { function repositoriesOf(p: RepositoryProject | undefined) { return p?.workspaces.filter(row => row.repoUrl) ?? []; } const regexEscape = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); async function connectRepositoryFixture(ctx: ChatCatalogContext) { - const config = readExternalConfig(); const receipt = externalPrerequisiteReceipt(ctx.scenario.id, config); - await ctx.input.evidence("chat-catalog-github-prerequisite.json", receipt); - const binding = requireExternalPrerequisites(ctx.scenario.id, config); + const binding = selectedExternalBinding(ctx); + await ctx.input.evidence("chat-catalog-github-prerequisite.json", { scenarioId: ctx.scenario.id, provider: "github", configured: true, missing: [] }); const envName = binding.prerequisites.tokenEnv!; if (!binding.credentialEnv.includes(envName) || !/^PAPERCLIP_CHAT_EVAL_[A-Z0-9_]+$/.test(envName)) throw new Error("Setup blocker: GitHub tokenEnv must name an explicitly authorized scoped credentialEnv binding"); - const token = process.env[envName]?.trim(); - if (!token || !ctx.input.privateSecrets) throw new Error("Setup blocker: disposable GitHub credential and attempt-private redaction sink are required"); - ctx.input.privateSecrets.push(token); + if (!ctx.input.privateSecrets) throw new Error("Setup blocker: disposable GitHub credential and attempt-private redaction sink are required"); + const token = selectedExternalCredential(ctx, binding, "tokenEnv"); const requested = ctx.scenario.id === "projects.add-repository" ? [{ id: binding.prerequisites.siteRepositoryId!, fullName: binding.prerequisites.siteRepositoryFullName! }] : [{ id: binding.prerequisites.oldRepositoryId!, fullName: binding.prerequisites.oldRepositoryFullName! }, { id: binding.prerequisites.newRepositoryId!, fullName: binding.prerequisites.newRepositoryFullName! }]; @@ -134,7 +132,7 @@ function repositoryCase(scenarioId: string): ChatCatalogCase { await ctx.input.api.delete(`/api/secrets/${fixture.secretId}`, { allowNotFound: true }); throw error; } }, gradeRepositories); - return { ...base, authentication: "human-session", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-repositories.ts", "chat-catalog-domains/external-fixtures.ts"], + return { ...base, authentication: "human-session", externalInput: externalRequirements[scenarioId], sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-repositories.ts", "chat-catalog-domains/external-fixtures.ts"], queryBindings: scenarioId === "projects.add-repository" ? [{ catalogToken: "acorn/site", stateKey: "siteRepository", resourceKind: "project-repository", valueField: "fullName" }] : [{ catalogToken: "acorn/site-old", stateKey: "oldRepository", resourceKind: "project-repository", valueField: "fullName" },