From c619ebbb7bde89ff5f4aba260ae619d6b03147a7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 22:54:26 -0600 Subject: [PATCH] test(evals): state team runtime fixture provenance accurately Record original and effective package identities, distinguish preserved non-runtime fields from rewritten sidecar bytes, and document one-shot supervisor bootstrap with non-overwrite coverage. Co-Authored-By: Paperclip --- .../external-team-runtime-fixture.test.ts | 8 ++++++++ .../external-team-runtime-fixture.ts | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-runtime-fixture.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-runtime-fixture.test.ts index f92e22101fb..747f57f2fc7 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-runtime-fixture.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-runtime-fixture.test.ts @@ -18,10 +18,18 @@ it("serves a derived runtime-only catalog while preserving shipped roles, permis const result = await normalizeTeamRuntimeCatalog({ repositoryRoot, temporaryRoot, team, adapters }); expect(result.originalContentHash).toBe(team.contentHash); expect(result.effectiveContentHash).not.toBe(team.contentHash); + expect(result.originalPackage).toEqual({ id: team.id, key: team.key, packageName: original.packageName, version: original.packageVersion, contentHash: team.contentHash }); + expect(result.effectivePackage).toEqual({ ...result.originalPackage, contentHash: result.effectiveContentHash }); + expect(result.nonRuntimeFilesUnchanged).toBe(true); + expect(result.nonRuntimeSidecarFieldsUnchanged).toBe(true); const sidecar = YAML.parse(await readFile(path.join(teamRuntimeCatalogRoot(temporaryRoot), team.path, ".paperclip.yaml"), "utf8")); expect(sidecar.agents.cto.permissions).toEqual({ canCreateAgents: true }); for (const slug of team.agentSlugs) expect(sidecar.agents[slug].adapter).toEqual({ type: "paperclip_runner", config: adapterConfig }); expect(result.team.requiredSkills).toEqual(team.requiredSkills); + const sidecarPath = path.join(teamRuntimeCatalogRoot(temporaryRoot), team.path, ".paperclip.yaml"); + const beforeBootstrap = await readFile(sidecarPath); + await expect(teamRuntimeCatalogServerEnvironment({ executionIds: ["chat-features.runner-opencode.local.teams.install-team"], temporaryRoot, repositoryRoot })).rejects.toThrow(); + expect((await readFile(sidecarPath)).equals(beforeBootstrap)).toBe(true); await expect(normalizeTeamRuntimeCatalog({ repositoryRoot, temporaryRoot, team: { ...team, contentHash: "foreign" }, adapters })).rejects.toThrow("exact unmodified"); } finally { await rm(temporaryRoot, { recursive: true, force: true }); } }); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-runtime-fixture.ts b/tests/runner-e2e/chat-catalog-domains/external-team-runtime-fixture.ts index 7dfaeaa33be..7474087e1e8 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-runtime-fixture.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-runtime-fixture.ts @@ -10,7 +10,11 @@ import { reviewedChatExecutionIds } from "../chat-runtime-profile.js"; const selectedIds = reviewedChatExecutionIds("teams.install-team"); export const teamRuntimeCatalogRoot = (temporaryRoot: string) => path.join(temporaryRoot, "team-runtime-catalog"); -/** Only the disposable installer fixture uses this catalog and native default. */ +/** Only the disposable installer fixture uses this catalog and native default. + * One-shot supervisor setup: server.ts constructs this environment once, and its + * child-server restarts reuse it. Recreating the supervisor in the same attempt + * root is unsupported and fails closed rather than replacing a normalized copy. + */ export async function teamRuntimeCatalogServerEnvironment(input: { executionIds: string[]; temporaryRoot: string; repositoryRoot: string }) { if (!input.executionIds.some(id => selectedIds.includes(id))) return {}; if (input.executionIds.length !== 1) throw new Error("Team runtime catalog requires its isolated execution"); @@ -20,7 +24,7 @@ export async function teamRuntimeCatalogServerEnvironment(input: { executionIds: return { PAPERCLIP_TEAMS_CATALOG_DIR: target, PAPERCLIP_TEAMS_CATALOG_DEFAULT_ADAPTER_TYPE: "paperclip_runner" }; } -/** Add runtime/account metadata only; all shipped team files remain byte-identical. */ +/** Add runtime/account metadata only; preserve other file bytes and non-runtime sidecar fields. */ export async function normalizeTeamRuntimeCatalog(input: { temporaryRoot: string; repositoryRoot: string; team: CatalogTeam; adapters: Record }> }) { const packageDir = teamRuntimeCatalogRoot(input.temporaryRoot); const original = JSON.parse(await readFile(path.join(input.repositoryRoot, "packages/teams-catalog/generated/catalog.json"), "utf8")); @@ -52,5 +56,13 @@ export async function normalizeTeamRuntimeCatalog(input: { temporaryRoot: string const withoutRuntime = (team: CatalogTeam) => ({ ...team, contentHash: "", fileCount: 0, files: team.files.filter(file => file.path !== ".paperclip.yaml") }); if (!isDeepStrictEqual(withoutRuntime(effective), withoutRuntime(source)) || effective.contentHash === source.contentHash || built.manifest.teams.some(team => team.id !== source.id && !isDeepStrictEqual(team, original.teams.find((row: CatalogTeam) => row.id === team.id)))) throw new Error("Derived catalog changed non-runtime team metadata"); await writeFile(path.join(packageDir, "generated/catalog.json"), JSON.stringify(built.manifest, null, 2) + "\n"); - return { originalContentHash: source.contentHash, effectiveContentHash: effective.contentHash, originalFilesUnchanged: true, team: effective }; + return { + originalPackage: { id: source.id, key: source.key, packageName: original.packageName, version: original.packageVersion, contentHash: source.contentHash }, + effectivePackage: { id: effective.id, key: effective.key, packageName: built.manifest.packageName, version: built.manifest.packageVersion, contentHash: effective.contentHash }, + originalContentHash: source.contentHash, + effectiveContentHash: effective.contentHash, + nonRuntimeFilesUnchanged: true, + nonRuntimeSidecarFieldsUnchanged: true, + team: effective, + }; }