From 872988beb1c444d3bb54ca9d17cd022aa4ec1587 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 16:00:03 -0500 Subject: [PATCH] test(evals): reconcile runtime descriptors before campaign admission --- tests/runner-e2e/CHAT-FEATURES.md | 11 + .../chat-catalog-consistency.test.ts | 36 +++ tests/runner-e2e/chat-catalog-consistency.ts | 25 +++ .../runner-e2e/chat-catalog-descriptors.json | 205 ++++++++++++++++-- 4 files changed, 253 insertions(+), 24 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-consistency.test.ts create mode 100644 tests/runner-e2e/chat-catalog-consistency.ts diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 86f675b2340..709627937c5 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -245,3 +245,14 @@ single ordinary lifecycle hello. The closed receipt records only HTTP status, recognition, expiry and key-budget availability; unlimited/unknown key budgets do not establish account credit or model access. Response bodies, labels, amounts, account identities and key values are never retained. There is no retry. + +Before freezing a campaign candidate, run the offline runtime-declaration guard: + +```sh +node --import ./cli/node_modules/tsx/dist/loader.mjs tests/runner-e2e/chat-catalog-consistency.ts +``` + +It imports all 242 lazy declarations through the existing runtime loaders without +calling fixture, browser, database, or provider callbacks. Catalog listing alone +reads the discovery JSON and cannot establish runtime declaration consistency. +The same guard also runs in the existing Product E2E unit-test command. diff --git a/tests/runner-e2e/chat-catalog-consistency.test.ts b/tests/runner-e2e/chat-catalog-consistency.test.ts new file mode 100644 index 00000000000..240945d92a6 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-consistency.test.ts @@ -0,0 +1,36 @@ +import { expect, it, vi } from "vitest"; +import { validateChatCatalogConsistency } from "./chat-catalog-consistency.js"; +import { chatCatalogCases } from "./chat-catalog-registry.js"; +import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; +import { validChatCatalogHumanCounts, chatCatalogHumanHistoryMatches } from "./chat-catalog-contract.js"; + +it("loads all 242 actual lazy declarations through the admission loaders without executing fixtures", async () => { + expect(chatCatalogCases).toHaveLength(242); + expect([...chatCatalogCases.map(row => row.scenarioId), "tasks.create-backlog", "tasks.delegate-announcement"].sort()) + .toEqual(chatCatalogScenarios.map(row => row.id).sort()); + expect(await validateChatCatalogConsistency()).toEqual({ descriptorCount: 242, fixtureCallbacksInvoked: 0 }); +}); +it("rejects actual declaration drift before any fixture callback executes", async () => { + const descriptor = chatCatalogCases.find(row => row.scenarioId === "agents.hire-denied")!; + const callback = vi.fn(() => { throw new Error("Fixture must not execute during preflight"); }); + const runtime = { ...descriptor, setup: callback, observe: callback, grade: callback, verifyResult: callback }; + const importer = async () => ({ [descriptor.exportName]: [runtime] }); + await expect(validateChatCatalogConsistency([descriptor], importer)).resolves.toMatchObject({ descriptorCount: 1 }); + await expect(validateChatCatalogConsistency([{ ...descriptor, sourceFiles: [...descriptor.sourceFiles, "unreviewed.ts"] }], importer)).rejects.toThrow("Runtime descriptor drift"); + await expect(validateChatCatalogConsistency([{ ...descriptor, expectedRunCount: descriptor.expectedRunCount + 1 }], importer)).rejects.toThrow("Runtime descriptor drift"); + expect(callback).not.toHaveBeenCalled(); +}); +it("preserves the effective one-human limit when optional declaration defaults are used", () => { + const choice = chatCatalogCases.find(row => row.scenarioId === "execution-environments.choose-capable-target")!; + const explicit = { ...choice, expectedHumanMessageCount: 1 }; + const effective = (row: typeof choice) => row.expectedHumanMessageCount ?? 1; + expect(effective(choice)).toBe(1); expect(effective(explicit)).toBe(1); + expect(validChatCatalogHumanCounts(choice)).toBe(true); expect(validChatCatalogHumanCounts(explicit)).toBe(true); + expect(validChatCatalogHumanCounts({ ...choice, expectedHumanMessageCount: 0 })).toBe(false); + const history = { human: [{ id: "canonical", body: "Original query" }], runIds: [], boundary: { humanCommentIds: [], runIds: [] }, renderedQuery: "Original query" }; + for (const implementation of [choice, explicit]) { + expect(chatCatalogHumanHistoryMatches({ ...history, implementation })).toBe(true); + expect(chatCatalogHumanHistoryMatches({ ...history, implementation, human: [] })).toBe(false); + expect(chatCatalogHumanHistoryMatches({ ...history, implementation, human: [...history.human, { id: "extra", body: "Hidden nudge" }] })).toBe(false); + } +}); diff --git a/tests/runner-e2e/chat-catalog-consistency.ts b/tests/runner-e2e/chat-catalog-consistency.ts new file mode 100644 index 00000000000..e165726dbc2 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-consistency.ts @@ -0,0 +1,25 @@ +import { pathToFileURL } from "node:url"; +import { chatCatalogCases, loadChatCatalogImplementation, loadIdentityBoundaryImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js"; + +/** Offline admission guard. Load declarations only; never invoke fixture callbacks. */ +export async function validateChatCatalogConsistency( + descriptors: readonly ChatCatalogDescriptor[] = chatCatalogCases, + importer?: (specifier: string) => Promise>, +) { + const failures: string[] = []; + for (const descriptor of descriptors) { + try { + if (descriptor.journeyKind === "browser-boundary") await loadIdentityBoundaryImplementation(descriptor, importer); + else await loadChatCatalogImplementation(descriptor, importer); + } catch (error) { + failures.push(`${descriptor.scenarioId}: ${error instanceof Error ? error.message : "Unknown declaration error"}`); + } + } + if (failures.length) throw new Error(`Chat catalog consistency failed (${failures.length}/${descriptors.length}):\n${failures.join("\n")}`); + return { descriptorCount: descriptors.length, fixtureCallbacksInvoked: 0 as const }; +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + validateChatCatalogConsistency().then(receipt => console.log(JSON.stringify(receipt))).catch(error => { + console.error(error instanceof Error ? error.message : "Chat catalog consistency failed"); process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 5d8c48cf8c3..8530516651b 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -38,7 +38,7 @@ ], "oracleVersion": "paperclip.chat-catalog.identity.v1", "coverageGaps": [ - "The current public grant-management route rejects agent membership principals. Required users:* agent authority cannot be established through the supported API; actual intended mutation and human acceptance/reuse callbacks remain unqualified until this setup capability is available." + "The current public grant-management route rejects agent membership principals. Required users:* agent authority cannot be established through the supported API; the actual intended mutation and human acceptance/reuse callbacks are authored but remain unqualified until this setup capability is available." ], "expectedRunCount": 1, "turnTimeoutMs": 120000, @@ -65,7 +65,7 @@ ], "oracleVersion": "paperclip.chat-catalog.identity.v1", "coverageGaps": [ - "The current public grant-management route rejects agent membership principals. Required users:* agent authority cannot be established through the supported API; actual intended mutation and human acceptance/reuse callbacks remain unqualified until this setup capability is available." + "The current public grant-management route rejects agent membership principals. Required users:* agent authority cannot be established through the supported API; the actual intended mutation and human acceptance/reuse callbacks are authored but remain unqualified until this setup capability is available." ], "expectedRunCount": 1, "turnTimeoutMs": 120000, @@ -147,7 +147,9 @@ "chat-scenarios/activity.yaml", "chat-catalog-domains/internal-agent-feedback.ts", "chat-catalog-domains/internal-privacy.ts", - "chat-catalog-domains/identity-auth.ts" + "chat-catalog-domains/identity-auth.ts", + "chat-catalog-domains/internal-commentary-reader.ts", + "chat-run-evidence-observer.ts" ], "oracleVersion": "paperclip.chat-catalog.internal.v1", "coverageGaps": [], @@ -805,6 +807,7 @@ "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-flow.ts", + "chat-query-attachments.ts", "run-observations.ts", "../../server/src/routes/issues.ts", "../../server/src/routes/tool-access.ts", @@ -823,7 +826,13 @@ "attemptTimeoutMs": 1140000, "authentication": "human-session", "modulePath": "./chat-catalog-domains/external-app-review.js", - "exportName": "appReviewExternalCases" + "exportName": "appReviewExternalCases", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "launch-update.md" + } + ] }, { "externalInput": { @@ -936,6 +945,11 @@ "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", + "../../ui/src/pages/apps/Browse.tsx", + "../../ui/src/pages/apps/ConnectorsToolbar.tsx", + "../../ui/src/pages/apps/connection-owner.tsx", + "../../ui/src/pages/apps/AppDetail.tsx", + "../../ui/src/pages/apps/app-detail/PermissionsPanel.tsx", "chat-scenarios/app-permissions.yaml" ], "oracleVersion": "paperclip.chat-catalog.external.v1", @@ -1031,6 +1045,13 @@ "scenarioId": "assistant-connections.device-login", "sourceFiles": [ "chat-catalog-domains/external-assistant-device.ts", + "chat-catalog-domains/external-private-browser.ts", + "chat-catalog-domains/external-private-browser-worker.ts", + "chat-catalog-domains/external-device-private-contract.ts", + "../../ui/src/pages/McpConnect.tsx", + "../../server/src/routes/public-mcp.ts", + "../../server/src/services/public-mcp/oauth.ts", + "../../packages/shared/src/public-mcp.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", @@ -1335,7 +1356,14 @@ "maximumActiveWorkerRuns": 1 }, "modulePath": "./chat-catalog-domains/integrator-case-create.js", - "exportName": "integratorCaseCreationCases" + "exportName": "integratorCaseCreationCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "customer-bug-brief.md" + } + ] }, { "respondingAgent": { @@ -1817,7 +1845,14 @@ "attemptTimeoutMs": 420000, "authentication": "human-session", "modulePath": "./chat-catalog-domains/privileged-cli.js", - "exportName": "privilegedCliCases" + "exportName": "privilegedCliCases", + "setupPolicy": { + "readiness": "preserve-scenario", + "invariantCheckIds": [ + "lead-ready", + "owned-manual-pause" + ] + } }, { "scenarioId": "cli-operations.export-document-cli", @@ -2138,11 +2173,14 @@ "scenarioId": "connection-setup.choose-provider", "sourceFiles": [ "chat-catalog-domains/external-provider-choice.ts", + "run-observations.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../packages/shared/src/app-definitions.ts", "../../packages/shared/src/app-definitions/notion.json", + "../../ui/src/features/connections/ConnectionIntentInteractionBody.tsx", + "../../ui/src/features/connections/ConnectionSetupFlow.tsx", "chat-scenarios/connection-setup.yaml" ], "oracleVersion": "paperclip.chat-catalog.external.v1", @@ -2232,6 +2270,8 @@ "chat-catalog-domains/external-connection-setup.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", + "../../ui/src/features/connections/ConnectionIntentInteractionBody.tsx", + "../../ui/src/api/connection-intents.ts", "chat-scenarios/connection-setup.yaml", "chat-catalog-domains/identity-support.ts" ], @@ -2523,6 +2563,10 @@ "scenarioId": "connection-setup.secure-api-key", "sourceFiles": [ "chat-catalog-domains/external-secure-key.ts", + "chat-query-attachments.ts", + "../../ui/src/App.tsx", + "../../ui/src/pages/apps/app-connect-policy.ts", + "../../ui/src/features/connections/ConnectionSetupFlow.tsx", "chat-catalog-domains/external-private-browser.ts", "chat-catalog-domains/external-private-browser-worker.ts", "chat-catalog-domains/external-credential-language.ts", @@ -2540,7 +2584,13 @@ "attemptTimeoutMs": 420000, "authentication": "human-session", "modulePath": "./chat-catalog-domains/external-secure-key.js", - "exportName": "secureKeyExternalCases" + "exportName": "secureKeyExternalCases", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "provider-account.md" + } + ] }, { "scenarioId": "connection-setup.unreachable-mcp", @@ -2616,7 +2666,8 @@ "chat-catalog-domains/privileged-production-ui.ts", "../../ui/vite.config.ts", "../../ui/src/App.tsx", - "../../ui/src/pages/NotFound.tsx" + "../../ui/src/pages/NotFound.tsx", + "chat-query-attachments.ts" ], "oracleVersion": "paperclip.chat-catalog.privileged.v1", "coverageGaps": [], @@ -2625,7 +2676,13 @@ "attemptTimeoutMs": 600000, "authentication": "human-session", "modulePath": "./chat-catalog-domains/privileged-gated-lab.js", - "exportName": "privilegedGatedLabCases" + "exportName": "privilegedGatedLabCases", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "historical-lab-link.md" + } + ] }, { "scenarioId": "developer-labs.interaction-fixture", @@ -2723,7 +2780,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 600000, "modulePath": "./chat-catalog-domains/internal-website-delivery.js", - "exportName": "websiteDeliveryCases" + "exportName": "websiteDeliveryCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "selected-source.md" + } + ] }, { "scenarioId": "documents-artifacts.resolve-feedback", @@ -2740,7 +2804,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, "modulePath": "./chat-catalog-domains/internal-documents-artifacts.js", - "exportName": "documents_artifactsCases" + "exportName": "documents_artifactsCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "selected-source.md" + } + ] }, { "scenarioId": "documents-artifacts.restore-document", @@ -2761,7 +2832,13 @@ "attemptTimeoutMs": 660000, "authentication": "human-session", "modulePath": "./chat-catalog-domains/internal-documents-artifacts.js", - "exportName": "documents_artifactsCases" + "exportName": "documents_artifactsCases", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "selected-source.md" + } + ] }, { "scenarioId": "documents-artifacts.revise-delivery", @@ -2780,7 +2857,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 600000, "modulePath": "./chat-catalog-domains/internal-website-delivery.js", - "exportName": "websiteDeliveryCases" + "exportName": "websiteDeliveryCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "selected-source.md" + } + ] }, { "scenarioId": "documents-artifacts.write-plan", @@ -2894,7 +2978,6 @@ "oracleVersion": "paperclip.chat-catalog.runtime.v2", "coverageGaps": [], "expectedRunCount": 1, - "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 900000, "authentication": "human-session", @@ -3081,6 +3164,11 @@ "scenarioId": "gateways-profiles.rotate-token", "sourceFiles": [ "chat-catalog-domains/external-gateway-rotation.ts", + "chat-catalog-domains/external-gateway-handoff.ts", + "../../ui/src/components/AppsSidebar.tsx", + "../../ui/src/components/IssueThreadInteractionCard.tsx", + "../../ui/src/pages/apps/gateways/GatewaysList.tsx", + "../../ui/src/pages/apps/gateways/GatewayDetail.tsx", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-secret-scan.ts", "chat-catalog-domains/external-support.ts", @@ -3322,6 +3410,8 @@ "chat-catalog-domains/privileged-support.ts", "chat-catalog-domains/privileged-backup.ts", "chat-scenarios/instance-operations.yaml", + "chat-catalog-domains/privileged-docker-database.ts", + "docker-postgres.ts", "../../packages/db/src/backup-lib.ts", "../../server/src/routes/instance-database-backups.ts" ], @@ -3342,6 +3432,8 @@ "chat-catalog-domains/privileged-diagnose-health.ts", "chat-scenarios/instance-operations.yaml", "chat-catalog-domains/privileged-database-fault.ts", + "chat-catalog-domains/privileged-docker-database.ts", + "docker-postgres.ts", "../../cli/src/checks/database-check.ts", "../../server/src/routes/health.ts" ], @@ -4591,7 +4683,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 720000, "modulePath": "./chat-catalog-domains/integrator-routine-webhook.js", - "exportName": "integratorRoutineWebhookCases" + "exportName": "integratorRoutineWebhookCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "selected-routine.md" + } + ] }, { "scenarioId": "runs-adapters.inspect-failed-run", @@ -4599,7 +4698,11 @@ "chat-catalog-domains/runtime-failed-runs.ts", "chat-catalog-domains/runtime-native-failure.ts", "chat-catalog-domains/runtime-support.ts", - "chat-scenarios/runs-adapters.yaml" + "chat-scenarios/runs-adapters.yaml", + "../../ui/src/components/TaskChatThread.tsx", + "../../ui/src/components/task-chat/TaskChatMarker.tsx", + "../../ui/src/components/task-chat/TaskChatThreadView.tsx", + "../../ui/src/pages/AgentDetail.tsx" ], "oracleVersion": "paperclip.chat-catalog.runtime.v2", "coverageGaps": [], @@ -4637,7 +4740,9 @@ "chat-catalog-domains/runtime-session.ts", "chat-catalog-domains/runtime-workspaces.ts", "chat-catalog-domains/runtime-support.ts", - "chat-scenarios/runs-adapters.yaml" + "chat-scenarios/runs-adapters.yaml", + "../../ui/src/components/AgentActionButtons.tsx", + "../../ui/src/pages/AgentDetail.tsx" ], "oracleVersion": "paperclip.chat-catalog.runtime.v2", "coverageGaps": [], @@ -4787,6 +4892,9 @@ "../../server/src/services/heartbeat/run-preparation.ts", "../../cli/dist/index.js", "../../cli/src/index.ts", + "../../cli/node_modules/tsx/dist/loader.mjs", + "chat-catalog-domains/external-plugin-sdk-mirror.ts", + "../../packages/plugins/sdk/package.json", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", @@ -4926,7 +5034,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, "modulePath": "./chat-catalog-domains/integrator-skills.js", - "exportName": "integratorSkillsCases" + "exportName": "integratorSkillsCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "skill-context.md" + } + ] }, { "scenarioId": "skills.enable-for-agent", @@ -4986,7 +5101,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, "modulePath": "./chat-catalog-domains/integrator-skill-imports.js", - "exportName": "integratorSkillImportCases" + "exportName": "integratorSkillImportCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "skill-context.md" + } + ] }, { "scenarioId": "skills.fork-read-only", @@ -5004,7 +5126,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, "modulePath": "./chat-catalog-domains/integrator-skill-imports.js", - "exportName": "integratorSkillImportCases" + "exportName": "integratorSkillImportCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "skill-context.md" + } + ] }, { "scenarioId": "skills.review-update", @@ -5021,7 +5150,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 540000, "modulePath": "./chat-catalog-domains/integrator-skill-update.js", - "exportName": "integratorSkillUpdateCases" + "exportName": "integratorSkillUpdateCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "skill-context.md" + } + ] }, { "scenarioId": "skills.studio-test", @@ -5038,7 +5174,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 540000, "modulePath": "./chat-catalog-domains/integrator-skill-studio.js", - "exportName": "integratorSkillStudioCases" + "exportName": "integratorSkillStudioCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "skill-context.md" + } + ] }, { "scenarioId": "status-cards.archive-card", @@ -5054,7 +5197,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 720000, "modulePath": "./chat-catalog-domains/integrator-status-cards.js", - "exportName": "integratorStatusCardsCases" + "exportName": "integratorStatusCardsCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "selected-status-card.md" + } + ] }, { "scenarioId": "status-cards.create-card", @@ -5088,7 +5238,14 @@ "turnTimeoutMs": 120000, "attemptTimeoutMs": 720000, "modulePath": "./chat-catalog-domains/integrator-status-cards.js", - "exportName": "integratorStatusCardsCases" + "exportName": "integratorStatusCardsCases", + "authentication": "human-session", + "queryAttachments": [ + { + "stateKey": "factualContextAttachment", + "filename": "selected-status-card.md" + } + ] }, { "scenarioId": "status-cards.other-authors-card",