import { reviewedChatProfile } from "./chat-runtime-profile.js"; import { publicOriginFingerprintIdentity } from "./chat-public-origin.js"; import { databaseFingerprintIdentity } from "./docker-postgres.js"; import { verifyRetainedChatOracleIdentity } from "./chat-private-inputs.js"; import { installedChatFingerprintIdentity, installedChatScenarioIds } from "./chat-installed-lane.js"; import { chatPrivateInputRequirements, chatPrivateInputNameIdentity } from "./chat-private-inputs.js"; import { createHash } from "node:crypto"; import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { buildSync } from "esbuild"; import type { MatrixExecution } from "./types.js"; import type { ChatCatalogDescriptor } from "./chat-catalog-registry.js"; // Spawned by path rather than imported, so name them explicitly; the Python // sandbox wraps every agent process and decides its filesystem isolation. export const chatCatalogSpawnedSourceEntries = ["tests/runner-e2e/server.ts", "tests/runner-e2e/server-entry.ts", "tests/runner-e2e/agent-filesystem-sandbox.py"] as const; const hash = (value: string) => createHash("sha256").update(value).digest("hex"); function resolveSource(specifier: string, parent: string): string { const absolute = path.resolve(path.dirname(parent), specifier); const candidates = [absolute.replace(/\.js$/, ".ts"), absolute.replace(/\.js$/, ".tsx"), absolute, `${absolute}.ts`, `${absolute}.tsx`, path.join(absolute, "index.ts"), path.join(absolute, "index.tsx")]; const found = candidates.find(candidate => existsSync(candidate) && statSync(candidate).isFile()); if (!found) throw new Error(`Unresolved fingerprint dependency: ${specifier} from ${parent}`); return found; } function conservativeSources(directory: string): string[] { return readdirSync(directory, { withFileTypes: true }).flatMap(entry => { if (entry.name.startsWith(".") || ["node_modules", "dist", "target", "results", "test-results", "playwright-report"].includes(entry.name)) return []; const child = path.join(directory, entry.name); return entry.isDirectory() ? conservativeSources(child) : /\.(ts|tsx|js|mjs|json|yaml)$/.test(entry.name) ? [child] : []; }); } /** Compile and inspect actual executable imports, rather than trusting a hand-maintained sourceFiles list. */ export function fingerprintChatDependencyClosure(input: { root: string; entries: readonly string[]; selected: unknown; /** Only pure metadata may be normalized; executable modules may never use this escape. */ normalizedMetadata?: Readonly>; /** The registry's one variable import is bound to selected.modulePath above. */ selectedDispatchFile?: string; reviewedDynamicResolutions?: Readonly>; }) { const files = new Map(); const unknownDynamicImports: string[] = []; const resolvedDynamicImports: Array<{ file: string; resolution: string }> = []; const visit = (absolute: string) => { const relative = path.relative(input.root, absolute).split(path.sep).join("/"); if (relative.startsWith("../")) throw new Error(`Fingerprint dependency escaped repository: ${relative}`); if (files.has(relative)) return; const normalized = input.normalizedMetadata?.[relative]; if (normalized !== undefined) { files.set(relative, JSON.stringify(normalized)); return; } const body = readFileSync(absolute, "utf8"); files.set(relative, body); if (!/\.[cm]?[jt]sx?$/.test(absolute)) return; const parsed = buildSync({ stdin: { contents: body, sourcefile: absolute, loader: absolute.endsWith(".tsx") ? "tsx" : /\.ts$/.test(absolute) ? "ts" : "js" }, bundle: false, metafile: true, write: false, format: "esm", platform: "node", minifyWhitespace: true, legalComments: "none", logLevel: "silent" }); const dependencies = Object.values(parsed.metafile!.outputs).flatMap(output => output.imports.map(item => item.path)); if (/\bimport\s*\(\s*(?!["'])/.test(parsed.outputFiles[0]!.text) && relative !== input.selectedDispatchFile) { const reviewed = input.reviewedDynamicResolutions?.[relative]; if (reviewed && reviewed.sourceSha256 === hash(body)) { resolvedDynamicImports.push({ file: relative, resolution: reviewed.resolution }); for (const dependency of reviewed.dependencies) visit(path.resolve(input.root, dependency)); } else unknownDynamicImports.push(relative); } for (const dependency of dependencies.filter(value => value.startsWith("."))) visit(resolveSource(dependency, absolute)); }; for (const entry of input.entries) visit(resolveSource(`./${entry}`, path.join(input.root, "entry.ts"))); // Unknown imports conservatively bind all source files. if (unknownDynamicImports.length) for (const absolute of conservativeSources(input.root)) { const relative = path.relative(input.root, absolute).split(path.sep).join("/"); if (!files.has(relative)) files.set(relative, input.normalizedMetadata?.[relative] !== undefined ? JSON.stringify(input.normalizedMetadata[relative]) : readFileSync(absolute, "utf8")); } for (const name of ["package.json", "pnpm-lock.yaml"]) if (existsSync(path.join(input.root, name))) files.set(name, readFileSync(path.join(input.root, name), "utf8")); const sources = [...files].sort(([a], [b]) => a.localeCompare(b)).map(([file, body]) => ({ file, sha256: hash(body) })); const digest = hash(JSON.stringify({ selected: input.selected, sources })); return { digest, definitionDigest: digest, graderDigest: digest, fingerprintKind: "complete-execution-and-oracle-closure", sources, resolvedDynamicImports, unknownDynamicImports: [...new Set(unknownDynamicImports)].sort() }; } export function assertChatCatalogFingerprintLane(selected: unknown, env: NodeJS.ProcessEnv = process.env, args: readonly string[] = process.argv) { const execution = selected as { suiteId?: string; flow?: string; environment?: string; executionId?: string; scenarioId?: string; installedSource?: string; databaseLane?: unknown; profile?: { id?: string; provider?: string; model?: string; adapterType?: string; credential?: string } }; const database = databaseFingerprintIdentity(env); if (execution.databaseLane !== undefined && JSON.stringify(execution.databaseLane) !== JSON.stringify(database)) throw new Error("Selected database lane differs from fingerprint environment"); const installed = execution.installedSource === "published-global-npm" && (installedChatScenarioIds as readonly string[]).includes(String(execution.scenarioId)); const installedKeys = ["PAPERCLIP_RUNNER_E2E_INSTALLED_CLI", "PAPERCLIP_RUNNER_E2E_INSTALLED_CLI_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_ROOT", "PAPERCLIP_RUNNER_E2E_INSTALLED_SERVER_SHA256", "PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY"]; const forbiddenArguments = ["--target-config", "--github-review-config", "--connection-config", "--installed-startup-only", "product-journeys"]; if (installed && !reviewedChatProfile(execution.profile) || execution.suiteId !== "chat-features" || execution.flow !== "agent_chat" || execution.environment !== "local" || !installed && installedKeys.some(key => env[key] && env[key] !== "0") || args.some(argument => forbiddenArguments.includes(argument))) throw new Error("Reviewed dynamic resolutions require source-local Agent Chat without external dispatch overrides"); return { suiteId: execution.suiteId, flow: execution.flow, environment: execution.environment, database, installedLane: installed, externalJourneyDispatch: false }; } export function chatCatalogCaseFingerprint(descriptor: ChatCatalogDescriptor, selectedExecution: unknown, retainedPrivateInputNames?: ReturnType, retainedInstalledIdentity?: Omit, "observedProcess">) { selectedExecution = { ...(selectedExecution as Record), journeyKind: descriptor.journeyKind ?? "agent-chat" }; const root = fileURLToPath(new URL("../../", import.meta.url)); const privateInputNameIdentity = retainedPrivateInputNames === undefined ? chatPrivateInputNameIdentity((selectedExecution as { executionId: string }).executionId) : retainedPrivateInputNames; if (retainedPrivateInputNames !== undefined) verifyRetainedChatOracleIdentity(retainedPrivateInputNames, descriptor); const lane = assertChatCatalogFingerprintLane({ ...(selectedExecution as Record), scenarioId: descriptor.scenarioId, installedSource: descriptor.installedSource }); const installedObservation = descriptor.installedSource && retainedInstalledIdentity === undefined ? installedChatFingerprintIdentity(process.env, (selectedExecution as { executionId: string }).executionId) : undefined; const installedIdentity = descriptor.installedSource ? retainedInstalledIdentity ?? (installedObservation && { kind: installedObservation.kind, sourceVersion: installedObservation.sourceVersion, cliSha256: installedObservation.cliSha256, serverSha256: installedObservation.serverSha256, treeSha256: installedObservation.treeSha256, publishedArtifacts: installedObservation.publishedArtifacts, }) : null; const reviewedDynamicResolutions = JSON.parse(readFileSync(new URL("./chat-catalog-dynamic-resolutions.json", import.meta.url), "utf8")); const selectedModule = `tests/runner-e2e/${descriptor.modulePath.slice(2)}`; const scenario = JSON.parse(readFileSync(new URL("./chat-campaign-manifest.json", import.meta.url), "utf8")).scenarios .find((row: { id: string }) => row.id === descriptor.scenarioId); if (!scenario) throw new Error(`Missing frozen scenario: ${descriptor.scenarioId}`); const { id, capabilities, context, query, expected_outcome } = scenario; const attachmentFormatter = descriptor.queryAttachments?.length ? ["ui/src/components/IssueChatThread.tsx", "ui/src/components/task-chat/TaskChatComposer.tsx"].map(file => ({ file, sha256: hash(readFileSync(new URL(`../../${file}`, import.meta.url), "utf8")), })) : null; const originIdentity = publicOriginFingerprintIdentity((selectedExecution as { executionId: string }).executionId, descriptor.scenarioId); const originReaders = originIdentity.selected ? ["server/src/config.ts", "server/src/auth/better-auth.ts", "server/src/routes/tool-access.ts", "server/src/services/chat-task-url.ts", "server/src/services/muse-setup.ts", "server/src/services/workspace-runtime.ts", "tests/runner-e2e/chat-catalog-domains/external-public-signin-worker.ts", "tests/runner-e2e/chat-catalog-domains/external-public-signin-contract.ts", "ui/src/pages/Auth.tsx"] : []; const result = fingerprintChatDependencyClosure({ root, entries: [selectedModule, ...chatCatalogSpawnedSourceEntries, "tests/runner-e2e/chat-catalog-flow.ts", "tests/runner-e2e/chat-catalog-fingerprint.ts", "tests/runner-e2e/chat-catalog-dynamic-resolutions.json", "tests/runner-e2e/chat-feature-flow.ts", "tests/runner-e2e/runner.spec.ts", "tests/runner-e2e/launch.ts", "tests/runner-e2e/chat-scenarios/defaults.yaml", ...originReaders, ...descriptor.sourceFiles.map(file => `tests/runner-e2e/${file}`)], selected: { descriptor, selectedExecution, lane, publicOriginIdentity: originIdentity, attachmentFormatter, privateInputRequirements: chatPrivateInputRequirements(descriptor), privateInputNameIdentity, installedIdentity, scenario: { id, capabilities, context, query, expected_outcome } }, reviewedDynamicResolutions, selectedDispatchFile: "tests/runner-e2e/chat-catalog-registry.ts", normalizedMetadata: { "tests/runner-e2e/chat-catalog-descriptors.json": descriptor, "tests/runner-e2e/chat-campaign-manifest.json": { id, capabilities, context, query, expected_outcome } }, }); return { ...result, databaseLaneIdentity: lane.database, attachmentFormatter, privateInputNameIdentity, installedIdentity, installedSourceProcess: installedObservation?.observedProcess, privateInputRequirements: chatPrivateInputRequirements(descriptor) }; } export function normalizedChatCatalogExecution(execution: MatrixExecution) { return { executionId: execution.id, suiteId: execution.suite.id, flow: execution.task.flow, journeyKind: "agent-chat", profile: { id: execution.profile.id, provider: execution.profile.provider, model: execution.profile.model, adapterType: execution.profile.adapterType, credential: execution.profile.credential }, environment: execution.environment.id, databaseLane: databaseFingerprintIdentity(), requiredCredentialNames: execution.requiredCredentials, task: { expectedRunCount: execution.task.expectedRunCount, minimumExpectedRunCount: execution.task.minimumExpectedRunCount, turnTimeoutMs: execution.task.turnTimeoutMs, ...(execution.task.optionalCompanySecretBindingApproval===true?{expectedHumanMessageCount:execution.task.expectedHumanMessageCount,optionalCompanySecretBindingApproval:true}:{}), attemptTimeoutMs: execution.task.attemptTimeoutMs } }; } export function legacyChatFeatureFingerprint(execution: MatrixExecution) { if (!["create-backlog", "delegate-announcement"].includes(execution.task.id)) throw new Error("Unknown legacy chat feature case"); return chatCatalogCaseFingerprint({ scenarioId: `tasks.${execution.task.id}`, modulePath: "./chat-feature-flow.js", exportName: "runChatFeatureFlow", sourceFiles: ["chat-feature-flow.ts", "chat-feature-scoring.ts", "chat-scenarios/tasks.yaml", "completion-updates.ts"], oracleVersion: "paperclip.chat-features.v1", expectedRunCount: execution.task.expectedRunCount, expectedHumanMessageCount: 1, turnTimeoutMs: execution.task.turnTimeoutMs!, attemptTimeoutMs: execution.task.attemptTimeoutMs.local, }, normalizedChatCatalogExecution(execution)); }