diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 53fc610d779..9e2ae1708ce 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -3770,7 +3770,7 @@ "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", - "../../cli/dist/index.js", + "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", @@ -3792,7 +3792,7 @@ "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", - "../../cli/dist/index.js", + "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", @@ -3814,7 +3814,7 @@ "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", - "../../cli/dist/index.js", + "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", @@ -3837,7 +3837,7 @@ "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-flow.ts", - "../../cli/dist/index.js", + "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", @@ -4611,7 +4611,7 @@ "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", - "../../cli/dist/index.js", + "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", @@ -4890,7 +4890,6 @@ "../../server/src/services/agent-instance-secrets.ts", "../../server/src/services/agent-lifecycle-driver.ts", "../../server/src/services/heartbeat/run-preparation.ts", - "../../cli/dist/index.js", "../../cli/src/index.ts", "../../cli/node_modules/tsx/dist/loader.mjs", "chat-catalog-domains/external-plugin-sdk-mirror.ts", diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-install.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-install.ts index 6fb12b8577c..16573c5dcad 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-plugin-install.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-install.ts @@ -33,7 +33,7 @@ export const pluginInstallExternalCases = [definition("plugins.install-plugin", return { pluginKey: fixture.key, syntheticPackage: true, agentInstallStatus }; } catch (error) { fixtures.delete(ctx); await fixture.close(); throw error; } }, gradePluginInstall, { - sourceFiles: ["chat-catalog-domains/external-plugin-install.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../cli/dist/index.js", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/plugins.yaml"], + sourceFiles: ["chat-catalog-domains/external-plugin-install.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/plugins.yaml"], async act(ctx, state) { await waitForReply(ctx); const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Vetted package fixture is missing"); diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts index 08471eb8dbd..1902222a7f4 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-plugin-tool.ts @@ -57,7 +57,7 @@ export const pluginToolExternalCases=[definition("plugins.use-plugin-tool",async }catch(error){fixtures.delete(ctx);try{if(profileId)await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profileId}/unbind`,{targetType:"agent",targetId:ctx.input.fixtures.agent.id});if(installedId)await ctx.input.api.delete(`/api/plugins/${installedId}`,{allowNotFound:true});}finally{await scaffold.close();}throw error;} },gradePluginTool,{ expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, - sourceFiles:["chat-catalog-domains/external-plugin-tool.ts","chat-catalog-domains/external-plugin-fixture.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","chat-flow.ts","../../cli/dist/index.js","../../cli/src/index.ts","../../packages/plugins/create-paperclip-plugin/src/index.ts","../../packages/plugins/sdk/src/index.ts","../../packages/plugins/sdk/dist/index.js","chat-scenarios/plugins.yaml"], + sourceFiles:["chat-catalog-domains/external-plugin-tool.ts","chat-catalog-domains/external-plugin-fixture.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","chat-flow.ts","../../cli/node_modules/tsx/dist/loader.mjs","../../cli/src/index.ts","../../packages/plugins/create-paperclip-plugin/src/index.ts","../../packages/plugins/sdk/src/index.ts","../../packages/plugins/sdk/dist/index.js","chat-scenarios/plugins.yaml"], async act(ctx,state){ await waitForReply(ctx);const first=await observeExternal(ctx,state),reply=first.phaseReplies.at(-1);if(!reply)throw new Error("Initial source preview reply missing");state.initialRunId=reply.runId; const href=previewLink(first.links,first.origin,state.companyPrefix,state.route);state.initialPreviewLinkMatched=!!href;if(!href)throw new Error("Actual plugin preview link missing from model reply");const source=ctx.input.page.url();await ctx.input.page.goto(new URL(href,first.origin).href,{waitUntil:"domcontentloaded"});await expect(ctx.input.page.getByRole("heading",{name:"Synthetic website preview",exact:true})).toBeVisible();await expect(ctx.input.page.frameLocator('iframe[title="Website preview"]').getByRole("heading",{name:"Synthetic website",exact:true})).toBeVisible();state.initialPreviewVisible=true;await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-plugins.ts b/tests/runner-e2e/chat-catalog-domains/external-plugins.ts index 068debfb62f..1fa2c26fc8e 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-plugins.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-plugins.ts @@ -58,7 +58,7 @@ export const pluginExternalCases = ["configure-plugin", "disable-plugin"].map(id resultPath: `/${ctx.input.fixtures.company.issuePrefix}/company/settings/instance/plugins/${plugin.id}`, resultLabel: "Website Preview Eval" }; } catch (error) { fixtures.delete(ctx); await fixture.close(); throw error; } }, gradePluginManagement, { - sourceFiles: ["chat-catalog-domains/external-plugins.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../cli/dist/index.js", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/plugins.yaml"], + sourceFiles: ["chat-catalog-domains/external-plugins.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/plugins.yaml"], async act(ctx, state) { await waitForReply(ctx); const before = await ctx.input.api.get(`/api/plugins/${state.pluginId}`); diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts index 4ff1fff51c2..df47eebc24b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-delivery.ts @@ -322,7 +322,6 @@ export const secretDeliveryExternalCases = [ "../../server/src/services/agent-instance-secrets.ts", "../../server/src/services/agent-lifecycle-driver.ts", "../../server/src/services/heartbeat/run-preparation.ts", - "../../cli/dist/index.js", "../../cli/src/index.ts", "../../cli/node_modules/tsx/dist/loader.mjs", "chat-catalog-domains/external-plugin-sdk-mirror.ts", diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-plugin-routine.ts b/tests/runner-e2e/chat-catalog-domains/integrator-plugin-routine.ts index 242b75b616b..66f041b55e1 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-plugin-routine.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-plugin-routine.ts @@ -73,5 +73,5 @@ export const integratorPluginRoutineCases = [integratorDefinition("routines", "r } catch (error) { await cleanup(ctx); throw error; } }, observeManaged, gradeIntegratorPluginRoutine, async (ctx, state) => { const routine = state.routine as RoutineDetail; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/routines/${routine.id}/history`, routine.title); -}, { sourceFiles: ["chat-catalog-domains/integrator-plugin-routine.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "../../cli/dist/index.js", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/routines.yaml"], cleanup, +}, { sourceFiles: ["chat-catalog-domains/integrator-plugin-routine.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/routines.yaml"], cleanup, })]; diff --git a/tests/runner-e2e/chat-plugin-source-fingerprint.test.ts b/tests/runner-e2e/chat-plugin-source-fingerprint.test.ts new file mode 100644 index 00000000000..c0b96a074ca --- /dev/null +++ b/tests/runner-e2e/chat-plugin-source-fingerprint.test.ts @@ -0,0 +1,41 @@ +import { createHash } from "node:crypto"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, expect, it } from "vitest"; +import { fingerprintChatDependencyClosure } from "./chat-catalog-fingerprint.js"; +import { findChatCatalogCase, loadChatCatalogImplementation } from "./chat-catalog-registry.js"; + +const selected = ["plugins.configure-plugin", "plugins.disable-plugin", "plugins.install-plugin", "plugins.use-plugin-tool", "routines.plugin-managed-routine", "secrets.least-scope-delivery"]; +const directories: string[] = []; +afterEach(() => { for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); }); + +it.each(selected)("%s binds its actual source CLI, loader and required built SDK without unused CLI dist", async id => { + const descriptor = findChatCatalogCase(id)!; + const implementation = await loadChatCatalogImplementation(descriptor); + expect(implementation.sourceFiles).toEqual(descriptor.sourceFiles); + expect(descriptor.sourceFiles).toEqual(expect.arrayContaining(["../../cli/src/index.ts", "../../cli/node_modules/tsx/dist/loader.mjs", "../../packages/plugins/sdk/dist/index.js"])); + expect(descriptor.sourceFiles).not.toContain("../../cli/dist/index.js"); +}); + +it("resolves and hashes the actual executed source CLI and loader dependency closure without running either", () => { + const root = fileURLToPath(new URL("../../", import.meta.url)); + const entries = ["cli/src/index.ts", "cli/node_modules/tsx/dist/loader.mjs"]; + const fingerprint = fingerprintChatDependencyClosure({ root, entries, selected: { fixture: "source-cli" } }); + for (const file of [...entries, "cli/src/commands/client/plugin.ts"]) { + expect(fingerprint.sources).toContainEqual({ file, sha256: createHash("sha256").update(readFileSync(path.join(root, file))).digest("hex") }); + } +}); + +it("still rejects missing actual source imports, the executed loader and required SDK artifact", () => { + const root = mkdtempSync(path.join(tmpdir(), "plugin-source-closure-")); directories.push(root); + mkdirSync(path.join(root, "cli/src"), { recursive: true }); + writeFileSync(path.join(root, "cli/src/index.ts"), 'export { command } from "./command.js";'); + const fingerprint = (entries: string[]) => fingerprintChatDependencyClosure({ root, entries, selected: {} }); + expect(() => fingerprint(["cli/src/index.ts"])).toThrow("Unresolved fingerprint dependency: ./command.js"); + writeFileSync(path.join(root, "cli/src/command.ts"), "export const command = 1;"); + expect(fingerprint(["cli/src/index.ts"]).sources.map(row => row.file)).toContain("cli/src/command.ts"); + expect(() => fingerprint(["cli/src/index.ts", "cli/node_modules/tsx/dist/loader.mjs"])).toThrow("Unresolved fingerprint dependency"); + expect(() => fingerprint(["cli/src/index.ts", "packages/plugins/sdk/dist/index.js"])).toThrow("Unresolved fingerprint dependency"); +});