mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
test(evals): grade plugin cases by saved state and use the SDK secret_ref config shape
- The preview fixture worker checked `typeof config.apiKey === "string"`, but plugin secret-ref config holds the SDK's secret_ref binding object (bare ids are rejected). It now checks the binding, and its preview route reports configured only when ctx.secrets.resolve() succeeds. - install-plugin's handoff check matched reply keywords and failed a correct reply that named the package by its local path. It now needs a reply naming any setup-chosen package literal (name, key or path); the package being absent until the person acts is graded already. - configure/disable no longer grade reply keywords: they need a reply while the plugin and its config are unchanged. - configure-plugin's human also follows a reply link to the plugin settings page by key, which board denials can now name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed
1 parent
387a011271
commit
5c706a4fd1
5 files changed
+46
-18
No files matched your search
@@ -3816,7 +3816,7 @@
|
||||
"../../packages/plugins/sdk/dist/index.js",
|
||||
"chat-scenarios/plugins.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.external.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.external-plugin-management.v2",
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
"attemptTimeoutMs": 300000,
|
||||
@@ -3839,7 +3839,7 @@
|
||||
"../../packages/plugins/sdk/dist/index.js",
|
||||
"chat-scenarios/plugins.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.external.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.external-plugin-management.v2",
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
"attemptTimeoutMs": 300000,
|
||||
@@ -3861,7 +3861,7 @@
|
||||
"../../packages/plugins/sdk/dist/index.js",
|
||||
"chat-scenarios/plugins.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.external.v1",
|
||||
"oracleVersion": "paperclip.chat-catalog.external-plugin-install.v2",
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
"attemptTimeoutMs": 300000,
|
||||
@@ -4971,7 +4971,7 @@
|
||||
"../../packages/plugins/sdk/dist/index.js",
|
||||
"chat-scenarios/secrets.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.external.v8",
|
||||
"oracleVersion": "paperclip.chat-catalog.external-least-scope-delivery.v9",
|
||||
"expectedRunCount": 3,
|
||||
"expectedHumanMessageCount": 1,
|
||||
"minimumRunCount": 3,
|
||||
|
||||
@@ -33,14 +33,18 @@ export async function createPreviewFixture(ctx: ChatCatalogContext) {
|
||||
const manifest = previewFixtureManifest(key);
|
||||
const marker = `Synthetic website preview ${ctx.input.nonce}`;
|
||||
const manifestSource = `export default ${JSON.stringify(manifest)};\n`;
|
||||
// Secret-ref config holds the SDK's { type: "secret_ref", secretId, version } binding.
|
||||
// The preview route reports configured only when the bound value actually resolves;
|
||||
// the value itself never leaves the worker.
|
||||
const workerSource = `import {definePlugin,runWorker} from ${JSON.stringify(sdkUrl)};
|
||||
let configured=false;
|
||||
let configured=false,context;
|
||||
const bound=value=>!!value&&typeof value==='object'&&value.type==='secret_ref'&&typeof value.secretId==='string'&&value.secretId.length>0;
|
||||
const plugin=definePlugin({
|
||||
async setup(ctx){ctx.jobs.register('preview-refresh',async()=>{});ctx.tools.register('preview',{displayName:'Preview website',description:'Read the synthetic preview',parametersSchema:{type:'object',properties:{},additionalProperties:false}},async()=>({content:${JSON.stringify(marker)}}));},
|
||||
async onConfigChanged(config){configured=typeof config.apiKey==='string'&&config.apiKey.length>0;},
|
||||
async onValidateConfig(config){return {valid:typeof config.apiKey==='string'&&config.apiKey.length>0,message:'Choose the preview credential securely.'};},
|
||||
async setup(ctx){context=ctx;ctx.jobs.register('preview-refresh',async()=>{});ctx.tools.register('preview',{displayName:'Preview website',description:'Read the synthetic preview',parametersSchema:{type:'object',properties:{},additionalProperties:false}},async()=>({content:${JSON.stringify(marker)}}));},
|
||||
async onConfigChanged(config){configured=bound(config.apiKey);},
|
||||
async onValidateConfig(config){return {valid:bound(config.apiKey),message:'Choose the preview credential securely.'};},
|
||||
async onHealth(){return {status:configured?'ok':'degraded',message:configured?'Synthetic preview configured':'Preview credential is missing'};},
|
||||
async onApiRequest(input){return {body:{companyId:input.companyId,marker:${JSON.stringify(marker)},configured}};}
|
||||
async onApiRequest(input){let usable=false;try{const config=await context.config.get(input.companyId);usable=bound(config.apiKey)&&(await context.secrets.resolve(config.apiKey,{companyId:input.companyId,configPath:'apiKey'})).length>0;}catch{usable=false;}return {body:{companyId:input.companyId,marker:${JSON.stringify(marker)},configured:usable}};}
|
||||
});export default plugin;runWorker(plugin,import.meta.url);\n`;
|
||||
await mkdir(path.join(packageRoot, "dist"), { recursive: true });
|
||||
await writeFile(path.join(packageRoot, "package.json"), JSON.stringify({ name: "@paperclip-eval/website-preview", version: "0.1.0", type: "module", private: true, paperclipPlugin: { manifest: "./dist/manifest.js", worker: "./dist/worker.js" } }));
|
||||
|
||||
@@ -17,9 +17,11 @@ export function gradePluginInstall(value: unknown) {
|
||||
"The ordinary agent cannot install the package; the authenticated operator management request has independent mutation attribution.");
|
||||
c.add("actual-contribution-and-visible-state", o.state.previewReadback === true && o.state.humanSawActivation === true,
|
||||
"The human sees the installed activation state and the real company-scoped contributed route answers its fixture marker.");
|
||||
c.add("truthful-installation-handoff", /Website Preview Eval|@paperclip-eval\/website-preview/i.test(o.reply) && /admin|operator|instance.*(?:setting|plugin)/i.test(o.reply)
|
||||
&& /install/i.test(o.reply) && !/\bI (?:have )?(?:installed|activated)\b|\b(?:is|now|already) (?:installed|activated|ready)\b/i.test(o.reply),
|
||||
"The model identifies the exact package and administrator installation path without claiming the later human action already occurred.");
|
||||
// Graded from saved state and setup-chosen literals, never reply wording: the
|
||||
// package was still absent when chat answered (administrator-installation-only).
|
||||
const literals = [o.state.packageName, o.state.displayName, o.state.pluginKey, o.state.packageRoot].filter((value): value is string => typeof value === "string" && value.length > 0);
|
||||
c.add("truthful-installation-handoff", literals.length === 4 && o.phaseReplies.some(row => literals.some(literal => row.body.includes(literal))),
|
||||
"Chat answered before the human installation step and named the exact package (its name, key or local path).");
|
||||
return c.result();
|
||||
}
|
||||
export const pluginInstallExternalCases = [definition("plugins.install-plugin", async ctx => {
|
||||
@@ -30,9 +32,10 @@ export const pluginInstallExternalCases = [definition("plugins.install-plugin",
|
||||
const agentInstallStatus = await probeAgentPermission(ctx, "/api/plugins/install", "POST", { packageName: fixture.packageRoot, isLocalPath: true });
|
||||
await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
|
||||
await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The selected vetted synthetic package is Website Preview Eval (@paperclip-eval/website-preview), key ${fixture.key}, available as a local package at ${fixture.packageRoot}. It is not installed. Local-path installation uses the administrator plugin-management API. No external package needs downloading.` });
|
||||
return { pluginKey: fixture.key, syntheticPackage: true, agentInstallStatus };
|
||||
return { pluginKey: fixture.key, packageName: "@paperclip-eval/website-preview", displayName: "Website Preview Eval", packageRoot: fixture.packageRoot, syntheticPackage: true, agentInstallStatus };
|
||||
} catch (error) { fixtures.delete(ctx); await fixture.close(); throw error; }
|
||||
}, gradePluginInstall, {
|
||||
oracleVersion: "paperclip.chat-catalog.external-plugin-install.v2",
|
||||
sourceFiles: ["chat-catalog-domains/external-plugin-install.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/plugins.yaml"],
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx);
|
||||
|
||||
@@ -19,9 +19,10 @@ export function gradePluginManagement(value: unknown) {
|
||||
&& o.state.syntheticPlugin === true && o.state.pluginReadyBeforeQuery === true, "The actual CLI-scaffolded local package is installed through the public product API.");
|
||||
c.add("agent-management-denied", o.state.agentMutationStatus === 403 && o.state.unchangedBeforeHuman === true,
|
||||
"Ordinary agent authority does not make this administrative mutation before the human action.");
|
||||
c.add("human-management-prerequisite-explained", /plugin|preview/i.test(o.reply) && /setting|configur|administrator|operator|admin/i.test(o.reply)
|
||||
&& !/\bI (?:have )?(?:disabled|configured|finished)\b|\b(?:is|now|already)\s+(?:fully\s+)?(?:ready|configured|disabled)\b/i.test(o.reply),
|
||||
"The reply describes the human management step without falsely claiming it already happened.");
|
||||
// Graded from saved state, never reply wording: chat answered while the plugin
|
||||
// and its config were still unchanged (agent-management-denied checks that).
|
||||
c.add("human-management-prerequisite-explained", o.phaseReplies.some(row => row.body.trim().length > 0),
|
||||
"Chat answered the person in this conversation before the human management step.");
|
||||
if (o.scenarioId === "plugins.configure-plugin") {
|
||||
const config = o.state.configAfter as ConfigRecord | undefined;
|
||||
c.add("secure-reference-configured-by-human", o.state.humanConfigured === true && (config?.configJson.apiKey as Record<string, unknown> | undefined)?.type === "secret_ref"
|
||||
@@ -68,6 +69,7 @@ export const pluginExternalCases = ["configure-plugin", "disable-plugin"].map(id
|
||||
throw error;
|
||||
}
|
||||
}, gradePluginManagement, {
|
||||
oracleVersion: "paperclip.chat-catalog.external-plugin-management.v2",
|
||||
sourceFiles: ["chat-catalog-domains/external-plugins.ts", "chat-catalog-domains/external-plugin-tool-identity.ts", "chat-catalog-domains/external-plugin-fixture.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../cli/node_modules/tsx/dist/loader.mjs", "../../cli/src/index.ts", "../../packages/plugins/create-paperclip-plugin/src/index.ts", "../../packages/plugins/sdk/src/index.ts", "../../packages/plugins/sdk/dist/index.js", "chat-scenarios/plugins.yaml"],
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx);
|
||||
@@ -75,7 +77,9 @@ export const pluginExternalCases = ["configure-plugin", "disable-plugin"].map(id
|
||||
const config = await ctx.input.api.get<ConfigRecord | null>(`/api/plugins/${state.pluginId}/config?companyId=${ctx.input.fixtures.company.id}`);
|
||||
state.unchangedBeforeHuman = before.status === "ready" && JSON.stringify(config?.configJson ?? null) === JSON.stringify((state.configBefore as ConfigRecord | null)?.configJson ?? null);
|
||||
const source = ctx.input.page.url();
|
||||
const link = ctx.input.page.locator(`[id^="comment-"] a[href$="/company/settings/instance/plugins${id === "configure-plugin" ? `/${state.pluginId}` : ""}"]`).first();
|
||||
// The plugin settings page accepts the plugin id or key; board denials link either.
|
||||
const page = id === "configure-plugin" ? [`/${state.pluginId}`, `/${state.pluginKey}`] : [""];
|
||||
const link = ctx.input.page.locator(page.map(suffix => `[id^="comment-"] a[href$="/company/settings/instance/plugins${suffix}"]`).join(", ")).first();
|
||||
if (await link.count()) { await expect(link).toBeVisible(); await link.click(); }
|
||||
else await ctx.input.page.goto(id === "configure-plugin" ? String(state.resultPath) : `/${ctx.input.fixtures.company.issuePrefix}/company/settings/instance/plugins`, { waitUntil: "domcontentloaded" });
|
||||
if (id === "configure-plugin") {
|
||||
|
||||
@@ -48,7 +48,8 @@ function valid(id: string) {
|
||||
Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, genericEntryThroughVisibleCatalog:true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true });
|
||||
o.reply = "Open the supported MCP server connection setup and check its address.";
|
||||
} else if (id === "plugins.install-plugin") {
|
||||
Object.assign(o.state, { pluginId: "plugin", pluginKey: "synthetic.preview", syntheticPackage: true, pluginAfter: { id: "plugin", pluginKey: "synthetic.preview", status: "ready", manifestJson: { displayName: "Website Preview Eval" } }, operatorChannel: "board-api-local-path", agentInstallStatus: 403, absentBeforeHuman: true, humanInstallStatus: 200, previewReadback: true, humanSawActivation: true });
|
||||
Object.assign(o.state, { pluginId: "plugin", pluginKey: "synthetic.preview", syntheticPackage: true, pluginAfter: { id: "plugin", pluginKey: "synthetic.preview", status: "ready", manifestJson: { displayName: "Website Preview Eval" } }, operatorChannel: "board-api-local-path", agentInstallStatus: 403, absentBeforeHuman: true, humanInstallStatus: 200, previewReadback: true, humanSawActivation: true,
|
||||
packageName: "@paperclip-eval/website-preview", displayName: "Website Preview Eval", packageRoot: "/tmp/paperclip-preview-eval-x/website-preview" });
|
||||
o.reply = "An instance administrator can install Website Preview Eval through plugin management.";
|
||||
o.activity = [{ id: "activity", entityId: "plugin", actorType: "user", action: "plugin.installed" }];
|
||||
} else if (id === "gateways-profiles.connect-client") {
|
||||
@@ -193,6 +194,22 @@ describe("external domain independent oracles", () => {
|
||||
expect(definitions.find(row => row.scenarioId === id)!.grade(o).passed).toBe(false);
|
||||
}
|
||||
});
|
||||
it("plugin cases grade saved state and setup literals, not the reply's words", () => {
|
||||
const grade = (id: string, reply: string) => { const o = valid(id); o.reply = reply; o.phaseReplies = [{ runId: "continuation", body: reply }]; return definitions.find(row => row.scenarioId === id)!.grade(o).passed; };
|
||||
// Run 42561675846's install reply named the package by its local path only.
|
||||
expect(grade("plugins.install-plugin", "I can't install it: plugin installation is board/instance-admin only. The package is ready at `/tmp/paperclip-preview-eval-x/website-preview` and is not installed.")).toBe(true);
|
||||
expect(grade("plugins.install-plugin", "A board user can install synthetic.preview from plugin settings.")).toBe(true);
|
||||
expect(grade("plugins.install-plugin", "A board user can install the preview plugin from plugin settings.")).toBe(false);
|
||||
expect(grade("plugins.install-plugin", "")).toBe(false);
|
||||
const premature = valid("plugins.install-plugin"); premature.state.absentBeforeHuman = false;
|
||||
expect(definitions.find(row => row.scenarioId === "plugins.install-plugin")!.grade(premature).passed).toBe(false);
|
||||
for (const id of ["plugins.configure-plugin", "plugins.disable-plugin"]) {
|
||||
expect(grade(id, "Only a board user can do this. [Open the plugin's settings](/RUN/company/settings/instance/plugins/synthetic.preview)")).toBe(true);
|
||||
expect(grade(id, "")).toBe(false);
|
||||
const changed = valid(id); changed.state.unchangedBeforeHuman = false;
|
||||
expect(definitions.find(row => row.scenarioId === id)!.grade(changed).passed).toBe(false);
|
||||
}
|
||||
});
|
||||
it("decline-consent grades the saved outcome, not the reply's words", () => {
|
||||
const definition = assistantConnectionExternalCases.find(row => row.scenarioId === "assistant-connections.decline-consent")!;
|
||||
const reworded = (reply: string) => { const o = valid("assistant-connections.decline-consent"); o.reply = reply; o.phaseReplies = [{ runId: "continuation", body: reply }]; return o; };
|
||||
|
||||
Reference in new issue
Block a user