mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
test(e2e): correct generic MCP entry and retain scoped admission diagnostics
This commit is contained in:
Dotta committed
1 parent
b65cce861b
commit
673586931f
4 files changed
+776
-134
No files matched your search
@@ -1,97 +1,434 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import type { ToolProfileWithDetails, ToolProfileEffectiveSummary, ToolRunDecisionLookup } from "../../../packages/shared/src/types/tool-access.js";
|
||||
import type {
|
||||
ToolProfileWithDetails,
|
||||
ToolProfileEffectiveSummary,
|
||||
ToolRunDecisionLookup,
|
||||
} from "../../../packages/shared/src/types/tool-access.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
|
||||
import { connectNotes, startNotesProvider, type NotesProvider } from "./external-provider.js";
|
||||
import {
|
||||
definition,
|
||||
externalObservation,
|
||||
observeExternal,
|
||||
root,
|
||||
checks,
|
||||
verifyRenderedReply,
|
||||
} from "./external-support.js";
|
||||
import {
|
||||
connectNotes,
|
||||
startNotesProvider,
|
||||
type NotesProvider,
|
||||
} from "./external-provider.js";
|
||||
|
||||
interface PrivateFixture { provider: NotesProvider; key?: { id: string; token: string }; session?: { sessionId: string; token: string } }
|
||||
interface PrivateFixture {
|
||||
provider: NotesProvider;
|
||||
key?: { id: string; token: string };
|
||||
session?: { sessionId: string; token: string };
|
||||
}
|
||||
const fixtures = new WeakMap<ChatCatalogContext, PrivateFixture>();
|
||||
async function privateRequest(ctx: ChatCatalogContext, pathname: string, token: string, body?: unknown, session = false) {
|
||||
async function privateRequest(
|
||||
ctx: ChatCatalogContext,
|
||||
pathname: string,
|
||||
token: string,
|
||||
body?: unknown,
|
||||
session = false,
|
||||
) {
|
||||
const url = new URL(pathname, ctx.input.api.baseURL);
|
||||
if (url.origin !== ctx.input.api.baseURL || !url.pathname.startsWith("/api/tool-gateway/")) throw new Error("Scoped session probe escaped its fixture API");
|
||||
return fetch(url, { method: body === undefined ? "GET" : "POST", redirect: "error", signal: AbortSignal.timeout(15_000), headers: {
|
||||
...(session ? { "x-paperclip-tool-gateway-token": token } : { Authorization: `Bearer ${token}` }), "content-type": "application/json",
|
||||
}, ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
|
||||
if (
|
||||
url.origin !== ctx.input.api.baseURL ||
|
||||
!url.pathname.startsWith("/api/tool-gateway/")
|
||||
)
|
||||
throw new Error("Scoped session probe escaped its fixture API");
|
||||
return fetch(url, {
|
||||
method: body === undefined ? "GET" : "POST",
|
||||
redirect: "error",
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
headers: {
|
||||
...(session
|
||||
? { "x-paperclip-tool-gateway-token": token }
|
||||
: { Authorization: `Bearer ${token}` }),
|
||||
"content-type": "application/json",
|
||||
},
|
||||
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
||||
});
|
||||
}
|
||||
/** Failure diagnostics are closed control-plane facts, never arbitrary upstream
|
||||
* error text. Unknown/secret-bearing fields are deliberately omitted. */
|
||||
export function scopedSessionFailureDiagnostic(
|
||||
status: number,
|
||||
context: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
runId: string;
|
||||
issueId: string;
|
||||
observedRunStatusAtAdmission: string;
|
||||
},
|
||||
response: unknown,
|
||||
) {
|
||||
const allowed = [
|
||||
"run_company_mismatch",
|
||||
"run_agent_mismatch",
|
||||
"run_inactive",
|
||||
"run_context_mismatch",
|
||||
];
|
||||
const reason =
|
||||
response && typeof response === "object"
|
||||
? (response as { reasonCode?: unknown }).reasonCode
|
||||
: undefined;
|
||||
return {
|
||||
httpStatus: status,
|
||||
companyId: context.companyId,
|
||||
agentId: context.agentId,
|
||||
runId: context.runId,
|
||||
issueId: context.issueId,
|
||||
observedRunStatusAtAdmission: context.observedRunStatusAtAdmission,
|
||||
reasonCode:
|
||||
typeof reason === "string" && allowed.includes(reason)
|
||||
? reason
|
||||
: "unclassified",
|
||||
responseBodyRetained: false,
|
||||
};
|
||||
}
|
||||
export function gradeGatewayClient(value: unknown) {
|
||||
const c = checks(), o = externalObservation(value);
|
||||
c.add("complete-scoped-client-evidence", !!o, "Actual assigned profile, run/task admission, independent discovery/denial and provider-read receipts are required."); if (!o) return c.result();
|
||||
const effective = o.state.effective as ToolProfileEffectiveSummary | undefined;
|
||||
c.add("intended-effective-profile", effective?.agentId === o.leadId && effective.profiles.some(row => row.id === o.state.profileId)
|
||||
&& effective.allowedTools.some(row => row.id === o.state.readEntryId) && !effective.allowedTools.some(row => row.id === o.state.writeEntryId),
|
||||
"The saved launch profile grants its read action to this agent while excluding the write action.");
|
||||
c.add("actual-active-run-scoped-session", o.state.sessionCreated === true && o.runs.some(run => run.id === o.state.probeRunId && run.agentId === o.leadId && run.companyId === o.companyId)
|
||||
&& o.state.discoveredRead === true && o.state.discoveredWrite === false,
|
||||
"A real agent bearer creates a session for the admitted source run/task and discovers only authorized actions.");
|
||||
const c = checks(),
|
||||
o = externalObservation(value);
|
||||
c.add(
|
||||
"complete-scoped-client-evidence",
|
||||
!!o,
|
||||
"Actual assigned profile, run/task admission, independent discovery/denial and provider-read receipts are required.",
|
||||
);
|
||||
if (!o) return c.result();
|
||||
const effective = o.state.effective as
|
||||
ToolProfileEffectiveSummary | undefined;
|
||||
c.add(
|
||||
"intended-effective-profile",
|
||||
effective?.agentId === o.leadId &&
|
||||
effective.profiles.some((row) => row.id === o.state.profileId) &&
|
||||
effective.allowedTools.some((row) => row.id === o.state.readEntryId) &&
|
||||
!effective.allowedTools.some((row) => row.id === o.state.writeEntryId),
|
||||
"The saved launch profile grants its read action to this agent while excluding the write action.",
|
||||
);
|
||||
c.add(
|
||||
"actual-active-run-scoped-session",
|
||||
o.state.sessionCreated === true &&
|
||||
o.runs.some(
|
||||
(run) =>
|
||||
run.id === o.state.probeRunId &&
|
||||
run.agentId === o.leadId &&
|
||||
run.companyId === o.companyId,
|
||||
) &&
|
||||
o.state.discoveredRead === true &&
|
||||
o.state.discoveredWrite === false,
|
||||
"A real agent bearer creates a session for the admitted source run/task and discovers only authorized actions.",
|
||||
);
|
||||
const decisions = o.state.decisions as ToolRunDecisionLookup[] | undefined;
|
||||
c.add("model-performed-bounded-read", !!decisions && decisions.some(row => row.decisions.some(decision => decision.invocation.runId === row.runId
|
||||
&& decision.invocation.agentId === o.leadId && decision.invocation.issueId === o.state.sourceIssueId && decision.invocation.connectionId === o.state.connectionId
|
||||
&& decision.invocation.catalogEntryId === o.state.readEntryId && decision.invocation.status === "succeeded"))
|
||||
&& Number(o.state.providerReadCalls) >= 1 && Number(o.state.providerReadCalls) <= 3 && typeof o.state.marker === "string" && o.reply.includes(o.state.marker),
|
||||
"The actual model run uses the real provider read and reports its unpredictable result; the oracle never performs that read.");
|
||||
c.add("disallowed-write-denied", o.state.writeProbeStatus === 403 && ["deny_default", "deny_policy_block"].includes(String(o.state.writeProbeReason)) && o.state.providerWriteCalls === 0 && o.state.written === null,
|
||||
"A real scoped-session write attempt is denied before provider execution.");
|
||||
c.add("no-private-token-disclosure", o.state.tokenScanComplete === true && o.state.privateTokenLeaked === false, "Scoped session and probe credentials never enter the persisted reply/run result.");
|
||||
c.add(
|
||||
"model-performed-bounded-read",
|
||||
!!decisions &&
|
||||
decisions.some((row) =>
|
||||
row.decisions.some(
|
||||
(decision) =>
|
||||
decision.invocation.runId === row.runId &&
|
||||
decision.invocation.agentId === o.leadId &&
|
||||
decision.invocation.issueId === o.state.sourceIssueId &&
|
||||
decision.invocation.connectionId === o.state.connectionId &&
|
||||
decision.invocation.catalogEntryId === o.state.readEntryId &&
|
||||
decision.invocation.status === "succeeded",
|
||||
),
|
||||
) &&
|
||||
Number(o.state.providerReadCalls) >= 1 &&
|
||||
Number(o.state.providerReadCalls) <= 3 &&
|
||||
typeof o.state.marker === "string" &&
|
||||
o.reply.includes(o.state.marker),
|
||||
"The actual model run uses the real provider read and reports its unpredictable result; the oracle never performs that read.",
|
||||
);
|
||||
c.add(
|
||||
"disallowed-write-denied",
|
||||
o.state.writeProbeStatus === 403 &&
|
||||
["deny_default", "deny_policy_block"].includes(
|
||||
String(o.state.writeProbeReason),
|
||||
) &&
|
||||
o.state.providerWriteCalls === 0 &&
|
||||
o.state.written === null,
|
||||
"A real scoped-session write attempt is denied before provider execution.",
|
||||
);
|
||||
c.add(
|
||||
"no-private-token-disclosure",
|
||||
o.state.tokenScanComplete === true && o.state.privateTokenLeaked === false,
|
||||
"Scoped session and probe credentials never enter the persisted reply/run result.",
|
||||
);
|
||||
return c.result();
|
||||
}
|
||||
export const gatewayClientExternalCases = [definition("gateways-profiles.connect-client", async ctx => {
|
||||
const marker = `Launch fixture result ${randomBytes(16).toString("hex")}`;
|
||||
const provider = await startNotesProvider(marker); const fixture: PrivateFixture = { provider }; fixtures.set(ctx, fixture);
|
||||
try {
|
||||
const connection = await connectNotes(ctx, provider, "read-only");
|
||||
const profile = await ctx.input.api.post<ToolProfileWithDetails>(`${root(ctx)}/tools/profiles`, { profileKey: "launch", name: "Launch profile", defaultAction: "deny", entries: [
|
||||
{ selectorType: "catalog_entry", effect: "include", connectionId: connection.connectionId, catalogEntryId: connection.read[0] },
|
||||
{ selectorType: "catalog_entry", effect: "exclude", connectionId: connection.connectionId, catalogEntryId: connection.write[0] },
|
||||
] });
|
||||
await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profile.id}/bind`, { targetType: "agent", targetId: ctx.input.fixtures.agent.id });
|
||||
const access = await ctx.input.api.get<{ access: { tools: Array<{ toolName: string; gatewayToolName: string }> } }>(`/api/tool-connections/${connection.connectionId}/test-agents/${ctx.input.fixtures.agent.id}/access`);
|
||||
const write = access.access.tools.find(row => row.toolName === "launch_update"); if (!write) throw new Error("The excluded write action has no independently discoverable catalog identity");
|
||||
return { connectionId: connection.connectionId, profileId: profile.id, readEntryId: connection.read[0], writeEntryId: connection.write[0], writeGatewayName: write.gatewayToolName, marker };
|
||||
} catch (error) { fixtures.delete(ctx); await provider.close(); throw error; }
|
||||
}, gradeGatewayClient, {
|
||||
sourceFiles: ["chat-catalog-domains/external-gateway-client.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/gateways-profiles.yaml"],
|
||||
async act(ctx, state) {
|
||||
const fixture = fixtures.get(ctx)!;
|
||||
const active = await pollUntil({ label: "admitted source run for independent scoped denial probe", deadlineAt: Date.now() + 60_000, intervalMs: 100,
|
||||
load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, accept: rows => rows.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "running"),
|
||||
reject: rows => rows.length && rows.every(row => ["succeeded", "failed", "cancelled", "timed_out"].includes(row.status)) ? "Source run settled before scoped-session probe admission" : undefined });
|
||||
const run = active.find(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "running")!;
|
||||
const source = ctx.issue(); if (!source || !ctx.input.privateSecrets) throw new Error("Scoped client requires source identity and private redaction");
|
||||
state.probeRunId = run.id; state.sourceIssueId = source.id;
|
||||
const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `scoped-client-probe-${ctx.input.nonce}` }); fixture.key = key; ctx.input.privateSecrets.push(key.token);
|
||||
const admitted = await privateRequest(ctx, "/api/tool-gateway/sessions", key.token, { companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id, runId: run.id, issueId: source.id });
|
||||
if (admitted.status !== 201) { await admitted.body?.cancel(); throw new Error("Active-run scoped session admission failed; response withheld"); }
|
||||
fixture.session = await admitted.json() as { sessionId: string; token: string }; ctx.input.privateSecrets.push(fixture.session.token); state.sessionCreated = true;
|
||||
const listed = await privateRequest(ctx, "/api/tool-gateway/tools", fixture.session.token, undefined, true);
|
||||
if (!listed.ok) { await listed.body?.cancel(); throw new Error("Scoped tool discovery failed; response withheld"); }
|
||||
const tools = await listed.json() as Array<{ catalogEntryId?: string }>;
|
||||
state.discoveredRead = tools.some(row => row.catalogEntryId === state.readEntryId); state.discoveredWrite = tools.some(row => row.catalogEntryId === state.writeEntryId);
|
||||
const denied = await privateRequest(ctx, "/api/tool-gateway/tools/call", fixture.session.token, { tool: state.writeGatewayName, parameters: { text: "This oracle write must be denied" } }, true);
|
||||
state.writeProbeStatus = denied.status;
|
||||
// Retain only the allowlisted policy code, never an arbitrary error body.
|
||||
const denial = await denied.json() as { reasonCode?: unknown };
|
||||
state.writeProbeReason = ["deny_default", "deny_policy_block"].includes(String(denial.reasonCode)) ? denial.reasonCode : "other";
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Scoped synthetic fixture missing");
|
||||
state.effective = await ctx.input.api.get<ToolProfileEffectiveSummary>(`${root(ctx)}/tools/profiles/effective/agents/${ctx.input.fixtures.agent.id}`);
|
||||
state.decisions = await Promise.all((await ctx.allRuns()).map(run => ctx.input.api.get<ToolRunDecisionLookup>(`${root(ctx)}/tools/runs/${run.id}/decisions`)));
|
||||
state.providerReadCalls = fixture.provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_read" && row.authorized).length;
|
||||
state.providerWriteCalls = fixture.provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_update").length; state.written = fixture.provider.written() ?? null;
|
||||
const o = await observeExternal(ctx, state);
|
||||
const tokens = [fixture.key?.token, fixture.session?.token].filter((token): token is string => typeof token === "string");
|
||||
state.tokenScanComplete = tokens.length === 2; state.privateTokenLeaked = tokens.some(token => JSON.stringify([o.reply, o.runs]).includes(token));
|
||||
return { ...o, state };
|
||||
},
|
||||
async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); },
|
||||
async cleanup(ctx, state) {
|
||||
const fixture = fixtures.get(ctx); fixtures.delete(ctx);
|
||||
try {
|
||||
if (fixture?.session && fixture.key) { const response = await privateRequest(ctx, `/api/tool-gateway/sessions/${fixture.session.sessionId}/revoke`, fixture.key.token, { companyId: ctx.input.fixtures.company.id }); if (!response.ok && response.status !== 404) throw new Error("Scoped fixture session revocation failed"); await response.body?.cancel(); }
|
||||
if (fixture?.key) await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${fixture.key.id}`, { allowNotFound: true });
|
||||
if (typeof state.profileId === "string") await ctx.input.api.post(`${root(ctx)}/tools/profiles/${state.profileId}/unbind`, { targetType: "agent", targetId: ctx.input.fixtures.agent.id });
|
||||
if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true });
|
||||
} finally { await fixture?.provider.close(); }
|
||||
},
|
||||
})];
|
||||
export const gatewayClientExternalCases = [
|
||||
definition(
|
||||
"gateways-profiles.connect-client",
|
||||
async (ctx) => {
|
||||
const marker = `Launch fixture result ${randomBytes(16).toString("hex")}`;
|
||||
const provider = await startNotesProvider(marker);
|
||||
const fixture: PrivateFixture = { provider };
|
||||
fixtures.set(ctx, fixture);
|
||||
try {
|
||||
const connection = await connectNotes(ctx, provider, "read-only");
|
||||
const profile = await ctx.input.api.post<ToolProfileWithDetails>(
|
||||
`${root(ctx)}/tools/profiles`,
|
||||
{
|
||||
profileKey: "launch",
|
||||
name: "Launch profile",
|
||||
defaultAction: "deny",
|
||||
entries: [
|
||||
{
|
||||
selectorType: "catalog_entry",
|
||||
effect: "include",
|
||||
connectionId: connection.connectionId,
|
||||
catalogEntryId: connection.read[0],
|
||||
},
|
||||
{
|
||||
selectorType: "catalog_entry",
|
||||
effect: "exclude",
|
||||
connectionId: connection.connectionId,
|
||||
catalogEntryId: connection.write[0],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
await ctx.input.api.post(
|
||||
`${root(ctx)}/tools/profiles/${profile.id}/bind`,
|
||||
{ targetType: "agent", targetId: ctx.input.fixtures.agent.id },
|
||||
);
|
||||
const access = await ctx.input.api.get<{
|
||||
access: {
|
||||
tools: Array<{ toolName: string; gatewayToolName: string }>;
|
||||
};
|
||||
}>(
|
||||
`/api/tool-connections/${connection.connectionId}/test-agents/${ctx.input.fixtures.agent.id}/access`,
|
||||
);
|
||||
const write = access.access.tools.find(
|
||||
(row) => row.toolName === "launch_update",
|
||||
);
|
||||
if (!write)
|
||||
throw new Error(
|
||||
"The excluded write action has no independently discoverable catalog identity",
|
||||
);
|
||||
return {
|
||||
connectionId: connection.connectionId,
|
||||
profileId: profile.id,
|
||||
readEntryId: connection.read[0],
|
||||
writeEntryId: connection.write[0],
|
||||
writeGatewayName: write.gatewayToolName,
|
||||
marker,
|
||||
};
|
||||
} catch (error) {
|
||||
fixtures.delete(ctx);
|
||||
await provider.close();
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
gradeGatewayClient,
|
||||
{
|
||||
sourceFiles: [
|
||||
"chat-catalog-domains/external-gateway-client.ts",
|
||||
"chat-catalog-domains/external-provider.ts",
|
||||
"chat-catalog-domains/external-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/gateways-profiles.yaml",
|
||||
],
|
||||
async act(ctx, state) {
|
||||
const fixture = fixtures.get(ctx)!;
|
||||
const active = await pollUntil({
|
||||
label: "admitted source run for independent scoped denial probe",
|
||||
deadlineAt: Date.now() + 60_000,
|
||||
intervalMs: 100,
|
||||
load: async () => {
|
||||
await ctx.refreshIssue();
|
||||
return ctx.allRuns();
|
||||
},
|
||||
accept: (rows) =>
|
||||
rows.some(
|
||||
(row) =>
|
||||
row.agentId === ctx.input.fixtures.agent.id &&
|
||||
row.status === "running",
|
||||
),
|
||||
reject: (rows) =>
|
||||
rows.length &&
|
||||
rows.every((row) =>
|
||||
["succeeded", "failed", "cancelled", "timed_out"].includes(
|
||||
row.status,
|
||||
),
|
||||
)
|
||||
? "Source run settled before scoped-session probe admission"
|
||||
: undefined,
|
||||
});
|
||||
const run = active.find(
|
||||
(row) =>
|
||||
row.agentId === ctx.input.fixtures.agent.id &&
|
||||
row.status === "running",
|
||||
)!;
|
||||
const source = ctx.issue();
|
||||
if (!source || !ctx.input.privateSecrets)
|
||||
throw new Error(
|
||||
"Scoped client requires source identity and private redaction",
|
||||
);
|
||||
state.probeRunId = run.id;
|
||||
state.sourceIssueId = source.id;
|
||||
const key = await ctx.input.api.postSensitive<{
|
||||
id: string;
|
||||
token: string;
|
||||
}>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, {
|
||||
name: `scoped-client-probe-${ctx.input.nonce}`,
|
||||
});
|
||||
fixture.key = key;
|
||||
ctx.input.privateSecrets.push(key.token);
|
||||
const admitted = await privateRequest(
|
||||
ctx,
|
||||
"/api/tool-gateway/sessions",
|
||||
key.token,
|
||||
{
|
||||
companyId: ctx.input.fixtures.company.id,
|
||||
agentId: ctx.input.fixtures.agent.id,
|
||||
runId: run.id,
|
||||
issueId: source.id,
|
||||
},
|
||||
);
|
||||
if (admitted.status !== 201) {
|
||||
// Parse only in private memory and retain only the exact finite classifier.
|
||||
// In particular, never serialize arbitrary body/error/token fields.
|
||||
const response = (await admitted
|
||||
.json()
|
||||
.catch(() => undefined)) as unknown;
|
||||
const diagnostic = scopedSessionFailureDiagnostic(
|
||||
admitted.status,
|
||||
{
|
||||
companyId: ctx.input.fixtures.company.id,
|
||||
agentId: ctx.input.fixtures.agent.id,
|
||||
runId: run.id,
|
||||
issueId: source.id,
|
||||
observedRunStatusAtAdmission: run.status,
|
||||
},
|
||||
response,
|
||||
);
|
||||
await ctx.input.evidence(
|
||||
"chat-catalog-scoped-session-admission-failure.json",
|
||||
diagnostic,
|
||||
);
|
||||
throw new Error(
|
||||
`Active-run scoped session admission failed (HTTP ${diagnostic.httpStatus}; ${diagnostic.reasonCode}); response withheld`,
|
||||
);
|
||||
}
|
||||
fixture.session = (await admitted.json()) as {
|
||||
sessionId: string;
|
||||
token: string;
|
||||
};
|
||||
ctx.input.privateSecrets.push(fixture.session.token);
|
||||
state.sessionCreated = true;
|
||||
const listed = await privateRequest(
|
||||
ctx,
|
||||
"/api/tool-gateway/tools",
|
||||
fixture.session.token,
|
||||
undefined,
|
||||
true,
|
||||
);
|
||||
if (!listed.ok) {
|
||||
await listed.body?.cancel();
|
||||
throw new Error("Scoped tool discovery failed; response withheld");
|
||||
}
|
||||
const tools = (await listed.json()) as Array<{
|
||||
catalogEntryId?: string;
|
||||
}>;
|
||||
state.discoveredRead = tools.some(
|
||||
(row) => row.catalogEntryId === state.readEntryId,
|
||||
);
|
||||
state.discoveredWrite = tools.some(
|
||||
(row) => row.catalogEntryId === state.writeEntryId,
|
||||
);
|
||||
const denied = await privateRequest(
|
||||
ctx,
|
||||
"/api/tool-gateway/tools/call",
|
||||
fixture.session.token,
|
||||
{
|
||||
tool: state.writeGatewayName,
|
||||
parameters: { text: "This oracle write must be denied" },
|
||||
},
|
||||
true,
|
||||
);
|
||||
state.writeProbeStatus = denied.status;
|
||||
// Retain only the allowlisted policy code, never an arbitrary error body.
|
||||
const denial = (await denied.json()) as { reasonCode?: unknown };
|
||||
state.writeProbeReason = ["deny_default", "deny_policy_block"].includes(
|
||||
String(denial.reasonCode),
|
||||
)
|
||||
? denial.reasonCode
|
||||
: "other";
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
const fixture = fixtures.get(ctx);
|
||||
if (!fixture) throw new Error("Scoped synthetic fixture missing");
|
||||
state.effective = await ctx.input.api.get<ToolProfileEffectiveSummary>(
|
||||
`${root(ctx)}/tools/profiles/effective/agents/${ctx.input.fixtures.agent.id}`,
|
||||
);
|
||||
state.decisions = await Promise.all(
|
||||
(await ctx.allRuns()).map((run) =>
|
||||
ctx.input.api.get<ToolRunDecisionLookup>(
|
||||
`${root(ctx)}/tools/runs/${run.id}/decisions`,
|
||||
),
|
||||
),
|
||||
);
|
||||
state.providerReadCalls = fixture.provider.calls.filter(
|
||||
(row) =>
|
||||
row.method === "tools/call" &&
|
||||
row.name === "launch_read" &&
|
||||
row.authorized,
|
||||
).length;
|
||||
state.providerWriteCalls = fixture.provider.calls.filter(
|
||||
(row) => row.method === "tools/call" && row.name === "launch_update",
|
||||
).length;
|
||||
state.written = fixture.provider.written() ?? null;
|
||||
const o = await observeExternal(ctx, state);
|
||||
const tokens = [fixture.key?.token, fixture.session?.token].filter(
|
||||
(token): token is string => typeof token === "string",
|
||||
);
|
||||
state.tokenScanComplete = tokens.length === 2;
|
||||
state.privateTokenLeaked = tokens.some((token) =>
|
||||
JSON.stringify([o.reply, o.runs]).includes(token),
|
||||
);
|
||||
return { ...o, state };
|
||||
},
|
||||
async verifyResult(ctx, _state, value) {
|
||||
await verifyRenderedReply(ctx, value);
|
||||
},
|
||||
async cleanup(ctx, state) {
|
||||
const fixture = fixtures.get(ctx);
|
||||
fixtures.delete(ctx);
|
||||
try {
|
||||
if (fixture?.session && fixture.key) {
|
||||
const response = await privateRequest(
|
||||
ctx,
|
||||
`/api/tool-gateway/sessions/${fixture.session.sessionId}/revoke`,
|
||||
fixture.key.token,
|
||||
{ companyId: ctx.input.fixtures.company.id },
|
||||
);
|
||||
if (!response.ok && response.status !== 404)
|
||||
throw new Error("Scoped fixture session revocation failed");
|
||||
await response.body?.cancel();
|
||||
}
|
||||
if (fixture?.key)
|
||||
await ctx.input.api.delete(
|
||||
`/api/agents/${ctx.input.fixtures.agent.id}/keys/${fixture.key.id}`,
|
||||
{ allowNotFound: true },
|
||||
);
|
||||
if (typeof state.profileId === "string")
|
||||
await ctx.input.api.post(
|
||||
`${root(ctx)}/tools/profiles/${state.profileId}/unbind`,
|
||||
{ targetType: "agent", targetId: ctx.input.fixtures.agent.id },
|
||||
);
|
||||
if (typeof state.connectionId === "string")
|
||||
await ctx.input.api.delete(
|
||||
`/api/tool-connections/${state.connectionId}`,
|
||||
{ allowNotFound: true },
|
||||
);
|
||||
} finally {
|
||||
await fixture?.provider.close();
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
];
|
||||
@@ -0,0 +1,153 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import type { Page } from "@playwright/test";
|
||||
import {
|
||||
enterGenericMcpThroughCatalog,
|
||||
gradeUnreachableMcp,
|
||||
} from "./external-mcp-failure.js";
|
||||
import { scopedSessionFailureDiagnostic } from "./external-gateway-client.js";
|
||||
import type { ExternalObservation } from "./external-support.js";
|
||||
function pageFixture(missing?: string) {
|
||||
const actions: string[] = [];
|
||||
const page = {
|
||||
getByRole: (role: string, options?: { name?: string }) => {
|
||||
const name = options?.name ?? "";
|
||||
const locator = {
|
||||
filter: () => locator,
|
||||
getByRole: (_r: string, o: { name: string }) => ({
|
||||
click: async () => {
|
||||
if (o.name === missing) throw new Error("Missing real control");
|
||||
actions.push(o.name);
|
||||
},
|
||||
}),
|
||||
click: async () => {
|
||||
if (name === missing) throw new Error("Missing real control");
|
||||
actions.push(name);
|
||||
},
|
||||
};
|
||||
return locator;
|
||||
},
|
||||
waitForURL: async (accept: (url: URL) => boolean) => {
|
||||
expect(accept(new URL("http://fixture.invalid/RUN/apps/byo"))).toBe(true);
|
||||
expect(accept(new URL("http://fixture.invalid/RUN/apps"))).toBe(false);
|
||||
actions.push("actual /RUN/apps/byo");
|
||||
},
|
||||
goto: () => {
|
||||
throw new Error("Direct navigation rescue is forbidden");
|
||||
},
|
||||
} as unknown as Page;
|
||||
return { page, actions };
|
||||
}
|
||||
describe("finite held external diagnostics and visible entry correction", () => {
|
||||
it("uses visible catalog controls including the expanded custom server option", async () => {
|
||||
const { page, actions } = pageFixture();
|
||||
await enterGenericMcpThroughCatalog(page, "RUN");
|
||||
expect(actions).toEqual([
|
||||
"Connectors",
|
||||
"Connect",
|
||||
"Connect your own MCP server",
|
||||
"actual /RUN/apps/byo",
|
||||
]);
|
||||
});
|
||||
it("does not rescue a missing catalog entry with direct navigation", async () => {
|
||||
for (const name of [
|
||||
"Connectors",
|
||||
"Connect",
|
||||
"Connect your own MCP server",
|
||||
]) {
|
||||
const { page, actions } = pageFixture(name);
|
||||
await expect(enterGenericMcpThroughCatalog(page, "RUN")).rejects.toThrow(
|
||||
"Missing real control",
|
||||
);
|
||||
expect(actions).not.toContain("actual /RUN/apps/byo");
|
||||
}
|
||||
});
|
||||
const context = {
|
||||
companyId: "owned-company",
|
||||
agentId: "owned-agent",
|
||||
runId: "owned-run",
|
||||
issueId: "owned-issue",
|
||||
observedRunStatusAtAdmission: "running",
|
||||
};
|
||||
it("retains actual HTTP500 and admitted run binding without inventing a cause or fixing admission", () => {
|
||||
expect(
|
||||
scopedSessionFailureDiagnostic(500, context, {
|
||||
error: "private diagnostic",
|
||||
}),
|
||||
).toEqual({
|
||||
...context,
|
||||
httpStatus: 500,
|
||||
reasonCode: "unclassified",
|
||||
responseBodyRetained: false,
|
||||
});
|
||||
});
|
||||
it("retains only a recognized policy/context code, never a secret-bearing error or extra response fields", () => {
|
||||
for (const reason of [
|
||||
"run_company_mismatch",
|
||||
"run_agent_mismatch",
|
||||
"run_inactive",
|
||||
"run_context_mismatch",
|
||||
]) {
|
||||
const d = scopedSessionFailureDiagnostic(403, context, {
|
||||
reasonCode: reason,
|
||||
token: "synthetic-credential",
|
||||
error: "private body",
|
||||
});
|
||||
expect(d.reasonCode).toBe(reason);
|
||||
expect(JSON.stringify(d)).not.toMatch(
|
||||
/synthetic-credential|private body/,
|
||||
);
|
||||
}
|
||||
});
|
||||
it("omits unknown, malformed, nested and secret-bearing reason values", () => {
|
||||
for (const body of [
|
||||
{ reasonCode: "Bearer synthetic-credential" },
|
||||
{ reasonCode: "new_upstream_message" },
|
||||
{ reasonCode: { token: "synthetic-credential" } },
|
||||
{ error: { reasonCode: "run_inactive" } },
|
||||
null,
|
||||
"private body",
|
||||
]) {
|
||||
const d = scopedSessionFailureDiagnostic(500, context, body);
|
||||
expect(d.reasonCode).toBe("unclassified");
|
||||
expect(JSON.stringify(d)).not.toMatch(
|
||||
/Bearer|synthetic-credential|private body|new_upstream_message/,
|
||||
);
|
||||
}
|
||||
});
|
||||
it("visible-entry correction does not turn the retained model missing-URL question into a completed setup pass", () => {
|
||||
const o = {
|
||||
scenarioId: "connection-setup.unreachable-mcp",
|
||||
companyId: "company",
|
||||
origin: "http://fixture.invalid",
|
||||
reply: "",
|
||||
connections: [],
|
||||
phaseReplies: [],
|
||||
profiles: [],
|
||||
secrets: [],
|
||||
endpoints: [],
|
||||
interactions: [],
|
||||
activity: [],
|
||||
links: [],
|
||||
runs: [],
|
||||
humanQueries: [],
|
||||
state: {
|
||||
fixtureServerClosed: true,
|
||||
serverUrl: "http://127.0.0.1:12345/",
|
||||
genericEntryThroughVisibleCatalog: true,
|
||||
setupStatus: 502,
|
||||
visibleConnectivityError: true,
|
||||
checkAgainAvailable: true,
|
||||
displayedServerPreserved: true,
|
||||
editableUrlPreserved: true,
|
||||
abandonedDraft: true,
|
||||
noUnexpectedConnection: true,
|
||||
},
|
||||
} as unknown as ExternalObservation;
|
||||
expect(gradeUnreachableMcp(o).passed).toBe(false);
|
||||
o.reply =
|
||||
"The MCP server is not connected; correct the address or abandon this setup.";
|
||||
expect(gradeUnreachableMcp(o).passed).toBe(true);
|
||||
o.state.genericEntryThroughVisibleCatalog = false;
|
||||
expect(gradeUnreachableMcp(o).passed).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,53 +1,205 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
import type { ToolConnection } from "../../../packages/shared/src/types/tool-access.js";
|
||||
import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
|
||||
import {
|
||||
definition,
|
||||
externalObservation,
|
||||
observeExternal,
|
||||
root,
|
||||
checks,
|
||||
verifyRenderedReply,
|
||||
} from "./external-support.js";
|
||||
import { startNotesProvider } from "./external-provider.js";
|
||||
import { waitForReply } from "./identity-support.js";
|
||||
/** Follow the actual catalog card rather than an unqualified route that the
|
||||
* product intentionally redirects to Browse. No direct /byo navigation fallback. */
|
||||
export async function enterGenericMcpThroughCatalog(
|
||||
page: Page,
|
||||
prefix: string,
|
||||
) {
|
||||
await page.getByRole("link", { name: "Connectors", exact: true }).click();
|
||||
const card = page
|
||||
.getByRole("listitem")
|
||||
.filter({
|
||||
has: page.getByRole("heading", {
|
||||
name: "Connect your own tool",
|
||||
exact: true,
|
||||
}),
|
||||
});
|
||||
await card.getByRole("button", { name: "Connect", exact: true }).click();
|
||||
await card
|
||||
.getByRole("button", { name: "Connect your own MCP server" })
|
||||
.click();
|
||||
await page.waitForURL((url) => url.pathname === `/${prefix}/apps/byo`);
|
||||
}
|
||||
export function gradeUnreachableMcp(value: unknown) {
|
||||
const c = checks(), o = externalObservation(value);
|
||||
c.add("complete-unreachable-server-evidence", !!o, "An actual unavailable fixture endpoint, rejected public setup request and visible recovery controls are required."); if (!o) return c.result();
|
||||
c.add("actual-unavailable-fixture", o.state.fixtureServerClosed === true && typeof o.state.serverUrl === "string" && /^http:\/\/127\.0\.0\.1:\d+\/$/.test(o.state.serverUrl),
|
||||
"The supplied synthetic endpoint is closed before the human query; it is not a mocked provider error.");
|
||||
c.add("supported-setup-failed-visibly", [400, 403, 422, 502, 503].includes(Number(o.state.setupStatus)) && o.state.visibleConnectivityError === true
|
||||
&& o.state.checkAgainAvailable === true, "The real setup endpoint rejects connectivity or destination policy and renders an actionable error.");
|
||||
c.add("safe-editable-choice-preserved", o.state.displayedServerPreserved === true && o.state.editableUrlPreserved === true && o.state.abandonedDraft === true,
|
||||
"The person can return to the exact safe address, edit it and abandon the unfinished setup.");
|
||||
c.add("no-usable-phantom-connection", !o.connections.some(row => row.status === "active") && o.state.noUnexpectedConnection === true,
|
||||
"Failed setup leaves no healthy or unexpected substitute connection.");
|
||||
c.add("no-false-connected-claim", /mcp|server|connect/i.test(o.reply) && !/\bI (?:have )?(?:connected|installed|enabled)\b|\b(?:is|now|already) (?:connected|healthy|ready to use)\b/i.test(o.reply),
|
||||
"Chat does not label the unavailable server healthy or silently substitute a different destination.");
|
||||
const c = checks(),
|
||||
o = externalObservation(value);
|
||||
c.add(
|
||||
"complete-unreachable-server-evidence",
|
||||
!!o,
|
||||
"An actual unavailable fixture endpoint, rejected public setup request and visible recovery controls are required.",
|
||||
);
|
||||
if (!o) return c.result();
|
||||
c.add(
|
||||
"actual-unavailable-fixture",
|
||||
o.state.fixtureServerClosed === true &&
|
||||
typeof o.state.serverUrl === "string" &&
|
||||
/^http:\/\/127\.0\.0\.1:\d+\/$/.test(o.state.serverUrl),
|
||||
"The supplied synthetic endpoint is closed before the human query; it is not a mocked provider error.",
|
||||
);
|
||||
c.add(
|
||||
"supported-setup-failed-visibly",
|
||||
[400, 403, 422, 502, 503].includes(Number(o.state.setupStatus)) &&
|
||||
o.state.visibleConnectivityError === true &&
|
||||
o.state.checkAgainAvailable === true &&
|
||||
o.state.genericEntryThroughVisibleCatalog === true,
|
||||
"The real setup endpoint rejects connectivity or destination policy and renders an actionable error.",
|
||||
);
|
||||
c.add(
|
||||
"safe-editable-choice-preserved",
|
||||
o.state.displayedServerPreserved === true &&
|
||||
o.state.editableUrlPreserved === true &&
|
||||
o.state.abandonedDraft === true,
|
||||
"The person can return to the exact safe address, edit it and abandon the unfinished setup.",
|
||||
);
|
||||
c.add(
|
||||
"no-usable-phantom-connection",
|
||||
!o.connections.some((row) => row.status === "active") &&
|
||||
o.state.noUnexpectedConnection === true,
|
||||
"Failed setup leaves no healthy or unexpected substitute connection.",
|
||||
);
|
||||
c.add(
|
||||
"no-false-connected-claim",
|
||||
/mcp|server|connect/i.test(o.reply) &&
|
||||
!/\bI (?:have )?(?:connected|installed|enabled)\b|\b(?:is|now|already) (?:connected|healthy|ready to use)\b/i.test(
|
||||
o.reply,
|
||||
),
|
||||
"Chat does not label the unavailable server healthy or silently substitute a different destination.",
|
||||
);
|
||||
return c.result();
|
||||
}
|
||||
export const mcpFailureExternalCases = [definition("connection-setup.unreachable-mcp", async ctx => {
|
||||
const server = await startNotesProvider("Never served by the closed fixture"); const serverUrl = server.url; await server.close();
|
||||
const before = await ctx.input.api.get<{ connections: unknown[] }>(`${root(ctx)}/tools/connections`); if (before.connections.length) throw new Error("Unavailable server journey requires an empty disposable connection catalog");
|
||||
await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
|
||||
await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The supplied synthetic generic MCP server URL is ${serverUrl}. No credential has been supplied.` });
|
||||
return { serverUrl, fixtureServerClosed: true };
|
||||
}, gradeUnreachableMcp, {
|
||||
sourceFiles: ["chat-catalog-domains/external-mcp-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml"],
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx); const source = ctx.input.page.url();
|
||||
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/apps/connect`, { waitUntil: "domcontentloaded" });
|
||||
const url = ctx.input.page.getByRole("textbox", { name: "MCP server URL", exact: true });
|
||||
await url.fill(String(state.serverUrl)); await ctx.input.page.getByRole("button", { name: "Continue", exact: true }).click();
|
||||
const checked = ctx.input.page.waitForResponse(response => new URL(response.url()).pathname === `${root(ctx)}/tools/apps/connect` && response.request().method() === "POST");
|
||||
await ctx.input.page.getByRole("button", { name: "Check link", exact: true }).click();
|
||||
state.setupStatus = (await checked).status();
|
||||
const error = ctx.input.page.getByRole("note").filter({ hasText: /couldn.t (?:reach|connect)|private network|couldn.t find that host/i });
|
||||
await expect(error).toBeVisible(); state.visibleConnectivityError = true;
|
||||
await expect(ctx.input.page.getByRole("button", { name: "Check link", exact: true })).toBeEnabled(); state.checkAgainAvailable = true;
|
||||
await expect(ctx.input.page.getByText(String(state.serverUrl), { exact: true })).toBeVisible(); state.displayedServerPreserved = true;
|
||||
await ctx.input.page.getByRole("button", { name: "Back", exact: true }).click();
|
||||
await expect(url).toHaveValue(String(state.serverUrl)); await url.fill(String(state.serverUrl) + "corrected"); state.editableUrlPreserved = await url.inputValue() === String(state.serverUrl) + "corrected";
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); state.abandonedDraft = true;
|
||||
await ctx.input.evidence("chat-catalog-unreachable-mcp.json", { setupStatus: state.setupStatus, visibleConnectivityError: true, editableUrlPreserved: state.editableUrlPreserved, abandonedDraft: true });
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
const rows = await ctx.input.api.get<{ connections: ToolConnection[] }>(`${root(ctx)}/tools/connections`);
|
||||
state.noUnexpectedConnection = rows.connections.every(row => row.config?.url === state.serverUrl && row.status !== "active");
|
||||
return observeExternal(ctx, state);
|
||||
},
|
||||
async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); },
|
||||
async cleanup(ctx) { const rows = await ctx.input.api.get<{ connections: Array<{ id: string }> }>(`${root(ctx)}/tools/connections`); for (const row of rows.connections) await ctx.input.api.delete(`/api/tool-connections/${row.id}`, { allowNotFound: true }); },
|
||||
})];
|
||||
export const mcpFailureExternalCases = [
|
||||
definition(
|
||||
"connection-setup.unreachable-mcp",
|
||||
async (ctx) => {
|
||||
const server = await startNotesProvider(
|
||||
"Never served by the closed fixture",
|
||||
);
|
||||
const serverUrl = server.url;
|
||||
await server.close();
|
||||
const before = await ctx.input.api.get<{ connections: unknown[] }>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
);
|
||||
if (before.connections.length)
|
||||
throw new Error(
|
||||
"Unavailable server journey requires an empty disposable connection catalog",
|
||||
);
|
||||
await ctx.input.api.post(
|
||||
`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`,
|
||||
);
|
||||
await ctx.refreshIssue();
|
||||
await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, {
|
||||
description: `The supplied synthetic generic MCP server URL is ${serverUrl}. No credential has been supplied.`,
|
||||
});
|
||||
return { serverUrl, fixtureServerClosed: true };
|
||||
},
|
||||
gradeUnreachableMcp,
|
||||
{
|
||||
sourceFiles: [
|
||||
"../../ui/src/App.tsx",
|
||||
"../../ui/src/pages/apps/Browse.tsx",
|
||||
"../../ui/src/features/connections/ConnectionSetupFlow.tsx",
|
||||
"../../ui/src/pages/apps/app-connect-policy.ts",
|
||||
"chat-catalog-domains/external-mcp-failure.ts",
|
||||
"chat-catalog-domains/external-provider.ts",
|
||||
"chat-catalog-domains/external-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-scenarios/connection-setup.yaml",
|
||||
],
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx);
|
||||
const source = ctx.input.page.url();
|
||||
await enterGenericMcpThroughCatalog(
|
||||
ctx.input.page,
|
||||
String(ctx.input.fixtures.company.issuePrefix),
|
||||
);
|
||||
state.genericEntryThroughVisibleCatalog = true;
|
||||
const url = ctx.input.page.getByRole("textbox", {
|
||||
name: "MCP server URL",
|
||||
exact: true,
|
||||
});
|
||||
await url.fill(String(state.serverUrl));
|
||||
await ctx.input.page
|
||||
.getByRole("button", { name: "Continue", exact: true })
|
||||
.click();
|
||||
const checked = ctx.input.page.waitForResponse(
|
||||
(response) =>
|
||||
new URL(response.url()).pathname ===
|
||||
`${root(ctx)}/tools/apps/connect` &&
|
||||
response.request().method() === "POST",
|
||||
);
|
||||
await ctx.input.page
|
||||
.getByRole("button", { name: "Check link", exact: true })
|
||||
.click();
|
||||
state.setupStatus = (await checked).status();
|
||||
const error = ctx.input.page
|
||||
.getByRole("note")
|
||||
.filter({
|
||||
hasText:
|
||||
/couldn.t (?:reach|connect)|private network|couldn.t find that host/i,
|
||||
});
|
||||
await expect(error).toBeVisible();
|
||||
state.visibleConnectivityError = true;
|
||||
await expect(
|
||||
ctx.input.page.getByRole("button", {
|
||||
name: "Check link",
|
||||
exact: true,
|
||||
}),
|
||||
).toBeEnabled();
|
||||
state.checkAgainAvailable = true;
|
||||
await expect(
|
||||
ctx.input.page.getByText(String(state.serverUrl), { exact: true }),
|
||||
).toBeVisible();
|
||||
state.displayedServerPreserved = true;
|
||||
await ctx.input.page
|
||||
.getByRole("button", { name: "Back", exact: true })
|
||||
.click();
|
||||
await expect(url).toHaveValue(String(state.serverUrl));
|
||||
await url.fill(String(state.serverUrl) + "corrected");
|
||||
state.editableUrlPreserved =
|
||||
(await url.inputValue()) === String(state.serverUrl) + "corrected";
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
state.abandonedDraft = true;
|
||||
await ctx.input.evidence("chat-catalog-unreachable-mcp.json", {
|
||||
setupStatus: state.setupStatus,
|
||||
visibleConnectivityError: true,
|
||||
editableUrlPreserved: state.editableUrlPreserved,
|
||||
abandonedDraft: true,
|
||||
});
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
const rows = await ctx.input.api.get<{ connections: ToolConnection[] }>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
);
|
||||
state.noUnexpectedConnection = rows.connections.every(
|
||||
(row) =>
|
||||
row.config?.url === state.serverUrl && row.status !== "active",
|
||||
);
|
||||
return observeExternal(ctx, state);
|
||||
},
|
||||
async verifyResult(ctx, _state, value) {
|
||||
await verifyRenderedReply(ctx, value);
|
||||
},
|
||||
async cleanup(ctx) {
|
||||
const rows = await ctx.input.api.get<{
|
||||
connections: Array<{ id: string }>;
|
||||
}>(`${root(ctx)}/tools/connections`);
|
||||
for (const row of rows.connections)
|
||||
await ctx.input.api.delete(`/api/tool-connections/${row.id}`, {
|
||||
allowNotFound: true,
|
||||
});
|
||||
},
|
||||
},
|
||||
),
|
||||
];
|
||||
@@ -45,7 +45,7 @@ function valid(id: string) {
|
||||
phaseBoundary: "operator-chat-to-viewer-browser-consent", chatCallerId: "operator", viewerSessionBefore: "viewer", viewerSessionAtConsent: "viewer", viewerSessionAfter: "viewer" });
|
||||
o.activity = [{ id: "authorization", action: "mcp.connection_authorized", entityId: "grant", actorType: "user", actorId: "viewer" }];
|
||||
} else if (id === "connection-setup.unreachable-mcp") {
|
||||
Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true });
|
||||
Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, genericEntryThroughVisibleCatalog:true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true });
|
||||
o.reply = "Open the supported MCP server connection setup and check its address.";
|
||||
} else if (id === "plugins.install-plugin") {
|
||||
Object.assign(o.state, { pluginId: "plugin", pluginKey: "synthetic.preview", syntheticPackage: true, pluginAfter: { id: "plugin", pluginKey: "synthetic.preview", status: "ready", manifestJson: { displayName: "Website Preview Eval" } }, operatorChannel: "board-api-local-path", agentInstallStatus: 403, absentBeforeHuman: true, humanInstallStatus: 200, previewReadback: true, humanSawActivation: true });
|
||||
|
||||
Reference in new issue
Block a user