diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts index eec7bd6ab1d..02b9752d8e4 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts @@ -5,7 +5,12 @@ import { gradeAppTest } from "./external-app-test.js"; const hash = (text: string) => createHash("sha256").update(text).digest("hex"); export function authorizedTestObservation() { const marker = "Synthetic sample page unit-492", requestId = "paperclip-tool-synthetic-unit"; - const result = JSON.stringify({ content: [{ text: marker, type: "text" }] }); + // Independently reproduce the gateway's stable normalized MCP result, not the + // raw provider response below. This synthetic fixture follows the public owner. + const result = JSON.stringify({ content: marker, data: { + content: [{ text: marker, type: "text" }], isError: false, + spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http", + } }); const invocation = { id: "invocation", companyId: "company", actorType: "agent", actorId: "ceo", agentId: "morgan", issueId: "source", runId: "native", connectionId: "connection", catalogEntryId: "read", toolName: "launch_read", status: "succeeded", policyDecision: "allow", errorCode: null, resultHash: hash(result), resultSizeBytes: Buffer.byteLength(result), startedAt: "2026-11-03T00:00:02Z", completedAt: "2026-11-03T00:00:03Z" }; const scope = { agentId: "morgan", allowed: ["read"], profiles: ["profile"], entries: ["entry"], bindings: ["binding"], connections: ["connection"] }; const reply = "I checked Morgan's read access to Synthetic Launch Notes. The sample-page test succeeded; it is a test result, not a guarantee about the next runtime."; @@ -80,3 +85,27 @@ it.each(["denied", "no provider read", "wrong target", "wrong caller", "wrong co if (fault === "false actor report") { o.reply = "I checked CEO read access and the sample-page test succeeded."; o.phaseReplies[0]!.body = o.reply; } expect(gradeAppTest(o).passed).toBe(false); }); + +it.each(["raw provider only", "wrong visible content", "wrong nested marker", "missing data", "error result", "local process", "wrong transport", "extra structured data", "wrong size"])("rejects %s even when durable invocation/audit hashes agree", fault => { + const o = authorizedTestObservation(); + const normalized: Record = { content: o.state.marker, data: { + content: [{ text: o.state.marker, type: "text" }], isError: false, + spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http", + } }; + let result: unknown = normalized; + if (fault === "raw provider only") result = { content: [{ text: o.state.marker, type: "text" }] }; + if (fault === "wrong visible content") normalized.content = "Different sample page"; + const data = normalized.data as Record; + if (fault === "wrong nested marker") data.content = [{ text: "Different sample page", type: "text" }]; + if (fault === "missing data") delete normalized.data; + if (fault === "error result") data.isError = true; + if (fault === "local process") data.spawnedLocalProcess = true; + if (fault === "wrong transport") data.transport = "local_stdio"; + if (fault === "extra structured data") data.structuredContent = { unexpected: true }; + const serialized = JSON.stringify(result); + const v = o.state.decisions[0]!.invocation; + v.resultHash = hash(serialized); v.resultSizeBytes = Buffer.byteLength(serialized) + (fault === "wrong size" ? 1 : 0); + o.state.decisions[0]!.events[0]!.resultHash = v.resultHash; + expect(o.state.providerCalls).toHaveLength(1); + expect(gradeAppTest(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts index b8a99ac23af..e660f79a480 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts @@ -68,7 +68,13 @@ export function gradeAppTest(value: unknown) { && o.humanQueries.length === 1 && o.humanQueries[0] === s.originalQuery && time(native.startedAt) >= time(s.setupFinishedAt); c.add("one-original-native-request", !!queryBound, "One unchanged human query and its actual successful CEO native turn must own the selected source conversation."); const decisions = Array.isArray(s.decisions) ? s.decisions.map(record) : [], decision = decisions[0], v = record(decision?.invocation); - const resultText = JSON.stringify({ content: [{ text: s.marker, type: "text" }] }), expectedResultHash = hash(resultText); + // The gateway persists its normalized MCP envelope, not the raw provider + // result. Keep this independently expected shape aligned with + // normalizeMcpToolResult + summarizeToolValue's stable key ordering. + const resultText = JSON.stringify({ content: s.marker, data: { + content: [{ text: s.marker, type: "text" }], isError: false, + spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http", + } }), expectedResultHash = hash(resultText); const call = (Array.isArray(s.providerCalls) ? s.providerCalls : []).map(record)[0], calls = s.providerCalls as unknown[] | undefined; const expectedProviderHash = hash(JSON.stringify({ content: [{ type: "text", text: s.marker }] })); const events = (Array.isArray(decision?.events) ? decision.events : []).map(record).filter(e => e.eventType === "call_completed"); @@ -117,7 +123,7 @@ export const appTestExternalCases = [definition("app-permissions.test-as-agent", connectionPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/${connection.connectionId}`, resultPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/${connection.connectionId}/permissions` }; } catch (error) { providers.delete(ctx); await provider.close(); throw error; } }, gradeAppTest, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.app-test.v2", + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.app-test.v3", sourceFiles: ["chat-catalog-domains/external-app-test.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../server/src/routes/tool-access.ts", "../../server/src/services/tool-gateway.ts", "../../server/src/services/tool-access/activity/service.ts", "../../server/src/services/tool-content-guards.ts", "../../server/src/services/tool-approval-execution.ts", "chat-scenarios/app-permissions.yaml"], async act(ctx) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); }, observe: observeAppTest,