From 459c5d784fcff2973c724cbb4f18f198a1edb2a2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 23:24:58 -0600 Subject: [PATCH] test(evals): bind app test evidence to normalized MCP result Reconstruct the documented gateway result independently, retaining raw provider and source attribution checks. Preserve the original failure and mark the oracle v3 changed condition. Co-Authored-By: Paperclip --- .../external-app-test-native.test.ts | 31 ++++++++++++++++++- .../chat-catalog-domains/external-app-test.ts | 10 ++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts index eec7bd6ab1d..02b9752d8e4 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts @@ -5,7 +5,12 @@ import { gradeAppTest } from "./external-app-test.js"; const hash = (text: string) => createHash("sha256").update(text).digest("hex"); export function authorizedTestObservation() { const marker = "Synthetic sample page unit-492", requestId = "paperclip-tool-synthetic-unit"; - const result = JSON.stringify({ content: [{ text: marker, type: "text" }] }); + // Independently reproduce the gateway's stable normalized MCP result, not the + // raw provider response below. This synthetic fixture follows the public owner. + const result = JSON.stringify({ content: marker, data: { + content: [{ text: marker, type: "text" }], isError: false, + spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http", + } }); const invocation = { id: "invocation", companyId: "company", actorType: "agent", actorId: "ceo", agentId: "morgan", issueId: "source", runId: "native", connectionId: "connection", catalogEntryId: "read", toolName: "launch_read", status: "succeeded", policyDecision: "allow", errorCode: null, resultHash: hash(result), resultSizeBytes: Buffer.byteLength(result), startedAt: "2026-11-03T00:00:02Z", completedAt: "2026-11-03T00:00:03Z" }; const scope = { agentId: "morgan", allowed: ["read"], profiles: ["profile"], entries: ["entry"], bindings: ["binding"], connections: ["connection"] }; const reply = "I checked Morgan's read access to Synthetic Launch Notes. The sample-page test succeeded; it is a test result, not a guarantee about the next runtime."; @@ -80,3 +85,27 @@ it.each(["denied", "no provider read", "wrong target", "wrong caller", "wrong co if (fault === "false actor report") { o.reply = "I checked CEO read access and the sample-page test succeeded."; o.phaseReplies[0]!.body = o.reply; } expect(gradeAppTest(o).passed).toBe(false); }); + +it.each(["raw provider only", "wrong visible content", "wrong nested marker", "missing data", "error result", "local process", "wrong transport", "extra structured data", "wrong size"])("rejects %s even when durable invocation/audit hashes agree", fault => { + const o = authorizedTestObservation(); + const normalized: Record = { content: o.state.marker, data: { + content: [{ text: o.state.marker, type: "text" }], isError: false, + spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http", + } }; + let result: unknown = normalized; + if (fault === "raw provider only") result = { content: [{ text: o.state.marker, type: "text" }] }; + if (fault === "wrong visible content") normalized.content = "Different sample page"; + const data = normalized.data as Record; + if (fault === "wrong nested marker") data.content = [{ text: "Different sample page", type: "text" }]; + if (fault === "missing data") delete normalized.data; + if (fault === "error result") data.isError = true; + if (fault === "local process") data.spawnedLocalProcess = true; + if (fault === "wrong transport") data.transport = "local_stdio"; + if (fault === "extra structured data") data.structuredContent = { unexpected: true }; + const serialized = JSON.stringify(result); + const v = o.state.decisions[0]!.invocation; + v.resultHash = hash(serialized); v.resultSizeBytes = Buffer.byteLength(serialized) + (fault === "wrong size" ? 1 : 0); + o.state.decisions[0]!.events[0]!.resultHash = v.resultHash; + expect(o.state.providerCalls).toHaveLength(1); + expect(gradeAppTest(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts index b8a99ac23af..e660f79a480 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-test.ts @@ -68,7 +68,13 @@ export function gradeAppTest(value: unknown) { && o.humanQueries.length === 1 && o.humanQueries[0] === s.originalQuery && time(native.startedAt) >= time(s.setupFinishedAt); c.add("one-original-native-request", !!queryBound, "One unchanged human query and its actual successful CEO native turn must own the selected source conversation."); const decisions = Array.isArray(s.decisions) ? s.decisions.map(record) : [], decision = decisions[0], v = record(decision?.invocation); - const resultText = JSON.stringify({ content: [{ text: s.marker, type: "text" }] }), expectedResultHash = hash(resultText); + // The gateway persists its normalized MCP envelope, not the raw provider + // result. Keep this independently expected shape aligned with + // normalizeMcpToolResult + summarizeToolValue's stable key ordering. + const resultText = JSON.stringify({ content: s.marker, data: { + content: [{ text: s.marker, type: "text" }], isError: false, + spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http", + } }), expectedResultHash = hash(resultText); const call = (Array.isArray(s.providerCalls) ? s.providerCalls : []).map(record)[0], calls = s.providerCalls as unknown[] | undefined; const expectedProviderHash = hash(JSON.stringify({ content: [{ type: "text", text: s.marker }] })); const events = (Array.isArray(decision?.events) ? decision.events : []).map(record).filter(e => e.eventType === "call_completed"); @@ -117,7 +123,7 @@ export const appTestExternalCases = [definition("app-permissions.test-as-agent", connectionPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/${connection.connectionId}`, resultPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/${connection.connectionId}/permissions` }; } catch (error) { providers.delete(ctx); await provider.close(); throw error; } }, gradeAppTest, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.app-test.v2", + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.app-test.v3", sourceFiles: ["chat-catalog-domains/external-app-test.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../server/src/routes/tool-access.ts", "../../server/src/services/tool-gateway.ts", "../../server/src/services/tool-access/activity/service.ts", "../../server/src/services/tool-content-guards.ts", "../../server/src/services/tool-approval-execution.ts", "chat-scenarios/app-permissions.yaml"], async act(ctx) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); }, observe: observeAppTest,