test(evals): bind app test evidence to normalized MCP result

Reconstruct the documented gateway result independently, retaining raw provider and source attribution checks. Preserve the original failure and mark the oracle v3 changed condition.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/external-app-test-native.test.ts

@@ -5,7 +5,12 @@ import { gradeAppTest } from "./external-app-test.js";
 const hash = (text: string) => createHash("sha256").update(text).digest("hex");
 export function authorizedTestObservation() {
   const marker = "Synthetic sample page unit-492", requestId = "paperclip-tool-synthetic-unit";
-  const result = JSON.stringify({ content: [{ text: marker, type: "text" }] });
+  // Independently reproduce the gateway's stable normalized MCP result, not the
+  // raw provider response below. This synthetic fixture follows the public owner.
+  const result = JSON.stringify({ content: marker, data: {
+    content: [{ text: marker, type: "text" }], isError: false,
+    spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http",
+  } });
   const invocation = { id: "invocation", companyId: "company", actorType: "agent", actorId: "ceo", agentId: "morgan", issueId: "source", runId: "native", connectionId: "connection", catalogEntryId: "read", toolName: "launch_read", status: "succeeded", policyDecision: "allow", errorCode: null, resultHash: hash(result), resultSizeBytes: Buffer.byteLength(result), startedAt: "2026-11-03T00:00:02Z", completedAt: "2026-11-03T00:00:03Z" };
   const scope = { agentId: "morgan", allowed: ["read"], profiles: ["profile"], entries: ["entry"], bindings: ["binding"], connections: ["connection"] };
   const reply = "I checked Morgan's read access to Synthetic Launch Notes. The sample-page test succeeded; it is a test result, not a guarantee about the next runtime.";
@@ -80,3 +85,27 @@ it.each(["denied", "no provider read", "wrong target", "wrong caller", "wrong co
   if (fault === "false actor report") { o.reply = "I checked CEO read access and the sample-page test succeeded."; o.phaseReplies[0]!.body = o.reply; }
   expect(gradeAppTest(o).passed).toBe(false);
 });
+
+it.each(["raw provider only", "wrong visible content", "wrong nested marker", "missing data", "error result", "local process", "wrong transport", "extra structured data", "wrong size"])("rejects %s even when durable invocation/audit hashes agree", fault => {
+  const o = authorizedTestObservation();
+  const normalized: Record<string, unknown> = { content: o.state.marker, data: {
+    content: [{ text: o.state.marker, type: "text" }], isError: false,
+    spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http",
+  } };
+  let result: unknown = normalized;
+  if (fault === "raw provider only") result = { content: [{ text: o.state.marker, type: "text" }] };
+  if (fault === "wrong visible content") normalized.content = "Different sample page";
+  const data = normalized.data as Record<string, unknown>;
+  if (fault === "wrong nested marker") data.content = [{ text: "Different sample page", type: "text" }];
+  if (fault === "missing data") delete normalized.data;
+  if (fault === "error result") data.isError = true;
+  if (fault === "local process") data.spawnedLocalProcess = true;
+  if (fault === "wrong transport") data.transport = "local_stdio";
+  if (fault === "extra structured data") data.structuredContent = { unexpected: true };
+  const serialized = JSON.stringify(result);
+  const v = o.state.decisions[0]!.invocation;
+  v.resultHash = hash(serialized); v.resultSizeBytes = Buffer.byteLength(serialized) + (fault === "wrong size" ? 1 : 0);
+  o.state.decisions[0]!.events[0]!.resultHash = v.resultHash;
+  expect(o.state.providerCalls).toHaveLength(1);
+  expect(gradeAppTest(o).passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/external-app-test.ts

@@ -68,7 +68,13 @@ export function gradeAppTest(value: unknown) {
     && o.humanQueries.length === 1 && o.humanQueries[0] === s.originalQuery && time(native.startedAt) >= time(s.setupFinishedAt);
   c.add("one-original-native-request", !!queryBound, "One unchanged human query and its actual successful CEO native turn must own the selected source conversation.");
   const decisions = Array.isArray(s.decisions) ? s.decisions.map(record) : [], decision = decisions[0], v = record(decision?.invocation);
-  const resultText = JSON.stringify({ content: [{ text: s.marker, type: "text" }] }), expectedResultHash = hash(resultText);
+  // The gateway persists its normalized MCP envelope, not the raw provider
+  // result. Keep this independently expected shape aligned with
+  // normalizeMcpToolResult + summarizeToolValue's stable key ordering.
+  const resultText = JSON.stringify({ content: s.marker, data: {
+    content: [{ text: s.marker, type: "text" }], isError: false,
+    spawnedLocalProcess: false, structuredContent: null, transport: "mcp_http",
+  } }), expectedResultHash = hash(resultText);
   const call = (Array.isArray(s.providerCalls) ? s.providerCalls : []).map(record)[0], calls = s.providerCalls as unknown[] | undefined;
   const expectedProviderHash = hash(JSON.stringify({ content: [{ type: "text", text: s.marker }] }));
   const events = (Array.isArray(decision?.events) ? decision.events : []).map(record).filter(e => e.eventType === "call_completed");
@@ -117,7 +123,7 @@ export const appTestExternalCases = [definition("app-permissions.test-as-agent",
       connectionPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/${connection.connectionId}`, resultPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/${connection.connectionId}/permissions` };
   } catch (error) { providers.delete(ctx); await provider.close(); throw error; }
 }, gradeAppTest, {
-  authentication: "human-session", oracleVersion: "paperclip.chat-catalog.app-test.v2",
+  authentication: "human-session", oracleVersion: "paperclip.chat-catalog.app-test.v3",
   sourceFiles: ["chat-catalog-domains/external-app-test.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "../../server/src/routes/tool-access.ts", "../../server/src/services/tool-gateway.ts", "../../server/src/services/tool-access/activity/service.ts", "../../server/src/services/tool-content-guards.ts", "../../server/src/services/tool-approval-execution.ts", "chat-scenarios/app-permissions.yaml"],
   async act(ctx) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); },
   observe: observeAppTest,