From 34d8e441a4aa9a80d65e7aa7407246c900c7b441 Mon Sep 17 00:00:00 2001
From: Dotta <bippadotta@protonmail.com>
Date: Wed, 4 Nov 2026 09:04:01 -0600
Subject: [PATCH] test(e2e): match the approved replay provider error contract

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 .../runner-e2e/chat-catalog-descriptors.json  |  2 +-
 .../external-app-review.test.ts               | 29 ++++++++++++++++++-
 .../external-app-review.ts                    |  5 ++--
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json
index 8163bf9bf1d..64cef4bbd4d 100644
--- a/tests/runner-e2e/chat-catalog-descriptors.json
+++ b/tests/runner-e2e/chat-catalog-descriptors.json
@@ -830,7 +830,7 @@
       "../../ui/src/components/task-chat/TaskChatComposer.tsx",
       "chat-scenarios/app-permissions.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.app-review.v4",
+    "oracleVersion": "paperclip.chat-catalog.app-review.v5",
     "expectedRunCount": 6,
     "expectedHumanMessageCount": 3,
     "minimumRunCount": 6,
diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review.test.ts
index 3fdc4b5e46a..ca957fe0fb8 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-app-review.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-app-review.test.ts
@@ -130,7 +130,7 @@ function valid() {
               catalogEntryId: "write",
               argumentsHash,
               status: ["denied", "failed", "succeeded"][index],
-              errorCode: index === 1 ? "tool_error" : null,
+              errorCode: index === 1 ? "tool_execution_failed" : null,
             },
             actionRequest: {
               id: phase.actionRequestId,
@@ -335,3 +335,30 @@ it.each(["attribution", "code", "mutation", "revocation"])("refuses invalid appr
   if (fault === "revocation") o.state.agentProbeKeyRevoked = false;
   expect(gradeAppReview(o).passed).toBe(false);
 });
+
+it("accepts the approved replay provider error with its persisted execution-failure code", () => {
+  const o = valid();
+  o.state.decisions[1]!.decisions[0]!.invocation.errorCode = "tool_execution_failed";
+  expect(gradeAppReview(o).passed).toBe(true);
+});
+
+it.each(["tool_error", "policy_denial", "approval_required", "action_declined", "tool_timeout", "tool_not_found", null])(
+  "rejects unrelated approved-outcome error code %s", errorCode => {
+    const o = valid();
+    o.state.decisions[1]!.decisions[0]!.invocation.errorCode = errorCode;
+    expect(gradeAppReview(o).passed).toBe(false);
+  },
+);
+it.each(["no human approval", "wrong human", "successful invocation", "successful request", "no provider call", "changed provider state", "wrong provider"])(
+  "the execution-failure code cannot hide %s", fault => {
+    const o = valid(), row = o.state.decisions[1]!.decisions[0]!, phase = o.state.phases[1]!;
+    if (fault === "no human approval") o.interactions[1]!.status = "rejected";
+    if (fault === "wrong human") o.interactions[1]!.resolvedByUserId = "another-person";
+    if (fault === "successful invocation") row.invocation.status = "succeeded";
+    if (fault === "successful request") row.actionRequest.status = "executed";
+    if (fault === "no provider call") phase.afterWrites = 0;
+    if (fault === "changed provider state") phase.writtenAfter = phase.reviewedText;
+    if (fault === "wrong provider") row.invocation.connectionId = "unrelated";
+    expect(gradeAppReview(o).passed).toBe(false);
+  },
+);
diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review.ts
index 18b27a61ef6..29f08bc5147 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-app-review.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-app-review.ts
@@ -210,7 +210,8 @@ export function gradeAppReview(value: unknown) {
           request?.id === phase.actionRequestId &&
           request.status === ["rejected", "failed", "executed"][index] &&
           v.status === ["denied", "failed", "succeeded"][index] &&
-          (index !== 1 || v.errorCode === "tool_error") &&
+          // Approved replay normalizes MCP error results to this code.
+          (index !== 1 || v.errorCode === "tool_execution_failed") &&
           phase.beforeWrites === [0, 0, 1][index] &&
           phase.afterWrites === [0, 1, 2][index] &&
           (index < 2
@@ -362,7 +363,7 @@ export const appReviewExternalCases = [
     gradeAppReview,
     {
       authentication: "human-session",
-      oracleVersion: "paperclip.chat-catalog.app-review.v4",
+      oracleVersion: "paperclip.chat-catalog.app-review.v5",
       queryAttachments: [{ stateKey: "factualContextAttachment", filename: "launch-update.md" }],
       expectedRunCount: 6,
       minimumRunCount: 6,