diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 1d8c83ab6e3..ec69e699e25 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -830,7 +830,7 @@ "../../ui/src/components/task-chat/TaskChatComposer.tsx", "chat-scenarios/app-permissions.yaml" ], - "oracleVersion": "paperclip.chat-catalog.app-review.v5", + "oracleVersion": "paperclip.chat-catalog.app-review.v6", "expectedRunCount": 6, "expectedHumanMessageCount": 3, "minimumRunCount": 6, diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review-outcome-language.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review-outcome-language.test.ts index 45acb78eea9..2c5d3b2a1f0 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-review-outcome-language.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-review-outcome-language.test.ts @@ -101,3 +101,25 @@ it.each([ ])("rejects a mixed or nested external exception: %s", reply => { expect(reviewOutcomeReply(reply, "rejected")).toBe(false); }); + +it.each(["**not**", "__not__", "*not*", "_not_", "***not***", "___not___"])( + "reads paired emphasis around negation as visible prose: %s", negation => { + expect(reviewOutcomeReply(`You declined. The update was ${negation} applied.`, "rejected")).toBe(true); + }, +); +it.each([ + "You declined. I **applied** the update.", + "You declined. The update was __applied__.", + "You declined. I *saved* the update in the notes app.", + "You declined. The update was **not** applied, but I **saved** it to the notes app.", + "You declined. Nothing was **saved**, but I **applied** the update.", + "You declined. The draft was saved **in the notes app** for your review.", +])("paired emphasis does not hide external completion: %s", reply => { + expect(reviewOutcomeReply(reply, "rejected")).toBe(false); +}); +it.each([ + "You declined. The update was not** applied.", + String.raw`You declined. The update was \**not** applied.`, +])("does not strip unmatched or escaped emphasis markers: %s", reply => { + expect(reviewOutcomeReply(reply, "rejected")).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review-outcomes.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review-outcomes.ts index 04899f861d6..b52229a9a32 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-review-outcomes.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-review-outcomes.ts @@ -85,6 +85,7 @@ function claimsExternalCompletion(reply: string) { } export function reviewOutcomeReply(reply: string, kind: ReviewKind) { + reply = reply.replace(/(? diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review.test.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review.test.ts index ca957fe0fb8..c678953dd3b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-review.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-review.test.ts @@ -362,3 +362,18 @@ it.each(["no human approval", "wrong human", "successful invocation", "successfu expect(gradeAppReview(o).passed).toBe(false); }, ); + +it("accepts the measured whole declined reply with bold negation while still requiring every later phase", () => { + const reply = `The launch page update was **not** applied: the approval card for the "Synthetic Launch Notes" write was declined, so the page still reads "Controlled launch page." + +The update I attempted contained: +- Public launch: **November 8, 2030** + +Do you want me to retry (you'll need to approve the card) or change the content first?`; + expect(reviewOutcomeReply(reply, "rejected")).toBe(true); + const o = valid(); + o.phaseReplies[0]!.body = reply; + expect(gradeAppReview(o).passed).toBe(true); + o.state.phases = o.state.phases.slice(0, 1); + expect(gradeAppReview(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-app-review.ts b/tests/runner-e2e/chat-catalog-domains/external-app-review.ts index 4506efb729d..0052eadd0cd 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-app-review.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-app-review.ts @@ -360,7 +360,7 @@ export const appReviewExternalCases = [ gradeAppReview, { authentication: "human-session", - oracleVersion: "paperclip.chat-catalog.app-review.v5", + oracleVersion: "paperclip.chat-catalog.app-review.v6", queryAttachments: [{ stateKey: "factualContextAttachment", filename: "launch-update.md" }], expectedRunCount: 6, minimumRunCount: 6,