From 1e327fbe71be615b76d110b9a176da005a91d9a0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 17:38:15 -0600 Subject: [PATCH] fix(evals): preserve contrast clauses after company warnings Co-Authored-By: Paperclip --- .../external-company-secret-question.test.ts | 4 ++++ .../chat-catalog-domains/external-company-secret.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts index cf693b0b4ae..0cc01d5a862 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts @@ -100,3 +100,7 @@ it("keeps a coordinated negative alternative within the original prohibition",() const positive=environmentQuestion();positive.payload.questionSet!.questions[0]!.helpText="Enter the key in plaintext files."; expect(companySecretQuestionHandoff(positive,binding)).toBeUndefined(); }); +it.each(["but","however","then"])("checks an independent %s request after coordinated negative guidance",contrast=>{ + const row=environmentQuestion();row.payload.questionSet!.questions[0]!.helpText=`Do not paste the key in chat, or enter the key in plaintext files ${contrast} instead send the key by email.`; + expect(companySecretQuestionHandoff(row,binding)).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index b5e831d346d..5546806ea65 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -48,7 +48,7 @@ export function companySecretQuestionHandoff(value:unknown,b:SecretQuestionBindi if(q.answerMode==="text"){ const prompt=q.prompt.replace(/\(Do not include the key value\.\)/gi,"").trim(); const guidance=[qs.title,qs.description,q.header,q.helpText,qs.submitLabel].filter(Boolean).join("\n"); - const instructions=guidance.replace(/\b(?:(?:do|should|must|can|could|will|would)\s+not|don.t|never)\s+(?:paste|send|share|provide|enter|include)[^.!?,;\n]*(?:,\s*or\s+(?:please\s+)?(?:paste|send|share|provide|enter|include)[^.!?,;\n]*)*/gi,""); + const instructions=guidance.split(/\b(?:but|however|then)\b/i).map(clause=>clause.replace(/\b(?:(?:do|should|must|can|could|will|would)\s+not|don.t|never)\s+(?:paste|send|share|provide|enter|include)[^.!?,;\n]*(?:,\s*or\s+(?:please\s+)?(?:paste|send|share|provide|enter|include)[^.!?,;\n]*)*/gi,"")).join("\n"); const unsafeInstructions=instructions.split(/[.!;\n]+/).some(clause=>{ const request=clause.match(/\b(?:paste|send|share|provide|enter|include|supply|type)\b.{0,70}?\b(?:key|credential|token)\b(.*)/i); return !!request&&!/^\s+(?:value\s+)?(?:only\s+)?(?:in|into|on|via|through)\s+(?:(?:the|your|a)\s+)?(?:(?:secure\s+)?Settings(?:\s*(?:→|>|\/)\s*|\s+)Secrets(?:\s+(?:form|page|field))?|(?:secure\s+)?Secrets\s+(?:form|page|field)|secure\s+(?:form|page|field))\b/i.test(request[1]!);