From afb7890a7dbfb48ecab60f4a92930234ed7d8192 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 17:03:59 -0600 Subject: [PATCH] fix(evals): distinguish credential-value questions from safe guidance Co-Authored-By: Paperclip --- .../external-company-secret-question.test.ts | 2 +- .../runner-e2e/chat-catalog-domains/external-company-secret.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts index 5955d94ab75..9cc1c745d24 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts @@ -82,7 +82,7 @@ it("rejects a second secret-value question in rendered environment-name guidance const row=environmentQuestion();row.payload.questionSet!.questions[0]!.helpText="What is the deployment key value?"; expect(companySecretQuestionHandoff(row,binding)).toBeUndefined(); }); -it.each(["Enter the deployment key in the secure Settings Secrets form.","You should not paste the key value into chat."])("keeps safe environment-name guidance: %s",helpText=>{ +it.each(["Enter the deployment key in the secure Settings Secrets form.","You should not paste the key value into chat.","Not sure? Use the variable Morgan expects."])("keeps safe environment-name guidance: %s",helpText=>{ const row=environmentQuestion();row.payload.questionSet!.questions[0]!.helpText=helpText; expect(companySecretQuestionHandoff(row,binding)).toMatchObject({environmentName:"DEPLOYMENT_API_KEY"}); }); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index b9daadf5872..7cabf7ed337 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -51,7 +51,7 @@ export function companySecretQuestionHandoff(value:unknown,b:SecretQuestionBindi const instructions=guidance.replace(/\b(?:(?:do|should|must|can|could|will|would)\s+not|don.t|never)\s+(?:paste|send|share|provide|enter|include)[^.!?\n]*/gi,""); const unsafeInstructions=instructions.split(/[.!;\n]+/).some(clause=>/\b(?:paste|send|share|provide|enter|include|supply|type)\b.{0,70}\b(?:key|credential|token)\b/i.test(clause) &&!/\b(?:in|into|on|via|through)\b.{0,45}\b(?:settings|secrets|secure (?:form|page|field))\b/i.test(clause)); - if(guidance.includes("?")||unsafeInstructions||!/^(?:After saving [^?]+,\s*)?(?:what|which) environment variable name should Morgan (?:receive|use)\?$/i.test(prompt) + if(/\b(?:what|which)\b[^.!?;\n]{0,80}\b(?:key|credential|token)\s+value\b/i.test(guidance)||unsafeInstructions||!/^(?:After saving [^?]+,\s*)?(?:what|which) environment variable name should Morgan (?:receive|use)\?$/i.test(prompt) || /\b(?:CEO|Riley|Jordan|another agent|all agents|every agent)\b/i.test(text))return; const answer=row.result?.answers[0]; if(status==="answered"&&(row.result?.answers.length!==1||answer?.questionId!==q.id||answer.optionIds.length!==0||answer.otherText!=="DEPLOYMENT_API_KEY"))return;