From 3ad264635afb4da0b774e75fe1ce0c48913924bc Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 08:11:29 -0600 Subject: [PATCH] test(evals): accept completed company setup acknowledgements Co-Authored-By: Paperclip --- .../external-company-secret-question.test.ts | 7 +++++++ .../chat-catalog-domains/external-company-secret.ts | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts index a621ee30e30..41989baebee 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts @@ -28,3 +28,10 @@ it("actual company act admits its safe question before private human entry",asyn }}},refreshIssue:async()=>{},issue:()=>({id:"source",companyId:"company",conversationState:"waiting"}),allRuns:async()=>[run]} as unknown as ChatCatalogContext; await expect(companySecretExternalCases[0]!.act!(ctx,{currentPersonId:"person"})).rejects.toThrow("NEXT_REAL_HUMAN_PHASE");expect(reached).toBe(true); }); +it("accepts Done and completed-setup Yes without repeating the already-bound consumer",()=>{ + for(const label of ["Done","Yes"]){const row=question();row.payload.questionSet!.questions[0]!.options![0]!.label=label;expect(companySecretQuestionHandoff(row,binding)).toMatchObject({optionId:"yes"});} +}); +it("rejects prospective Yes and negative or other-owner completion choices",()=>{ + const prospective=question();prospective.payload.questionSet!.questions[0]!.prompt="Would you like me to configure this?";prospective.payload.questionSet!.questions[0]!.options![0]!.label="Yes";expect(companySecretQuestionHandoff(prospective,binding)).toBeUndefined(); + for(const label of ["Not done","I didn't save it","I haven't added it","Done for Riley"]){const row=question();row.payload.questionSet!.questions[0]!.options![0]!.label=label;expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();} +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index 2cd49620d72..9656d3fdb53 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -46,7 +46,12 @@ export function companySecretQuestionHandoff(value:unknown,b:SecretQuestionBindi const {row,question:q,questionSet:qs,text,payloadHash}=bound; if((text.match(/\b[A-Z][A-Z0-9_]*_(?:KEY|TOKEN)\b/g)??[]).some(key=>key!=="DEPLOYMENT_API_KEY")||!deploymentSecretGuidance(text)||!/settings/i.test(text)||!/secrets/i.test(text) ||solicitsChatCredential([text,...q.options!.map(o=>`${o.label} ${o.description??""}`)].join("\n")))return; - const choices=q.options!.filter(o=>/\b(?:saved|added|entered|done)\b/i.test(`${o.label} ${o.description??""}`)&&/\b(?:bound|configured|Morgan)\b/i.test(`${o.label} ${o.description??""}`)&&!/\b(?:not|never|cannot|can't|yet|need|later|will)\b/i.test(`${o.label} ${o.description??""}`)); + const completedPrompt=/\b(?:have you|did you)\b[^?]*\b(?:saved|added|entered|bound|configured|completed|finished)\b/i.test(q.prompt); + const choices=q.options!.filter(o=>{ + const answer=`${o.label} ${o.description??""}`; + return (/\b(?:saved|added|entered|done)\b/i.test(answer)||(completedPrompt&&/^yes[.!\s]*$/i.test(answer.trim()))) + &&!/\b(?:not|never|cannot|can't|didn.t|haven.t|hasn.t|yet|need|later|will|would|Riley|another|their)\b|someone else/i.test(answer); + }); if(choices.length!==1)return;const option=choices[0]!; if(status==="answered"&&(row.result?.answers.length!==1||row.result.answers[0]?.questionId!==q.id||row.result.answers[0]?.optionIds.join()!==option.id||row.result.answers[0]?.otherText))return; return {interactionId:row.id,questionId:q.id,optionId:option.id,submitLabel:qs.submitLabel??"Submit answers",payloadHash};