fix(evals): distinguish own-value completion from permission answers

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/external-personal-secret-question.test.ts

@@ -74,3 +74,20 @@ it.each(["header", "title", "submitLabel"] as const)("rejects unsafe chat instru
   else qs[field] = unsafe;
   expect(personalSecretQuestionHandoff(row, binding)).toBeUndefined();
 });
+
+it("accepts explicit own-value completion and contextual Yes while rejecting foreign or prospective answers", () => {
+  const choose = (prompt: string, label: string, description?: string, header?: string, helpText?: string) => {
+    const row = question(), qs = row.payload.questionSet!;
+    row.payload = questionSetToAskUserQuestionsPayload({ ...qs, questions: [{ ...qs.questions[0]!, prompt, header,
+      helpText: [qs.questions[0]!.helpText, helpText].filter(Boolean).join("\n"),
+      options: [{ id: "yes", label, description }, { id: "later", label: "Not yet" }] }] });
+    return personalSecretQuestionHandoff(row, binding);
+  };
+  expect(choose("Is your personal value saved?", "Yes, entered", "My personal value is saved in My secrets.")).toMatchObject({ optionId: "yes" });
+  expect(choose("Did you save your own value?", "Yes")).toMatchObject({ optionId: "yes" });
+  expect(choose("Is your personal value saved?", "Yes")).toMatchObject({ optionId: "yes" });
+  expect(choose("Confirm your setup", "Yes", undefined, "Have you saved your own value?")).toMatchObject({ optionId: "yes" });
+  expect(choose("Confirm your setup", "Yes", undefined, undefined, "Did you save your own value?")).toMatchObject({ optionId: "yes" });
+  expect(choose("Have you saved your own value?", "Added to company secrets")).toBeUndefined();
+  expect(choose("May I save the value for you?", "Yes, saved", "My personal value is saved.")).toBeUndefined();
+});

tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts

@@ -121,13 +121,19 @@ export function personalSecretQuestionHandoff(value: unknown, b: PersonalSecretQ
   if (!text.split(/[^A-Za-z0-9_-]+/).includes(b.definitionId) || !text.split(/[^A-Za-z0-9_-]+/).includes("PERSONAL_PUBLISHING_KEY") || !personalSecretGuidance(text)
     || !/settings/i.test(text) || !/my secrets/i.test(text) || !/set my value/i.test(text)
     || solicitsChatCredential([text, ...q.options.map(o => `${o.label} ${o.description ?? ""}`)].join("\n"))) return;
-  const completedSetupQuestion = /\b(?:have you|did you)\b[^?]{0,80}\b(?:saved|set|entered|added|configured|completed)\b/i.test(q.prompt)
-    && !/\b(?:not|never|haven't|didn't|want|permission)\b|\b(?:may|can|should|could) (?:i|we)\b/i.test(q.prompt);
+  if (/\b(?:may|can|should|could) (?:i|we)\b|\b(?:permission|want|will|would)\b/i.test(q.prompt)) return;
+  const completedSetupQuestion = [q.prompt, q.header, q.helpText].some(value => {
+    const completion = /\b(?:have you|did you|is your|has your)\b[^?\n]{0,80}\b(?:sav(?:e|ed)|set|enter(?:ed)?|add(?:ed)?|configur(?:e|ed)|complet(?:e|ed)|stor(?:e|ed)|finish(?:ed)?|done)\b[^?\n]*/i.exec(value ?? "")?.[0];
+    return !!completion && !/\b(?:not|never|haven't|didn't|Riley|another|other|their|company|shared)\b/i.test(completion);
+  });
   const confirmations = q.options.filter(o => {
     const answer = `${o.label} ${o.description ?? ""}`;
-    return (!/^yes\b/i.test(o.label.trim()) || completedSetupQuestion)
-      && (/\b(?:saved|entered|added|done)\b/i.test(answer) || (completedSetupQuestion && /^yes[.!]?$/i.test(o.label.trim())))
-      && !/\b(?:no|not|never|cannot|can.t|didn.t|haven.t|hasn.t|yet|need|later|will|would|Riley|another|other|their)\b/i.test(answer);
+    const explicitOwnCompletion = /\b(?:saved|entered|added|done|stored|configured|completed)\b/i.test(answer)
+      && /\b(?:my|own|personal)\b/i.test(answer);
+    const terseCompletion = /^done[.!]?$/i.test(o.label.trim())
+      || (completedSetupQuestion && /^yes[.!]?$/i.test(o.label.trim()));
+    return (explicitOwnCompletion || terseCompletion)
+      && !/\b(?:no|not|never|cannot|can.t|didn.t|haven.t|hasn.t|yet|need|later|will|would|Riley|another|other|their|company|shared)\b/i.test(answer);
   });
   if (confirmations.length !== 1) return;
   const option = confirmations[0]!;