fix(evals): inspect all rendered credential question instructions

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-04 08:24:49 -06:00
1 parent 6b38cf596c
commit 5d91105991
2 files changed
+9 -1

No files matched your search

@@ -66,3 +66,11 @@ it("uses bound completion context for terse answers without treating permission
expect(withChoice("Done for Riley")).toBeUndefined();
expect(withChoice("Yes", "May I save the value for you?")).toBeUndefined();
});
it.each(["header", "title", "submitLabel"] as const)("rejects unsafe chat instructions in rendered %s", field => {
const row = question(), qs = row.payload.questionSet!;
const unsafe = "Enter your personal publishing credential here in chat.";
if (field === "header") qs.questions[0]!.header = unsafe;
else qs[field] = unsafe;
expect(personalSecretQuestionHandoff(row, binding)).toBeUndefined();
});
@@ -18,6 +18,6 @@ export function boundSecretQuestion(value: unknown, b: SecretQuestionBinding, st
const q = qs.questions[0]!;
if (q.answerMode !== "single_select" || q.customAnswer?.enabled || !q.required || !q.options?.length) return;
return { row: { ...row, id: row.id }, questionSet: qs, question: { ...q, options: q.options },
text: [qs.description, q.prompt, q.helpText].filter(Boolean).join("\n"),
text: [qs.title, qs.description, q.header, q.prompt, q.helpText, qs.submitLabel].filter(Boolean).join("\n"),
payloadHash: createHash("sha256").update(JSON.stringify(row.payload)).digest("hex") };
}