fix(evals): admit saved-secret-name metadata defaults

Co-Authored-By: Paperclip <paperclip@paperclip.ing>

tests/runner-e2e/chat-catalog-contract.ts

@@ -332,7 +332,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: {
   } else if(savedName){
     const parsed=askUserQuestionsPayloadSchema.safeParse(row.payload),qs=parsed.success?parsed.data.questionSet:undefined,q=qs?.questions[0],answer=row.result.answers[0];
     const secrets=input.companySecrets?.filter(secret=>secret.id===savedName.secretId),secret=secrets?.[0];
-    if(!qs||qs.questions.length!==1||!q||q.id!==savedName.questionId||q.answerMode!=="text"||!q.required||q.options?.length||q.customAnswer?.enabled||q.initialText!==undefined
+    if(!qs||qs.questions.length!==1||!q||q.id!==savedName.questionId||q.answerMode!=="text"||!q.required||q.options?.length||q.customAnswer?.enabled||(q.initialText!==undefined&&q.initialText!==""&&!/^[a-zA-Z0-9_.-]{1,120}$/.test(q.initialText))
       ||!chatCatalogSavedSecretNameIntent(q.prompt)
       ||secrets?.length!==1||!secret||secret.companyId!==input.companyId||secret.scope!=="company"||secret.status!=="active"||secret.createdByUserId!==b.userId
       ||!Number.isFinite(Date.parse(String(row.createdAt)))||!Number.isFinite(Date.parse(source?.startedAt??""))||Date.parse(secret.createdAt)<=Math.max(Date.parse(String(row.createdAt)),Date.parse(source?.startedAt??""))

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4909,7 +4909,7 @@
       "chat-flow.ts",
       "chat-scenarios/secrets.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v10",
+    "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v11",
     "expectedRunCount": 2,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts

@@ -121,7 +121,7 @@ it.each(["value","mixed","owner","source","help","initial"])("rejects unsafe sav
  if(fault==="owner")row.addresseeUserId="other";
  if(fault==="source")row.sourceRunId="other";
  if(fault==="help")q.helpText+=" Paste the key here in chat.";
- if(fault==="initial")q.initialText="deployment_api_key";
+ if(fault==="initial")q.initialText="not metadata value";
  expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();
 });
 it("checks saved-name answers against actual saved metadata rather than arbitrary text",()=>{
@@ -190,3 +190,15 @@ it("keeps a generic secure-save completion question with a later agent binding p
  expect(companyOperatorBindingRequested("Have you configured the deployment secret in Settings Secrets? Once saved, I'll add its reference to Morgan's environment.")).toBe(false);
  expect(companyOperatorBindingRequested("Have you configured the secret reference in Morgan's environment?")).toBe(true);
 });
+
+it("admits the actual secure saved-name card with a proposed metadata default, not a human answer",()=>{
+ const row=question();row.payload=questionSetToAskUserQuestionsPayload({schema:"paperclip.question_set.v1",title:"Deployment API key setup",description:"Add the secret in the Paperclip UI, then confirm the secret key name. Never paste the value here.",submitLabel:"Save answer",questions:[{id:"secret_key",prompt:"Which secret key name did you save the deployment API key under?",helpText:"Enter the key name you used in Company Settings → Secrets. I will verify the secret exists, then add a DEPLOYMENT_API_KEY secret reference to Morgan's environment.",required:true,answerMode:"text",initialText:"deployment_api_key",textValidation:{minLength:1,maxLength:120,inputType:"text"}}]});
+ expect(companySecretQuestionHandoff(row,binding)).toMatchObject({savedSecretName:true,questionId:"secret_key"});
+ expect(companySecretQuestionHandoff(row,binding,"answered","deployment_api_key")).toBeUndefined();
+ const blank=structuredClone(row);blank.payload.questionSet!.questions[0]!.initialText="";expect(companySecretQuestionHandoff(blank,binding)).toMatchObject({savedSecretName:true});
+ for(const initialText of ["not metadata value", "x".repeat(121)]){const bad=structuredClone(row);bad.payload.questionSet!.questions[0]!.initialText=initialText;expect(companySecretQuestionHandoff(bad,binding)).toBeUndefined();}
+ const wrongActor={...row,createdByAgentId:"other"};expect(companySecretQuestionHandoff(wrongActor,binding)).toBeUndefined();
+ const rawValue=structuredClone(row);rawValue.payload.questionSet!.questions[0]!.prompt="What is the deployment key value?";expect(companySecretQuestionHandoff(rawValue,binding)).toBeUndefined();
+ const environment=structuredClone(row);environment.payload.questionSet!.questions[0]!.prompt="What environment variable name should Morgan receive?";expect(boundSecretQuestion(environment,binding,"pending","company-environment-name")).toBeUndefined();
+ const choice=question();choice.payload.questionSet!.questions[0]!.initialText="deployment_api_key";expect(companySecretQuestionHandoff(choice,binding)).toBeUndefined();
+});

tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { questionSetToAskUserQuestionsPayload } from "../../../packages/shared/src/question-set.js";
 import { chatCatalogHumanHistoryMatches,verifyChatCatalogHumanQuestionAnswer } from "../chat-catalog-contract.js";
 import { describe,expect,it,vi } from "vitest";
@@ -196,6 +197,13 @@ it("requires native saved-name binding and exact metadata-only question accounti
  const nativeRuns=[{id:"first",companyId:"company",agentId:"ceo",status:"succeeded",runtimeMode:"native",startedAt:"2026-11-04T13:00:00Z",contextSnapshot:{issueId:"source",commentId:"human"}},{id:"second",companyId:"company",agentId:"ceo",status:"succeeded",runtimeMode:"native",startedAt:"2026-11-04T13:41:00Z",contextSnapshot:{issueId:"source",interactionId:"question"}}];
  const evidence={expected,companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"human",authorUserId:"person"},interactions:[row],companySecrets:o.secrets,activity:[{companyId:"company",entityId:"source",action:"issue.thread_interaction_answered",actorType:"user",actorId:"person",details:{interactionId:"question",interactionKind:"ask_user_questions",interactionStatus:"answered",answeredQuestionCount:1}}],runs:nativeRuns};
  expect(verifyChatCatalogHumanQuestionAnswer(evidence)).toBeDefined();
+ for(const initialText of ["deployment_api_key", "", "not metadata value"]){
+  const changed=structuredClone(row);changed.payload.questionSet!.questions[0]!.initialText=initialText;
+  const next={...expected,companySavedSecretNameAnswer:{...expected.companySavedSecretNameAnswer!,payloadHash:createHash("sha256").update(JSON.stringify(changed.payload)).digest("hex")}};
+  expect(!!verifyChatCatalogHumanQuestionAnswer({...evidence,expected:next,interactions:[changed]})).toBe(initialText!=="not metadata value");
+  expect(verifyChatCatalogHumanQuestionAnswer({...evidence,interactions:[changed]})).toBeUndefined();
+ }
+
  for(const createdAt of ["2026-11-04T13:33:00Z","2026-11-04T12:59:00Z"])
   expect(verifyChatCatalogHumanQuestionAnswer({...evidence,companySecrets:[{...o.secrets[0]!,createdAt}]})).toBeUndefined();
  for(const prompt of ["Which name did you use for the saved deployment secret?","What alias identifies the deployment credential you created?"]){

tests/runner-e2e/chat-catalog-domains/external-company-secret.ts

@@ -165,7 +165,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s
   return {interactionId:String(o.state.initialQuestionId),sourceRunId:String(o.state.initialRunId),userId:String(o.state.currentPersonId),
    ...(o.state.initialQuestionSavedSecretName===true?{companySavedSecretNameAnswer:{questionId:String(o.state.initialQuestionFieldId),payloadHash:String(o.state.initialQuestionPayloadHash),value:String(o.secrets.find(row=>row.id===o.state.secretId)?.key??""),secretId:String(o.state.secretId)}}:{}),
    ...(o.state.initialQuestionEnvironmentName==="DEPLOYMENT_API_KEY"?{companyEnvironmentNameAnswer:{questionId:String(o.state.initialQuestionFieldId),payloadHash:String(o.state.initialQuestionPayloadHash),value:"DEPLOYMENT_API_KEY" as const}}:{})};
- },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v10",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
+ },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v11",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
  sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"],
  async act(ctx,state){
   const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx);

tests/runner-e2e/chat-catalog-domains/external-secret-question.ts

@@ -17,7 +17,7 @@ export function boundSecretQuestion(value: unknown, b: SecretQuestionBinding, st
   if (!qs || qs.questions.length !== 1) return;
   const q = qs.questions[0]!;
   if (!q.required || (mode
-    ? q.answerMode !== "text" || !!q.options?.length || !!q.customAnswer?.enabled || q.initialText !== undefined
+    ? q.answerMode !== "text" || !!q.options?.length || !!q.customAnswer?.enabled || (q.initialText !== undefined && (mode !== "company-saved-secret-name" || (q.initialText !== "" && !/^[a-zA-Z0-9_.-]{1,120}$/.test(q.initialText))))
       || (mode === "company-environment-name" && (q.textValidation?.pattern !== "^[A-Za-z_][A-Za-z0-9_]*$" || q.textValidation.minLength !== 1
       || !q.textValidation.maxLength || q.textValidation.maxLength > 120 || q.textValidation.maxLength < "DEPLOYMENT_API_KEY".length))
     : q.answerMode !== "single_select" || !q.options?.length)) return;