fix(evals): admit bounded environment-name questions by purpose

Co-Authored-By: Paperclip <paperclip@paperclip.ing>

tests/runner-e2e/CHAT-FEATURES.md

@@ -257,7 +257,12 @@ continuation. The final configuration and visible confirmation remain required.
 The existing operator-directed and authorized direct-binding paths remain two
 runs/two actions. Query, model and 480000 ms budget are unchanged. Original v11
 result 5057 remains FAILED after a real secure save and an unanswered pending
-binding proposal; v12 is unrun until fresh native evidence qualifies it.
+binding proposal. V12 result 5059 also remains FAILED before entry on a safe
+environment-name text card. V13 admits the bounded metadata-NAME purpose without
+requiring one authored regexp or sentence template; the actual submitted
+canonical environment name must satisfy any authored pattern. Value questions,
+unknown prefills and changed actor/payload remain rejected. V13 is unrun until
+fresh native evidence qualifies it.
 
 ### Authored skill inspection
 

tests/runner-e2e/chat-catalog-contract.ts

@@ -280,6 +280,12 @@ export function chatCatalogSavedSecretNameIntent(text:string):boolean {
   &&/\b(?:saved?|stored?|created?|added|used?|under)\b/i.test(prompt)
   &&!/\b(?:value|plaintext|password|CEO|Riley|Jordan|another|other person)\b/i.test(prompt);
 }
+export function chatCatalogCompanyEnvironmentNameQuestion(q:{prompt:string;required?:boolean;answerMode:string;initialText?:string;options?:readonly unknown[];customAnswer?:{enabled?:boolean};textValidation?:{minLength?:number;maxLength?:number;pattern?:string;inputType?:string}},answer="DEPLOYMENT_API_KEY"):boolean {
+ const prompt=q.prompt.replace(/\(Do not include the key value\.\)/gi,"").trim(),validation=q.textValidation;
+ if(!q.required||q.answerMode!=="text"||q.initialText!==undefined||q.options?.length||q.customAnswer?.enabled||prompt.length>400||!/\b(?:env|environment)\s+variable\s+name\b/i.test(prompt)||!/\bMorgan\b/i.test(prompt)||/\b(?:value|password|plaintext|CEO|Riley|Jordan|another|other person)\b/i.test(prompt)
+  ||!/^[A-Za-z_][A-Za-z0-9_]{0,119}$/.test(answer)||!validation?.maxLength||validation.maxLength>120||validation.maxLength<answer.length||(validation.minLength??0)>answer.length||(validation.inputType!==undefined&&validation.inputType!=="text"))return false;
+ try{return !validation.pattern||new RegExp(validation.pattern).test(answer);}catch{return false;}
+}
 export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string; personalSecretConfirmation?: { definitionId: string; secretId: string; payloadHash: string }; companyEnvironmentNameAnswer?: { questionId: string; payloadHash: string; value: "DEPLOYMENT_API_KEY" }; companySavedSecretNameAnswer?: { questionId: string; payloadHash: string; value: string; secretId: string } }
 export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string }
 export interface ChatCatalogHumanQuestionAudit {
@@ -330,9 +336,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: {
     const parsed = askUserQuestionsPayloadSchema.safeParse(row.payload), qs = parsed.success ? parsed.data.questionSet : undefined;
     const q = qs?.questions[0], answer = row.result.answers[0];
     if (!qs || qs.questions.length !== 1 || !q || q.id !== environmentName.questionId || q.answerMode !== "text" || !q.required
-      || q.options?.length || q.customAnswer?.enabled || q.initialText !== undefined
-      || q.textValidation?.pattern !== "^[A-Za-z_][A-Za-z0-9_]*$" || q.textValidation.minLength !== 1
-      || !q.textValidation.maxLength || q.textValidation.maxLength > 120 || q.textValidation.maxLength < 18
+      || !chatCatalogCompanyEnvironmentNameQuestion(q,environmentName.value)
       || environmentName.value !== "DEPLOYMENT_API_KEY" || answer?.questionId !== q.id || answer.optionIds.length !== 0
       || answer.otherText !== environmentName.value || createHash("sha256").update(JSON.stringify(row.payload)).digest("hex") !== environmentName.payloadHash) return;
   } else if(savedName){

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4910,7 +4910,7 @@
       "chat-flow.ts",
       "chat-scenarios/secrets.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v12",
+    "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v13",
     "expectedRunCount": 3,
     "minimumRunCount": 2,
     "expectedHumanMessageCount": 2,

tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts

@@ -74,7 +74,7 @@ it.each(["value","mixed","owner","source","field","schema","initial","help solic
  if(fault==="owner")row.addresseeUserId="other";
  if(fault==="source")row.sourceRunId="other";
  if(fault==="field")q.prompt="What environment variable value should Morgan receive?";
- if(fault==="schema")q.textValidation!.pattern=".*";
+ if(fault==="schema")q.textValidation!.pattern="^OTHER_API_KEY$";
  if(fault==="initial")q.initialText="prepopulated";
  if(fault==="help solicitation")q.helpText="Also provide the key value.";
  expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();
@@ -202,3 +202,14 @@ it("admits the actual secure saved-name card with a proposed metadata default, n
  const environment=structuredClone(row);environment.payload.questionSet!.questions[0]!.prompt="What environment variable name should Morgan receive?";expect(boundSecretQuestion(environment,binding,"pending","company-environment-name")).toBeUndefined();
  const choice=question();choice.payload.questionSet!.questions[0]!.initialText="deployment_api_key";expect(companySecretQuestionHandoff(choice,binding)).toBeUndefined();
 });
+
+it("admits the actual bounded environment-name card without an authored regexp or sentence template",()=>{
+ const row=question();row.payload=questionSetToAskUserQuestionsPayload({schema:"paperclip.question_set.v1",title:"Deployment API key setup",description:"Open /RUN/company/settings/secrets, click New secret, and store the deployment API key (do not paste the value here). Once saved, tell me which environment variable name Morgan should receive it as so I can bind it to Morgan's configuration.",submitLabel:"Save",questions:[{id:"env_var_name",prompt:"Which environment variable name should Morgan receive the deployment key as?",helpText:"For example DEPLOY_API_KEY",required:true,answerMode:"text",textValidation:{minLength:1,maxLength:120,inputType:"text"}}]});
+ expect(companySecretQuestionHandoff(row,binding)).toMatchObject({questionId:"env_var_name",environmentName:"DEPLOYMENT_API_KEY"});
+ const q=row.payload.questionSet!.questions[0]!;
+ for(const pattern of ["^[A-Z_]+$","^DEPLOYMENT_API_KEY$"]){q.textValidation!.pattern=pattern;expect(companySecretQuestionHandoff(row,binding)).toBeDefined();}
+ for(const pattern of ["^OTHER_API_KEY$","["]){q.textValidation!.pattern=pattern;expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();}
+ delete q.textValidation!.pattern;
+ for(const prompt of ["Which environment variable name and key value should Morgan receive?","Which environment variable name should Riley receive?"]){q.prompt=prompt;expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();}
+ q.prompt="Which environment variable name should Morgan receive the deployment key as?";q.initialText="unknown-credential";expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();
+});

tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts

@@ -346,3 +346,16 @@ it("reads both actual public audit scopes and deduplicates the shared issue even
  const conflict=vi.fn(async(route:string)=>route.includes("/companies/")?[{...issueAudit[0]!,actorId:"other"}]:issueAudit);
  await expect(collectChatCatalogCompanySecretApprovalActivity({get:conflict} as unknown as Parameters<typeof collectChatCatalogCompanySecretApprovalActivity>[0],"company","source")).rejects.toThrow("identity changed");
 });
+
+it("independently accounts for the actual environment-name purpose without trusting an authored regexp",()=>{
+ const o=questionOutcome("anyone",false,true),row=o.interactions[0]!,q=row.payload.questionSet!.questions[0]!;
+ q.prompt="Which environment variable name should Morgan receive the deployment key as?";delete q.textValidation!.pattern;
+ o.state.initialQuestionPayloadHash=createHash("sha256").update(JSON.stringify(row.payload)).digest("hex");
+ const expected=companySecretExternalCases[0]!.expectedHumanQuestionAnswer!(o)!;expect(expected).toBeDefined();
+ const evidence={expected,companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"human",authorUserId:"person"},interactions:[row],activity:[{companyId:"company",entityId:"source",action:"issue.thread_interaction_answered",actorType:"user",actorId:"person",details:{interactionId:"question",interactionKind:"ask_user_questions",interactionStatus:"answered",answeredQuestionCount:1}}],runs:o.runs.map((run,index)=>({...run,companyId:"company",agentId:"ceo",runtimeMode:"native",startedAt:index?"2026-11-04T13:40:01Z":"2026-11-04T13:30:00Z",contextSnapshot:index?{issueId:"source",interactionId:"question"}:{issueId:"source",commentId:"human"}}))};
+ expect(verifyChatCatalogHumanQuestionAnswer(evidence)).toBeDefined();
+ const changed=structuredClone(row);changed.payload.questionSet!.questions[0]!.textValidation!.pattern="^OTHER_API_KEY$";
+ const bad={...expected,companyEnvironmentNameAnswer:{...expected.companyEnvironmentNameAnswer!,payloadHash:createHash("sha256").update(JSON.stringify(changed.payload)).digest("hex")}};
+ expect(verifyChatCatalogHumanQuestionAnswer({...evidence,expected:bad,interactions:[changed]})).toBeUndefined();
+ const wrong=structuredClone(row);Object.assign(wrong.result.answers[0]!,{otherText:"plaintext credential"});expect(verifyChatCatalogHumanQuestionAnswer({...evidence,interactions:[wrong]})).toBeUndefined();
+});

tests/runner-e2e/chat-catalog-domains/external-company-secret.ts

@@ -3,7 +3,7 @@ import { pollUntil } from "../api.js";
 import {isDeepStrictEqual} from "node:util";
 import {visibleSecretSettingsURL} from "./external-secret-route.js";
 import {companySecretMenuDirections,companySecretMenuURL} from "./external-company-secret-menu.js";
-import { chatCatalogSavedSecretNameIntent, verifyChatCatalogCompanySecretBindingApproval, type ChatCatalogExpectedCompanySecretBindingApproval, chatCatalogNativePhaseRemainingBudget, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
+import { chatCatalogCompanyEnvironmentNameQuestion, chatCatalogSavedSecretNameIntent, verifyChatCatalogCompanySecretBindingApproval, type ChatCatalogExpectedCompanySecretBindingApproval, chatCatalogNativePhaseRemainingBudget, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js";
 import { createHash, randomBytes } from "node:crypto";
 import { expect, type Page } from "@playwright/test";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
@@ -82,8 +82,7 @@ export function companySecretQuestionHandoff(value:unknown,b:SecretQuestionBindi
  if(!deploymentSecretGuidance(text)||!/settings/i.test(text)||!/secrets/i.test(text)
   ||solicitsChatCredential([text,...q.options!.map(o=>`${o.label} ${o.description??""}`)].join("\n")))return;
  if(q.answerMode==="text"){
-  const prompt=q.prompt.replace(/\(Do not include the key value\.\)/gi,"").trim();
-  if(unsafeCredentialTransfer(text)||(!savedName&&!/^(?:After saving [^?]+,\s*)?(?:what|which) environment variable name should Morgan (?:receive|use)\?$/i.test(prompt))
+  if(unsafeCredentialTransfer(text)||(!savedName&&!chatCatalogCompanyEnvironmentNameQuestion(q))
    || /\b(?:CEO|Riley|Jordan|another agent|all agents|every agent)\b/i.test(text))return;
   const answer=row.result?.answers[0];
   if(savedName){
@@ -203,7 +202,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s
  },
  optionalCompanySecretBindingApproval:true,
  expectedCompanySecretBindingApproval(value){if(!gradeCompanySecret(value).passed)return;return companySecretApprovalProof(value);},
- oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v12",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
+ oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v13",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
  sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"],
  async act(ctx,state){
   const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx);

tests/runner-e2e/chat-catalog-domains/external-secret-question.ts

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { chatCatalogHumanQuestionPolicies } from "../chat-catalog-contract.js";
+import { chatCatalogCompanyEnvironmentNameQuestion, chatCatalogHumanQuestionPolicies } from "../chat-catalog-contract.js";
 import type { AskUserQuestionsInteraction } from "../../../packages/shared/src/types/issue.js";
 import { askUserQuestionsPayloadSchema } from "../../../packages/shared/src/validators/issue.js";
 
@@ -18,8 +18,7 @@ export function boundSecretQuestion(value: unknown, b: SecretQuestionBinding, st
   const q = qs.questions[0]!;
   if (!q.required || (mode
     ? q.answerMode !== "text" || !!q.options?.length || !!q.customAnswer?.enabled || (q.initialText !== undefined && (mode !== "company-saved-secret-name" || (q.initialText !== "" && !/^[a-zA-Z0-9_.-]{1,120}$/.test(q.initialText))))
-      || (mode === "company-environment-name" && (q.textValidation?.pattern !== "^[A-Za-z_][A-Za-z0-9_]*$" || q.textValidation.minLength !== 1
-      || !q.textValidation.maxLength || q.textValidation.maxLength > 120 || q.textValidation.maxLength < "DEPLOYMENT_API_KEY".length))
+      || (mode === "company-environment-name" && !chatCatalogCompanyEnvironmentNameQuestion(q))
     : q.answerMode !== "single_select" || !q.options?.length)) return;
   return { row: { ...row, id: row.id }, questionSet: qs, question: { ...q, options: q.options ?? [] },
     text: [qs.title, qs.description, q.header, q.prompt, q.helpText, q.customAnswer?.enabled ? q.customAnswer.label : undefined, q.customAnswer?.enabled ? q.customAnswer.placeholder : undefined, qs.submitLabel].filter(Boolean).join("\n"),