From 1b0ee51877748ce6d04bcb759c48eb33ef209531 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 19:29:56 -0600 Subject: [PATCH] fix(evals): admit bounded environment-name questions by purpose Co-Authored-By: Paperclip --- tests/runner-e2e/CHAT-FEATURES.md | 7 ++++++- tests/runner-e2e/chat-catalog-contract.ts | 10 +++++++--- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../external-company-secret-question.test.ts | 13 ++++++++++++- .../external-company-secret.test.ts | 13 +++++++++++++ .../chat-catalog-domains/external-company-secret.ts | 7 +++---- .../external-secret-question.ts | 5 ++--- 7 files changed, 44 insertions(+), 13 deletions(-) diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 90d39ddfb77..dc2367513e2 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -257,7 +257,12 @@ continuation. The final configuration and visible confirmation remain required. The existing operator-directed and authorized direct-binding paths remain two runs/two actions. Query, model and 480000 ms budget are unchanged. Original v11 result 5057 remains FAILED after a real secure save and an unanswered pending -binding proposal; v12 is unrun until fresh native evidence qualifies it. +binding proposal. V12 result 5059 also remains FAILED before entry on a safe +environment-name text card. V13 admits the bounded metadata-NAME purpose without +requiring one authored regexp or sentence template; the actual submitted +canonical environment name must satisfy any authored pattern. Value questions, +unknown prefills and changed actor/payload remain rejected. V13 is unrun until +fresh native evidence qualifies it. ### Authored skill inspection diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index e49ebd192b6..ed1cd44b7f2 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -280,6 +280,12 @@ export function chatCatalogSavedSecretNameIntent(text:string):boolean { &&/\b(?:saved?|stored?|created?|added|used?|under)\b/i.test(prompt) &&!/\b(?:value|plaintext|password|CEO|Riley|Jordan|another|other person)\b/i.test(prompt); } +export function chatCatalogCompanyEnvironmentNameQuestion(q:{prompt:string;required?:boolean;answerMode:string;initialText?:string;options?:readonly unknown[];customAnswer?:{enabled?:boolean};textValidation?:{minLength?:number;maxLength?:number;pattern?:string;inputType?:string}},answer="DEPLOYMENT_API_KEY"):boolean { + const prompt=q.prompt.replace(/\(Do not include the key value\.\)/gi,"").trim(),validation=q.textValidation; + if(!q.required||q.answerMode!=="text"||q.initialText!==undefined||q.options?.length||q.customAnswer?.enabled||prompt.length>400||!/\b(?:env|environment)\s+variable\s+name\b/i.test(prompt)||!/\bMorgan\b/i.test(prompt)||/\b(?:value|password|plaintext|CEO|Riley|Jordan|another|other person)\b/i.test(prompt) + ||!/^[A-Za-z_][A-Za-z0-9_]{0,119}$/.test(answer)||!validation?.maxLength||validation.maxLength>120||validation.maxLengthanswer.length||(validation.inputType!==undefined&&validation.inputType!=="text"))return false; + try{return !validation.pattern||new RegExp(validation.pattern).test(answer);}catch{return false;} +} export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string; personalSecretConfirmation?: { definitionId: string; secretId: string; payloadHash: string }; companyEnvironmentNameAnswer?: { questionId: string; payloadHash: string; value: "DEPLOYMENT_API_KEY" }; companySavedSecretNameAnswer?: { questionId: string; payloadHash: string; value: string; secretId: string } } export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string } export interface ChatCatalogHumanQuestionAudit { @@ -330,9 +336,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { const parsed = askUserQuestionsPayloadSchema.safeParse(row.payload), qs = parsed.success ? parsed.data.questionSet : undefined; const q = qs?.questions[0], answer = row.result.answers[0]; if (!qs || qs.questions.length !== 1 || !q || q.id !== environmentName.questionId || q.answerMode !== "text" || !q.required - || q.options?.length || q.customAnswer?.enabled || q.initialText !== undefined - || q.textValidation?.pattern !== "^[A-Za-z_][A-Za-z0-9_]*$" || q.textValidation.minLength !== 1 - || !q.textValidation.maxLength || q.textValidation.maxLength > 120 || q.textValidation.maxLength < 18 + || !chatCatalogCompanyEnvironmentNameQuestion(q,environmentName.value) || environmentName.value !== "DEPLOYMENT_API_KEY" || answer?.questionId !== q.id || answer.optionIds.length !== 0 || answer.otherText !== environmentName.value || createHash("sha256").update(JSON.stringify(row.payload)).digest("hex") !== environmentName.payloadHash) return; } else if(savedName){ diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 2ec5b97539e..c7132b60bd6 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4910,7 +4910,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v12", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v13", "expectedRunCount": 3, "minimumRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts index 103461c5c05..c87f720d2b3 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts @@ -74,7 +74,7 @@ it.each(["value","mixed","owner","source","field","schema","initial","help solic if(fault==="owner")row.addresseeUserId="other"; if(fault==="source")row.sourceRunId="other"; if(fault==="field")q.prompt="What environment variable value should Morgan receive?"; - if(fault==="schema")q.textValidation!.pattern=".*"; + if(fault==="schema")q.textValidation!.pattern="^OTHER_API_KEY$"; if(fault==="initial")q.initialText="prepopulated"; if(fault==="help solicitation")q.helpText="Also provide the key value."; expect(companySecretQuestionHandoff(row,binding)).toBeUndefined(); @@ -202,3 +202,14 @@ it("admits the actual secure saved-name card with a proposed metadata default, n const environment=structuredClone(row);environment.payload.questionSet!.questions[0]!.prompt="What environment variable name should Morgan receive?";expect(boundSecretQuestion(environment,binding,"pending","company-environment-name")).toBeUndefined(); const choice=question();choice.payload.questionSet!.questions[0]!.initialText="deployment_api_key";expect(companySecretQuestionHandoff(choice,binding)).toBeUndefined(); }); + +it("admits the actual bounded environment-name card without an authored regexp or sentence template",()=>{ + const row=question();row.payload=questionSetToAskUserQuestionsPayload({schema:"paperclip.question_set.v1",title:"Deployment API key setup",description:"Open /RUN/company/settings/secrets, click New secret, and store the deployment API key (do not paste the value here). Once saved, tell me which environment variable name Morgan should receive it as so I can bind it to Morgan's configuration.",submitLabel:"Save",questions:[{id:"env_var_name",prompt:"Which environment variable name should Morgan receive the deployment key as?",helpText:"For example DEPLOY_API_KEY",required:true,answerMode:"text",textValidation:{minLength:1,maxLength:120,inputType:"text"}}]}); + expect(companySecretQuestionHandoff(row,binding)).toMatchObject({questionId:"env_var_name",environmentName:"DEPLOYMENT_API_KEY"}); + const q=row.payload.questionSet!.questions[0]!; + for(const pattern of ["^[A-Z_]+$","^DEPLOYMENT_API_KEY$"]){q.textValidation!.pattern=pattern;expect(companySecretQuestionHandoff(row,binding)).toBeDefined();} + for(const pattern of ["^OTHER_API_KEY$","["]){q.textValidation!.pattern=pattern;expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();} + delete q.textValidation!.pattern; + for(const prompt of ["Which environment variable name and key value should Morgan receive?","Which environment variable name should Riley receive?"]){q.prompt=prompt;expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();} + q.prompt="Which environment variable name should Morgan receive the deployment key as?";q.initialText="unknown-credential";expect(companySecretQuestionHandoff(row,binding)).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts index 985b30a1157..fbc4a547858 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts @@ -346,3 +346,16 @@ it("reads both actual public audit scopes and deduplicates the shared issue even const conflict=vi.fn(async(route:string)=>route.includes("/companies/")?[{...issueAudit[0]!,actorId:"other"}]:issueAudit); await expect(collectChatCatalogCompanySecretApprovalActivity({get:conflict} as unknown as Parameters[0],"company","source")).rejects.toThrow("identity changed"); }); + +it("independently accounts for the actual environment-name purpose without trusting an authored regexp",()=>{ + const o=questionOutcome("anyone",false,true),row=o.interactions[0]!,q=row.payload.questionSet!.questions[0]!; + q.prompt="Which environment variable name should Morgan receive the deployment key as?";delete q.textValidation!.pattern; + o.state.initialQuestionPayloadHash=createHash("sha256").update(JSON.stringify(row.payload)).digest("hex"); + const expected=companySecretExternalCases[0]!.expectedHumanQuestionAnswer!(o)!;expect(expected).toBeDefined(); + const evidence={expected,companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"human",authorUserId:"person"},interactions:[row],activity:[{companyId:"company",entityId:"source",action:"issue.thread_interaction_answered",actorType:"user",actorId:"person",details:{interactionId:"question",interactionKind:"ask_user_questions",interactionStatus:"answered",answeredQuestionCount:1}}],runs:o.runs.map((run,index)=>({...run,companyId:"company",agentId:"ceo",runtimeMode:"native",startedAt:index?"2026-11-04T13:40:01Z":"2026-11-04T13:30:00Z",contextSnapshot:index?{issueId:"source",interactionId:"question"}:{issueId:"source",commentId:"human"}}))}; + expect(verifyChatCatalogHumanQuestionAnswer(evidence)).toBeDefined(); + const changed=structuredClone(row);changed.payload.questionSet!.questions[0]!.textValidation!.pattern="^OTHER_API_KEY$"; + const bad={...expected,companyEnvironmentNameAnswer:{...expected.companyEnvironmentNameAnswer!,payloadHash:createHash("sha256").update(JSON.stringify(changed.payload)).digest("hex")}}; + expect(verifyChatCatalogHumanQuestionAnswer({...evidence,expected:bad,interactions:[changed]})).toBeUndefined(); + const wrong=structuredClone(row);Object.assign(wrong.result.answers[0]!,{otherText:"plaintext credential"});expect(verifyChatCatalogHumanQuestionAnswer({...evidence,interactions:[wrong]})).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index ebafba39eb1..4d1a891e68b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -3,7 +3,7 @@ import { pollUntil } from "../api.js"; import {isDeepStrictEqual} from "node:util"; import {visibleSecretSettingsURL} from "./external-secret-route.js"; import {companySecretMenuDirections,companySecretMenuURL} from "./external-company-secret-menu.js"; -import { chatCatalogSavedSecretNameIntent, verifyChatCatalogCompanySecretBindingApproval, type ChatCatalogExpectedCompanySecretBindingApproval, chatCatalogNativePhaseRemainingBudget, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; +import { chatCatalogCompanyEnvironmentNameQuestion, chatCatalogSavedSecretNameIntent, verifyChatCatalogCompanySecretBindingApproval, type ChatCatalogExpectedCompanySecretBindingApproval, chatCatalogNativePhaseRemainingBudget, chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; import { createHash, randomBytes } from "node:crypto"; import { expect, type Page } from "@playwright/test"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; @@ -82,8 +82,7 @@ export function companySecretQuestionHandoff(value:unknown,b:SecretQuestionBindi if(!deploymentSecretGuidance(text)||!/settings/i.test(text)||!/secrets/i.test(text) ||solicitsChatCredential([text,...q.options!.map(o=>`${o.label} ${o.description??""}`)].join("\n")))return; if(q.answerMode==="text"){ - const prompt=q.prompt.replace(/\(Do not include the key value\.\)/gi,"").trim(); - if(unsafeCredentialTransfer(text)||(!savedName&&!/^(?:After saving [^?]+,\s*)?(?:what|which) environment variable name should Morgan (?:receive|use)\?$/i.test(prompt)) + if(unsafeCredentialTransfer(text)||(!savedName&&!chatCatalogCompanyEnvironmentNameQuestion(q)) || /\b(?:CEO|Riley|Jordan|another agent|all agents|every agent)\b/i.test(text))return; const answer=row.result?.answers[0]; if(savedName){ @@ -203,7 +202,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s }, optionalCompanySecretBindingApproval:true, expectedCompanySecretBindingApproval(value){if(!gradeCompanySecret(value).passed)return;return companySecretApprovalProof(value);}, - oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v12",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, + oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v13",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts index acf1f386f13..ab4213a3c5e 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { chatCatalogHumanQuestionPolicies } from "../chat-catalog-contract.js"; +import { chatCatalogCompanyEnvironmentNameQuestion, chatCatalogHumanQuestionPolicies } from "../chat-catalog-contract.js"; import type { AskUserQuestionsInteraction } from "../../../packages/shared/src/types/issue.js"; import { askUserQuestionsPayloadSchema } from "../../../packages/shared/src/validators/issue.js"; @@ -18,8 +18,7 @@ export function boundSecretQuestion(value: unknown, b: SecretQuestionBinding, st const q = qs.questions[0]!; if (!q.required || (mode ? q.answerMode !== "text" || !!q.options?.length || !!q.customAnswer?.enabled || (q.initialText !== undefined && (mode !== "company-saved-secret-name" || (q.initialText !== "" && !/^[a-zA-Z0-9_.-]{1,120}$/.test(q.initialText)))) - || (mode === "company-environment-name" && (q.textValidation?.pattern !== "^[A-Za-z_][A-Za-z0-9_]*$" || q.textValidation.minLength !== 1 - || !q.textValidation.maxLength || q.textValidation.maxLength > 120 || q.textValidation.maxLength < "DEPLOYMENT_API_KEY".length)) + || (mode === "company-environment-name" && !chatCatalogCompanyEnvironmentNameQuestion(q)) : q.answerMode !== "single_select" || !q.options?.length)) return; return { row: { ...row, id: row.id }, questionSet: qs, question: { ...q, options: q.options ?? [] }, text: [qs.title, qs.description, q.header, q.prompt, q.helpText, q.customAnswer?.enabled ? q.customAnswer.label : undefined, q.customAnswer?.enabled ? q.customAnswer.placeholder : undefined, qs.submitLabel].filter(Boolean).join("\n"),