From 4b277227e75d066775da86deec720d81a45fe730 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 17:07:38 -0600 Subject: [PATCH] fix(evals): admit saved-secret-name metadata defaults Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-contract.ts | 2 +- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../external-company-secret-question.test.ts | 14 +++++++++++++- .../external-company-secret.test.ts | 8 ++++++++ .../external-company-secret.ts | 2 +- .../external-secret-question.ts | 2 +- 6 files changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 773ac84335f..22a9510d706 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -332,7 +332,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { } else if(savedName){ const parsed=askUserQuestionsPayloadSchema.safeParse(row.payload),qs=parsed.success?parsed.data.questionSet:undefined,q=qs?.questions[0],answer=row.result.answers[0]; const secrets=input.companySecrets?.filter(secret=>secret.id===savedName.secretId),secret=secrets?.[0]; - if(!qs||qs.questions.length!==1||!q||q.id!==savedName.questionId||q.answerMode!=="text"||!q.required||q.options?.length||q.customAnswer?.enabled||q.initialText!==undefined + if(!qs||qs.questions.length!==1||!q||q.id!==savedName.questionId||q.answerMode!=="text"||!q.required||q.options?.length||q.customAnswer?.enabled||(q.initialText!==undefined&&q.initialText!==""&&!/^[a-zA-Z0-9_.-]{1,120}$/.test(q.initialText)) ||!chatCatalogSavedSecretNameIntent(q.prompt) ||secrets?.length!==1||!secret||secret.companyId!==input.companyId||secret.scope!=="company"||secret.status!=="active"||secret.createdByUserId!==b.userId ||!Number.isFinite(Date.parse(String(row.createdAt)))||!Number.isFinite(Date.parse(source?.startedAt??""))||Date.parse(secret.createdAt)<=Math.max(Date.parse(String(row.createdAt)),Date.parse(source?.startedAt??"")) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index ffd3a95982f..c38dccd72b7 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4909,7 +4909,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v10", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v11", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts index 3546e94c320..103461c5c05 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts @@ -121,7 +121,7 @@ it.each(["value","mixed","owner","source","help","initial"])("rejects unsafe sav if(fault==="owner")row.addresseeUserId="other"; if(fault==="source")row.sourceRunId="other"; if(fault==="help")q.helpText+=" Paste the key here in chat."; - if(fault==="initial")q.initialText="deployment_api_key"; + if(fault==="initial")q.initialText="not metadata value"; expect(companySecretQuestionHandoff(row,binding)).toBeUndefined(); }); it("checks saved-name answers against actual saved metadata rather than arbitrary text",()=>{ @@ -190,3 +190,15 @@ it("keeps a generic secure-save completion question with a later agent binding p expect(companyOperatorBindingRequested("Have you configured the deployment secret in Settings Secrets? Once saved, I'll add its reference to Morgan's environment.")).toBe(false); expect(companyOperatorBindingRequested("Have you configured the secret reference in Morgan's environment?")).toBe(true); }); + +it("admits the actual secure saved-name card with a proposed metadata default, not a human answer",()=>{ + const row=question();row.payload=questionSetToAskUserQuestionsPayload({schema:"paperclip.question_set.v1",title:"Deployment API key setup",description:"Add the secret in the Paperclip UI, then confirm the secret key name. Never paste the value here.",submitLabel:"Save answer",questions:[{id:"secret_key",prompt:"Which secret key name did you save the deployment API key under?",helpText:"Enter the key name you used in Company Settings → Secrets. I will verify the secret exists, then add a DEPLOYMENT_API_KEY secret reference to Morgan's environment.",required:true,answerMode:"text",initialText:"deployment_api_key",textValidation:{minLength:1,maxLength:120,inputType:"text"}}]}); + expect(companySecretQuestionHandoff(row,binding)).toMatchObject({savedSecretName:true,questionId:"secret_key"}); + expect(companySecretQuestionHandoff(row,binding,"answered","deployment_api_key")).toBeUndefined(); + const blank=structuredClone(row);blank.payload.questionSet!.questions[0]!.initialText="";expect(companySecretQuestionHandoff(blank,binding)).toMatchObject({savedSecretName:true}); + for(const initialText of ["not metadata value", "x".repeat(121)]){const bad=structuredClone(row);bad.payload.questionSet!.questions[0]!.initialText=initialText;expect(companySecretQuestionHandoff(bad,binding)).toBeUndefined();} + const wrongActor={...row,createdByAgentId:"other"};expect(companySecretQuestionHandoff(wrongActor,binding)).toBeUndefined(); + const rawValue=structuredClone(row);rawValue.payload.questionSet!.questions[0]!.prompt="What is the deployment key value?";expect(companySecretQuestionHandoff(rawValue,binding)).toBeUndefined(); + const environment=structuredClone(row);environment.payload.questionSet!.questions[0]!.prompt="What environment variable name should Morgan receive?";expect(boundSecretQuestion(environment,binding,"pending","company-environment-name")).toBeUndefined(); + const choice=question();choice.payload.questionSet!.questions[0]!.initialText="deployment_api_key";expect(companySecretQuestionHandoff(choice,binding)).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts index ba2d149398d..95beec4a10a 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { questionSetToAskUserQuestionsPayload } from "../../../packages/shared/src/question-set.js"; import { chatCatalogHumanHistoryMatches,verifyChatCatalogHumanQuestionAnswer } from "../chat-catalog-contract.js"; import { describe,expect,it,vi } from "vitest"; @@ -196,6 +197,13 @@ it("requires native saved-name binding and exact metadata-only question accounti const nativeRuns=[{id:"first",companyId:"company",agentId:"ceo",status:"succeeded",runtimeMode:"native",startedAt:"2026-11-04T13:00:00Z",contextSnapshot:{issueId:"source",commentId:"human"}},{id:"second",companyId:"company",agentId:"ceo",status:"succeeded",runtimeMode:"native",startedAt:"2026-11-04T13:41:00Z",contextSnapshot:{issueId:"source",interactionId:"question"}}]; const evidence={expected,companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"human",authorUserId:"person"},interactions:[row],companySecrets:o.secrets,activity:[{companyId:"company",entityId:"source",action:"issue.thread_interaction_answered",actorType:"user",actorId:"person",details:{interactionId:"question",interactionKind:"ask_user_questions",interactionStatus:"answered",answeredQuestionCount:1}}],runs:nativeRuns}; expect(verifyChatCatalogHumanQuestionAnswer(evidence)).toBeDefined(); + for(const initialText of ["deployment_api_key", "", "not metadata value"]){ + const changed=structuredClone(row);changed.payload.questionSet!.questions[0]!.initialText=initialText; + const next={...expected,companySavedSecretNameAnswer:{...expected.companySavedSecretNameAnswer!,payloadHash:createHash("sha256").update(JSON.stringify(changed.payload)).digest("hex")}}; + expect(!!verifyChatCatalogHumanQuestionAnswer({...evidence,expected:next,interactions:[changed]})).toBe(initialText!=="not metadata value"); + expect(verifyChatCatalogHumanQuestionAnswer({...evidence,interactions:[changed]})).toBeUndefined(); + } + for(const createdAt of ["2026-11-04T13:33:00Z","2026-11-04T12:59:00Z"]) expect(verifyChatCatalogHumanQuestionAnswer({...evidence,companySecrets:[{...o.secrets[0]!,createdAt}]})).toBeUndefined(); for(const prompt of ["Which name did you use for the saved deployment secret?","What alias identifies the deployment credential you created?"]){ diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index a68e28038b3..14ccde5e10b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -165,7 +165,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s return {interactionId:String(o.state.initialQuestionId),sourceRunId:String(o.state.initialRunId),userId:String(o.state.currentPersonId), ...(o.state.initialQuestionSavedSecretName===true?{companySavedSecretNameAnswer:{questionId:String(o.state.initialQuestionFieldId),payloadHash:String(o.state.initialQuestionPayloadHash),value:String(o.secrets.find(row=>row.id===o.state.secretId)?.key??""),secretId:String(o.state.secretId)}}:{}), ...(o.state.initialQuestionEnvironmentName==="DEPLOYMENT_API_KEY"?{companyEnvironmentNameAnswer:{questionId:String(o.state.initialQuestionFieldId),payloadHash:String(o.state.initialQuestionPayloadHash),value:"DEPLOYMENT_API_KEY" as const}}:{})}; - },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v10",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, + },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v11",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts index 78c88dabecf..acf1f386f13 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-question.ts @@ -17,7 +17,7 @@ export function boundSecretQuestion(value: unknown, b: SecretQuestionBinding, st if (!qs || qs.questions.length !== 1) return; const q = qs.questions[0]!; if (!q.required || (mode - ? q.answerMode !== "text" || !!q.options?.length || !!q.customAnswer?.enabled || q.initialText !== undefined + ? q.answerMode !== "text" || !!q.options?.length || !!q.customAnswer?.enabled || (q.initialText !== undefined && (mode !== "company-saved-secret-name" || (q.initialText !== "" && !/^[a-zA-Z0-9_.-]{1,120}$/.test(q.initialText)))) || (mode === "company-environment-name" && (q.textValidation?.pattern !== "^[A-Za-z_][A-Za-z0-9_]*$" || q.textValidation.minLength !== 1 || !q.textValidation.maxLength || q.textValidation.maxLength > 120 || q.textValidation.maxLength < "DEPLOYMENT_API_KEY".length)) : q.answerMode !== "single_select" || !q.options?.length)) return;