From 9b17aa2c98df7f420dc2e7fbd46b286c6996c8e3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 11:05:24 -0600 Subject: [PATCH] test(evals): declare company addressed-human acknowledgement v4 Exercise canonical anyone audience and safe optional-note affordance while retaining enumerated answers and saved-reference proof. Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../external-company-secret-question.test.ts | 12 ++++++++++-- .../external-company-secret.test.ts | 16 ++++++++++++---- .../external-company-secret.ts | 2 +- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 461be40b25e..1d8c83ab6e3 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4905,7 +4905,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v3", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v4", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts index 9aceae4d36c..a9acf414b42 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts @@ -7,7 +7,7 @@ import { companySecretExternalCases,companySecretQuestionHandoff } from "./exter const binding={companyId:"company",issueId:"source",agentId:"ceo",userId:"person",runId:"initial"}; function question(){return {id:"question",companyId:"company",issueId:"source",sourceRunId:"initial",createdByAgentId:"ceo",addresseeUserId:"person",resolverPolicy:"human_only",continuationPolicy:"wake_assignee",kind:"ask_user_questions",status:"pending",payload:questionSetToAskUserQuestionsPayload({schema:"paperclip.question_set.v1",description:"Use secure Settings Secrets to add the deployment credential, then bind its reference in Morgan's environment as DEPLOYMENT_API_KEY. Never paste the key in chat.",questions:[{id:"saved",prompt:"Have you saved and bound the deployment key for Morgan?",answerMode:"single_select",required:true,options:[{id:"yes",label:"Saved and bound for Morgan"},{id:"later",label:"Not yet"}]}]})};} it("admits a bound secure company-entry confirmation",()=>expect(companySecretQuestionHandoff(question(),binding)).toMatchObject({interactionId:"question",optionId:"yes"})); -it.each(["key","owner","run","actor","company","source","unsafe chat","text value"])("rejects %s",fault=>{const row=question(); +it.each(["key","owner","run","actor","company","source","unsafe chat","unsafe optional note"])("rejects %s",fault=>{const row=question(); if(fault==="key")row.payload.questionSet!.description=row.payload.questionSet!.description!.replace("DEPLOYMENT_API_KEY","OTHER_API_KEY"); if(fault==="owner")row.addresseeUserId="other"; if(fault==="run")row.sourceRunId="other"; @@ -15,7 +15,7 @@ it.each(["key","owner","run","actor","company","source","unsafe chat","text valu if(fault==="company")row.companyId="other"; if(fault==="source")row.issueId="other"; if(fault==="unsafe chat")row.payload.questionSet!.description+=" Paste the deployment key here in chat."; - if(fault==="text value")row.payload.questionSet!.questions[0]!.customAnswer={enabled:true}; + if(fault==="unsafe optional note")row.payload.questionSet!.questions[0]!.customAnswer={enabled:true,label:"Paste the deployment key here in chat"}; expect(companySecretQuestionHandoff(row,binding)).toBeUndefined();}); it("actual company act admits its safe question before private human entry",async()=>{ @@ -42,3 +42,11 @@ it("keeps an explicit Morgan acknowledgement and rejects other named completion expect(!!companySecretQuestionHandoff(row,binding)).toBe(accepted); } }); + +it("accepts an anyone-policy question addressed to the current human with a safe optional note",()=>{ + const row=question();row.resolverPolicy="anyone";Object.assign(row,{requestedResolverPolicy:"anyone",effectiveResolverPolicy:"anyone"}); + row.payload.questionSet!.questions[0]!.customAnswer={enabled:true,label:"Optional note"}; + expect(companySecretQuestionHandoff(row,binding)).toMatchObject({interactionId:"question",optionId:"yes"}); + const foreign={...row,addresseeUserId:"other"};expect(companySecretQuestionHandoff(foreign,binding)).toBeUndefined(); + const unknown={...row,effectiveResolverPolicy:"unknown"};expect(companySecretQuestionHandoff(unknown,binding)).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts index 29a1b7e0750..cc7e40693e3 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts @@ -61,15 +61,16 @@ it("requires the original controller metadata and version before and after secur const missing=valid();missing.state.controllerSecretBaseline=[];expect(companySecretExternalCases[0]!.grade(missing).passed).toBe(false); }); -function questionOutcome(){ - const row={id:"question",companyId:"company",issueId:"source",sourceRunId:"first",createdByAgentId:"ceo",addresseeUserId:"person",resolverPolicy:"human_only" as const,continuationPolicy:"wake_assignee" as const,kind:"ask_user_questions" as const,status:"answered" as const,resolvedByUserId:"person",resolvedAt:"2026-11-04T13:40:00Z",result:{version:1 as const,answers:[{questionId:"saved",optionIds:["yes"]}],summaryMarkdown:null},payload:questionSetToAskUserQuestionsPayload({schema:"paperclip.question_set.v1",description:"Use secure Settings Secrets for the deployment credential and bind its reference in Morgan's environment as DEPLOYMENT_API_KEY. Never paste the key here.",questions:[{id:"saved",prompt:"Have you saved and bound it?",answerMode:"single_select",required:true,options:[{id:"yes",label:"Saved and bound for Morgan"}]}]})}; +function questionOutcome(policy:"human_only"|"anyone"="human_only",optionalNote=false){ + const row={id:"question",companyId:"company",issueId:"source",sourceRunId:"first",createdByAgentId:"ceo",addresseeUserId:"person",resolverPolicy:policy,continuationPolicy:"wake_assignee" as const,kind:"ask_user_questions" as const,status:"answered" as const,resolvedByUserId:"person",resolvedAt:"2026-11-04T13:40:00Z",result:{version:1 as const,answers:[{questionId:"saved",optionIds:["yes"]}],summaryMarkdown:null},payload:questionSetToAskUserQuestionsPayload({schema:"paperclip.question_set.v1",description:"Use secure Settings Secrets for the deployment credential and bind its reference in Morgan's environment as DEPLOYMENT_API_KEY. Never paste the key here.",questions:[{id:"saved",prompt:"Have you saved and bound it?",answerMode:"single_select",required:true,options:[{id:"yes",label:"Saved and bound for Morgan"}]}]})}; + if(optionalNote)row.payload.questionSet!.questions[0]!.customAnswer={enabled:true,label:"Optional note"}; const b={companyId:"company",issueId:"source",agentId:"ceo",userId:"person",runId:"first"},q=companySecretQuestionHandoff(row,b,"answered")!; const o=valid(); return {...o,phaseReplies:o.phaseReplies.slice(1),runs:o.runs.map(run=>({...run,contextSnapshot:{interactionId:"question"}})),interactions:[row],secrets:[{...o.secrets[0],createdAt:"2026-11-04T13:35:00Z"}],activity:[...o.activity,{entityId:"morgan",action:"agent.updated",actorType:"user",createdAt:"2026-11-04T13:36:00Z"}],state:{...o.state,sourceIssueId:"source",currentPersonId:"person",initialQuestionId:q.interactionId,initialQuestionPayloadHash:q.payloadHash,initialQuestionOptionId:q.optionId,questionAnsweredThroughUi:true}}; } it("accepts the same question answered through UI after actual saved company entry and Morgan binding",()=>expect(companySecretExternalCases[0]!.grade(questionOutcome()).passed).toBe(true)); it.each(["foreign owner","changed payload","before creation","before binding","wrong continuation","missing UI","missing reference"])("rejects conditional completion with %s",fault=>{ - const o=questionOutcome(); + const o=questionOutcome("anyone",true); if(fault==="foreign owner")o.interactions[0]!.resolvedByUserId="other"; if(fault==="changed payload")o.interactions[0]!.payload.questionSet!.description+=" Changed"; if(fault==="before creation")o.interactions[0]!.resolvedAt="2026-11-04T13:34:00Z"; @@ -81,7 +82,7 @@ it.each(["foreign owner","changed payload","before creation","before binding","w }); it("accounts for one real company question answer plus one chat request only after the full proof",()=>{ - const o=questionOutcome(),definition=companySecretExternalCases[0]!,expected=definition.expectedHumanQuestionAnswer!(o); + const o=questionOutcome("anyone",true),definition=companySecretExternalCases[0]!,expected=definition.expectedHumanQuestionAnswer!(o); expect(expected).toEqual({interactionId:"question",sourceRunId:"first",userId:"person"}); const raw={expected:expected!,companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"human",authorUserId:"person"},interactions:o.interactions, activity:[{companyId:"company",entityId:"source",action:"issue.thread_interaction_answered",actorType:"user",actorId:"person",details:{interactionId:"question",interactionKind:"ask_user_questions",interactionStatus:"answered"}}], @@ -94,3 +95,10 @@ it("accounts for one real company question answer plus one chat request only aft o.state.savedReferenceMatches=false;expect(definition.expectedHumanQuestionAnswer!(o)).toBeUndefined(); expect(definition.expectedHumanQuestionAnswer!(valid())).toBeUndefined(); }); + +it("accepts a saved company binding acknowledged by its addressed human under anyone",()=>{ + const o=questionOutcome("anyone",true);expect(companySecretExternalCases[0]!.grade(o).passed).toBe(true); + expect(companySecretExternalCases[0]!.expectedHumanQuestionAnswer!(o)).toEqual({interactionId:"question",sourceRunId:"first",userId:"person"}); + Object.assign(o.interactions[0]!.result.answers[0]!,{otherText:"a submitted note"}); + expect(companySecretExternalCases[0]!.grade(o).passed).toBe(false);expect(companySecretExternalCases[0]!.expectedHumanQuestionAnswer!(o)).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index a77c8bdb323..0a3181f5482 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -89,7 +89,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s const o=externalObservation(value); if(!o?.state.initialQuestionId||!gradeCompanySecret(value).passed)return; return {interactionId:String(o.state.initialQuestionId),sourceRunId:String(o.state.initialRunId),userId:String(o.state.currentPersonId)}; - },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v3",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, + },oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v4",expectedRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, sourceFiles:["chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx);