From 8cb77e382f9e428c07f8c20e1c83f47a0e567250 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 22:33:56 -0600 Subject: [PATCH] fix(evals): retain direct ACK identity and wake policy Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-contract.ts | 8 ++++---- ...external-company-secret-confirmation.test.ts | 5 +++++ .../external-company-secret-confirmation.ts | 4 ++-- .../external-company-secret-observe.test.ts | 16 ++++++++++++++++ .../external-company-secret.test.ts | 17 ++++++++++++++--- .../external-company-secret.ts | 14 +++++++------- 6 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-company-secret-observe.test.ts diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 618f3e55a54..6b0ae58aaf2 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -287,7 +287,7 @@ export function chatCatalogCompanyEnvironmentNameQuestion(q:{prompt:string;requi ||!/^[A-Za-z_][A-Za-z0-9_]{0,119}$/.test(answer)||!validation?.maxLength||validation.maxLength>120||validation.maxLengthanswer.length||(validation.inputType!==undefined&&validation.inputType!=="text"))return false; try{return !validation.pattern||new RegExp(validation.pattern).test(answer);}catch{return false;} } -export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string; personalSecretConfirmation?: { definitionId: string; secretId: string; payloadHash: string }; companySecretConfirmation?: { secretId: string; key: string; payloadHash: string }; companyEnvironmentNameAnswer?: { questionId: string; payloadHash: string; value: "DEPLOYMENT_API_KEY" }; companySavedSecretNameAnswer?: { questionId: string; payloadHash: string; value: string; secretId: string } } +export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string; personalSecretConfirmation?: { definitionId: string; secretId: string; payloadHash: string }; companySecretConfirmation?: { secretId: string; key: string; payloadHash: string; continuationPolicy: "wake_assignee" | "wake_assignee_on_accept" }; companyEnvironmentNameAnswer?: { questionId: string; payloadHash: string; value: "DEPLOYMENT_API_KEY" }; companySavedSecretNameAnswer?: { questionId: string; payloadHash: string; value: string; secretId: string } } export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string } export interface ChatCatalogHumanQuestionAudit { companyId?: string; entityId?: string; action?: string; actorType?: string; actorId?: string; @@ -307,7 +307,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { const secrets=input.companySecrets?.filter(secret=>secret.id===binding.secretId),secret=secrets?.[0],resolvedAt=Date.parse(String(row?.resolvedAt)),createdAt=Date.parse(String(row?.createdAt)); const audits=input.activity.filter(event=>event.action==="issue.thread_interaction_accepted"&&event.details?.interactionId===b.interactionId),audit=audits[0]; const saves=input.activity.filter(event=>event.action==="secret.created"&&event.entityId===binding.secretId),save=saves[0]; - if(b.personalSecretConfirmation||b.companyEnvironmentNameAnswer||b.companySavedSecretNameAnswer||rows.length!==1||!row||row.kind!=="request_confirmation"||row.status!=="accepted"||row.companyId!==input.companyId||row.issueId!==input.issueId||row.sourceRunId!==b.sourceRunId||row.createdByAgentId!==input.agentId||row.createdByUserId||row.addresseeAgentId||(row.addresseeUserId!=null&&row.addresseeUserId!==b.userId)||!row.originCommentIds?.includes(input.canonicalComment.id)||b.userId!==input.canonicalComment.authorUserId||!chatCatalogHumanQuestionPolicies(row)||row.continuationPolicy!=="wake_assignee"||row.resolvedByUserId!==b.userId||row.resolvedByAgentId||row.result?.outcome!=="accepted"||row.result.resumeFailure||row.result.toolAction||row.result.secretProposal||row.dismissedAt + if(b.personalSecretConfirmation||b.companyEnvironmentNameAnswer||b.companySavedSecretNameAnswer||rows.length!==1||!row||row.kind!=="request_confirmation"||row.status!=="accepted"||row.companyId!==input.companyId||row.issueId!==input.issueId||row.sourceRunId!==b.sourceRunId||row.createdByAgentId!==input.agentId||row.createdByUserId||row.addresseeAgentId||(row.addresseeUserId!=null&&row.addresseeUserId!==b.userId)||!row.originCommentIds?.includes(input.canonicalComment.id)||b.userId!==input.canonicalComment.authorUserId||!chatCatalogHumanQuestionPolicies(row)||(row.continuationPolicy!=="wake_assignee"&&row.continuationPolicy!=="wake_assignee_on_accept")||row.continuationPolicy!==binding.continuationPolicy||row.resolvedByUserId!==b.userId||row.resolvedByAgentId||row.result?.outcome!=="accepted"||row.result.resumeFailure||row.result.toolAction||row.result.secretProposal||row.dismissedAt ||!row.payload||!requestConfirmationPayloadSchema.safeParse(row.payload).success||row.payload.target||row.payload.toolAction||row.payload.secretProposal||row.payload.connectionAuthorization||row.payload.qualityReview||createHash("sha256").update(JSON.stringify(row.payload)).digest("hex")!==binding.payloadHash ||secrets?.length!==1||!secret||secret.companyId!==input.companyId||secret.scope!=="company"||secret.status!=="active"||secret.createdByUserId!==b.userId||secret.key!==binding.key||!/^[a-zA-Z0-9_.-]{1,120}$/.test(binding.key)||![createdAt,resolvedAt,Date.parse(secret.createdAt)].every(Number.isFinite)||Date.parse(secret.createdAt)<=createdAt||Date.parse(secret.createdAt)>resolvedAt ||audits.length!==1||audit?.companyId!==input.companyId||audit.entityId!==input.issueId||audit.actorType!=="user"||audit.actorId!==b.userId||audit.details?.interactionKind!=="request_confirmation"||audit.details.interactionStatus!=="accepted" @@ -382,7 +382,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { } export interface ChatCatalogExpectedCompanySecretBindingApproval { - interactionId:string;proposalId:string;sourceRunId:string;initialRunId:string;userId:string;secretId:string;targetAgentId:string;configPath:string;payloadHash:string;humanInputId:string;humanInputKind:"comment"|"question"|"confirmation";humanInputPayloadHash?:string;humanInputSecretKey?:string; + interactionId:string;proposalId:string;sourceRunId:string;initialRunId:string;userId:string;secretId:string;targetAgentId:string;configPath:string;payloadHash:string;humanInputId:string;humanInputKind:"comment"|"question"|"confirmation";humanInputPayloadHash?:string;humanInputSecretKey?:string;humanInputContinuationPolicy?:"wake_assignee"|"wake_assignee_on_accept"; } export interface ChatCatalogVerifiedCompanySecretBindingApproval extends ChatCatalogExpectedCompanySecretBindingApproval {continuationRunId:string;resolvedAt:string} export function verifyChatCatalogCompanySecretBindingApproval(input:{ @@ -408,7 +408,7 @@ export function verifyChatCatalogCompanySecretBindingApproval(input:{ ||proposal.targetType!=="agent"||proposal.target?.id!==b.targetAgentId||proposal.configPath!==b.configPath||proposal.appliedBindingConfigPath!==b.configPath||proposal.originIssue?.id!==input.issueId||proposal.originRunId!==b.sourceRunId||proposal.proposedBy?.id!==input.agentId||proposal.resolvedByUserId!==b.userId ||!initialHuman||initialHuman.authorUserId!==b.userId||initialHuman.authorAgentId||initialHuman.createdByRunId||initialHuman.companyId!==input.companyId||initialHuman.issueId!==input.issueId||!first||first.companyId!==input.companyId||first.agentId!==input.agentId||first.status!=="succeeded"||first.runtimeMode!=="native"||first.contextSnapshot?.issueId!==input.issueId ||!secret||secret.companyId!==input.companyId||secret.scope!=="company"||secret.status!=="active"||secret.createdByUserId!==b.userId||!first||Date.parse(secret.createdAt)<=Date.parse(String(first.startedAt))||Date.parse(secret.createdAt)>=inputAt - ||!Number.isFinite(inputAt)||(b.humanInputKind==="confirmation"?!verifyChatCatalogHumanQuestionAnswer({expected:{interactionId:b.humanInputId,sourceRunId:b.initialRunId,userId:b.userId,companySecretConfirmation:{secretId:b.secretId,key:b.humanInputSecretKey??"",payloadHash:b.humanInputPayloadHash??""}},companyId:input.companyId,issueId:input.issueId,agentId:input.agentId,canonicalComment:initialHuman!,interactions:input.interactions as Array>,activity:input.activity,companySecrets:input.secrets.map(secret=>({...secret,key:secret.key??""})),runs:input.runs}):(human?(human.companyId!==input.companyId||human.issueId!==input.issueId||human.authorUserId!==b.userId||!!human.authorAgentId||!!human.createdByRunId):!question||question.kind!=="ask_user_questions"||question.sourceRunId!==b.initialRunId||question.createdByAgentId!==input.agentId||question.status!=="answered"||question.resolvedByUserId!==b.userId||question.companyId!==input.companyId||question.issueId!==input.issueId)) + ||!Number.isFinite(inputAt)||(b.humanInputKind==="confirmation"?!verifyChatCatalogHumanQuestionAnswer({expected:{interactionId:b.humanInputId,sourceRunId:b.initialRunId,userId:b.userId,companySecretConfirmation:{secretId:b.secretId,key:b.humanInputSecretKey??"",payloadHash:b.humanInputPayloadHash??"",continuationPolicy:b.humanInputContinuationPolicy!}},companyId:input.companyId,issueId:input.issueId,agentId:input.agentId,canonicalComment:initialHuman!,interactions:input.interactions as Array>,activity:input.activity,companySecrets:input.secrets.map(secret=>({...secret,key:secret.key??""})),runs:input.runs}):(human?(human.companyId!==input.companyId||human.issueId!==input.issueId||human.authorUserId!==b.userId||!!human.authorAgentId||!!human.createdByRunId):!question||question.kind!=="ask_user_questions"||question.sourceRunId!==b.initialRunId||question.createdByAgentId!==input.agentId||question.status!=="answered"||question.resolvedByUserId!==b.userId||question.companyId!==input.companyId||question.issueId!==input.issueId)) ||!source||source.id===first.id||source.companyId!==input.companyId||source.agentId!==input.agentId||source.runtimeMode!=="native"||source.status!=="succeeded"||source.contextSnapshot?.issueId!==input.issueId ||source.contextSnapshot[b.humanInputKind==="comment"?"commentId":"interactionId"]!==b.humanInputId||Date.parse(String(source.startedAt))r.contextSnapshot?.interactionId===b.interactionId),run=continuations[0]; diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.test.ts index be09c9c9dab..a3731d0ee8d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.test.ts @@ -29,3 +29,8 @@ it("keeps optional metadata bounded and matches the production canonical-key fun const actual=new Function(source.slice(start,end+2).replace("export function normalizeSecretKey(input: string)","function normalizeSecretKey(input)")+";return normalizeSecretKey")() as (key:string)=>string; for(const key of ["DEPLOYMENT_API_KEY","deployment.api-key"])expect(canonicalCompanySecretKey(key)).toBe(actual(key)); }); + +it("preserves either supported accept-response policy and rejects no continuation",()=>{ + for(const continuationPolicy of ["wake_assignee","wake_assignee_on_accept"])expect(companySecretConfirmationHandoff({...card(),continuationPolicy},binding,guidance)?.continuationPolicy).toBe(continuationPolicy); + expect(companySecretConfirmationHandoff({...card(),continuationPolicy:"none"},binding,guidance)).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.ts index 09fe113c068..e63fbde653b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.ts @@ -14,13 +14,13 @@ export function validCompanySecretMetadata(value:unknown):value is {name?:string export interface CompanyConfirmationBinding extends SecretQuestionBinding {canonicalCommentId:string} export function companySecretConfirmationHandoff(value:unknown,b:CompanyConfirmationBinding,guidance:(text:string)=>boolean,status:"pending"|"accepted"="pending"){ const row=value as Partial|undefined,p=row?.payload; - if(!row?.id||row.kind!=="request_confirmation"||row.status!==status||row.companyId!==b.companyId||row.issueId!==b.issueId||row.sourceRunId!==b.runId||row.createdByAgentId!==b.agentId||row.createdByUserId||row.addresseeAgentId||(row.addresseeUserId!=null&&row.addresseeUserId!==b.userId)||!row.originCommentIds?.includes(b.canonicalCommentId)||!chatCatalogHumanQuestionPolicies(row)||row.continuationPolicy!=="wake_assignee"||row.dismissedAt||!p||!requestConfirmationPayloadSchema.safeParse(p).success||p.target||p.toolAction||p.secretProposal||p.connectionAuthorization||p.qualityReview + if(!row?.id||row.kind!=="request_confirmation"||row.status!==status||row.companyId!==b.companyId||row.issueId!==b.issueId||row.sourceRunId!==b.runId||row.createdByAgentId!==b.agentId||row.createdByUserId||row.addresseeAgentId||(row.addresseeUserId!=null&&row.addresseeUserId!==b.userId)||!row.originCommentIds?.includes(b.canonicalCommentId)||!chatCatalogHumanQuestionPolicies(row)||(row.continuationPolicy!=="wake_assignee"&&row.continuationPolicy!=="wake_assignee_on_accept")||row.dismissedAt||!p||!requestConfirmationPayloadSchema.safeParse(p).success||p.target||p.toolAction||p.secretProposal||p.connectionAuthorization||p.qualityReview ||(status==="pending"?row.resolvedByUserId||row.resolvedByAgentId||row.result||row.resolvedAt:row.resolvedByUserId!==b.userId||row.resolvedByAgentId||row.result?.outcome!=="accepted"||row.result.resumeFailure||row.result.toolAction||row.result.secretProposal||!Number.isFinite(Date.parse(String(row.resolvedAt)))))return; const text=[row.title,p.prompt,p.detailsMarkdown,p.acceptLabel,p.rejectLabel].filter(Boolean).join("\n"); if(!guidance(text)||!/\b(?:save|saved|set|entered)\b[^.!?\n]{0,100}\b(?:accept|confirm)\b/i.test(p.prompt)||!/^(?:Confirm|Accept|Done|Saved)$/i.test(p.acceptLabel??""))return; const names=[...p.prompt.matchAll(/\bname\s+(?:it|the secret)\s+["“]([^"”\n]{1,120})["”]/gi)],keys=[...p.prompt.matchAll(/\b(?:set\s+)?(?:secret\s+)?key\s+`([a-zA-Z0-9_.-]{1,120})`/gi)]; if(names.length>1||keys.length>1||/\bname\s+(?:it|the secret)\b/i.test(p.prompt)&&names.length!==1||/\bset\s+(?:the\s+)?key\b/i.test(p.prompt)&&keys.length!==1||!validCompanySecretMetadata({name:names[0]?.[1],key:keys[0]?.[1]}))return; - return {interactionId:row.id,createdAt:String(row.createdAt),payloadHash:createHash("sha256").update(JSON.stringify(p)).digest("hex"),acceptLabel:p.acceptLabel!,prompt:p.prompt,metadata:{name:names[0]?.[1],key:keys[0]?.[1]}}; + return {interactionId:row.id,continuationPolicy:row.continuationPolicy,createdAt:String(row.createdAt),payloadHash:createHash("sha256").update(JSON.stringify(p)).digest("hex"),acceptLabel:p.acceptLabel!,prompt:p.prompt,metadata:{name:names[0]?.[1],key:keys[0]?.[1]}}; } export async function visibleCompanySecretConfirmationURL(page:Page,handoff:{prompt:string},origin:string,prefix:string,remaining:()=>number){ const card=page.getByTestId("task-chat-composer-takeover"),prompt=card.getByText(handoff.prompt,{exact:true}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-observe.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-observe.test.ts new file mode 100644 index 00000000000..5f46f6990db --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-observe.test.ts @@ -0,0 +1,16 @@ +import {expect,it,vi} from "vitest"; +import type {ChatCatalogContext} from "../chat-catalog-contract.js"; +const port=vi.hoisted(()=>({observation:{} as Record})); +vi.mock("./identity-support.js",async original=>({...await original(),identityWorker:async()=>({id:"morgan"}),probeAgentPermission:async()=>403})); +vi.mock("./external-support.js",async original=>({...await original(),observeExternal:async(_ctx:unknown,state:unknown)=>({...port.observation,state})})); +vi.mock("./external-secret-scan.js",async original=>({...await original(),scanReachableSecretOutputs:async()=>({complete:true,leaked:false})})); +import {companySecretExternalCases} from "./external-company-secret.js"; +it("the actual direct-ACK observer fetches its canonical human without preseeded source comments",async()=>{ + const controller={id:"controller",companyId:"company",scope:"company",ownerUserId:null,userSecretDefinitionId:null,key:"openrouter_api_key",name:"Runner E2E controller",provider:"local_encrypted",status:"active",managedMode:"paperclip_managed",latestVersion:1},human={id:"human",companyId:"company",issueId:"source",authorUserId:"person",body:"Add our deployment API key so Morgan can use it."}; + const get=vi.fn(async(path:string)=>{if(path.endsWith("/secret-providers"))return[{id:"local_encrypted"}];if(path==="/api/auth/get-session")return{user:{id:"person"}};if(path==="/api/agents/morgan")return{adapterConfig:{env:{}}};if(path.endsWith("/secrets"))return[controller];if(path.endsWith("/usage"))return{bindings:[]};if(path==="/api/issues/source/comments?order=asc")return[human];throw new Error("Unexpected public metadata read");}); + const ctx={input:{privateSecrets:[],fixtures:{company:{id:"company",issuePrefix:"RUN"},secretRefs:{OPENROUTER_API_KEY:{type:"secret_ref",secretId:"controller",version:"latest"}}},api:{get}},issue:()=>({id:"source"})} as unknown as ChatCatalogContext; + const definition=companySecretExternalCases[0]!,state=await definition.setup(ctx);Object.assign(state,{sourceIssueId:"source",initialConfirmationId:"ack",secretId:"new",nativeBindingRequired:true}); + expect(Object.hasOwn(state,"companySourceComments")).toBe(false);expect(state.followupCommentId).toBeUndefined();expect(state.companySecretApproval).toBeUndefined(); + port.observation={secrets:[controller],tasks:[],runs:[],activity:[]};await definition.observe(ctx,state); + expect(state.companySourceComments).toEqual([human]);expect(get).toHaveBeenCalledWith("/api/issues/source/comments?order=asc"); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts index 0265aebfc7a..2ca983149ff 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts @@ -369,7 +369,7 @@ function confirmationOutcome(){ Object.assign(o.runs[0]!,{exitCode:0});Object.assign(o.runs[1]!,{exitCode:0,contextSnapshot:{issueId:"source",interactionId:"ack"}}); o.activity[0]={id:"save",companyId:"company",entityId:"secret",action:"secret.created",actorType:"user",actorId:"person",createdAt:"2026-11-04T13:35:00Z"}; o.activity.push({id:"accepted",companyId:"company",entityId:"source",action:"issue.thread_interaction_accepted",actorType:"user",actorId:"person",createdAt:"2026-11-04T13:40:00Z",details:{interactionId:"ack",interactionKind:"request_confirmation",interactionStatus:"accepted"}}); - Object.assign(o.state,{initialConfirmationId:"ack",initialConfirmationPayloadHash:createHash("sha256").update(JSON.stringify(payload)).digest("hex"),canonicalHumanCommentId:"initial-human",confirmationVisibleBeforeEntry:true,confirmationAnsweredThroughUi:true,confirmationResponseVerified:true,requestedCompanyName:"Deployment API Key",requestedCompanyKey:"DEPLOYMENT_API_KEY",savedCompanyName:"Deployment API Key",savedCompanyKey:"deployment_api_key",companySourceComments:[{id:"initial-human",companyId:"company",issueId:"source",authorUserId:"person",createdAt:"2026-11-04T12:59:00Z",body:"Add our deployment API key so Morgan can use it."}]}); + Object.assign(o.state,{initialConfirmationId:"ack",initialConfirmationContinuationPolicy:"wake_assignee",initialConfirmationPayloadHash:createHash("sha256").update(JSON.stringify(payload)).digest("hex"),canonicalHumanCommentId:"initial-human",confirmationVisibleBeforeEntry:true,confirmationAnsweredThroughUi:true,confirmationResponseVerified:true,requestedCompanyName:"Deployment API Key",requestedCompanyKey:"DEPLOYMENT_API_KEY",savedCompanyName:"Deployment API Key",savedCompanyKey:"deployment_api_key",companySourceComments:[{id:"initial-human",companyId:"company",issueId:"source",authorUserId:"person",createdAt:"2026-11-04T12:59:00Z",body:"Add our deployment API key so Morgan can use it."}]}); return o; } it("counts only a saved current-human confirmation and proves its actual bound native binding",()=>{ @@ -388,12 +388,23 @@ it.each(["pre-save","wrong person","wrong source","wrong scope","effect","unsafe it("keeps secure-save ACK and the later executed binding approval as distinct real human actions",()=>{ const {o,card,marker}=governedCompanyOutcome(),direct=confirmationOutcome(),ack=direct.interactions[0]!; o.interactions.push(ack);o.phaseReplies=o.phaseReplies.filter(reply=>reply.runId!=="first"); - for(const key of ["initialConfirmationId","initialConfirmationPayloadHash","canonicalHumanCommentId","confirmationVisibleBeforeEntry","confirmationAnsweredThroughUi","confirmationResponseVerified","requestedCompanyName","requestedCompanyKey","savedCompanyName","savedCompanyKey","companySourceComments"])o.state[key]=direct.state[key]; + for(const key of ["initialConfirmationId","initialConfirmationPayloadHash","initialConfirmationContinuationPolicy","canonicalHumanCommentId","confirmationVisibleBeforeEntry","confirmationAnsweredThroughUi","confirmationResponseVerified","requestedCompanyName","requestedCompanyKey","savedCompanyName","savedCompanyKey","companySourceComments"])o.state[key]=direct.state[key]; Object.assign(o.secrets[0]!,{name:"Deployment API Key"});Object.assign(o.runs[0]!,{exitCode:0});Object.assign(o.runs[1]!,{exitCode:0,contextSnapshot:{issueId:"source",interactionId:"ack"}}); o.activity[0]=direct.activity[0]!;o.activity.push(direct.activity.at(-1)!); card.payload.secretProposal!.sourceSecretLabel="Deployment API Key"; - Object.assign(marker,{humanInputKind:"confirmation",humanInputId:"ack",humanInputPayloadHash:o.state.initialConfirmationPayloadHash,humanInputSecretKey:"deployment_api_key",payloadHash:createHash("sha256").update(JSON.stringify(card.payload)).digest("hex")}); + Object.assign(marker,{humanInputKind:"confirmation",humanInputId:"ack",humanInputPayloadHash:o.state.initialConfirmationPayloadHash,humanInputSecretKey:"deployment_api_key",humanInputContinuationPolicy:"wake_assignee",payloadHash:createHash("sha256").update(JSON.stringify(card.payload)).digest("hex")}); const grade=companySecretExternalCases[0]!.grade(o);expect(grade.passed,JSON.stringify(grade.checks.filter(c=>!c.passed))).toBe(true); expect(companySecretExternalCases[0]!.expectedHumanQuestionAnswer!(o)?.companySecretConfirmation).toBeDefined();expect(companySecretExternalCases[0]!.expectedCompanySecretBindingApproval!(o)).toBeDefined(); Object.assign(ack,{status:"pending",result:null,resolvedAt:null,resolvedByUserId:null});expect(companySecretExternalCases[0]!.grade(o).passed).toBe(false); }); + +it("retains either supported ACK continuation policy and rejects a changed or absent continuation",()=>{ + for(const policy of ["wake_assignee","wake_assignee_on_accept"]){ + const o=confirmationOutcome();Object.assign(o.interactions[0]!,{continuationPolicy:policy});o.state.initialConfirmationContinuationPolicy=policy; + const definition=companySecretExternalCases[0]!;expect(definition.grade(o).passed,policy).toBe(true);expect(definition.expectedHumanQuestionAnswer!(o)?.companySecretConfirmation?.continuationPolicy).toBe(policy); + } + const o=confirmationOutcome(),expected=companySecretExternalCases[0]!.expectedHumanQuestionAnswer!(o)!; + Object.assign(o.interactions[0]!,{continuationPolicy:"wake_assignee_on_accept"});expect(companySecretExternalCases[0]!.grade(o).passed).toBe(false); + expect(verifyChatCatalogHumanQuestionAnswer({expected,companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"initial-human",authorUserId:"person"},interactions:o.interactions as never,activity:o.activity,companySecrets:o.secrets as never,runs:o.runs})).toBeUndefined(); + Object.assign(o.interactions[0]!,{continuationPolicy:"none"});expect(companySecretExternalCases[0]!.grade(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index 33db5dfc315..42245c5b082 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -110,10 +110,10 @@ export function companySecretConfirmationProof(value:unknown){ const o=value as ExternalObservation,row=o.interactions.find(q=>q.id===o.state.initialConfirmationId),secret=o.secrets.find(q=>q.id===o.state.secretId); const binding={companyId:o.companyId,issueId:String(o.state.sourceIssueId),agentId:o.leadId,userId:String(o.state.currentPersonId),runId:String(o.state.initialRunId),canonicalCommentId:String(o.state.canonicalHumanCommentId)}; const handoff=companySecretConfirmationHandoff(row,binding,text=>deploymentSecretGuidance(text)&&!companyOperatorBindingRequested(text),"accepted"); - if(!handoff||!secret||o.state.requestedCompanyName!==(handoff.metadata.name??"Deployment API key")||handoff.metadata.key&&o.state.requestedCompanyKey!==handoff.metadata.key||o.state.savedCompanyName!==secret.name||o.state.savedCompanyKey!==secret.key||typeof secret.key!=="string"||!secret.key||handoff.payloadHash!==o.state.initialConfirmationPayloadHash||o.state.confirmationVisibleBeforeEntry!==true||o.state.confirmationAnsweredThroughUi!==true||o.state.confirmationResponseVerified!==true||handoff.metadata.key&&secret.key!==canonicalCompanySecretKey(handoff.metadata.key)||handoff.metadata.name&&secret.name!==handoff.metadata.name)return; + if(!handoff||!secret||o.state.requestedCompanyName!==(handoff.metadata.name??"Deployment API key")||handoff.metadata.key&&o.state.requestedCompanyKey!==handoff.metadata.key||o.state.savedCompanyName!==secret.name||o.state.savedCompanyKey!==secret.key||typeof secret.key!=="string"||!secret.key||handoff.payloadHash!==o.state.initialConfirmationPayloadHash||handoff.continuationPolicy!==o.state.initialConfirmationContinuationPolicy||o.state.confirmationVisibleBeforeEntry!==true||o.state.confirmationAnsweredThroughUi!==true||o.state.confirmationResponseVerified!==true||handoff.metadata.key&&secret.key!==canonicalCompanySecretKey(handoff.metadata.key)||handoff.metadata.name&&secret.name!==handoff.metadata.name)return; const canonical=(o.state.companySourceComments as Array<{id:string;authorUserId?:string}>|undefined)?.find(c=>c.id===binding.canonicalCommentId); if(!canonical)return; - return verifyChatCatalogHumanQuestionAnswer({expected:{interactionId:handoff.interactionId,sourceRunId:binding.runId,userId:binding.userId,companySecretConfirmation:{secretId:secret.id,key:secret.key,payloadHash:handoff.payloadHash}},companyId:o.companyId,issueId:binding.issueId,agentId:o.leadId,canonicalComment:canonical,interactions:o.interactions as never,activity:o.activity,companySecrets:o.secrets as never,runs:o.runs}); + return verifyChatCatalogHumanQuestionAnswer({expected:{interactionId:handoff.interactionId,sourceRunId:binding.runId,userId:binding.userId,companySecretConfirmation:{secretId:secret.id,key:secret.key,payloadHash:handoff.payloadHash,continuationPolicy:handoff.continuationPolicy}},companyId:o.companyId,issueId:binding.issueId,agentId:o.leadId,canonicalComment:canonical,interactions:o.interactions as never,activity:o.activity,companySecrets:o.secrets as never,runs:o.runs}); } export function companySavedNameBinding(config:Record,before:unknown,secretId:unknown,bindings:unknown,morganId:unknown,expectedEnvironmentName?:string){ if(!before||typeof before!=="object"||!config.env||typeof config.env!=="object"||Array.isArray(config.env)||!Array.isArray(bindings)||bindings.length!==1)return; @@ -149,7 +149,7 @@ export function companySecretApprovalCandidate(proposal:SecretProposalView,cardV ||card.kind!=="request_confirmation"||card.status!=="pending"||card.companyId!==o.companyId||card.issueId!==o.state.sourceIssueId||card.sourceRunId!==sourceRunId||card.createdByAgentId!==o.leadId||card.createdByUserId||card.addresseeAgentId||(card.addresseeUserId&&card.addresseeUserId!==o.state.currentPersonId)||card.resolvedByUserId||card.resolvedByAgentId ||card.requestedResolverPolicy!=="human_only"||card.effectiveResolverPolicy!=="human_only"||card.continuationPolicy!=="wake_assignee"||card.idempotencyKey!==`secret-proposal:${proposal.id}`||p.proposalId!==proposal.id||p.targetAgentId!==o.state.morganId||p.targetAgentName!=="Morgan"||p.configPath!==proposal.configPath||p.sourceSecretLabel!==secret.name||Date.parse(p.expiresAt)<=Date.parse(String(card.createdAt)) ||secret.companyId!==o.companyId||secret.scope!=="company"||secret.status!=="active"||secret.createdByUserId!==o.state.currentPersonId||Date.parse(String(secret.createdAt))>=inputAt||solicitsChatCredential(JSON.stringify(card.payload))||unsafeCredentialTransfer(JSON.stringify(card.payload)))return; - return {interactionId:card.id,proposalId:proposal.id,sourceRunId,initialRunId:String(o.state.initialRunId),userId:String(o.state.currentPersonId),secretId:String(o.state.secretId),targetAgentId:String(o.state.morganId),configPath:proposal.configPath!,payloadHash:createHash("sha256").update(JSON.stringify(card.payload)).digest("hex"),humanInputId:inputId,humanInputKind:inputKind,...(inputKind==="confirmation"?{humanInputPayloadHash:String(o.state.initialConfirmationPayloadHash),humanInputSecretKey:secret.key}:{})} satisfies ChatCatalogExpectedCompanySecretBindingApproval; + return {interactionId:card.id,proposalId:proposal.id,sourceRunId,initialRunId:String(o.state.initialRunId),userId:String(o.state.currentPersonId),secretId:String(o.state.secretId),targetAgentId:String(o.state.morganId),configPath:proposal.configPath!,payloadHash:createHash("sha256").update(JSON.stringify(card.payload)).digest("hex"),humanInputId:inputId,humanInputKind:inputKind,...(inputKind==="confirmation"?{humanInputPayloadHash:String(o.state.initialConfirmationPayloadHash),humanInputSecretKey:secret.key,humanInputContinuationPolicy:o.state.initialConfirmationContinuationPolicy as "wake_assignee"|"wake_assignee_on_accept"}:{})} satisfies ChatCatalogExpectedCompanySecretBindingApproval; } export function companySecretApprovalProof(value:unknown){ const o=externalObservation(value),expected=o?.state.companySecretApproval as ChatCatalogExpectedCompanySecretBindingApproval|undefined; @@ -234,7 +234,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s if(initial.companyId!==source.companyId||(pending.length?(!question&&!confirmation):!commentHandoff))throw new Error("Actual reply must direct secure deployment-key entry and Morgan-only reference binding before human action"); state.initialRunId=run.id;state.sourceIssueId=source.id; state.nativeBindingRequired=confirmation?true:question?question.savedSecretName===true||!question.operatorBinding:!companyOperatorBindingRequested(reply!.body); - if(confirmation)Object.assign(state,{initialConfirmationId:confirmation.interactionId,initialConfirmationPayloadHash:confirmation.payloadHash,canonicalHumanCommentId:canonical[0]!.id}); + if(confirmation)Object.assign(state,{initialConfirmationId:confirmation.interactionId,initialConfirmationPayloadHash:confirmation.payloadHash,initialConfirmationContinuationPolicy:confirmation.continuationPolicy,canonicalHumanCommentId:canonical[0]!.id}); if(question)Object.assign(state,{initialQuestionId:question.interactionId,initialQuestionOptionId:question.optionId,initialQuestionEnvironmentName:question.environmentName,initialQuestionSavedSecretName:question.savedSecretName,initialQuestionFieldId:question.questionId,initialQuestionPayloadHash:question.payloadHash}); remaining(); const beforeHuman=await ctx.input.api.get(`${root(ctx)}/secrets`); @@ -264,10 +264,10 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s const savedSecret=(await ctx.input.api.get(`${root(ctx)}/secrets`)).find(row=>row.id===state.secretId),createdAt=Date.parse(String((pending[0] as unknown as RequestConfirmationInteraction).createdAt)); if(!savedSecret||state.requestedCompanyName!==(confirmation.metadata.name??"Deployment API key")||confirmation.metadata.key&&state.requestedCompanyKey!==confirmation.metadata.key||state.savedCompanyName!==savedSecret.name||state.savedCompanyKey!==savedSecret.key||savedSecret.companyId!==source.companyId||savedSecret.scope!=="company"||savedSecret.status!=="active"||savedSecret.createdByUserId!==state.currentPersonId||!Number.isFinite(Date.parse(String(savedSecret.createdAt)))||Date.parse(String(savedSecret.createdAt))<=createdAt||confirmation.metadata.key&&savedSecret.key!==canonicalCompanySecretKey(confirmation.metadata.key)||confirmation.metadata.name&&savedSecret.name!==confirmation.metadata.name)throw new Error("Company confirmation requires the actual new current-human saved metadata"); const rows=await ctx.input.api.get(`/api/issues/${source.id}/interactions`),current=companySecretConfirmationHandoff(rows.find(row=>row.id===confirmation.interactionId),confirmationBinding,text=>deploymentSecretGuidance(text)&&!companyOperatorBindingRequested(text)); - if(!current||current.payloadHash!==confirmation.payloadHash)throw new Error("Company confirmation changed before acknowledgment"); + if(!current||current.payloadHash!==confirmation.payloadHash||current.continuationPolicy!==confirmation.continuationPolicy)throw new Error("Company confirmation changed before acknowledgment"); await visibleCompanySecretConfirmationURL(ctx.input.page,current,initial.origin,prefix,remaining); const response=await acceptCompanySecretConfirmation(ctx.input.page,source.id,current,{secret:savedSecret,binding:confirmationBinding},remaining),accepted=companySecretConfirmationHandoff(response,confirmationBinding,text=>deploymentSecretGuidance(text)&&!companyOperatorBindingRequested(text),"accepted"); - if(!accepted||accepted.payloadHash!==current.payloadHash||Date.parse(String((response as RequestConfirmationInteraction).resolvedAt))}}>(`/api/agents/${state.morganId}`),ref=savedAgent.adapterConfig.env?.[String(state.bindingEnvironmentName)] as {type?:string;secretId?:string;version?:string}|undefined; @@ -314,7 +314,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s async observe(ctx,state){ const o=await observeExternal(ctx,state),value=values.get(ctx); if(state.companySecretApproval)state.companyBindingProposals=await ctx.input.api.get(`${root(ctx)}/secret-proposals`); - if(state.followupCommentId||state.companySecretApproval)state.companySourceComments=await ctx.input.api.get(`/api/issues/${state.sourceIssueId}/comments?order=asc`); + if(state.followupCommentId||state.initialConfirmationId||state.companySecretApproval)state.companySourceComments=await ctx.input.api.get(`/api/issues/${state.sourceIssueId}/comments?order=asc`); state.controllerSecretsPreserved=Array.isArray(state.controllerSecretBaseline)&&o.secrets.length===state.controllerSecretBaseline.length+1&&o.secrets.filter(row=>row.id===state.secretId).length===1&&companySecretControllersUnchanged(o.secrets as unknown as CompanySecret[],state.controllerSecretBaseline); if(!value||!ctx.issue())throw new Error("Private secret output scan fixture missing"); state.bindings=(await ctx.input.api.get<{bindings:CompanySecretUsageBinding[]}>(`/api/secrets/${state.secretId}/usage`)).bindings;