From 58449c4ecd7fd374f21f175411ef3249f55c18a3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 6 Nov 2026 00:51:52 -0600 Subject: [PATCH] fix(evals): admit completed Company secure-entry questions --- tests/runner-e2e/CHAT-FEATURES.md | 7 +++++++ tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../external-company-secret-confirmation.test.ts | 8 ++++++++ .../external-company-secret-confirmation.ts | 4 +++- .../external-company-secret.test.ts | 10 ++++++++++ .../chat-catalog-domains/external-company-secret.ts | 2 +- 6 files changed, 30 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index e29e8c61fda..68b57175504 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -498,3 +498,10 @@ unresolved" to the already supported "remains blocked" and passed the owning gra This is an assertion correction, with no new native pass or runtime-migration credit. The derived runtime fixture, protected local skill, actor/phase/resource/link checks, original query, two native/human turns and 480000ms attempt are unchanged. + +Company secret v15 admits a bound question asking whether deployment-key secure +entry has been completed, as well as the existing save-then-confirm instruction. +This is an acknowledgment of an actual prior current-person secure save, not +permission to execute a binding. The separate saved-value, payload, actor, +continuation and governed-approval proofs remain mandatory. Original failures +remain unchanged; v15 has not run natively. Counts, queries and clocks are unchanged. diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 3d8f7ff900e..1c9ea392098 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4923,7 +4923,7 @@ "chat-flow.ts", "chat-scenarios/secrets.yaml" ], - "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v14", + "oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v15", "expectedRunCount": 3, "minimumRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.test.ts index a3731d0ee8d..6bb6141040f 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.test.ts @@ -34,3 +34,11 @@ it("preserves either supported accept-response policy and rejects no continuatio for(const continuationPolicy of ["wake_assignee","wake_assignee_on_accept"])expect(companySecretConfirmationHandoff({...card(),continuationPolicy},binding,guidance)?.continuationPolicy).toBe(continuationPolicy); expect(companySecretConfirmationHandoff({...card(),continuationPolicy:"none"},binding,guidance)).toBeUndefined(); }); + +it("admits a bound completed-entry question with native Confirm and rejects other purposes",()=>{ + const completed="Have you saved the deployment API key in Settings > Secrets (`/RUN/company/settings/secrets`) so I can bind its reference to Morgan's environment?"; + expect(companySecretConfirmationHandoff({...card(),payload:{...card().payload,prompt:completed}},binding,guidance)).toBeDefined(); + for(const prompt of ["May I bind the deployment key in Settings > Secrets to Morgan's environment?","Will you save the deployment key in Settings > Secrets so I can bind it for Morgan?","Have you saved your profile? I can bind the deployment key in Settings > Secrets for Morgan.",completed+" What is the key value?"]){ + expect(companySecretConfirmationHandoff({...card(),payload:{...card().payload,prompt}},binding,guidance),prompt).toBeUndefined(); + } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.ts index f981057279c..bb2f8249bce 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-confirmation.ts @@ -17,7 +17,9 @@ export function companySecretConfirmationHandoff(value:unknown,b:CompanyConfirma if(!row?.id||row.kind!=="request_confirmation"||row.status!==status||row.companyId!==b.companyId||row.issueId!==b.issueId||row.sourceRunId!==b.runId||row.createdByAgentId!==b.agentId||row.createdByUserId||row.addresseeAgentId||(row.addresseeUserId!=null&&row.addresseeUserId!==b.userId)||!row.originCommentIds?.includes(b.canonicalCommentId)||!chatCatalogHumanQuestionPolicies(row)||(row.continuationPolicy!=="wake_assignee"&&row.continuationPolicy!=="wake_assignee_on_accept")||row.dismissedAt||!p||!requestConfirmationPayloadSchema.safeParse(p).success||p.target||p.toolAction||p.secretProposal||p.connectionAuthorization||p.qualityReview ||(status==="pending"?row.resolvedByUserId||row.resolvedByAgentId||row.result||row.resolvedAt:row.resolvedByUserId!==b.userId||row.resolvedByAgentId||row.result?.outcome!=="accepted"||row.result.resumeFailure||row.result.toolAction||row.result.secretProposal||!Number.isFinite(Date.parse(String(row.resolvedAt)))))return; const text=[row.title,p.prompt,p.detailsMarkdown,p.acceptLabel,p.rejectLabel].filter(Boolean).join("\n"); - if(!guidance(text)||!/\b(?:save|saved|set|entered)\b[^.!?\n]{0,100}\b(?:accept|confirm)\b/i.test(p.prompt)||!/^(?:Confirm|Accept|Done|Saved)$/i.test(p.acceptLabel??""))return; + const completedQuestion=p.prompt.match(/\bhave\s+you\s+(?:already\s+)?(?:saved|set|entered|completed)\b[^?!\n]{0,240}\?/i)?.[0]; + const completedEntry=!!completedQuestion&&/\bdeployment\s+(?:api\s+)?(?:key|credential|secret)\b/i.test(completedQuestion); + if(!guidance(text)||(!/\b(?:save|saved|set|entered)\b[^.!?\n]{0,100}\b(?:accept|confirm)\b/i.test(p.prompt)&&!completedEntry)||!/^(?:Confirm|Accept|Done|Saved)$/i.test(p.acceptLabel??""))return; const names=[...p.prompt.matchAll(/\bname\s+(?:it|the secret)\s+["“]([^"”\n]{1,120})["”]/gi)],keys=[...p.prompt.matchAll(/\b(?:set\s+)?(?:secret\s+)?key\s+`([a-zA-Z0-9_.-]{1,120})`/gi)]; if(names.length>1||keys.length>1||/\bname\s+(?:it|the secret)\b/i.test(p.prompt)&&names.length!==1||/\bset\s+(?:the\s+)?key\b/i.test(p.prompt)&&keys.length!==1||!validCompanySecretMetadata({name:names[0]?.[1],key:keys[0]?.[1]}))return; return {interactionId:row.id,continuationPolicy:row.continuationPolicy,createdAt:String(row.createdAt),payloadHash:createHash("sha256").update(JSON.stringify(p)).digest("hex"),acceptLabel:p.acceptLabel!,prompt:p.prompt,metadata:{name:names[0]?.[1],key:keys[0]?.[1]}}; diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts index 2ca983149ff..913ee088a5d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.test.ts @@ -408,3 +408,13 @@ it("retains either supported ACK continuation policy and rejects a changed or ab expect(verifyChatCatalogHumanQuestionAnswer({expected,companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"initial-human",authorUserId:"person"},interactions:o.interactions as never,activity:o.activity,companySecrets:o.secrets as never,runs:o.runs})).toBeUndefined(); Object.assign(o.interactions[0]!,{continuationPolicy:"none"});expect(companySecretExternalCases[0]!.grade(o).passed).toBe(false); }); + +it("retains actual save, actor, payload and continuation proof for a completed-entry question",()=>{ + const o=confirmationOutcome(),card=o.interactions[0]! as unknown as RequestConfirmationInteraction; + card.payload.prompt='In Settings > Secrets (`/RUN/company/settings/secrets`), name it "Deployment API Key", set key `DEPLOYMENT_API_KEY`. Have you saved the deployment API key so I can bind its reference to Morgan\'s environment?'; + o.state.initialConfirmationPayloadHash=createHash("sha256").update(JSON.stringify(card.payload)).digest("hex"); + expect(companySecretExternalCases[0]!.grade(o).passed).toBe(true); + expect(companySecretExternalCases[0]!.expectedHumanQuestionAnswer!(o)?.companySecretConfirmation).toBeDefined(); + Object.assign(o.secrets[0]!,{createdAt:"2026-11-04T13:28:00Z"}); + expect(companySecretExternalCases[0]!.grade(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index 87537f68bf6..0ca993cfdfb 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -215,7 +215,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s }, optionalCompanySecretBindingApproval:true, expectedCompanySecretBindingApproval(value){if(!gradeCompanySecret(value).passed)return;return companySecretApprovalProof(value);}, - oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v14",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, + oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v15",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000, sourceFiles:["../../server/src/routes/secrets.ts","../../packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts","../../packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.ts","../../packages/paperclip-runner/src/protocol-actions/request-human-input.ts","chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-company-secret-confirmation.ts","chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","../../server/src/services/native-runtime/runner-api-catalog.ts","../../server/src/routes/issues.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"], async act(ctx,state){ const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx);