diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 6b0ae58aaf2..c6349d8be91 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -287,7 +287,7 @@ export function chatCatalogCompanyEnvironmentNameQuestion(q:{prompt:string;requi ||!/^[A-Za-z_][A-Za-z0-9_]{0,119}$/.test(answer)||!validation?.maxLength||validation.maxLength>120||validation.maxLengthanswer.length||(validation.inputType!==undefined&&validation.inputType!=="text"))return false; try{return !validation.pattern||new RegExp(validation.pattern).test(answer);}catch{return false;} } -export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string; personalSecretConfirmation?: { definitionId: string; secretId: string; payloadHash: string }; companySecretConfirmation?: { secretId: string; key: string; payloadHash: string; continuationPolicy: "wake_assignee" | "wake_assignee_on_accept" }; companyEnvironmentNameAnswer?: { questionId: string; payloadHash: string; value: "DEPLOYMENT_API_KEY" }; companySavedSecretNameAnswer?: { questionId: string; payloadHash: string; value: string; secretId: string } } +export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string; personalSecretConfirmation?: { definitionId: string; secretId: string; payloadHash: string; continuationPolicy: "wake_assignee" | "wake_assignee_on_accept" }; companySecretConfirmation?: { secretId: string; key: string; payloadHash: string; continuationPolicy: "wake_assignee" | "wake_assignee_on_accept" }; companyEnvironmentNameAnswer?: { questionId: string; payloadHash: string; value: "DEPLOYMENT_API_KEY" }; companySavedSecretNameAnswer?: { questionId: string; payloadHash: string; value: string; secretId: string } } export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string } export interface ChatCatalogHumanQuestionAudit { companyId?: string; entityId?: string; action?: string; actorType?: string; actorId?: string; @@ -326,7 +326,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { if (b.companySecretConfirmation || b.companyEnvironmentNameAnswer || b.companySavedSecretNameAnswer || rows.length !== 1 || !row || row.kind !== "request_confirmation" || row.status !== "accepted" || row.companyId !== input.companyId || row.issueId !== input.issueId || row.sourceRunId !== b.sourceRunId || row.createdByAgentId !== input.agentId || row.createdByUserId || (row.addresseeUserId != null && row.addresseeUserId !== b.userId) || row.addresseeAgentId || !row.originCommentIds?.includes(input.canonicalComment.id) - || b.userId !== input.canonicalComment.authorUserId || !chatCatalogHumanQuestionPolicies(row) || row.continuationPolicy !== "wake_assignee" + || b.userId !== input.canonicalComment.authorUserId || !chatCatalogHumanQuestionPolicies(row) || (row.continuationPolicy !== "wake_assignee" && row.continuationPolicy !== "wake_assignee_on_accept") || row.continuationPolicy !== binding.continuationPolicy || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.outcome !== "accepted" || row.result.resumeFailure || row.dismissedAt || !row.payload || row.payload.target || row.payload.toolAction || row.payload.secretProposal || row.payload.connectionAuthorization || row.payload.qualityReview || !chatCatalogHasExactDefinitionId([row.payload.prompt, row.payload.detailsMarkdown].filter(Boolean).join("\n"), binding.definitionId) || createHash("sha256").update(JSON.stringify(row.payload)).digest("hex") !== binding.payloadHash diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.test.ts index d69b3233f9f..6fca9b8ff43 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.test.ts @@ -19,7 +19,7 @@ it.each(["company", "issue", "run", "creator", "origin", "definition", "approval function accounting(){ const row={...card(),status:"accepted",resolvedByUserId:"person",resolvedAt:"2026-11-05T12:01:00Z",result:{version:1,outcome:"accepted"}}; const payloadHash=createHash("sha256").update(JSON.stringify(row.payload)).digest("hex"); - return {expected:{interactionId:"confirmation",sourceRunId:"initial",userId:"person",personalSecretConfirmation:{definitionId:"current-definition",secretId:"secret",payloadHash}},companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"human",authorUserId:"person"},interactions:[row],personalSecrets:[{definition:{id:"current-definition"},secret:{id:"secret",companyId:"company",scope:"user",ownerUserId:"person",userSecretDefinitionId:"current-definition",status:"active",createdAt:"2026-11-05T12:00:50Z"}}],activity:[{companyId:"company",entityId:"source",action:"issue.thread_interaction_accepted",actorType:"user",actorId:"person",details:{interactionId:"confirmation",interactionKind:"request_confirmation",interactionStatus:"accepted"}}],runs:[{id:"initial",companyId:"company",agentId:"ceo",status:"succeeded",runtimeMode:"native",startedAt:"2026-11-05T12:00:00Z",contextSnapshot:{issueId:"source",commentId:"human"}},{id:"continuation",companyId:"company",agentId:"ceo",status:"succeeded",runtimeMode:"native",startedAt:"2026-11-05T12:01:01Z",contextSnapshot:{issueId:"source",interactionId:"confirmation"}}]}; + return {expected:{interactionId:"confirmation",sourceRunId:"initial",userId:"person",personalSecretConfirmation:{definitionId:"current-definition",secretId:"secret",payloadHash,continuationPolicy:"wake_assignee"}},companyId:"company",issueId:"source",agentId:"ceo",canonicalComment:{id:"human",authorUserId:"person"},interactions:[row],personalSecrets:[{definition:{id:"current-definition"},secret:{id:"secret",companyId:"company",scope:"user",ownerUserId:"person",userSecretDefinitionId:"current-definition",status:"active",createdAt:"2026-11-05T12:00:50Z"}}],activity:[{companyId:"company",entityId:"source",action:"issue.thread_interaction_accepted",actorType:"user",actorId:"person",details:{interactionId:"confirmation",interactionKind:"request_confirmation",interactionStatus:"accepted"}}],runs:[{id:"initial",companyId:"company",agentId:"ceo",status:"succeeded",runtimeMode:"native",startedAt:"2026-11-05T12:00:00Z",contextSnapshot:{issueId:"source",commentId:"human"}},{id:"continuation",companyId:"company",agentId:"ceo",status:"succeeded",runtimeMode:"native",startedAt:"2026-11-05T12:01:01Z",contextSnapshot:{issueId:"source",interactionId:"confirmation"}}]}; } it("counts initial chat plus one bound actual human confirmation exactly once",()=>{ const input=accounting(),answer=verifyChatCatalogHumanQuestionAnswer(input as never);expect(answer).toMatchObject({continuationRunId:"continuation"}); @@ -50,3 +50,30 @@ it("binds an exact definition URI in visible details through admission and indep Object.assign(input.interactions[0]!.payload,row.payload);input.expected.personalSecretConfirmation.payloadHash=createHash("sha256").update(JSON.stringify(row.payload)).digest("hex"); expect(verifyChatCatalogHumanQuestionAnswer(input as never)).toBeUndefined(); }); + +it.each(["After saving, confirm here so I can pick up RUN-1.", "Have you saved your personal credential? Confirm here.", "Save your personal credential. Definition current-definition uses PERSONAL_PUBLISHING_KEY. Then confirm here."])("accepts a bound post-save acknowledgement: %s", ending => { + const row=card();row.continuationPolicy="wake_assignee_on_accept"; + row.payload.prompt=row.payload.prompt.replace("When you have saved it, confirm this card so the requirement is recorded.",ending); + expect(personalSecretConfirmationHandoff(row,binding,personalSecretGuidance)).toMatchObject({continuationPolicy:"wake_assignee_on_accept"}); +}); +it.each(["Confirm before saving your personal credential.", "Confirm so you can save your personal credential.", "You saved your personal credential yesterday. Confirm permission to enter a new value.", "After saving your profile, confirm here.", "After saving OTHER_KEY, confirm here.", "Do not save your personal credential. Then confirm here.", "Confirm here before saving. After saving, I will resume."])("rejects a non-post-save acknowledgement: %s", ending => { + const row=card();row.payload.prompt=row.payload.prompt.replace("When you have saved it, confirm this card so the requirement is recorded.",ending); + expect(personalSecretConfirmationHandoff(row,binding,personalSecretGuidance)).toBeUndefined(); +}); +it("independently credits either immutable accepted-response policy only with actual save and continuation",()=>{ + const input=accounting();input.interactions[0]!.continuationPolicy="wake_assignee_on_accept";input.expected.personalSecretConfirmation.continuationPolicy="wake_assignee_on_accept"; + expect(verifyChatCatalogHumanQuestionAnswer(input as never)).toMatchObject({continuationRunId:"continuation"}); + input.interactions[0]!.continuationPolicy="wake_assignee"; + expect(verifyChatCatalogHumanQuestionAnswer(input as never)).toBeUndefined(); +}); +it.each(["pending","declined","historical save","foreign owner","wrong runtime","missing continuation","effect"])("rejects nonqualifying accepted-response %s",fault=>{ + const input=accounting();input.interactions[0]!.continuationPolicy="wake_assignee_on_accept";input.expected.personalSecretConfirmation.continuationPolicy="wake_assignee_on_accept"; + if(fault==="pending")input.interactions[0]!.status="pending"; + if(fault==="declined")input.interactions[0]!.result.outcome="declined"; + if(fault==="historical save")input.personalSecrets[0]!.secret.createdAt="2026-11-04T12:00:00Z"; + if(fault==="foreign owner")input.personalSecrets[0]!.secret.ownerUserId="other"; + if(fault==="wrong runtime")input.runs[1]!.runtimeMode="process"; + if(fault==="missing continuation")input.runs.pop(); + if(fault==="effect")Object.assign(input.interactions[0]!.payload,{toolAction:{toolName:"deploy"}}); + expect(verifyChatCatalogHumanQuestionAnswer(input as never)).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts index ce8671d6f83..3aa6e52cc30 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-confirmation.ts @@ -9,16 +9,27 @@ export function personalSecretConfirmationHandoff(value: unknown, b: PersonalCon if (!row?.id || row.kind !== "request_confirmation" || row.status !== status || row.companyId !== b.companyId || row.issueId !== b.issueId || row.sourceRunId !== b.runId || row.createdByAgentId !== b.agentId || row.createdByUserId || row.addresseeAgentId || (row.addresseeUserId != null && row.addresseeUserId !== b.userId) || !row.originCommentIds?.includes(b.canonicalCommentId) - || !chatCatalogHumanQuestionPolicies(row) || row.continuationPolicy !== "wake_assignee" || !p + || !chatCatalogHumanQuestionPolicies(row) || (row.continuationPolicy !== "wake_assignee" && row.continuationPolicy !== "wake_assignee_on_accept") || !p || p.target || p.toolAction || p.secretProposal || p.connectionAuthorization || p.qualityReview || !requestConfirmationPayloadSchema.safeParse(p).success || row.dismissedAt || (status === "pending" ? row.resolvedByUserId || row.resolvedByAgentId : row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.outcome !== "accepted")) return; const actionText = [p.prompt, p.detailsMarkdown].filter(Boolean).join("\n"); const text = [actionText, p.acceptLabel].filter(Boolean).join("\n"); + const entry = String.raw`(?:it|(?:your|the|this)\s+(?:personal\s+)?(?:publishing\s+)?(?:key|value|credential|secret))\b`; + const postSave = new RegExp(String.raw`\b(?:when|once|after)\s+(?:you(?:\s+have|['’]ve)?\s+)?(?:sav(?:ed|ing)|set(?:ting)?|enter(?:ed|ing))(?=\s*(?:[,.;!?]|$)|\s+${entry})`, "i").exec(actionText); + const completedQuestion = new RegExp(String.raw`\b(?:have|has)\s+(?:you|the\s+user)\s+(?:saved|set|entered)\s+${entry}[^?]*\?`, "i").test(actionText); + const saveThen = new RegExp(String.raw`\b(?:save|set|enter)\s+${entry}[^!?]{0,400}\bthen\s+confirm\b`, "i").exec(actionText); + const saveThenConfirm = !!saveThen && !/(?:\b(?:do|should|must|can|could|will|would)\s+not|\bdon.t|\bnever|\b(?:skip|avoid))\s*$/i.test(actionText.slice(0, saveThen.index)); + const confirmsAfter = (tail: string) => [...tail.matchAll(/\bconfirm\b[^.!?]*/gi)].some(match => + !/(?:\b(?:do|should|must)\s+not|\bdon.t|\bnever|\b(?:skip|avoid))\s*$/i.test(tail.slice(0, match.index)) + && !/\b(?:before|permission|authorize)\b/i.test(match[0])); + const confirmsSavedEntry = /\bconfirm\b/i.test(text) && (completedQuestion + || saveThenConfirm && confirmsAfter(actionText.slice(saveThen!.index)) + || !!postSave && confirmsAfter(actionText.slice(postSave.index + postSave[0].length))); if (!guidance(text) || !chatCatalogHasExactDefinitionId(actionText, b.definitionId) || !/\bPERSONAL_PUBLISHING_KEY\b/.test(text) || !/my secrets/i.test(text) - || !/\b(?:when|once|after)\b[^.!?]*\b(?:saved|set|entered)\b[^.!?]*\bconfirm\b|\bconfirm\b[^.!?]*\b(?:saved|set|entered)\b/i.test(text) + || !confirmsSavedEntry || /\b(?:deploy(?:ment)?|delet(?:e|ion)|execut(?:e|ion)|purchas(?:e|ing))\b/i.test(actionText)) return; - return { interactionId: row.id, payloadHash: createHash("sha256").update(JSON.stringify(p)).digest("hex"), acceptLabel: p.acceptLabel ?? "Approve", prompt: p.prompt }; + return { interactionId: row.id, payloadHash: createHash("sha256").update(JSON.stringify(p)).digest("hex"), acceptLabel: p.acceptLabel ?? "Approve", prompt: p.prompt, continuationPolicy: row.continuationPolicy }; } export async function acceptPersonalSecretConfirmation(page: Page, issueId: string, interactionId: string, label: string, prompt: string, remaining: () => number) { const card = page.getByTestId("task-chat-composer-takeover"); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts index ad2ab4d5ee6..7a1a27987f0 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts @@ -288,7 +288,7 @@ it("accepts a saved-value confirmation without a fabricated initial reply and re const o = valid(); const payload = { version: 1, prompt: "Open Settings Secrets, My secrets, Set my value for your personal publishing credential PERSONAL_PUBLISHING_KEY definition definition. Never paste the value in chat. When you have saved it, confirm this card.", acceptLabel: "Confirm" }; const row = { id: "confirmation", companyId: "company", issueId: "source", sourceRunId: "initial", createdByAgentId: "ceo", originCommentIds: ["human"], kind: "request_confirmation", status: "accepted", resolverPolicy: "anyone", continuationPolicy: "wake_assignee", resolvedByUserId: "person", resolvedAt: "2026-11-03T23:59:59Z", result: { version: 1, outcome: "accepted" }, payload }; - Object.assign(o.state, { sourceIssueId: "source", confirmationVisibleBeforeEntry: true, initialConfirmationId: row.id, canonicalHumanCommentId: "human", initialConfirmationPayloadHash: createHash("sha256").update(JSON.stringify(payload)).digest("hex") }); + Object.assign(o.state, { sourceIssueId: "source", confirmationVisibleBeforeEntry: true, initialConfirmationId: row.id, initialConfirmationContinuationPolicy: row.continuationPolicy, canonicalHumanCommentId: "human", initialConfirmationPayloadHash: createHash("sha256").update(JSON.stringify(payload)).digest("hex") }); Object.assign(o.state.currentSecret, { createdAt: "2026-11-03T23:59:58Z" }); Object.assign(o.runs[2]!, { agentId: "ceo", contextSnapshot: { issueId: "source", interactionId: row.id } }); o.phaseReplies.shift(); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index d74b70f8ea3..8c54305bc1f 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -236,7 +236,7 @@ export function gradePersonalSecret(value: unknown) { .at(-1); const confirmationRow = o.interactions.find(row => row.id === o.state.initialConfirmationId); const confirmation = personalSecretConfirmationHandoff(confirmationRow, { companyId: o.companyId, issueId: String(o.state.sourceIssueId), agentId: o.leadId, userId: String(o.state.currentPersonId), runId: String(o.state.initialRunId), definitionId: String(o.state.definitionId), canonicalCommentId: String(o.state.canonicalHumanCommentId) }, personalSecretGuidance, "accepted"); - const confirmationAccepted = o.state.confirmationVisibleBeforeEntry === true && !!confirmation && confirmation.payloadHash === o.state.initialConfirmationPayloadHash + const confirmationAccepted = o.state.confirmationVisibleBeforeEntry === true && !!confirmation && confirmation.payloadHash === o.state.initialConfirmationPayloadHash && confirmation.continuationPolicy === o.state.initialConfirmationContinuationPolicy && o.activity.some(event => event.action === "issue.thread_interaction_accepted" && event.companyId === o.companyId && event.entityId === o.state.sourceIssueId && event.actorType === "user" && event.actorId === o.state.currentPersonId && event.details?.interactionId === confirmation.interactionId && event.details.interactionStatus === "accepted"); const secret = o.state.currentSecret as Partial | undefined; c.add( @@ -458,7 +458,7 @@ export const personalSecretExternalCases = [ expectedHumanQuestionAnswer(value) { const o = externalObservation(value); if (!o || !gradePersonalSecret(value).passed) return; - if (o.state.initialConfirmationId) return { interactionId: String(o.state.initialConfirmationId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId), personalSecretConfirmation: { definitionId: String(o.state.definitionId), secretId: String(o.state.secretId), payloadHash: String(o.state.initialConfirmationPayloadHash) } }; + if (o.state.initialConfirmationId) return { interactionId: String(o.state.initialConfirmationId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId), personalSecretConfirmation: { definitionId: String(o.state.definitionId), secretId: String(o.state.secretId), payloadHash: String(o.state.initialConfirmationPayloadHash), continuationPolicy: o.state.initialConfirmationContinuationPolicy as "wake_assignee" | "wake_assignee_on_accept" } }; if (!o.state.initialQuestionId) return; return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) }; }, @@ -608,7 +608,7 @@ export const personalSecretExternalCases = [ if (initial.companyId !== source.companyId || (pendingQuestions.length ? !question && !confirmation : !commentHandoff)) throw new Error("Personal setup must direct the current person's secure publishing value before human action"); state.initialRunId = run.id; - if (confirmation) Object.assign(state, { initialConfirmationId: confirmation.interactionId, initialConfirmationPayloadHash: confirmation.payloadHash, canonicalHumanCommentId: canonical[0]!.id }); + if (confirmation) Object.assign(state, { initialConfirmationId: confirmation.interactionId, initialConfirmationPayloadHash: confirmation.payloadHash, initialConfirmationContinuationPolicy: confirmation.continuationPolicy, canonicalHumanCommentId: canonical[0]!.id }); if (confirmation) { const visible = await readVisiblePersonalSecretConfirmation(ctx.input.page, confirmation, { origin: initial.origin, prefix: String(ctx.input.fixtures.company.issuePrefix), definitionId: String(state.definitionId) }, personalSecretGuidance, remaining); state.confirmationVisibleBeforeEntry = visible.definitionVisible; @@ -684,13 +684,13 @@ export const personalSecretExternalCases = [ const current = personalSecretConfirmationHandoff(cards.find(row => (row as { id?: string }).id === confirmation.interactionId), confirmationBinding, personalSecretGuidance); if (!own || own.id !== state.secretId || own.ownerUserId !== state.currentPersonId || own.userSecretDefinitionId !== state.definitionId || own.status !== "active" || cards.some(row => (row as { id?: string; status?: string }).status === "pending" && (row as { id?: string }).id !== confirmation.interactionId) - || !current || current.payloadHash !== confirmation.payloadHash) throw new Error("Personal confirmation requires unchanged card and actual saved own value"); + || !current || current.payloadHash !== confirmation.payloadHash || current.continuationPolicy !== confirmation.continuationPolicy) throw new Error("Personal confirmation requires unchanged card and actual saved own value"); await confirmThenStartPersonalConsumer( async () => { const response = await acceptPersonalSecretConfirmation(ctx.input.page, source.id, confirmation.interactionId, confirmation.acceptLabel, confirmation.prompt, remaining); const accepted = personalSecretConfirmationHandoff(response, confirmationBinding, personalSecretGuidance, "accepted"); const audit = await ctx.input.api.get }>>(`/api/issues/${source.id}/activity`); - if (!accepted || accepted.payloadHash !== confirmation.payloadHash || !audit.some(event => event.companyId === source.companyId && event.entityId === source.id && event.actorType === "user" && event.actorId === state.currentPersonId && event.action === "issue.thread_interaction_accepted" && event.details?.interactionId === confirmation.interactionId && event.details.interactionStatus === "accepted")) throw new Error("Personal secure-entry confirmation lacks actual human acceptance audit"); + if (!accepted || accepted.payloadHash !== confirmation.payloadHash || accepted.continuationPolicy !== confirmation.continuationPolicy || !audit.some(event => event.companyId === source.companyId && event.entityId === source.id && event.actorType === "user" && event.actorId === state.currentPersonId && event.action === "issue.thread_interaction_accepted" && event.details?.interactionId === confirmation.interactionId && event.details.interactionStatus === "accepted")) throw new Error("Personal secure-entry confirmation lacks actual human acceptance audit"); }, () => ctx.input.api.patch(`/api/issues/${state.taskId}`, { status: "todo" }).then(() => undefined), );