From 244facf29db5fb9f09cd8518b4818e14445549a8 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 08:01:20 -0600 Subject: [PATCH] fix(evals): verify secure answers from resolver audit and delivery Co-Authored-By: Paperclip --- .../external-personal-secret.test.ts | 17 ++++++++++++----- .../external-personal-secret.ts | 19 +++++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts index b934a4dbc1a..b2884468376 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.test.ts @@ -177,18 +177,25 @@ function answeredQuestionValue() { const base = valid(); const row = { id: "question", companyId: "company", issueId: "source", sourceRunId: "initial", createdByAgentId: "ceo", addresseeUserId: "person", resolverPolicy: "human_only", continuationPolicy: "wake_assignee", kind: "ask_user_questions", status: "answered", - resolvedByUserId: "person", resolvedAt: "2026-11-04T13:30:00Z", result: { version: 1, commentId: "answer", answers: [{ questionId: "saved", optionIds: ["yes"] }] }, + resolvedByUserId: "person", resolvedAt: "2026-11-04T13:30:00Z", result: { version: 1, summaryMarkdown: null, answers: [{ questionId: "saved", optionIds: ["yes"] }] }, payload: questionSetToAskUserQuestionsPayload({ schema: "paperclip.question_set.v1", description: "Set your own personal publishing value securely in My secrets. Never paste your credential in chat.", questions: [{ id: "saved", prompt: "Have you saved your own value?", helpText: "Settings Secrets My secrets Set my value definition PERSONAL_PUBLISHING_KEY", answerMode: "single_select", required: true, options: [{ id: "yes", label: "Yes", description: "My personal value is saved." }, { id: "later", label: "Not yet" }] }] }), }; const admitted = personalSecretQuestionHandoff(row, { companyId: "company", issueId: "source", agentId: "ceo", userId: "person", runId: "initial", definitionId: "definition" }, "answered"); if (!admitted) throw Error("Invalid question test prerequisite"); - return { ...base, interactions: [row], comments: [{ id: "answer", companyId: "company", issueId: "source", authorUserId: "person" }], - phaseReplies: base.phaseReplies.slice(1), runs: base.runs.map(run => run.id === "confirmed" ? { ...run, contextSnapshot: { interactionId: "question" } } : run), + return { ...base, interactions: [row], comments: [ + { id: "query", companyId: "company", issueId: "source", authorUserId: "person", body: "Set up my personal publishing credential for this task." }, + { id: "reply", companyId: "company", issueId: "source", authorAgentId: "ceo", createdByRunId: "confirmed", body: base.reply }, + ], activity: [...base.activity, + { companyId: "company", action: "issue.thread_interaction_answered", entityId: "source", actorType: "user", actorId: "person", details: { interactionId: "question", interactionKind: "ask_user_questions", interactionStatus: "answered" } }, + { companyId: "company", action: "issue.question_response_delivered", entityId: "source", actorType: "system", actorId: "question-response-delivery", details: { interactionId: "question", sourceRunId: "initial", targetRunId: "confirmed", deliveryStatus: "delivered" } }, + ], + phaseReplies: base.phaseReplies.slice(1), runs: base.runs.map(run => run.id === "confirmed" ? { ...run, companyId: "company", agentId: "ceo", contextSnapshot: { issueId: "source", interactionId: "question" } } : run), state: { ...base.state, sourceIssueId: "source", definitionId: "definition", initialQuestionId: row.id, initialQuestionOptionId: "yes", initialQuestionPayloadHash: admitted.payloadHash, currentSecret: { ...base.state.currentSecret, createdAt: "2026-11-04T13:29:00Z" } } }; } -it("accepts the same human question answered only after actual own-value save and bound worker/continuation evidence", () => { +it("accepts the production answer shape without a commentId or human answer comment, with owner audit and delivery evidence", () => { + const row = answeredQuestionValue(); expect(row.comments).toHaveLength(2); expect(row.comments.filter(comment => comment.authorUserId === "person")).toHaveLength(1); expect(row.interactions[0]!.result).not.toHaveProperty("commentId"); expect(gradePersonalSecret(answeredQuestionValue()).passed).toBe(true); }); it.each(["foreign resolver", "changed payload", "before save", "wrong continuation", "missing UI"])("rejects question completion with %s", fault => { @@ -196,7 +203,7 @@ it.each(["foreign resolver", "changed payload", "before save", "wrong continuati if (fault === "foreign resolver") value.interactions[0]!.resolvedByUserId = "other-person"; if (fault === "changed payload") value.state.initialQuestionPayloadHash = "wrong"; if (fault === "before save") value.interactions[0]!.resolvedAt = "2026-11-04T13:28:00Z"; - if (fault === "wrong continuation") value.runs = value.runs.map(run => run.id === "confirmed" ? { ...run, contextSnapshot: { interactionId: "other" } } : run); + if (fault === "wrong continuation") { const run = value.runs.find(row => row.id === "confirmed"); if (run?.contextSnapshot && "interactionId" in run.contextSnapshot) run.contextSnapshot.interactionId = "other"; } if (fault === "missing UI") value.state.humanSubmittedSecurely = false; expect(gradePersonalSecret(value).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index 55f743a6ab5..cc290624f55 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -165,11 +165,12 @@ export function gradePersonalSecret(value: unknown) { questionRow = o.interactions.find(row => row.id === o.state.initialQuestionId), question = personalSecretQuestionHandoff(questionRow, { companyId: o.companyId, issueId: String(o.state.sourceIssueId), agentId: o.leadId, userId: String(o.state.currentPersonId), runId: String(o.state.initialRunId), definitionId: String(o.state.definitionId) }, "answered"), - answeredQuestion = questionRow as Partial | undefined, questionAnswered = !!question && question.payloadHash === o.state.initialQuestionPayloadHash && question.optionId === o.state.initialQuestionOptionId - && o.comments.some(comment => comment.id === answeredQuestion?.result?.commentId && comment.companyId === o.companyId - && comment.issueId === o.state.sourceIssueId && comment.authorUserId === o.state.currentPersonId && !comment.authorAgentId), + && o.activity.some(event => event.action === "issue.thread_interaction_answered" && event.companyId === o.companyId + && event.entityId === o.state.sourceIssueId && event.actorType === "user" && event.actorId === o.state.currentPersonId + && event.details?.interactionId === question.interactionId && event.details.interactionKind === "ask_user_questions" + && event.details.interactionStatus === "answered"), last = o.phaseReplies .filter((row) => row.runId !== o.state.initialRunId) .at(-1); @@ -241,7 +242,12 @@ export function gradePersonalSecret(value: unknown) { ) && personalSecretConfirmation(last.body) && (!o.state.initialQuestionId || (questionAnswered - && o.runs.some(run => run.id === last.runId && run.contextSnapshot?.interactionId === question?.interactionId) + && o.runs.some(run => run.id === last.runId && run.companyId === o.companyId && run.agentId === o.leadId + && run.contextSnapshot?.issueId === o.state.sourceIssueId && run.contextSnapshot?.interactionId === question?.interactionId) + && o.activity.some(event => event.action === "issue.question_response_delivered" && event.companyId === o.companyId + && event.entityId === o.state.sourceIssueId && event.actorType === "system" && event.actorId === "question-response-delivery" + && event.details?.interactionId === question?.interactionId && event.details.sourceRunId === o.state.initialRunId + && ["delivered", "fallback_queued"].includes(String(event.details.deliveryStatus))) && Number.isFinite(Date.parse(String(secret?.createdAt))) && Number.isFinite(Date.parse(String(questionRow?.resolvedAt))) && Date.parse(String(secret?.createdAt)) <= Date.parse(String(questionRow?.resolvedAt)))) && o.state.secretScanComplete === true && @@ -380,6 +386,11 @@ export const personalSecretExternalCases = [ gradePersonalSecret, { authentication: "human-session", + expectedHumanQuestionAnswer(value) { + const o = externalObservation(value); + if (!o?.state.initialQuestionId || !gradePersonalSecret(value).passed) return; + return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) }; + }, oracleVersion: "paperclip.chat-catalog.personal-secret.v3", setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }, expectedRunCount: 3,