diff --git a/tests/runner-e2e/chat-catalog-consistency.test.ts b/tests/runner-e2e/chat-catalog-consistency.test.ts index 745ba02e390..02f8049b067 100644 --- a/tests/runner-e2e/chat-catalog-consistency.test.ts +++ b/tests/runner-e2e/chat-catalog-consistency.test.ts @@ -49,3 +49,54 @@ it("declares company approval as a static optional extra action without invoking expect(validChatCatalogHumanCounts({...descriptor,scenarioId:"secrets.personal-secret"})).toBe(false); expect(callback).not.toHaveBeenCalled(); }); + +// Full cohort 12 fix run (tasks.ambiguous-task): the person chose the Launch option on the CEO's question card. +// A chosen option saves no comment, so the flow must count the independently verified answer as the second action. +it("counts a verified clarification choice as the ambiguous-task person's second action", async () => { + const { verifyChatCatalogHumanQuestionAnswer } = await import("./chat-catalog-contract.js"); + const { answeredClarification } = await import("./chat-catalog-domains/internal-tasks.js"); + const implementation = chatCatalogCases.find(row => row.scenarioId === "tasks.ambiguous-task")!; + const expected = answeredClarification({ state: { answeredQuestion: { interactionId: "card", sourceRunId: "first", userId: "person", questionId: "landing_page_task", optionId: "com-1" } } })!; + expect(expected).toEqual({ interactionId: "card", sourceRunId: "first", userId: "person", clarificationChoice: { questionId: "landing_page_task", optionId: "com-1" } }); + expect(answeredClarification({ state: { answeredQuestion: { interactionId: "card", sourceRunId: "first", userId: null } } })).toBeUndefined(); + expect(answeredClarification({ state: {} })).toBeUndefined(); + const card = { id: "card", kind: "ask_user_questions", status: "answered", companyId: "co", issueId: "chat", sourceRunId: "first", createdByAgentId: "ceo", + createdByUserId: null, addresseeUserId: "person", addresseeAgentId: null, resolverPolicy: "anyone", effectiveResolverPolicy: "anyone", + resolvedByUserId: "person", resolvedByAgentId: null, resolvedAt: "2026-11-13T15:01:47Z", + result: { answers: [{ questionId: "landing_page_task", optionIds: ["com-1"] }] } }; + const audit = { companyId: "co", entityId: "chat", action: "issue.thread_interaction_answered", actorType: "user", actorId: "person", + details: { interactionId: "card", interactionKind: "ask_user_questions", interactionStatus: "answered" } }; + const runs = [ + { id: "first", companyId: "co", agentId: "ceo", status: "succeeded", runtimeMode: "native", startedAt: "2026-11-13T15:00:30Z", contextSnapshot: { issueId: "chat", commentId: "query" } }, + { id: "second", companyId: "co", agentId: "ceo", status: "succeeded", runtimeMode: "native", startedAt: "2026-11-13T15:01:48Z", contextSnapshot: { issueId: "chat", interactionId: "card" } }, + ]; + const verify = (overrides: Record = {}, rows = runs) => verifyChatCatalogHumanQuestionAnswer({ expected, companyId: "co", issueId: "chat", agentId: "ceo", + canonicalComment: { id: "query", authorUserId: "person" }, interactions: [{ ...card, ...overrides } as never], activity: [audit], runs: rows }); + const answer = verify(); + expect(answer).toMatchObject({ continuationRunId: "second" }); + // A typed "Other" answer, an unanswered card, or no continuation run is not a verified choice. + expect(verify({ result: { answers: [{ questionId: "landing_page_task", optionIds: [], otherText: "COM-1" }] } })).toBeUndefined(); + expect(verify({ status: "pending" })).toBeUndefined(); + // The product lets the person answer an unaddressed card, or one any resolver but its agent creator may answer. + expect(verify({ addresseeUserId: null, resolverPolicy: "not_creator", effectiveResolverPolicy: "not_creator" })).toMatchObject({ continuationRunId: "second" }); + expect(verify({ resolverPolicy: "agent_only", effectiveResolverPolicy: "agent_only" })).toBeUndefined(); + expect(verify({ addresseeUserId: "someone-else" })).toBeUndefined(); + // The saved choice must be the Launch option alone. + expect(verify({ result: { answers: [{ questionId: "landing_page_task", optionIds: ["com-2"] }] } })).toBeUndefined(); + expect(verify({ result: { answers: [{ questionId: "landing_page_task", optionIds: ["com-1", "com-2"] }] } })).toBeUndefined(); + expect(verify({}, runs.slice(0, 1))).toBeUndefined(); + const history = { human: [{ id: "query", body: "Mark the landing-page task as high priority." }], runIds: ["first", "second"], + boundary: { humanCommentIds: [], runIds: [] }, renderedQuery: "Mark the landing-page task as high priority." }; + expect(chatCatalogHumanHistoryMatches({ ...history, implementation, questionAnswer: answer })).toBe(true); + expect(chatCatalogHumanHistoryMatches({ ...history, implementation })).toBe(false); + expect(chatCatalogHumanHistoryMatches({ ...history, implementation: { ...implementation, scenarioId: "tasks.edit-properties" }, questionAnswer: answer })).toBe(false); +}); +it("accepts a not_creator card resolved by a person other than its creator, as the server does", async () => { + const { chatCatalogHumanQuestionPolicies } = await import("./chat-catalog-contract.js"); + const notCreator = { resolverPolicy: "not_creator", requestedResolverPolicy: "not_creator", effectiveResolverPolicy: "not_creator" }; + expect(chatCatalogHumanQuestionPolicies({ ...notCreator, createdByUserId: null }, { userId: "person" })).toBe(true); + expect(chatCatalogHumanQuestionPolicies({ ...notCreator, createdByUserId: "person" }, { userId: "person" })).toBe(false); + expect(chatCatalogHumanQuestionPolicies({ ...notCreator, createdByUserId: null })).toBe(false); + expect(chatCatalogHumanQuestionPolicies({ resolverPolicy: "anyone", effectiveResolverPolicy: "human_only" })).toBe(true); + expect(chatCatalogHumanQuestionPolicies({ resolverPolicy: "agent_only" }, { userId: "person" })).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index b2ff82b7946..333faf9c1fc 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -257,7 +257,8 @@ export function validChatCatalogHumanCounts(implementation: Pick= setup + (implementation.queryOutcome === "rejected" ? 0 : 1); } export function chatCatalogAllowsHumanQuestionAnswer(scenarioId: string): boolean { - return ["secrets.personal-secret", "secrets.configure-company-secret"].includes(scenarioId); + // tasks.ambiguous-task: the person can answer the clarification by choosing one of the card's saved options. + return ["secrets.personal-secret", "secrets.configure-company-secret", "tasks.ambiguous-task"].includes(scenarioId); } export function chatCatalogHumanHistoryMatches(input: { implementation: Pick & Partial>; @@ -282,9 +283,16 @@ export function chatCatalogHumanHistoryMatches(input: { && input.human.length + (input.questionAnswer ? 1 : 0) === (input.implementation.expectedHumanMessageCount ?? 1)); } -export function chatCatalogHumanQuestionPolicies(row: { resolverPolicy?: unknown; requestedResolverPolicy?: unknown; effectiveResolverPolicy?: unknown }): boolean { +/** + * Whether a person may resolve this card under its resolver policies, mirroring the server's user-actor rule in + * evaluateIssueThreadInteractionResolverAudience: `anyone` and `human_only` admit the person, and `not_creator` + * admits anyone except the card's creator. Without a resolver (a card still pending) `not_creator` is not accepted. + */ +export function chatCatalogHumanQuestionPolicies(row: { resolverPolicy?: unknown; requestedResolverPolicy?: unknown; effectiveResolverPolicy?: unknown; createdByUserId?: unknown }, + resolver?: { userId: string }): boolean { return [row.resolverPolicy, row.requestedResolverPolicy ?? row.resolverPolicy, row.effectiveResolverPolicy ?? row.resolverPolicy] - .every(policy => policy === "anyone" || policy === "human_only"); + .every(policy => policy === "anyone" || policy === "human_only" + || policy === "not_creator" && !!resolver?.userId && row.createdByUserId !== resolver.userId); } export function chatCatalogHasExactDefinitionId(text: string, definitionId: string): boolean { @@ -305,7 +313,9 @@ export function chatCatalogCompanyEnvironmentNameQuestion(q:{prompt:string;requi ||!/^[A-Za-z_][A-Za-z0-9_]{0,119}$/.test(answer)||!validation?.maxLength||validation.maxLength>120||validation.maxLengthanswer.length||(validation.inputType!==undefined&&validation.inputType!=="text"))return false; try{return !validation.pattern||new RegExp(validation.pattern).test(answer);}catch{return false;} } -export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string; personalSecretConfirmation?: { definitionId: string; secretId: string; payloadHash: string; continuationPolicy: "wake_assignee" | "wake_assignee_on_accept" }; companySecretConfirmation?: { secretId: string; key: string; payloadHash: string; continuationPolicy: "wake_assignee" | "wake_assignee_on_accept" }; companyEnvironmentNameAnswer?: { questionId: string; payloadHash: string; value: "DEPLOYMENT_API_KEY" }; companySavedSecretNameAnswer?: { questionId: string; payloadHash: string; value: string; secretId: string } } +export interface ChatCatalogExpectedHumanQuestionAnswer { interactionId: string; sourceRunId: string; userId: string; personalSecretConfirmation?: { definitionId: string; secretId: string; payloadHash: string; continuationPolicy: "wake_assignee" | "wake_assignee_on_accept" }; companySecretConfirmation?: { secretId: string; key: string; payloadHash: string; continuationPolicy: "wake_assignee" | "wake_assignee_on_accept" }; companyEnvironmentNameAnswer?: { questionId: string; payloadHash: string; value: "DEPLOYMENT_API_KEY" }; companySavedSecretNameAnswer?: { questionId: string; payloadHash: string; value: string; secretId: string }; + /** A clarification answered by choosing one saved option of the card's single question. */ + clarificationChoice?: { questionId: string; optionId: string } } export interface ChatCatalogVerifiedHumanQuestionAnswer extends ChatCatalogExpectedHumanQuestionAnswer { continuationRunId: string; resolvedAt: string } export interface ChatCatalogHumanQuestionAudit { companyId?: string; entityId?: string; action?: string; actorType?: string; actorId?: string; @@ -325,7 +335,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { const secrets=input.companySecrets?.filter(secret=>secret.id===binding.secretId),secret=secrets?.[0],resolvedAt=Date.parse(String(row?.resolvedAt)),createdAt=Date.parse(String(row?.createdAt)); const audits=input.activity.filter(event=>event.action==="issue.thread_interaction_accepted"&&event.details?.interactionId===b.interactionId),audit=audits[0]; const saves=input.activity.filter(event=>event.action==="secret.created"&&event.entityId===binding.secretId),save=saves[0]; - if(b.personalSecretConfirmation||b.companyEnvironmentNameAnswer||b.companySavedSecretNameAnswer||rows.length!==1||!row||row.kind!=="request_confirmation"||row.status!=="accepted"||row.companyId!==input.companyId||row.issueId!==input.issueId||row.sourceRunId!==b.sourceRunId||row.createdByAgentId!==input.agentId||row.createdByUserId||row.addresseeAgentId||(row.addresseeUserId!=null&&row.addresseeUserId!==b.userId)||!row.originCommentIds?.includes(input.canonicalComment.id)||b.userId!==input.canonicalComment.authorUserId||!chatCatalogHumanQuestionPolicies(row)||(row.continuationPolicy!=="wake_assignee"&&row.continuationPolicy!=="wake_assignee_on_accept")||row.continuationPolicy!==binding.continuationPolicy||row.resolvedByUserId!==b.userId||row.resolvedByAgentId||row.result?.outcome!=="accepted"||row.result.resumeFailure||row.result.toolAction||row.result.secretProposal||row.dismissedAt + if(b.personalSecretConfirmation||b.companyEnvironmentNameAnswer||b.companySavedSecretNameAnswer||rows.length!==1||!row||row.kind!=="request_confirmation"||row.status!=="accepted"||row.companyId!==input.companyId||row.issueId!==input.issueId||row.sourceRunId!==b.sourceRunId||row.createdByAgentId!==input.agentId||row.createdByUserId||row.addresseeAgentId||(row.addresseeUserId!=null&&row.addresseeUserId!==b.userId)||!row.originCommentIds?.includes(input.canonicalComment.id)||b.userId!==input.canonicalComment.authorUserId||!chatCatalogHumanQuestionPolicies(row,{userId:b.userId})||(row.continuationPolicy!=="wake_assignee"&&row.continuationPolicy!=="wake_assignee_on_accept")||row.continuationPolicy!==binding.continuationPolicy||row.resolvedByUserId!==b.userId||row.resolvedByAgentId||row.result?.outcome!=="accepted"||row.result.resumeFailure||row.result.toolAction||row.result.secretProposal||row.dismissedAt ||!row.payload||!requestConfirmationPayloadSchema.safeParse(row.payload).success||row.payload.target||row.payload.toolAction||row.payload.secretProposal||row.payload.connectionAuthorization||row.payload.qualityReview||createHash("sha256").update(JSON.stringify(row.payload)).digest("hex")!==binding.payloadHash ||secrets?.length!==1||!secret||secret.companyId!==input.companyId||secret.scope!=="company"||secret.status!=="active"||secret.createdByUserId!==b.userId||secret.key!==binding.key||!/^[a-zA-Z0-9_.-]{1,120}$/.test(binding.key)||![createdAt,resolvedAt,Date.parse(secret.createdAt)].every(Number.isFinite)||Date.parse(secret.createdAt)<=createdAt||Date.parse(secret.createdAt)>resolvedAt ||audits.length!==1||audit?.companyId!==input.companyId||audit.entityId!==input.issueId||audit.actorType!=="user"||audit.actorId!==b.userId||audit.details?.interactionKind!=="request_confirmation"||audit.details.interactionStatus!=="accepted" @@ -344,7 +354,7 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { if (b.companySecretConfirmation || b.companyEnvironmentNameAnswer || b.companySavedSecretNameAnswer || rows.length !== 1 || !row || row.kind !== "request_confirmation" || row.status !== "accepted" || row.companyId !== input.companyId || row.issueId !== input.issueId || row.sourceRunId !== b.sourceRunId || row.createdByAgentId !== input.agentId || row.createdByUserId || (row.addresseeUserId != null && row.addresseeUserId !== b.userId) || row.addresseeAgentId || !row.originCommentIds?.includes(input.canonicalComment.id) - || b.userId !== input.canonicalComment.authorUserId || !chatCatalogHumanQuestionPolicies(row) || (row.continuationPolicy !== "wake_assignee" && row.continuationPolicy !== "wake_assignee_on_accept") || row.continuationPolicy !== binding.continuationPolicy + || b.userId !== input.canonicalComment.authorUserId || !chatCatalogHumanQuestionPolicies(row, { userId: b.userId }) || (row.continuationPolicy !== "wake_assignee" && row.continuationPolicy !== "wake_assignee_on_accept") || row.continuationPolicy !== binding.continuationPolicy || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.outcome !== "accepted" || row.result.resumeFailure || row.dismissedAt || !row.payload || row.payload.target || row.payload.toolAction || row.payload.secretProposal || row.payload.connectionAuthorization || row.payload.qualityReview || !chatCatalogHasExactDefinitionId([row.payload.prompt, row.payload.detailsMarkdown].filter(Boolean).join("\n"), binding.definitionId) || createHash("sha256").update(JSON.stringify(row.payload)).digest("hex") !== binding.payloadHash @@ -357,15 +367,22 @@ export function verifyChatCatalogHumanQuestionAnswer(input: { || !Number.isFinite(Date.parse(continuation.startedAt ?? "")) || Date.parse(continuation.startedAt!) < resolvedAt) return; return { ...b, continuationRunId: continuation.id, resolvedAt: new Date(resolvedAt).toISOString() }; } - const b = input.expected, rows = input.interactions.filter(row => row.id === b.interactionId), row = rows[0]; + const b = input.expected, rows = input.interactions.filter(row => row.id === b.interactionId), row = rows[0], choice = b.clarificationChoice; + // As the server allows: an unaddressed card may be answered by the person; an addressed one only by its addressee. + const addressed = choice ? row?.addresseeUserId == null || row.addresseeUserId === b.userId : row?.addresseeUserId === b.userId; + const policies = !!row && chatCatalogHumanQuestionPolicies(row, { userId: b.userId }); if (rows.length !== 1 || !row || b.userId !== input.canonicalComment.authorUserId || row.companyId !== input.companyId || row.issueId !== input.issueId || row.sourceRunId !== b.sourceRunId - || row.createdByAgentId !== input.agentId || row.createdByUserId || row.addresseeUserId !== b.userId || row.addresseeAgentId - || row.kind !== "ask_user_questions" || row.status !== "answered" || !chatCatalogHumanQuestionPolicies(row) + || row.createdByAgentId !== input.agentId || row.createdByUserId || !addressed || row.addresseeAgentId + || row.kind !== "ask_user_questions" || row.status !== "answered" || !policies || row.resolvedByUserId !== b.userId || row.resolvedByAgentId || row.result?.answers?.length !== 1) return; const source = input.runs.find(run => run.id === b.sourceRunId); const environmentName = b.companyEnvironmentNameAnswer, savedName=b.companySavedSecretNameAnswer; - if(environmentName&&savedName)return; + if((environmentName?1:0)+(savedName?1:0)+(choice?1:0)>1)return; + if (choice) { + const answer = row.result.answers[0]; + if (answer?.questionId !== choice.questionId || answer.optionIds.length !== 1 || answer.optionIds[0] !== choice.optionId || answer.otherText) return; + } if (environmentName) { const parsed = askUserQuestionsPayloadSchema.safeParse(row.payload), qs = parsed.success ? parsed.data.questionSet : undefined; const q = qs?.questions[0], answer = row.result.answers[0]; diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index a117bbf0f84..4fca4c84fa8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -488,7 +488,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-primary.v5", + "oracleVersion": "paperclip.chat-catalog.runtime-primary.v6", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, @@ -5535,7 +5535,7 @@ "chat-rendered-links.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.edit-queued-message.structural.v6", + "oracleVersion": "paperclip.chat-catalog.edit-queued-message.structural.v7", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, @@ -5843,7 +5843,7 @@ "chat-catalog-domains/identity-support.ts", "chat-rendered-links.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.ambiguous-task.v5", + "oracleVersion": "paperclip.chat-catalog.internal.ambiguous-task.v8", "coverageGaps": [], "expectedRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts index 31b2e6fbd29..3cbe10259b1 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts @@ -1,5 +1,6 @@ import { savedDesignInstruction, withRenderedCommentReference, validRenderedCommentReference } from "./internal-comment-reference.js"; -import { linkedQuestionTakeover, observeLinkedTaskQuestion, questionCardNamesCandidates, type ClarificationObservation } from "../chat-linked-question.js"; +import { chosenTaskOption, linkedQuestionTakeover, observeLinkedTaskQuestion, questionCardNamesCandidates, type ClarificationObservation } from "../chat-linked-question.js"; +import { expect } from "@playwright/test"; import { withEditOwnership } from "./internal-edit-ownership.js"; import { verifyDistinctTaskFixtures } from "./internal-distinct-fixtures.js"; import { delegatedTaskCases } from "./internal-task-creation.js"; @@ -85,6 +86,14 @@ export const tasksCases: readonly ChatCatalogCase[] = [ ]; +/** The person's answer through the clarification card's Launch choice, for the flow to verify independently. */ +export function answeredClarification(value: unknown) { + const answered = (value as { state?: { answeredQuestion?: Record } } | undefined)?.state?.answeredQuestion; + const text = (key: string) => typeof answered?.[key] === "string" && !!answered[key] ? answered[key] as string : undefined; + const interactionId = text("interactionId"), sourceRunId = text("sourceRunId"), userId = text("userId"), questionId = text("questionId"), optionId = text("optionId"); + if (!interactionId || !sourceRunId || !userId || !questionId || !optionId) return; + return { interactionId, sourceRunId, userId, clarificationChoice: { questionId, optionId } }; +} function ambiguousTask(): ChatCatalogCase { const base = define("tasks.ambiguous-task", async ctx => { const p = await project(ctx); const otherProject = await project(ctx, "Experiments"); @@ -94,7 +103,8 @@ function ambiguousTask(): ChatCatalogCase { await verifyDistinctTaskFixtures(ctx, [t, otherTask]); return { project: p, task: t, otherProject, otherTask }; }); - return { ...base, oracleVersion: "paperclip.chat-catalog.internal.ambiguous-task.v5", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts", "chat-catalog-domains/identity-support.ts", "chat-rendered-links.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, + return { ...base, oracleVersion: "paperclip.chat-catalog.internal.ambiguous-task.v8", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts", "chat-catalog-domains/identity-support.ts", "chat-rendered-links.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, + expectedHumanQuestionAnswer: answeredClarification, async act(ctx, state) { await pollUntil({ label: "Task ambiguity clarification", deadlineAt: Date.now() + 150_000, load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs.every(row => row.status === "succeeded") @@ -116,6 +126,23 @@ function ambiguousTask(): ChatCatalogCase { throw new Error("The actual linked clarification question changed before the person's selection"); } const answerRoot = card ? linkedQuestionTakeover(ctx.input.page, card) : ctx.input.page; + // A card that offers the Launch task as its own choice is answered by selecting that choice, found by the + // saved option structure and the task identifier rather than by the option's wording. + const choice = card ? chosenTaskOption((first as ClarificationObservation).questionCards ?? [], card, selected, [selected, other]) : undefined; + if (card && choice) { + await answerRoot.locator(`[id=${JSON.stringify(choice.domId)}]`).click(); + await expect(answerRoot.locator(`[id=${JSON.stringify(choice.domId)}]`)).toHaveAttribute("aria-checked", "true"); + const responded = ctx.input.page.waitForResponse(response => new URL(response.url()).pathname.endsWith(`/interactions/${card.interactionId}/respond`) + && response.request().method() === "POST"); + await answerRoot.getByRole("button", { name: choice.submitLabel, exact: true }).click(); + const response = await responded; + if (!response.ok()) throw new Error(`The person's clarification answer returned ${response.status()}`); + // A chosen option saves no comment; the flow counts it as the person's second action only after it + // independently verifies the answered card, its audit event and the one continuation run. + const anchor = (await ctx.input.api.get(`/api/issues/${card.sourceId}/comments?order=asc`)).find(row => row.id === card.anchorCommentId); + state.answeredQuestion = { interactionId: card.interactionId, sourceRunId: card.sourceRunId, userId: anchor?.authorUserId ?? null, questionId: card.questionId, optionId: choice.optionId }; + return; + } const textInput = answerRoot.getByTestId("question-text-answer-composer").last(); if (await textInput.isVisible()) { await textInput.locator('[contenteditable="true"],textarea').first().fill(answer); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-primary.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-primary.test.ts index 8ab5e477513..b0318515d99 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-primary.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-primary.test.ts @@ -1,5 +1,5 @@ import { it, expect } from "vitest"; -import { gradePrimary } from "./runtime-primary.js"; +import { gradePrimary, isPrimarySaveRequest } from "./runtime-primary.js"; function valid() { const primaryBefore = { companyId: "company", userId: "person", primaryAgentId: "ceo" }; const other = { companyId: "company", userId: "other-person", primaryAgentId: null as string | null }; @@ -27,3 +27,10 @@ it("rejects a preference changed for another person", () => { it("rejects navigation that starts a new execution", () => { const o = valid(); o.state.runsAfterNavigation = 2; expect(gradePrimary(o).passed).toBe(false); }); +it("waits only for the person's save that names the chosen agent", () => { + const url = "http://fixture.invalid/api/companies/company/primary-agent/me"; + expect(isPrimarySaveRequest({ method: "PUT", url, body: JSON.stringify({ primaryAgentId: "morgan" }) }, "company", "morgan")).toBe(true); + expect(isPrimarySaveRequest({ method: "GET", url, body: null }, "company", "morgan")).toBe(false); + expect(isPrimarySaveRequest({ method: "PUT", url, body: JSON.stringify({ primaryAgentId: "ceo" }) }, "company", "morgan")).toBe(false); + expect(isPrimarySaveRequest({ method: "PUT", url: "http://fixture.invalid/api/companies/other/primary-agent/me", body: JSON.stringify({ primaryAgentId: "morgan" }) }, "company", "morgan")).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts b/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts index e6e1496c63f..2ec5f97c20f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts @@ -13,6 +13,11 @@ async function personPrimary(person: Person, companyId: string): Promise; } +/** Whether a browser request is this person's primary-agent save naming the chosen agent. */ +export function isPrimarySaveRequest(request: { method: string; url: string; body: string | null }, companyId: string, agentId: string) { + if (request.method !== "PUT" || new URL(request.url).pathname !== endpoint(companyId)) return false; + try { return (JSON.parse(request.body ?? "null") as { primaryAgentId?: unknown } | null)?.primaryAgentId === agentId; } catch { return false; } +} export function gradePrimary(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("complete-human-primary-evidence", !!o, "Real user-scoped saved preferences and human replacement control are required."); if (!o) return c.result(); @@ -105,7 +110,7 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a return { worker, primaryBefore, otherCompany, otherPersonBefore: await personPrimary(person, ctx.input.fixtures.company.id), otherCompanyBefore: await ctx.input.api.get(endpoint(otherCompany.id)) }; }, gradePrimary, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.runtime-primary.v5", + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.runtime-primary.v6", sourceFiles: ["chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"], async act(ctx, state) { remainingIdentityObservationBudget(ctx); @@ -125,8 +130,19 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a await ctx.input.page.getByRole("button", { name: "Set as my primary", exact: true }).click(); const dialog = ctx.input.page.getByRole("alertdialog"); await expect(dialog).toContainText("Change your primary agent?"); await expect(dialog).toContainText("Morgan"); + // The crown fills optimistically, so the page alone cannot show the save + // finished. Wait for the browser's own save to answer for Morgan before + // leaving, or the next navigation would abort it. + const companyId = ctx.input.fixtures.company.id; + const saved = ctx.input.page.waitForResponse(response => isPrimarySaveRequest( + { method: response.request().method(), url: response.url(), body: response.request().postData() }, companyId, worker.id)); await dialog.getByRole("button", { name: "Set as my primary", exact: true }).click(); - await expect(ctx.input.page.getByRole("img", { name: "My primary agent", exact: true }).first()).toBeVisible(); + const save = await saved; + if (!save.ok()) throw new Error(`The person's primary agent save returned ${save.status()}`); + await expect.poll(async () => (await ctx.input.api.get(endpoint(companyId))).primaryAgentId).toBe(worker.id); + // The sidebar already shows the CEO's crown, so look only in Morgan's profile header. + const header = ctx.input.page.locator("header").filter({ has: ctx.input.page.getByRole("img", { name: `${worker.name} avatar`, exact: true }) }); + await expect(header.getByRole("img", { name: "My primary agent", exact: true })).toBeVisible(); state.replacementConfirmed = true; // Revisit the valid recent source before opening the normal Chat landing. await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts index b4bdc7ca957..9628bdc36fb 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue-controls.ts @@ -85,7 +85,7 @@ export function gradeQueueEdit(value: unknown) { export const runtimeQueueControlCases = [runtimeDefinition("steering.edit-queued-message", ctx => humanQueuedRuntime(ctx), gradeQueueEdit, { sourceFiles: ["chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/steering.yaml"], - oracleVersion: "paperclip.chat-catalog.edit-queued-message.structural.v6", + oracleVersion: "paperclip.chat-catalog.edit-queued-message.structural.v7", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 720_000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 2 }, maximumTerminalStatusCounts: { succeeded: 3 } }, setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts index 7a6997b9f03..733bfb43357 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts @@ -63,13 +63,16 @@ it("finds the saved comment when a continuation envelope nests the events JSON a function reorderPage(enableAfterMs: number) { const enabledAt = Date.now() + enableAfterMs, pressed: Array<{ key: string; enabled: boolean }> = []; let resolveResponse!: (value: unknown) => void; + let dragging = false; const page = { waitForResponse: (match: (response: unknown) => boolean, options: { timeout: number }) => new Promise((resolve, reject) => { resolveResponse = value => { if (match(value)) resolve(value); }; setTimeout(() => reject(new Error("timeout")), options.timeout); - }) }; + }), evaluate: async () => undefined }; const handle = { isEnabled: async () => Date.now() >= enabledAt, focus: async () => undefined, + getAttribute: async (name: string) => name === "aria-pressed" && dragging ? "true" : null, press: async (key: string) => { const enabled = Date.now() >= enabledAt; pressed.push({ key, enabled }); + if (enabled && key === "Space") dragging = !dragging; if (enabled && key === "Space" && pressed.filter(row => row.key === "Space").length === 2) resolveResponse({ request: () => ({ method: () => "PUT" }), url: () => "http://127.0.0.1/api/issues/t/queued-comments/order", status: () => 200 }); } }; @@ -136,3 +139,37 @@ describe("queued follow-up brief", () => { }); it("does not pre-complete the queued contact requirement", () => expect(brief).not.toMatch(/contact/i)); }); + +// Full cohort 12 (steering.edit-queued-message, as in cohort 9): the edit saved, but Space, ArrowUp, Space pressed +// back to back sent no PUT /queued-comments/order in 2 gestures. dnd-kit starts a keyboard drag on Space, listens +// for later keys only after a timer, and moves only after measuring the rows on a later render, so a key pressed +// before then is dropped and the closing Space drops the item where it started. +function asyncDragPage() { + let resolveResponse!: (value: unknown) => void, dragging = false, activatedAt = 0, moved = false; + const page = { waitForResponse: (match: (response: unknown) => boolean, options: { timeout: number }) => new Promise((resolve, reject) => { + resolveResponse = value => { if (match(value)) resolve(value); }; + setTimeout(() => reject(new Error("timeout")), options.timeout); + }), evaluate: () => new Promise(resolve => setTimeout(resolve, 20)) }; + const ready = () => dragging && Date.now() - activatedAt >= 15; + const handle = { isEnabled: async () => true, focus: async () => undefined, + getAttribute: async (name: string) => name === "aria-pressed" && Date.now() >= activatedAt && dragging ? "true" : null, + press: async (key: string) => { + if (key === "Space" && !dragging) { dragging = true; activatedAt = Date.now() + 10; moved = false; return; } + if (!ready()) return; + if (key === "ArrowUp") moved = true; + if (key === "Space") { + dragging = false; + if (moved) resolveResponse({ request: () => ({ method: () => "PUT" }), url: () => "http://127.0.0.1/api/issues/t/queued-comments/order", status: () => 200 }); + } + } }; + return { page: page as never, handle: handle as never }; +} +it("waits for the keyboard drag to start and the page to settle between keys", async () => { + const fake = asyncDragPage(); + await expect(keyboardMoveQueuedMessageUp(fake.page, fake.handle, 500, 5)).resolves.toMatchObject({ status: 200, gestures: 1, dragStarted: [true] }); +}); +it("reports a gesture whose drag never starts instead of pressing keys blind", async () => { + const fake = asyncDragPage(); + const handle = { ...(fake.handle as object), getAttribute: async () => null }; + await expect(keyboardMoveQueuedMessageUp(fake.page, handle as never, 200, 5)).rejects.toThrow(/drag started: false/); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts index 69987bd7b14..46037285229 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts @@ -17,8 +17,21 @@ import type { IssueWorkProduct } from "../../../packages/shared/src/types/work-p * request; its status is returned so the caller can tell an accepted move from a refused one. When a gesture * sends nothing, `beforeRetry` (for example a reload that refetches the queue) runs and the gesture is pressed * again, up to `attempts` gestures in all; every gesture is reported. */ -export async function keyboardMoveQueuedMessageUp(page: Pick, handle: Pick, - timeoutMs = 15_000, intervalMs = 100, options: { attempts?: number; beforeRetry?: () => Promise } = {}): Promise<{ status: number; waitedForEnabledMs: number; gestures: number }> { +export async function keyboardMoveQueuedMessageUp(page: Pick, handle: Pick, + timeoutMs = 15_000, intervalMs = 100, options: { attempts?: number; beforeRetry?: () => Promise } = {}): Promise<{ status: number; waitedForEnabledMs: number; gestures: number; dragStarted: boolean[] }> { + const dragStarted: boolean[] = []; + // The drag library starts a keyboard drag on Space but listens for the next keys only after a timer, and moves + // only once it has measured the rows on a later render. A person's keystrokes are far apart; automated ones are + // not, so wait for the handle to report the drag (aria-pressed) and let the page paint between keys. + const settle = () => page.evaluate(() => new Promise(resolve => setTimeout(() => requestAnimationFrame(() => requestAnimationFrame(() => resolve())), 0))); + const waitForDrag = async () => { + const started = Date.now(); + while (await handle.getAttribute("aria-pressed") !== "true") { + if (Date.now() - started >= Math.min(timeoutMs, 3_000)) return false; + await new Promise(resolve => setTimeout(resolve, Math.min(intervalMs, 50))); + } + return true; + }; const attempts = Math.max(1, options.attempts ?? 1); for (let gesture = 1; ; gesture++) { const started = Date.now(); @@ -30,10 +43,12 @@ export async function keyboardMoveQueuedMessageUp(page: Pick response.request().method() === "PUT" && new URL(response.url()).pathname.endsWith("/queued-comments/order"), { timeout: timeoutMs }); sent.catch(() => undefined); - await handle.focus(); await handle.press("Space"); await handle.press("ArrowUp"); await handle.press("Space"); - const response = await sent.catch(() => null); - if (response) return { status: response.status(), waitedForEnabledMs, gestures: gesture }; - if (gesture >= attempts) throw new Error(`The keyboard reorder gesture sent no queue order request after ${gesture} gesture(s)`); + await handle.focus(); await handle.press("Space"); + const dragging = await waitForDrag(); dragStarted.push(dragging); + if (dragging) { await settle(); await handle.press("ArrowUp"); await settle(); await handle.press("Space"); } + const response = dragging ? await sent.catch(() => null) : null; + if (response) return { status: response.status(), waitedForEnabledMs, gestures: gesture, dragStarted }; + if (gesture >= attempts) throw new Error(`The keyboard reorder gesture sent no queue order request after ${gesture} gesture(s) (drag started: ${dragStarted.join(", ")})`); await options.beforeRetry?.(); } } diff --git a/tests/runner-e2e/chat-linked-question.test.ts b/tests/runner-e2e/chat-linked-question.test.ts index 0797c9e1190..5214896ac27 100644 --- a/tests/runner-e2e/chat-linked-question.test.ts +++ b/tests/runner-e2e/chat-linked-question.test.ts @@ -1,6 +1,6 @@ // @vitest-environment jsdom import { afterEach, beforeEach, expect, it, vi } from "vitest"; -import { linkedQuestionBinding, readLinkedQuestionDom } from "./chat-linked-question.js"; +import { chosenTaskOption, linkedQuestionBinding, readLinkedQuestionDom } from "./chat-linked-question.js"; import { valid, dom, type Input } from "./chat-linked-question.test-fixture.js"; beforeEach(() => { document.body.innerHTML = dom(); vi.spyOn(Element.prototype, "getClientRects").mockReturnValue([{ width: 20, height: 20 }] as unknown as DOMRectList); }); @@ -92,3 +92,25 @@ it.each(domNegatives)("rejects %s DOM evidence", (_, change) => { change(); expect(collect()).toBeUndefined(); }); +it("selects the saved choice that offers only the Launch task, by option id and task identifier", () => { + const launch = { id: "launch-task", identifier: "COM-1" }, experiments = { id: "experiment-task", identifier: "COM-2" }; + // The card shape cohort 12's agent asked: one choice per task plus a choice naming both. + const card = (options: Array<{ id: string; label: string; description?: string }>, extra: Record = {}) => [{ + id: "question", kind: "ask_user_questions", + payload: { version: 1, questions: [], questionSet: { schema: "paperclip.question_set.v1", ...extra, + questions: [{ id: "which-task", prompt: "Which landing-page task should become high priority?", required: true, answerMode: "single_select", options }] } }, + }] as unknown as Parameters[0]; + const evidence = { interactionId: "question", questionId: "which-task" }; + const offered = [{ id: "com-1", label: "COM-1 (Launch project)" }, { id: "com-2", label: "COM-2 (Experiments project)" }, { id: "both", label: "Both tasks" }]; + expect(chosenTaskOption(card(offered), evidence, launch, [launch, experiments])) + .toEqual({ optionId: "com-1", domId: "question-which-task-com-1", submitLabel: "Submit answers" }); + // Wording does not matter; the identifier may sit in the description, and a custom submit label is used. + expect(chosenTaskOption(card([{ id: "b", label: "Experiments one", description: "COM-2" }, { id: "a", label: "The other one", description: "COM-1" }], { submitLabel: "Choose" }), + evidence, launch, [launch, experiments])).toEqual({ optionId: "a", domId: "question-which-task-a", submitLabel: "Choose" }); + // No single unambiguous choice: an option naming both tasks, duplicates, or neither. + expect(chosenTaskOption(card([{ id: "both", label: "COM-1 and COM-2" }]), evidence, launch, [launch, experiments])).toBeUndefined(); + expect(chosenTaskOption(card([{ id: "x", label: "COM-1" }, { id: "y", label: "COM-1 again" }]), evidence, launch, [launch, experiments])).toBeUndefined(); + expect(chosenTaskOption(card([{ id: "x", label: "Launch" }, { id: "y", label: "Experiments" }]), evidence, launch, [launch, experiments])).toBeUndefined(); + expect(chosenTaskOption(card([{ id: "x", label: "COM-10" }]), evidence, launch, [launch, experiments])).toBeUndefined(); + expect(chosenTaskOption(card(offered), { ...evidence, interactionId: "other" }, launch, [launch, experiments])).toBeUndefined(); +}); diff --git a/tests/runner-e2e/chat-linked-question.ts b/tests/runner-e2e/chat-linked-question.ts index 898121974b8..50ca5a1d571 100644 --- a/tests/runner-e2e/chat-linked-question.ts +++ b/tests/runner-e2e/chat-linked-question.ts @@ -79,20 +79,38 @@ export function readLinkedQuestionDom(input: NonNullable>; export type ClarificationObservation = InternalObservation & { linkedQuestionEvidence?: LinkedQuestionEvidence; questionCards?: AskUserQuestionsInteraction[] }; +type Candidate = { id: string; identifier?: string | null }; +type QuestionOption = { id?: string; label?: string; description?: string | null }; +/** Whether an option's plain text names this candidate's identifier (a setup literal) and no other candidate's. */ +function optionNamesOnly(option: QuestionOption, task: Candidate, tasks: readonly Candidate[]) { + const names = (text: string, row: Candidate) => !!row.identifier + && new RegExp(`(? other.id === task.id || !names(text, other)); +} +/** + * The saved option of the bound question that offers exactly the chosen task, found by option structure and the + * task identifier, never by the option's wording. Returns nothing unless exactly one option offers that task alone. + */ +export function chosenTaskOption(cards: readonly AskUserQuestionsInteraction[], evidence: Pick, + chosen: Candidate, tasks: readonly Candidate[]) { + const card = cards.find(row => row.id === evidence.interactionId && row.kind === "ask_user_questions"); + const set = card?.payload.questionSet, question = set?.questions.find(row => row.id === evidence.questionId); + if (!set || set.questions.length !== 1 || !question || question.answerMode !== "single_select" && question.answerMode !== "multi_select" || !tasks.some(task => task.id === chosen.id)) return; + const matches = (question.options ?? []).filter(option => !!option.id && optionNamesOnly(option, chosen, tasks)); + if (matches.length !== 1) return; + return { optionId: matches[0]!.id, domId: `${evidence.interactionId}-${evidence.questionId}-${matches[0]!.id}`, + submitLabel: set.submitLabel ?? "Submit answers" }; +} /** * Whether the run's saved question card offers each candidate task as its own choice. Option text is plain * text inside a choice button, so it cannot carry links; a candidate counts when one option names its identifier * (the setup literal) and no other candidate's. The option wording is left to human review. */ export function questionCardNamesCandidates(cards: readonly AskUserQuestionsInteraction[], tasks: ReadonlyArray<{ id: string; identifier?: string | null }>, runId: string | undefined, leadId: string) { - const names = (text: string, task: { identifier?: string | null }) => !!task.identifier - && new RegExp(`(? 1 && new Set(tasks.map(task => task.identifier)).size === tasks.length && cards.some(card => card.kind === "ask_user_questions" && card.status === "pending" && !card.dismissedAt && !card.resolvedAt && card.createdByAgentId === leadId && card.sourceRunId === runId - && (card.payload.questionSet?.questions ?? card.payload.questions ?? []).some(question => tasks.every(task => (question.options ?? []).some(option => { - const text = `${option.label ?? ""}\n${option.description ?? ""}`; - return names(text, task) && tasks.every(other => other.id === task.id || !names(text, other)); - })))); + && (card.payload.questionSet?.questions ?? card.payload.questions ?? []).some(question => tasks.every(task => (question.options ?? []).some(option => optionNamesOnly(option, task, tasks))))); } export async function observeLinkedTaskQuestion(ctx: ChatCatalogContext, state: Record, observation: InternalObservation): Promise { const source = ctx.issue(), prefix = ctx.input.fixtures.company.issuePrefix;