diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts index db659a43459..ec6bf564db6 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts @@ -1,5 +1,7 @@ -import { describe, expect, it } from "vitest"; -import { confirmationWorkflowIdentityCases, gradeConfirmationWorkflow } from "./identity-confirmation-workflows.js"; +import { describe, expect, it, vi } from "vitest"; +import { answerContinuationCompleted, confirmationWorkflowIdentityCases, gradeConfirmationWorkflow, waitForAnswerContinuation } from "./identity-confirmation-workflows.js"; +import * as api from "../api.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { IdentityObservation, Interaction } from "./identity-support.js"; function valid(id: string): IdentityObservation { const task = { id: "draft", companyId: "acorn", title: "Acorn launch checklist n42", description: "Draft checklist for later", status: "backlog" }; @@ -14,15 +16,17 @@ function valid(id: string): IdentityObservation { { id: "created", action: "issue.created", entityId: "draft", agentId: "ceo" }]; } else { original.kind = "ask_user_questions"; original.payload = { questionSet: { questions: [{ id: "product", prompt: "Product name?", answerMode: "text", required: true }] } }; - o.interactions = [{ ...original, status: "answered", resolvedByUserId: "alex", result: { answers: [{ questionId: "product", otherText: "Acorn", optionIds: [] }] } }]; - Object.assign(o.state, { humanId: "alex", worker: { id: "morgan" }, submitStatuses: [409, 200], actualSubmitRequests: 2, noTaskBeforeAnswers: true, bothTabsResolved: true }); + o.interactions = [{ ...original, status: "answered", resolvedByUserId: "alex", resolvedAt: "2026-10-28T10:00:02Z", result: { answers: [{ questionId: "product", otherText: "Acorn", optionIds: [] }] } }]; + Object.assign(o.state, { humanId: "alex", sourceIssueId: "source", sourceConversationState: "waiting", canonicalCommentId: "canonical", worker: { id: "morgan" }, submitStatuses: [409, 200], actualSubmitRequests: 2, noTaskBeforeAnswers: true, bothTabsResolved: true }); o.activity = [{ id: "answer", action: "issue.thread_interaction_answered", actorType: "user", details: { interactionId: "request" } }, - { id: "document", action: "issue.document_created", entityId: "draft", agentId: "morgan" }]; - o.documents = [{ id: "introduction", body: "Acorn helps small teams write clear product introductions.", latestRevisionId: "revision" }]; + { id: "delivery", action: "issue.question_response_delivered", entityId: "source", actorType: "system", actorId: "question-response-delivery", details: { interactionId: "request", targetRunId: "continue", deliveryStatus: "fallback_queued" } }, + { id: "document", action: "issue.document_created", entityId: "draft", agentId: "morgan", details: { documentId: "introduction", key: "introduction" } }]; + o.documents = [{ id: "introduction", key: "introduction", issueId: "draft", body: "Acorn helps small teams write clear product introductions.", latestRevisionId: "revision" }]; o.revisions = [{ id: "revision", documentId: "introduction", createdByRunId: "write", createdByAgentId: "morgan" }]; - o.runs = [{ id: "query", companyId: "acorn", agentId: "ceo", status: "succeeded" }, - { id: "continue", companyId: "acorn", agentId: "ceo", status: "succeeded", contextSnapshot: { interactionId: "request" } }, - { id: "write", companyId: "acorn", agentId: "morgan", status: "succeeded" }]; + o.tasks[0]!.status = "done"; + o.runs = [{ id: "query", companyId: "acorn", agentId: "ceo", status: "succeeded", startedAt: "2026-10-28T10:00:00Z", contextSnapshot: { issueId: "source", commentId: "canonical" } }, + { id: "continue", companyId: "acorn", agentId: "ceo", status: "succeeded", startedAt: "2026-10-28T10:00:03Z", contextSnapshot: { interactionId: "request", issueId: "source" } }, + { id: "write", companyId: "acorn", agentId: "morgan", status: "succeeded", startedAt: "2026-10-28T10:00:04Z" }]; } return o; } @@ -59,3 +63,58 @@ describe("conversational proposal and simultaneous human answer oracles", () => const wrong = valid("questions-and-approvals.duplicate-answer"); wrong.revisions[0]!.documentId = "unrelated-document"; expect(gradeConfirmationWorkflow(wrong).passed).toBe(false); }); }); + +describe("answer continuation and ordinary deliverable custody", () => { + it("retains one mandatory plan beside one actual introduction without hiding duplicate deliverables", () => { + const o = valid("questions-and-approvals.duplicate-answer"); + o.documents.push({ id: "plan", key: "plan", issueId: "draft", body: "Draft an introduction", latestRevisionId: "plan-revision" }); + o.revisions.push({ id: "plan-revision", documentId: "plan", createdByRunId: "query", createdByAgentId: "ceo" }); + expect(gradeConfirmationWorkflow(o).passed).toBe(true); + o.documents.push({ ...o.documents[0]!, id: "duplicate", key: "second-introduction" }); + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + }); + it.each(["foreign-task", "wrong-key", "unfinished", "wrong-company", "unstarted", "unattributed-plan"])("rejects %s output evidence", kind => { + const o = valid("questions-and-approvals.duplicate-answer"); + if (kind === "foreign-task") o.documents[0]!.issueId = "foreign"; + if (kind === "wrong-key") o.activity.find(row => row.id === "document")!.details!.key = "other"; + if (kind === "unfinished") o.tasks[0]!.status = "in_progress"; + if (kind === "wrong-company") o.runs[2]!.companyId = "foreign"; + if (kind === "unstarted") o.runs[2]!.startedAt = null; + if (kind === "unattributed-plan") o.documents.push({ id: "plan", key: "plan", issueId: "draft", latestRevisionId: "missing" }); + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + }); + it.each(["null-target", "wrong-target", "queued", "unstarted", "before-answer", "foreign-company", "foreign-issue", "unrelated-canonical", "missing-canonical", "missing-resolution", "running-source"])("does not infer consumption from %s", kind => { + const o = valid("questions-and-approvals.duplicate-answer"); + const delivery = o.activity.find(row => row.id === "delivery")!; + if (kind === "null-target") delivery.details!.targetRunId = null; + if (kind === "wrong-target") delivery.details!.targetRunId = "unrelated"; + if (kind === "queued") o.runs[1]!.status = "queued"; + if (kind === "unstarted") o.runs[1]!.startedAt = null; + if (kind === "before-answer") o.runs[1]!.startedAt = "2026-10-28T10:00:01Z"; + if (kind === "foreign-company") o.runs[1]!.companyId = "foreign"; + if (kind === "foreign-issue") o.runs[1]!.contextSnapshot!.issueId = "foreign"; + if (kind === "unrelated-canonical") o.runs[0]!.contextSnapshot!.commentId = "other"; + if (kind === "missing-canonical") delete o.state.canonicalCommentId; + if (kind === "missing-resolution") delete o.interactions[0]!.resolvedAt; + if (kind === "running-source") o.state.sourceConversationState = "active"; + expect(answerContinuationCompleted(o)).toBe(false); + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + }); + it("uses the original setup-inclusive clock and refuses expired or late results", async () => { + const clock = vi.spyOn(Date, "now").mockReturnValue(200_000); + const poll = vi.spyOn(api, "pollUntil").mockImplementation(async options => { + expect(options.deadlineAt).toBe(570_000); + expect(options.accept(valid("questions-and-approvals.duplicate-answer"))).toBe(true); + clock.mockReturnValue(570_000); + expect(options.accept(valid("questions-and-approvals.duplicate-answer"))).toBe(false); + return undefined as never; + }); + const ctx = { observationDeadlineAt: 570_000 } as ChatCatalogContext; + try { + await waitForAnswerContinuation(ctx, {}); + expect(poll).toHaveBeenCalledTimes(1); + await expect(waitForAnswerContinuation(ctx, {})).rejects.toThrow("exhausted"); + expect(poll).toHaveBeenCalledTimes(1); + } finally { clock.mockRestore(); poll.mockRestore(); } + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts index 9ffa7aaffcd..a3d7b15dd9d 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts @@ -5,11 +5,29 @@ import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contra import { pollUntil } from "../api.js"; import { sendChatMessage } from "../chat-flow.js"; import { checks, entity, hasLink, verifyInternalResult, type Entity } from "./internal-support.js"; -import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, waitForReply, remainingIdentityObservationBudget, type Interaction } from "./identity-support.js"; +import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, waitForReply, remainingIdentityObservationBudget, type IdentityObservation, type Interaction } from "./identity-support.js"; interface SourceComment extends Entity { authorType?: string } const tabs = new WeakMap(); const sources = ["chat-catalog-domains/identity-confirmation-workflows.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"]; +export function answerContinuationCompleted(o: IdentityObservation): boolean { + const original = o.state.original as Interaction | undefined; + const current = o.interactions.find(row => row.id === original?.id); + const resolvedAt = Date.parse(current?.resolvedAt ?? ""); + const continuations = o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id); + const continuation = continuations[0]; + return !!original && typeof o.state.sourceIssueId === "string" && typeof o.state.canonicalCommentId === "string" && o.state.sourceConversationState === "waiting" + && continuations.length === 1 && !!continuation && continuation.companyId === o.companyId && continuation.agentId === o.leadId + && continuation.contextSnapshot?.issueId === o.state.sourceIssueId && continuation.status === "succeeded" + && Number.isFinite(resolvedAt) && Number.isFinite(Date.parse(continuation.startedAt ?? "")) + && Date.parse(continuation.startedAt!) >= resolvedAt + && o.runs.some(run => run.id !== continuation.id && run.companyId === o.companyId && run.agentId === o.leadId + && run.status === "succeeded" && !!run.startedAt && run.contextSnapshot?.issueId === o.state.sourceIssueId + && run.contextSnapshot?.commentId === o.state.canonicalCommentId) + && o.activity.some(row => row.action === "issue.question_response_delivered" && row.entityId === o.state.sourceIssueId + && row.actorType === "system" && row.actorId === "question-response-delivery" && row.details?.interactionId === original.id + && row.details.targetRunId === continuation.id && ["delivered", "fallback_queued"].includes(String(row.details.deliveryStatus))); +} export function gradeConfirmationWorkflow(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-confirmation-workflow", !!o, "Persisted request version, real human events, native run identities and actual output records are independently required."); if (!o) return c.result(); @@ -37,14 +55,22 @@ export function gradeConfirmationWorkflow(value: unknown) { && o.state.noTaskBeforeAnswers === true && o.state.bothTabsResolved === true, "Both actual visible submit controls race, yielding one accepted response and one real conflict; both views converge on the same saved receipt."); c.add("single-answer-delivery", o.activity.filter(row => row.action === "issue.thread_interaction_answered" && row.details?.interactionId === original?.id && row.actorType === "user").length === 1 - && o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id).length === 1, + && answerContinuationCompleted(o), "Exactly one answer audit and one interaction-bound continuation are observed; the losing tab cannot add a run or a new human chat comment."); - c.add("one-actual-answer-artifact", o.tasks.length === 1 && o.tasks[0]?.companyId === o.companyId && hasLink(o, "issues", o.tasks[0]) - && o.documents.length === 1 && (o.documents[0]?.body ?? "").includes("Acorn") && !!o.documents[0]?.latestRevisionId - && o.activity.some(row => row.action === "issue.document_created" && row.entityId === o.tasks[0]?.id && [o.leadId, (o.state.worker as Entity | undefined)?.id].includes(row.agentId ?? "")) - && o.revisions.some(revision => revision.id === o.documents[0]?.latestRevisionId && revision.documentId === o.documents[0]?.id - && o.runs.some(run => run.id === revision.createdByRunId && run.agentId === revision.createdByAgentId && [o.leadId, (o.state.worker as Entity | undefined)?.id].includes(run.agentId))), - "The one continuation produces one linked ordinary draft task and one real saved introduction document using Acorn; an empty task or a prose promise is insufficient."); + const introductions = o.documents.filter(document => document.key !== "plan"); + const introduction = introductions[0]; + const authored = (document: Entity) => document.issueId === o.tasks[0]?.id && !!document.latestRevisionId + && o.revisions.some(revision => revision.id === document.latestRevisionId && revision.documentId === document.id + && o.runs.some(run => run.id === revision.createdByRunId && run.status === "succeeded" && !!run.startedAt + && run.companyId === o.companyId && run.agentId === revision.createdByAgentId + && [o.leadId, (o.state.worker as Entity | undefined)?.id].includes(run.agentId))); + c.add("one-actual-answer-artifact", o.tasks.length === 1 && o.tasks[0]?.companyId === o.companyId && o.tasks[0]?.status === "done" && hasLink(o, "issues", o.tasks[0]) + && introductions.length === 1 && !!introduction && (introduction.body ?? "").includes("Acorn") && authored(introduction) + && o.documents.filter(document => document.key === "plan").length <= 1 && o.documents.every(authored) + && o.activity.some(row => row.action === "issue.document_created" && row.entityId === o.tasks[0]?.id + && row.details?.documentId === introduction?.id && row.details.key === introduction?.key + && [o.leadId, (o.state.worker as Entity | undefined)?.id].includes(row.agentId ?? "")), + "One linked completed task has one attributed saved introduction using Acorn; its separately keyed platform plan is not another introduction."); } else c.add("known-workflow", false, "This workflow has no independent oracle."); return c.result(); } @@ -52,6 +78,9 @@ async function sourceComments(ctx: ChatCatalogContext) { return ctx.input.api.ge async function observeWorkflow(ctx: ChatCatalogContext, state: Record) { const o = await observeIdentity(ctx, state); const comments = await sourceComments(ctx); + if (ctx.scenario.id === "questions-and-approvals.duplicate-answer") { + state.sourceIssueId = ctx.issue()?.id; state.sourceConversationState = ctx.issue()?.conversationState; + } state.canonicalCommentId = comments.find(row => row.authorUserId && !row.createdByRunId && row.body?.trim() === ctx.scenario.query)?.id; if (ctx.scenario.id === "questions-and-approvals.accept-one-proposal") state.afterRepeatResolvedCommentId = o.interactions.find(row => row.id === entity(state, "original").id)?.result?.commentId; if (ctx.scenario.id === "questions-and-approvals.duplicate-answer" && o.tasks.length === 1) { @@ -63,6 +92,12 @@ async function observeWorkflow(ctx: ChatCatalogContext, state: Record) { + remainingIdentityObservationBudget(ctx); + await pollUntil({ label: "One answer-race continuation completed", deadlineAt: ctx.observationDeadlineAt!, + load: () => observeWorkflow(ctx, state), + accept: value => Date.now() < ctx.observationDeadlineAt! && answerContinuationCompleted(value) }); +} export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ identityDefinition("questions-and-approvals.accept-one-proposal", async ctx => { const source = await openStandingChat(ctx); const taskTitle = `Acorn launch checklist ${ctx.input.nonce}`; @@ -113,16 +148,18 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ observe: observeWorkflow, async act(ctx, state) { const other = tabs.get(ctx); if (!other) throw new Error("The second real browser tab is missing"); - await pollUntil({ label: "Canonical name reply has an admitted native run", deadlineAt: Date.now() + 120_000, + remainingIdentityObservationBudget(ctx); + await pollUntil({ label: "Canonical name reply has an admitted native run", deadlineAt: ctx.observationDeadlineAt!, load: () => ctx.allRuns(), accept: runs => runs.some(run => run.agentId === ctx.input.fixtures.agent.id && ["running", "succeeded"].includes(run.status)) }); + remainingIdentityObservationBudget(ctx); const pending = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`); if (pending.find(row => row.id === entity(state, "original").id)?.status !== "pending") throw new Error("The original human-only question is not pending before the two-tab answer race"); const before = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true`); state.noTaskBeforeAnswers = before.filter(row => row.id !== ctx.issue()!.id).length === 0; if (!state.noTaskBeforeAnswers) throw new Error("Work started before the protected human question was actually answered"); for (const page of [ctx.input.page, other]) { const form = page.getByTestId("task-chat-composer-takeover"); - if (!(await form.isVisible())) await page.locator(`#interaction-${entity(state, "original").id}`).click(); - await expect(form).toBeVisible(); await form.getByTestId("question-text-answer-composer").locator('[contenteditable="true"],textarea').first().fill("Acorn"); + if (!(await form.isVisible())) await page.locator(`#interaction-${entity(state, "original").id}`).click({ timeout: remainingIdentityObservationBudget(ctx) }); + await expect(form).toBeVisible({ timeout: remainingIdentityObservationBudget(ctx) }); await form.getByTestId("question-text-answer-composer").locator('[contenteditable="true"],textarea').first().fill("Acorn", { timeout: remainingIdentityObservationBudget(ctx) }); } const endpoint = `/api/issues/${ctx.issue()!.id}/interactions/${entity(state, "original").id}/respond`; let release!: () => void; const bothActualRequests = new Promise(resolve => { release = resolve; }); let submitted = 0; @@ -132,24 +169,21 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ await bothActualRequests; await route.continue(); }; const url = ctx.input.api.baseURL + endpoint; const context = ctx.input.page.context(); await context.route(url, handler); - const timer = setTimeout(release, 30_000); + const timer = setTimeout(release, Math.min(30_000, remainingIdentityObservationBudget(ctx))); try { // Hold only transport admission until both genuine visible buttons have // submitted. Both unmodified requests then reach the real public route; // no response is fulfilled, replayed, or manufactured by this barrier. - const responses = [ctx.input.page, other].map(page => page.waitForResponse(response => response.request().method() === "POST" && new URL(response.url()).pathname === endpoint)); - await Promise.all([ctx.input.page, other].map(page => page.getByTestId("task-chat-composer-takeover").getByRole("button", { name: "Submit answers", exact: true }).click())); + const responses = [ctx.input.page, other].map(page => page.waitForResponse(response => response.request().method() === "POST" && new URL(response.url()).pathname === endpoint, { timeout: remainingIdentityObservationBudget(ctx) })); + await Promise.all([ctx.input.page, other].map(page => page.getByTestId("task-chat-composer-takeover").getByRole("button", { name: "Submit answers", exact: true }).click({ timeout: remainingIdentityObservationBudget(ctx) }))); state.submitStatuses = (await Promise.all(responses)).map(response => response.status()); state.actualSubmitRequests = submitted; } finally { release(); clearTimeout(timer); await context.unroute(url, handler); } - await pollUntil({ label: "One answer-race continuation completed", deadlineAt: Date.now() + 180_000, - load: async () => { await ctx.refreshIssue(); return { runs: await ctx.allRuns(), source: ctx.issue() }; }, - accept: value => value.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id).length >= 2 - && value.runs.filter(run => run.agentId === ctx.input.fixtures.agent.id).every(run => run.status === "succeeded") && value.source?.conversationState === "waiting" }); + await waitForAnswerContinuation(ctx, state); for (const page of [ctx.input.page, other]) { - await page.reload({ waitUntil: "domcontentloaded" }); + await page.reload({ waitUntil: "domcontentloaded", timeout: remainingIdentityObservationBudget(ctx) }); const receipt = page.locator(`#interaction-${entity(state, "original").id}`).getByTestId("task-chat-answered-questions-receipt"); - await expect(receipt).toBeVisible(); await receipt.locator("summary").click(); await expect(receipt).toContainText("Acorn"); - await expect(page.locator(`#interaction-${entity(state, "original").id}`).getByTestId("task-chat-unanswered-question")).toHaveCount(0); + await expect(receipt).toBeVisible({ timeout: remainingIdentityObservationBudget(ctx) }); await receipt.locator("summary").click({ timeout: remainingIdentityObservationBudget(ctx) }); await expect(receipt).toContainText("Acorn", { timeout: remainingIdentityObservationBudget(ctx) }); + await expect(page.locator(`#interaction-${entity(state, "original").id}`).getByTestId("task-chat-unanswered-question")).toHaveCount(0, { timeout: remainingIdentityObservationBudget(ctx) }); } state.bothTabsResolved = true; await ctx.input.evidence("identity-simultaneous-question-answer.json", { questionId: entity(state, "original").id, userId: state.humanId, submitStatuses: state.submitStatuses, bothTabsResolved: true });