diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 98e7b3817a4..b42ad644fe6 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4134,7 +4134,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.accept-one-proposal.v2", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts index 2123ce25808..48fb975594d 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; const repeated = vi.hoisted(() => vi.fn()); vi.mock("../chat-flow.js", async importOriginal => ({ ...await importOriginal(), sendChatMessage: repeated })); -import { confirmationWorkflowIdentityCases } from "./identity-confirmation-workflows.js"; +import { confirmationWorkflowIdentityCases, repeatedProposalAcceptance } from "./identity-confirmation-workflows.js"; const definition = confirmationWorkflowIdentityCases.find(row => row.scenarioId === "questions-and-approvals.accept-one-proposal")!; function context(settlesAt = 180_000): ChatCatalogContext { return { observationDeadlineAt: 390_000, scenario: { query: "Yes, go ahead with that." }, refreshIssue: async () => {}, @@ -45,6 +45,6 @@ describe("confirmation first reply and declared repeat share the original420s ou ctx.input = { api: { get }, fixtures: { company: { id: "acorn" } }, page: { reload: async () => {} }, evidence: async () => {} } as never; repeated.mockImplementation(async (_page, _query, remaining) => { expect(remaining()).toBe(10_000); vi.setSystemTime(391_000); expect(() => remaining()).toThrow(/exhausted/); }); await expect(definition.act!(ctx, { taskTitle: "Acorn checklist", original: { id: "proposal" } })).rejects.toThrow(/exhausted/); - expect(repeated).toHaveBeenCalledOnce(); expect(repeated.mock.calls[0]![1]).toBe(ctx.scenario.query); + expect(repeated).toHaveBeenCalledOnce(); expect(repeated.mock.calls[0]![1]).toBe(repeatedProposalAcceptance); expect(repeatedProposalAcceptance).not.toBe(ctx.scenario.query); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts index eac541f8517..992aca5e647 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { answerContinuationCompleted, confirmationWorkflowIdentityCases, gradeConfirmationWorkflow, waitForAnswerContinuation } from "./identity-confirmation-workflows.js"; +import { answerContinuationCompleted, confirmationWorkflowIdentityCases, gradeConfirmationWorkflow, repeatedProposalAcceptance, waitForAnswerContinuation } from "./identity-confirmation-workflows.js"; import * as api from "../api.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { IdentityObservation, Interaction } from "./identity-support.js"; @@ -12,7 +12,7 @@ function valid(id: string): IdentityObservation { if (id === "questions-and-approvals.accept-one-proposal") { Object.assign(o.state, { firstTask: { ...task }, taskTitle: task.title, canonicalCommentId: "human-first", afterReloadTaskId: "draft", afterRepeatResolvedCommentId: "human-first" }); o.interactions = [{ ...original, status: "accepted", resolvedByAgentId: "ceo", result: { outcome: "accepted", commentId: "human-first" } }]; - o.humanQueries = ["Yes, go ahead with that.", "Yes, go ahead with that."]; + o.humanQueries = ["Yes, go ahead with that.", repeatedProposalAcceptance]; o.activity = [{ id: "decision", action: "issue.thread_interaction_accepted", agentId: "ceo", details: { interactionId: "request", source: "conversation_reply", responseCommentId: "human-first" } }, { id: "created", action: "issue.created", entityId: "draft", agentId: "ceo" }]; } else { @@ -45,6 +45,20 @@ describe("conversational proposal and simultaneous human answer oracles", () => expect(gradeConfirmationWorkflow(o).passed).toBe(false); } }); + it("the repeat names the already accepted proposal, never a bare yes that could answer a newer offer", () => { + expect(repeatedProposalAcceptance).toContain("Create Acorn launch checklist"); + for (const second of ["Yes, go ahead with that.", "Yes, move it into todo and execute it."]) { + const o = valid("questions-and-approvals.accept-one-proposal"); o.humanQueries[1] = second; + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + } + const missing = valid("questions-and-approvals.accept-one-proposal"); missing.humanQueries.pop(); + expect(gradeConfirmationWorkflow(missing).passed).toBe(false); + }); + it("acting on the repeat by moving the accepted backlog task into execution fails", () => { + const o = valid("questions-and-approvals.accept-one-proposal"); o.tasks[0]!.status = "todo"; + o.activity.push({ id: "moved", action: "issue.updated", entityId: "draft", agentId: "ceo", details: { status: "todo" } }); + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + }); it("accepting the proposal twice cannot produce two durable tasks", () => { const o = valid("questions-and-approvals.accept-one-proposal"); o.tasks.push({ ...o.tasks[0]!, id: "duplicate" }); expect(gradeConfirmationWorkflow(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts index 2f90d28fef7..4da6893633b 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts @@ -10,6 +10,12 @@ import { identityDefinition, identityObservation, identityWorker, observeIdentit interface SourceComment extends Entity { authorType?: string } const tabs = new WeakMap(); const sources = ["chat-catalog-domains/identity-confirmation-workflows.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"]; +/** + * The person's repeated acceptance names the proposal card they already + * accepted (its visible title), so it cannot be read as a go-ahead for a newer + * offer in the agent's reply. It says nothing about how to handle the repeat. + */ +export const repeatedProposalAcceptance = 'Yes, go ahead with the "Create Acorn launch checklist" request.'; function savedProductAnswerMatches(o: IdentityObservation): boolean { const original = o.state.original as Interaction | undefined; const current = o.interactions.find(row => row.id === original?.id), answers = current?.result?.answers; @@ -67,7 +73,8 @@ export function gradeConfirmationWorkflow(value: unknown) { && o.activity.filter(row => row.action === "issue.created" && row.entityId === first.id && row.agentId === o.leadId).length === 1 && o.state.afterReloadTaskId === first.id, "Acceptance is credited only when the named backlog task actually exists once, remains the same after repeat/reload and has a rendered result link."); c.add("repeat-did-not-replace-decision", o.state.afterRepeatResolvedCommentId === o.state.canonicalCommentId && o.humanQueries.length === 2 - && o.humanQueries.every(query => query.trim() === "Yes, go ahead with that."), "Repeating the actual human reply does not replace the saved decision or create a duplicate instruction side effect."); + && o.humanQueries[0]?.trim() === "Yes, go ahead with that." && o.humanQueries[1]?.trim() === repeatedProposalAcceptance, + "Repeating acceptance of the same named proposal does not replace the saved decision or create a duplicate instruction side effect."); } else if (o.scenarioId === "questions-and-approvals.duplicate-answer") { c.add("one-real-human-answer", savedProductAnswerMatches(o), "Both real browser tabs target the same original company/task question and only one saved human answer exists."); c.add("real-simultaneous-conflict", o.state.actualSubmitRequests === 2 && Array.isArray(o.state.submitStatuses) && [...o.state.submitStatuses].sort().join(",") === "200,409" @@ -126,7 +133,7 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ })); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { original, taskTitle }; }, gradeConfirmationWorkflow, { - authentication: "human-session", sourceFiles: sources, expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, + authentication: "human-session", sourceFiles: sources, oracleVersion: "paperclip.chat-catalog.accept-one-proposal.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, observe: observeWorkflow, async act(ctx, state) { const remaining = () => remainingIdentityObservationBudget(ctx); @@ -139,8 +146,8 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`, { timeout: remaining() }); state.canonicalCommentId = comments.find(row => row.authorUserId && row.body?.trim() === ctx.scenario.query)?.id; await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); state.afterReloadTaskId = (await ctx.input.api.get(`/api/issues/${first[0]!.id}`, { timeout: remaining() })).id; - await sendChatMessage(ctx.input.page, ctx.scenario.query, remaining); - await ctx.input.evidence("identity-repeat-original-confirmation.json", { requestId: entity(state, "original").id, firstTaskId: first[0]!.id, canonicalCommentId: state.canonicalCommentId, repeatedOriginalReply: true }); + await sendChatMessage(ctx.input.page, repeatedProposalAcceptance, remaining); + await ctx.input.evidence("identity-repeat-original-confirmation.json", { requestId: entity(state, "original").id, firstTaskId: first[0]!.id, canonicalCommentId: state.canonicalCommentId, repeatedAcceptance: repeatedProposalAcceptance }); remaining(); }, async verifyResult(ctx, state, value) {