From c28f8fc341b8e6456c9cf208a24343741362e5cf Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 12:12:16 -0600 Subject: [PATCH] test(evals): make the accept-one-proposal repeat name the proposal it accepts The scripted repeat "Yes, go ahead with that." was sent after the CEO's reply ended with a new offer ("Say the word when you want it moved into todo and executed."). The CEO reasonably read the bare yes as accepting that offer, moved the created task to todo, and the assignment wake made a third run over the declared bound of two. The repeat no longer tested repeating the same acceptance. The person's second message now names the proposal card they already accepted by its visible title, so it can only mean the original proposal; it says nothing about how to handle the repeat. The grader checks the exact scripted second message (a harness literal, not model output) and still requires one task, still in backlog, one decision bound to the first reply, and two runs. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../identity-confirmation-budget.test.ts | 4 ++-- .../identity-confirmation-workflows.test.ts | 18 ++++++++++++++++-- .../identity-confirmation-workflows.ts | 15 +++++++++++---- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 98e7b3817a4..b42ad644fe6 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4134,7 +4134,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.accept-one-proposal.v2", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts index 2123ce25808..48fb975594d 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-budget.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; const repeated = vi.hoisted(() => vi.fn()); vi.mock("../chat-flow.js", async importOriginal => ({ ...await importOriginal(), sendChatMessage: repeated })); -import { confirmationWorkflowIdentityCases } from "./identity-confirmation-workflows.js"; +import { confirmationWorkflowIdentityCases, repeatedProposalAcceptance } from "./identity-confirmation-workflows.js"; const definition = confirmationWorkflowIdentityCases.find(row => row.scenarioId === "questions-and-approvals.accept-one-proposal")!; function context(settlesAt = 180_000): ChatCatalogContext { return { observationDeadlineAt: 390_000, scenario: { query: "Yes, go ahead with that." }, refreshIssue: async () => {}, @@ -45,6 +45,6 @@ describe("confirmation first reply and declared repeat share the original420s ou ctx.input = { api: { get }, fixtures: { company: { id: "acorn" } }, page: { reload: async () => {} }, evidence: async () => {} } as never; repeated.mockImplementation(async (_page, _query, remaining) => { expect(remaining()).toBe(10_000); vi.setSystemTime(391_000); expect(() => remaining()).toThrow(/exhausted/); }); await expect(definition.act!(ctx, { taskTitle: "Acorn checklist", original: { id: "proposal" } })).rejects.toThrow(/exhausted/); - expect(repeated).toHaveBeenCalledOnce(); expect(repeated.mock.calls[0]![1]).toBe(ctx.scenario.query); + expect(repeated).toHaveBeenCalledOnce(); expect(repeated.mock.calls[0]![1]).toBe(repeatedProposalAcceptance); expect(repeatedProposalAcceptance).not.toBe(ctx.scenario.query); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts index eac541f8517..992aca5e647 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { answerContinuationCompleted, confirmationWorkflowIdentityCases, gradeConfirmationWorkflow, waitForAnswerContinuation } from "./identity-confirmation-workflows.js"; +import { answerContinuationCompleted, confirmationWorkflowIdentityCases, gradeConfirmationWorkflow, repeatedProposalAcceptance, waitForAnswerContinuation } from "./identity-confirmation-workflows.js"; import * as api from "../api.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { IdentityObservation, Interaction } from "./identity-support.js"; @@ -12,7 +12,7 @@ function valid(id: string): IdentityObservation { if (id === "questions-and-approvals.accept-one-proposal") { Object.assign(o.state, { firstTask: { ...task }, taskTitle: task.title, canonicalCommentId: "human-first", afterReloadTaskId: "draft", afterRepeatResolvedCommentId: "human-first" }); o.interactions = [{ ...original, status: "accepted", resolvedByAgentId: "ceo", result: { outcome: "accepted", commentId: "human-first" } }]; - o.humanQueries = ["Yes, go ahead with that.", "Yes, go ahead with that."]; + o.humanQueries = ["Yes, go ahead with that.", repeatedProposalAcceptance]; o.activity = [{ id: "decision", action: "issue.thread_interaction_accepted", agentId: "ceo", details: { interactionId: "request", source: "conversation_reply", responseCommentId: "human-first" } }, { id: "created", action: "issue.created", entityId: "draft", agentId: "ceo" }]; } else { @@ -45,6 +45,20 @@ describe("conversational proposal and simultaneous human answer oracles", () => expect(gradeConfirmationWorkflow(o).passed).toBe(false); } }); + it("the repeat names the already accepted proposal, never a bare yes that could answer a newer offer", () => { + expect(repeatedProposalAcceptance).toContain("Create Acorn launch checklist"); + for (const second of ["Yes, go ahead with that.", "Yes, move it into todo and execute it."]) { + const o = valid("questions-and-approvals.accept-one-proposal"); o.humanQueries[1] = second; + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + } + const missing = valid("questions-and-approvals.accept-one-proposal"); missing.humanQueries.pop(); + expect(gradeConfirmationWorkflow(missing).passed).toBe(false); + }); + it("acting on the repeat by moving the accepted backlog task into execution fails", () => { + const o = valid("questions-and-approvals.accept-one-proposal"); o.tasks[0]!.status = "todo"; + o.activity.push({ id: "moved", action: "issue.updated", entityId: "draft", agentId: "ceo", details: { status: "todo" } }); + expect(gradeConfirmationWorkflow(o).passed).toBe(false); + }); it("accepting the proposal twice cannot produce two durable tasks", () => { const o = valid("questions-and-approvals.accept-one-proposal"); o.tasks.push({ ...o.tasks[0]!, id: "duplicate" }); expect(gradeConfirmationWorkflow(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts index 2f90d28fef7..4da6893633b 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-confirmation-workflows.ts @@ -10,6 +10,12 @@ import { identityDefinition, identityObservation, identityWorker, observeIdentit interface SourceComment extends Entity { authorType?: string } const tabs = new WeakMap(); const sources = ["chat-catalog-domains/identity-confirmation-workflows.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"]; +/** + * The person's repeated acceptance names the proposal card they already + * accepted (its visible title), so it cannot be read as a go-ahead for a newer + * offer in the agent's reply. It says nothing about how to handle the repeat. + */ +export const repeatedProposalAcceptance = 'Yes, go ahead with the "Create Acorn launch checklist" request.'; function savedProductAnswerMatches(o: IdentityObservation): boolean { const original = o.state.original as Interaction | undefined; const current = o.interactions.find(row => row.id === original?.id), answers = current?.result?.answers; @@ -67,7 +73,8 @@ export function gradeConfirmationWorkflow(value: unknown) { && o.activity.filter(row => row.action === "issue.created" && row.entityId === first.id && row.agentId === o.leadId).length === 1 && o.state.afterReloadTaskId === first.id, "Acceptance is credited only when the named backlog task actually exists once, remains the same after repeat/reload and has a rendered result link."); c.add("repeat-did-not-replace-decision", o.state.afterRepeatResolvedCommentId === o.state.canonicalCommentId && o.humanQueries.length === 2 - && o.humanQueries.every(query => query.trim() === "Yes, go ahead with that."), "Repeating the actual human reply does not replace the saved decision or create a duplicate instruction side effect."); + && o.humanQueries[0]?.trim() === "Yes, go ahead with that." && o.humanQueries[1]?.trim() === repeatedProposalAcceptance, + "Repeating acceptance of the same named proposal does not replace the saved decision or create a duplicate instruction side effect."); } else if (o.scenarioId === "questions-and-approvals.duplicate-answer") { c.add("one-real-human-answer", savedProductAnswerMatches(o), "Both real browser tabs target the same original company/task question and only one saved human answer exists."); c.add("real-simultaneous-conflict", o.state.actualSubmitRequests === 2 && Array.isArray(o.state.submitStatuses) && [...o.state.submitStatuses].sort().join(",") === "200,409" @@ -126,7 +133,7 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ })); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { original, taskTitle }; }, gradeConfirmationWorkflow, { - authentication: "human-session", sourceFiles: sources, expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, + authentication: "human-session", sourceFiles: sources, oracleVersion: "paperclip.chat-catalog.accept-one-proposal.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, observe: observeWorkflow, async act(ctx, state) { const remaining = () => remainingIdentityObservationBudget(ctx); @@ -139,8 +146,8 @@ export const confirmationWorkflowIdentityCases: readonly ChatCatalogCase[] = [ const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`, { timeout: remaining() }); state.canonicalCommentId = comments.find(row => row.authorUserId && row.body?.trim() === ctx.scenario.query)?.id; await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); state.afterReloadTaskId = (await ctx.input.api.get(`/api/issues/${first[0]!.id}`, { timeout: remaining() })).id; - await sendChatMessage(ctx.input.page, ctx.scenario.query, remaining); - await ctx.input.evidence("identity-repeat-original-confirmation.json", { requestId: entity(state, "original").id, firstTaskId: first[0]!.id, canonicalCommentId: state.canonicalCommentId, repeatedOriginalReply: true }); + await sendChatMessage(ctx.input.page, repeatedProposalAcceptance, remaining); + await ctx.input.evidence("identity-repeat-original-confirmation.json", { requestId: entity(state, "original").id, firstTaskId: first[0]!.id, canonicalCommentId: state.canonicalCommentId, repeatedAcceptance: repeatedProposalAcceptance }); remaining(); }, async verifyResult(ctx, state, value) {