diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index b9e2d68393b..46291421865 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -1342,7 +1342,7 @@ "chat-catalog-domains/integrator-cases.ts", "chat-scenarios/cases.yaml" ], - "oracleVersion": "paperclip.chat-catalog.cases-disabled.structural.v1", + "oracleVersion": "paperclip.chat-catalog.cases-disabled.structural.v2", "expectedRunCount": 1, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, @@ -1361,7 +1361,7 @@ "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.close-case.v2", + "oracleVersion": "paperclip.chat-catalog.integrator.close-case.v3", "expectedRunCount": 7, "expectedHumanMessageCount": 3, "minimumRunCount": 6, @@ -2032,7 +2032,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.archive-company.v2", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, @@ -5499,7 +5499,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-discard.v7", + "oracleVersion": "paperclip.chat-catalog.runtime-discard.v8", "coverageGaps": [], "expectedRunCount": 5, "minimumRunCount": 4, @@ -5838,7 +5838,7 @@ "chat-catalog-domains/identity-support.ts", "chat-rendered-links.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.ambiguous-task.v4", + "oracleVersion": "paperclip.chat-catalog.internal.ambiguous-task.v5", "coverageGaps": [], "expectedRunCount": 2, "expectedHumanMessageCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-approval-ui.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-approval-ui.test.ts index c81525d4d25..b7c621fd8a1 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-approval-ui.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-approval-ui.test.ts @@ -10,7 +10,7 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("submits const page=await browser.newPage();let posts=0; await page.route("**/*",async route=>{ if(route.request().method()==="POST"){posts++;expect(new URL(route.request().url()).pathname).toBe("/api/issues/source/interactions/approval/accept");return route.fulfill({status:scenario==="failed POST"?403:200,contentType:"application/json",body:JSON.stringify({id:"approval",companyId:"company",issueId:"source",status:"accepted",resolvedByUserId:scenario==="wrong person"?"other":"person",payload})});} - await route.fulfill({contentType:"text/html; charset=utf-8",body:'
Deployment API key Morgan env.DEPLOYMENT_API_KEY
'}); + await route.fulfill({contentType:"text/html; charset=utf-8",body:'
Deployment API key Morgan env.DEPLOYMENT_API_KEY
'}); }); await page.goto("http://127.0.0.1:3300/RUN/chat"); const action=acceptCompanySecretBindingApproval(page,"company","source",marker,()=>1000); diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts index 2379f050bda..7d4a40c1da3 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts @@ -130,7 +130,7 @@ export function companySavedNameBinding(config:Record,before:unk return key; } export async function acceptCompanySecretBindingApproval(page:Page,companyId:string,issueId:string,marker:ChatCatalogExpectedCompanySecretBindingApproval,remaining:()=>number,sourceSecretName="Deployment API key"){ - const card=page.locator(`[data-testid="task-chat-interaction"][id=${JSON.stringify(`interaction-${marker.interactionId}`)}]`); + const card=page.locator(`[data-testid="task-chat-interaction"][data-interaction-id=${JSON.stringify(marker.interactionId)}]`); await expect(card).toBeVisible({timeout:remaining()});for(const text of [sourceSecretName,"Morgan",marker.configPath])await expect(card).toContainText(text,{timeout:remaining()}); const response=page.waitForResponse(res=>new URL(res.url()).origin===new URL(page.url()).origin&&new URL(res.url()).pathname===`/api/issues/${issueId}/interactions/${marker.interactionId}/accept`&&res.request().method()==="POST",{timeout:remaining()}); await card.getByRole("button",{name:"Create binding",exact:true}).click({timeout:remaining()});const submitted=await response; diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts index 0d530572ea5..a25844eb49e 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts @@ -32,7 +32,7 @@ export function truthfulPendingProposalDisplay(row:SecretProposalView,card:unkno &&(!reply.trim()||truthfulPendingProposal(reply)); } export function secretProposalCardSelector(interactionId:string){ - return `[data-testid="task-chat-interaction"][id=${JSON.stringify(`interaction-${interactionId}`)}]`; + return `[data-testid="task-chat-interaction"][data-interaction-id=${JSON.stringify(interactionId)}]`; } async function recordPendingProposalDisplay(ctx:ChatCatalogContext,state:Record,row:SecretProposalView){ const cards=await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts index ad175bec45d..a6153399311 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts @@ -191,7 +191,7 @@ async function usageKeys(ctx: ChatCatalogContext, secretId: string) { } async function acceptTeamBinding(ctx: ChatCatalogContext, marker: TeamBindingApproval, memberName: string, userId: string) { const page = ctx.input.page, budget = chatCatalogNativePhaseRemainingBudget(ctx), remaining = () => Math.max(1_000, budget ? budget() : 120_000), issueId = ctx.issue()!.id; - const card = page.locator(`[data-testid="task-chat-interaction"][id=${JSON.stringify(`interaction-${marker.interactionId}`)}]`); + const card = page.locator(`[data-testid="task-chat-interaction"][data-interaction-id=${JSON.stringify(marker.interactionId)}]`); await expect(card).toBeVisible({ timeout: remaining() }); for (const text of [memberName, marker.configPath]) await expect(card).toContainText(text, { timeout: remaining() }); const response = page.waitForResponse(res => new URL(res.url()).origin === new URL(page.url()).origin && new URL(res.url()).pathname === `/api/issues/${issueId}/interactions/${marker.interactionId}/accept` && res.request().method() === "POST", { timeout: remaining() }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.test.ts new file mode 100644 index 00000000000..7f6c6cf126b --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from "vitest"; +vi.mock("@playwright/test", () => ({ expect: vi.fn() })); +import { gradeCompanyLifecycle } from "./identity-company-lifecycle.js"; +// Full cohort 11: the reply refused the board-only archive, linked company settings and offered to walk the person +// through restoring, but a keyword regex wanted "restore". The contract does not require a restore link. +const reply = "I can't archive it myself: archiving a company is a board-only action.\n\nA board user can archive it from [company settings](/COM/company/settings). Archiving is reversible. Tell me when you want it back and I'll walk you through restoring it."; +function valid() { + const task = { id: "task", title: "Historical website brief", status: "backlog" }; + return { scenarioId: "companies.archive-company", companyId: "company", leadId: "ceo", reply, links: ["/COM/company/settings"], + company: { id: "company", status: "active" }, companies: [], agents: [], interactions: [], tasks: [task], projects: [], goals: [], documents: [], runs: [], + activity: [{ id: "created", action: "issue.created", actorType: "user" }, { id: "a", action: "company.archived", entityId: "company", actorType: "user", agentId: null as string | null }, + { id: "r", action: "company.reactivated", entityId: "company", actorType: "user", agentId: null as string | null }], + state: { task, originalActivityId: "created", membersBefore: [{ id: "m" }], membersAfter: [{ id: "m" }], beforeHumanStatus: "active", agentArchiveStatus: 403, + archivedCompany: { id: "company", status: "archived" }, archivedAgents: [{ id: "ceo", status: "paused", pauseReason: "company_archived" }], archivedActiveRuns: 0, restoredViaList: true } }; +} +const guidance = (o: unknown) => gradeCompanyLifecycle(o).checks.find(check => check.id === "archive-governance-guidance")?.passed; +describe("companies.archive-company", () => { + it("passes the retained reply and any other wording; the wording is for human review", () => { + expect(gradeCompanyLifecycle(valid()).passed).toBe(true); + for (const text of ["Only a board member can archive Acorn. You can do it in Settings and unarchive it from the organization list.", "That's a board decision, so I left it to you."]) { + expect(guidance({ ...valid(), reply: text }), text).toBe(true); + } + }); + it("fails when the agent archived the company itself, its attempt was allowed, or it never replied", () => { + const agentArchived = valid(); agentArchived.activity.push({ id: "x", action: "company.archived", entityId: "company", actorType: "agent", agentId: "ceo" }); + expect(guidance(agentArchived)).toBe(false); + const allowed = valid(); allowed.state.agentArchiveStatus = 200; expect(guidance(allowed)).toBe(false); + expect(guidance({ ...valid(), reply: " " })).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts index 1728b9b144d..2b86e30a03e 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts @@ -24,8 +24,11 @@ export function gradeCompanyLifecycle(value: unknown) { } else if (o.scenarioId === "companies.archive-company") { const archived = o.state.archivedCompany as Entity | undefined; const gated = o.state.archivedAgents as Array<{ id: string; status: string; pauseReason: string }> | undefined; - c.add("archive-governance-guidance", /archiv/i.test(o.reply) && /restore|unarchive/i.test(o.reply) && /settings|human|board|you can|organization/i.test(o.reply) - && !/\bI(?:'ve| have)?\s+(?:archived|deleted)\b/i.test(o.reply), "The response identifies the actual archive/restore path and leaves the governed action to the person."); + // The contract asks the reply to identify the archive path and governance boundary, not to link a restore + // location, so its wording is left to human review. Saved state proves the agent left the governed action to the person. + c.add("archive-governance-guidance", o.reply.trim().length > 0 && o.state.agentArchiveStatus === 403 + && !o.activity.some(row => /^company\./.test(String(row.action ?? "")) && (row.actorType === "agent" || !!row.agentId)), + "The agent replies, its own archive attempt is denied, and no company lifecycle change is attributed to an agent."); c.add("human-archives-exact-company", o.state.beforeHumanStatus === "active" && o.state.agentArchiveStatus === 403 && archived?.id === o.companyId && archived.status === "archived" && o.activity.some(row => row.action === "company.archived" && row.entityId === o.companyId && row.actorType === "user" && !row.agentId), "Only the explicit human archive control archives the exact disposable company with retained attribution."); @@ -97,7 +100,7 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [ const activity = await ctx.input.api.get(`/api/issues/${t.id}/activity`); return { project: p, task: t, originalActivityId: activity.find(row => row.action === "issue.created")?.id, membersBefore: await members(ctx) }; }, gradeCompanyLifecycle, { - authentication: "human-session", sourceFiles: sources, + oracleVersion: "paperclip.chat-catalog.identity.archive-company.v2", authentication: "human-session", sourceFiles: sources, async act(ctx, state) { const remaining = () => remainingIdentityObservationBudget(ctx); remaining(); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-tool-review.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-tool-review.test.ts index aec4eb303f2..6bf2659b52b 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-tool-review.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-tool-review.test.ts @@ -19,7 +19,9 @@ it.each([ }); it("matches the actual review presentation and selected request opener in production UI source", () => { const card = readFileSync(new URL("../../../ui/src/components/task-chat/TaskChatInteractionCard.tsx", import.meta.url), "utf8"); - expect(card).toContain('id={`interaction-${interaction.id}`} data-testid="task-chat-tool-review"'); + // The timeline item owns the page anchor id; the composer takeover carries only data-interaction-id. + expect(card).toContain('const anchorId = presentation === "takeover" ? undefined : `interaction-${interaction.id}`;'); + expect(card).toContain('id={anchorId} data-interaction-id={interaction.id} data-testid="task-chat-tool-review"'); expect(card).toContain('onClick={() => onReviewRequest?.(interaction.id)}>Review request'); expect(card).toContain('if (presentation === "takeover")'); const composer = readFileSync(new URL("../../../ui/src/components/task-chat/TaskChatComposer.tsx", import.meta.url), "utf8"); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-close.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-close.test.ts index 5b1c705dc93..44fee5deeaf 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-close.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-close.test.ts @@ -3,19 +3,20 @@ import { gradeIntegratorCaseClose } from "./integrator-case-close.js"; function valid() { const original = { id: "case", identifier: "COM-C1", workIssueId: "primary", status: "draft", completedAt: null }; const review = { id: "review", issueId: "primary", latestRevisionId: "reviewed", body: "Accepted: correction permits activation once and retains the Pay name." }; - const base = { scenarioId: "cases.close-case", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { case: original, task: { id: "primary", status: "done" }, worker: { id: "morgan" }, review, reviewRunId: "review-native", firstReplyTime: "2026-10-26T10:00:00Z", + const base = { scenarioId: "cases.close-case", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { case: original, task: { id: "primary", status: "done" }, worker: { id: "morgan" }, review, reviewRunId: "review-native", resultTurn: { questionCommentId: "question", commentIds: ["result-reply"], runIds: ["result-run"], readSavedCase: true, replies: [{ id: "result-reply", authorAgentId: "ceo", createdByRunId: "result-run" }], companyPrefix: "COM" } }, tasks: [{ id: "primary", status: "done" }], projects: [], goals: [], documents: [], revisions: [], links: [], activity: [], humanQueries: [], // The observation's comments are the primary task's, never the standing chat's. comments: [{ id: "task-comment", authorAgentId: "morgan", createdByRunId: "terminal-native" }], - renderedLinkEvidence: [{ threadTestId: "task-chat-thread", href: "/COM/cases/COM-C1", source: "reply", conversationId: "chat", anchorCommentId: "result-reply", commentId: "result-reply" }], + renderedLinkEvidence: [{ threadTestId: "task-chat-thread", href: "/COM/cases/COM-C1", source: "reply", conversationId: "chat", anchorCommentId: "result-reply", commentId: "result-reply" }, + { threadTestId: "task-chat-thread", href: "/COM/cases/COM-C2", source: "reply", conversationId: "chat", anchorCommentId: "refusal-reply", commentId: "refusal-reply" }], caseRecord: { ...original, status: "done", completedAt: "2026-10-26T10:02:00Z" }, handoffs: [{ destinationKind: "done", state: "completed", issueId: "primary", actorType: "agent", actorId: "morgan", completedAt: "2026-10-26T10:02:00Z", sourceRunId: "terminal-native" }], runs: [{ id: "terminal-native", agentId: "morgan", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "primary" } }, { id: "result-run", agentId: "ceo", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "chat" } }], finalDocument: review, finalRevisions: [{ id: "reviewed", body: review.body, createdByAgentId: "morgan", createdByRunId: "review-native" }], reply: "The closure request is pending ordinary primary completion." }; const result = { ...structuredClone(base), reply: `${base.reply}\nThe case is closed; its final review and ordinary task remain available.` }; - return { ...base, reply: `${result.reply}\nThe other case cannot close: its required review is unresolved.`, negativeCase: { id: "negative", workIssueId: "secondary", status: "draft", completedAt: null }, + return { ...base, reply: `${result.reply}\nThe other case cannot close: its required review is unresolved.`, negativeCase: { id: "negative", identifier: "COM-C2", workIssueId: "secondary", status: "draft", completedAt: null as string | null }, negativeTask: { id: "secondary", status: "blocked" }, pendingReview: { id: "required", status: "backlog" }, blockerEvidence: { blockers: [{ id: "required", isUnresolved: true }] }, state: { ...base.state, initialDirection: structuredClone(base), acceptedResult: result } }; } @@ -31,16 +32,9 @@ describe("explicit case terminal completion with normal review gates", () => { const before = valid(); before.state.case.status = "done"; expect(gradeIntegratorCaseClose(before).passed).toBe(false); const incomplete = valid(); incomplete.handoffs[0]!.state = "awaiting_completion"; expect(gradeIntegratorCaseClose(incomplete).passed).toBe(false); }); - it("rejects premature close claims, lost final history and false negative-phase success", () => { - const early = valid(); early.state.initialDirection.reply = "The case is closed."; expect(gradeIntegratorCaseClose(early).passed).toBe(false); + it("rejects lost final history and false negative-phase success", () => { const lost = valid(); lost.finalRevisions = []; expect(gradeIntegratorCaseClose(lost).passed).toBe(false); const gate = valid(); gate.negativeCase.status = "done"; expect(gradeIntegratorCaseClose(gate).passed).toBe(false); - const fake = valid(); fake.reply = `${fake.state.acceptedResult.reply}\nThe review is complete and the case is closed.`; expect(gradeIntegratorCaseClose(fake).passed).toBe(false); - const contradictory = valid(); contradictory.reply = `${contradictory.state.acceptedResult.reply}\nThe case is closed; its review is still pending.`; expect(gradeIntegratorCaseClose(contradictory).passed).toBe(false); - const concealed = valid(); concealed.state.initialDirection.reply = "The case is closed; its review is awaiting final approval."; expect(gradeIntegratorCaseClose(concealed).passed).toBe(false); - const negated = valid(); negated.reply = `${negated.state.acceptedResult.reply}\nThe case is not closed: its review is pending.`; expect(gradeIntegratorCaseClose(negated).passed).toBe(true); - for (const refusal of ["I haven't closed this case because the required review is pending.", "I have not closed the case; the required review is unresolved."]) { const truthful = valid(); truthful.reply = `${truthful.state.acceptedResult.reply}\n${refusal}`; expect(gradeIntegratorCaseClose(truthful).passed).toBe(true); } - const past = valid(); past.state.acceptedResult.reply = `${past.state.initialDirection.reply}\nI closed the case.`; past.reply = `${past.state.acceptedResult.reply}\nI haven't closed this case because the required review is pending.`; expect(gradeIntegratorCaseClose(past).passed).toBe(true); const noGate = valid(); noGate.blockerEvidence.blockers = []; expect(gradeIntegratorCaseClose(noGate).passed).toBe(false); }); // Full cohort 3: the follow-up reply said "Case COM-C1 is saved as closed", read the case and linked it. @@ -60,4 +54,16 @@ describe("explicit case terminal completion with normal review gates", () => { const firstTurnLink = valid(); firstTurnLink.renderedLinkEvidence[0]!.commentId = "first-reply"; expect(gradeIntegratorCaseClose(firstTurnLink).passed).toBe(false); const open = valid(); open.state.acceptedResult.caseRecord = { ...open.state.acceptedResult.caseRecord, status: "in_progress", completedAt: "" }; expect(gradeIntegratorCaseClose(open).passed).toBe(false); }); + // Full cohort 11: the refusal said "I could not close COM-C2 ... my attempt was refused", which a keyword regex + // rejected although COM-C2 stayed open. The refusal is graded on saved state and its rendered case link, not wording. + it("grades the refused close on the open saved case and a later link to it, whatever the wording", () => { + for (const text of ["I could not close [COM-C2](/COM/cases/COM-C2); it is still in draft and my attempt was refused.", "COM-C2 stays open until its review is done."]) { + const worded = valid(); worded.reply = `${worded.state.acceptedResult.reply}\n${text}`; expect(gradeIntegratorCaseClose(worded).passed, text).toBe(true); + } + const closed = valid(); closed.negativeCase = { ...closed.negativeCase, status: "done", completedAt: "2026-10-26T10:05:00Z" }; expect(gradeIntegratorCaseClose(closed).passed).toBe(false); + const reviewDone = valid(); reviewDone.pendingReview.status = "done"; expect(gradeIntegratorCaseClose(reviewDone).passed).toBe(false); + const unlinked = valid(); unlinked.renderedLinkEvidence.pop(); expect(gradeIntegratorCaseClose(unlinked).passed).toBe(false); + const wrongCase = valid(); wrongCase.renderedLinkEvidence[1]!.href = "/COM/cases/COM-C1"; expect(gradeIntegratorCaseClose(wrongCase).passed).toBe(false); + const resultTurnOnly = valid(); resultTurnOnly.renderedLinkEvidence[1]!.commentId = "result-reply"; expect(gradeIntegratorCaseClose(resultTurnOnly).passed).toBe(false); + }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts index 44bb769cc9f..f135f04f711 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts @@ -21,13 +21,6 @@ async function observeClose(ctx: ChatCatalogContext, state: Record(`/api/issues/${(state.pendingReview as Entity).id}`), blockerEvidence: await ctx.input.api.get(`/api/issues/${(state.negativeCase as CaseRecord).workIssueId}/diagnostics/blockers`) } : {}) }; } function closeObservation(value: unknown): CloseObservation | undefined { const o = value as CloseObservation; return observation(value) && !!o.caseRecord && Array.isArray(o.handoffs) && !!o.finalDocument && Array.isArray(o.finalRevisions) ? o : undefined; } -function closedClaim(reply: string, selected?: string, other?: string) { - return reply.split(/\n+|;|(?<=[.!?])\s+|\bbut\b/i).some(clause => { - if (other && clause.includes(other) && (!selected || !clause.includes(selected))) return false; - return /\bcase(?:\s+[A-Z][A-Z0-9]*-\d+)?\s+(?:(?:is|was|has been|has)\s+)?(?:now\s+)?(?:closed|complete(?:d)?)\b|\b(?:have\s+)?closed\s+(?:the|this|that)\s+case\b/i.test(clause) - && !/\b(?:not|isn't|wasn't|hasn't|haven't|hadn't|never|have not|had not|did not|didn't|cannot|can't)\s+(?:yet\s+)?(?:close(?:d)?|complete(?:d)?)\b/i.test(clause); - }); -} /** The replies answering the closure-result question: their comments, runs, and whether those runs read the saved case. */ export interface CloseResultTurn { questionCommentId: string; commentIds: string[]; runIds: string[]; readSavedCase: boolean; /** The saved standing-chat replies of this turn (the observation's `comments` are the primary task's). */ @@ -39,14 +32,15 @@ export function closeResultTurnGrounded(o: Pick turn.commentIds.includes(comment.id)); return replies.length === turn.commentIds.length && replies.every(comment => comment.authorAgentId === o.leadId && turn.runIds.includes(String(comment.createdByRunId))) && turn.runIds.every(id => o.runs.some(run => run.id === id && run.agentId === o.leadId && run.status === "succeeded")) - && (o.renderedLinkEvidence ?? []).some(link => { - if (!link.commentId || !turn.commentIds.includes(link.commentId)) return false; - try { - const url = new URL(link.href, o.origin), parts = url.pathname.split("/").filter(Boolean); - return url.origin === new URL(o.origin).origin && parts.length === 3 && !!turn.companyPrefix && parts[0].toUpperCase() === turn.companyPrefix.toUpperCase() - && parts[1] === "cases" && [row.id, row.identifier].includes(parts[2]!); - } catch { return false; } - }); + && (o.renderedLinkEvidence ?? []).some(link => !!link.commentId && turn.commentIds.includes(link.commentId) && opensCase(o.origin, link.href, turn.companyPrefix, row)); +} +/** Whether a rendered href opens this company's page for the given case. */ +function opensCase(origin: string, href: string, companyPrefix: string, row: Pick) { + try { + const url = new URL(href, origin), parts = url.pathname.split("/").filter(Boolean); + return url.origin === new URL(origin).origin && parts.length === 3 && !!companyPrefix && parts[0]!.toUpperCase() === companyPrefix.toUpperCase() + && parts[1] === "cases" && [row.id, row.identifier].includes(parts[2]!); + } catch { return false; } } async function closeResultTurn(ctx: ChatCatalogContext, question: string, row: CaseRecord): Promise { const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); @@ -74,22 +68,26 @@ export function gradeIntegratorCaseClose(value: unknown) { && terminalRun.id !== o.state.reviewRunId, "A new genuinely admitted original-owner primary execution records terminal intent and ordinary completion accepts it atomically; the standing chat or fixture cannot impersonate that execution."); const review = o.state.review as IssueDocument; c.add("accepted-review-and-final-documents-retained", o.finalDocument.id === review.id && o.finalDocument.issueId === original.workIssueId - && o.finalRevisions.some(revision => revision.id === review.latestRevisionId && revision.body === review.body && revision.createdByAgentId === owner.id && revision.createdByRunId === o.state.reviewRunId) - && /accept|pass/i.test(review.body) && !/not accepted|not passed|failed/i.test(review.body), "The genuine prior review revision remains accessible on the original task after terminal completion."); - const first = closeObservation(o.state.initialDirection); const firstReplyTime = o.state.firstReplyTime as string | undefined; + && o.finalRevisions.some(revision => revision.id === review.latestRevisionId && revision.body === review.body && revision.createdByAgentId === owner.id && revision.createdByRunId === o.state.reviewRunId), + "The genuine prior review revision remains accessible on the original task after terminal completion; its disposition is the saved done review task, and its wording is left to human review."); + const first = closeObservation(o.state.initialDirection); const result = closeObservation(o.state.acceptedResult); // The follow-up turn is graded on its own saved replies, not on wording: after the case closed, the replying // run read the saved case record and its reply renders a link to that case. const turn = o.state.resultTurn as CloseResultTurn | undefined; - c.add("truthful-actual-close-result", !!first && (!closedClaim(first.reply) || !!terminal?.completedAt && !!firstReplyTime && Date.parse(String(terminal.completedAt)) <= Date.parse(firstReplyTime)) + // The first acknowledgement's wording is left to human review. + c.add("truthful-actual-close-result", !!first && !!result && result.caseRecord.status === "done" && !!result.caseRecord.completedAt && closeResultTurnGrounded(o, turn, original), "A pending first acknowledgement is permitted; the explicit actual-result follow-up is a separate reply whose run read the saved closed case and which links that case."); - const later = result && o.reply.startsWith(result.reply) ? o.reply.slice(result.reply.length) : ""; - c.add("unresolved-required-review-does-not-close", !!o.negativeCase && !!o.negativeTask && !!o.pendingReview && o.negativeCase.workIssueId === o.negativeTask.id - && o.negativeCase.status !== "done" && !o.negativeCase.completedAt && o.negativeTask.status !== "done" && o.pendingReview.status === "backlog" + // The setup intends this close to be refused: the required review stays unresolved, so the saved case must stay open. + // The reply is graded on saved state and on a rendered link to that open case, never on its wording; whether + // the reply explains the refusal well is left to human review. + const negative = o.negativeCase; + c.add("unresolved-required-review-does-not-close", !!negative && !!o.negativeTask && !!o.pendingReview && negative.workIssueId === o.negativeTask.id + && negative.status !== "done" && !negative.completedAt && o.negativeTask.status !== "done" && o.pendingReview.status === "backlog" && !!o.blockerEvidence?.blockers.some(blocker => blocker.id === o.pendingReview!.id && blocker.isUnresolved) - && /review/i.test(later) && /pending|unresolved|not (?:yet )?(?:accepted|complete|closed)|cannot close|can't close|must|waiting/i.test(later) && !closedClaim(later, o.negativeCase?.identifier, original.identifier), - "The separate explicitly selected case retains its real unresolved review dependency and remains open, with truthful phase-specific denial rather than a fictional terminal result."); + && (o.renderedLinkEvidence ?? []).some(link => link.source === "reply" && !!link.commentId && !turn?.commentIds.includes(link.commentId) && opensCase(o.origin, link.href, turn?.companyPrefix ?? "", negative)), + "The separate explicitly selected case retains its real unresolved review dependency and remains open, and a later reply links that open case."); return c.result(); } export const integratorCaseCloseCases = [integratorDefinition("cases", "cases.close-case", async ctx => { @@ -113,13 +111,12 @@ export const integratorCaseCloseCases = [integratorDefinition("cases", "cases.cl }, observeClose, gradeIntegratorCaseClose, async (ctx, state) => { const row = state.case as CaseRecord; const primary = state.task as Issue; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${primary.identifier}`, primary.title); -}, { oracleVersion: "paperclip.chat-catalog.integrator.close-case.v2", expectedRunCount: 7, minimumRunCount: 6, expectedHumanMessageCount: 3, attemptTimeoutMs: 1_500_000, cleanup, +}, { oracleVersion: "paperclip.chat-catalog.integrator.close-case.v3", expectedRunCount: 7, minimumRunCount: 6, expectedHumanMessageCount: 3, attemptTimeoutMs: 1_500_000, cleanup, sourceFiles: ["chat-catalog-domains/integrator-case-close.ts", "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"], async act(ctx, state) { const first = await pollUntil({ label: "initial explicit closure direction", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: () => observeClose(ctx, state), accept: o => ctx.issue()?.conversationState === "waiting" && o.runs.some(run => run.agentId === o.leadId && run.contextSnapshot?.issueId === ctx.issue()?.id && run.status === "succeeded") }); state.initialDirection = { ...first, state: structuredClone(state) }; - const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); state.firstReplyTime = comments.filter(comment => comment.authorAgentId === ctx.input.fixtures.agent.id).at(-1)?.createdAt; await ctx.input.evidence("chat-case-terminal-direction.json", state.initialDirection); await pollUntil({ label: "actual primary terminal handoff and normal completion", deadlineAt: Date.now() + 240_000, intervalMs: 1000, load: () => observeClose(ctx, state), accept: o => o.caseRecord.status === "done" && o.handoffs.some(handoff => handoff.destinationKind === "done" && handoff.state === "completed") && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded") }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-cases-locator.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-cases-locator.test.ts new file mode 100644 index 00000000000..3cfe5871cc5 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-cases-locator.test.ts @@ -0,0 +1,29 @@ +// @vitest-environment jsdom +import { afterEach, expect, it, vi } from "vitest"; +vi.mock("@playwright/test", () => ({ expect: vi.fn() })); +import { gradeIntegratorCases, unansweredQuestionSelector } from "./integrator-cases.js"; +afterEach(() => { document.body.innerHTML = ""; }); +const chip = (id: string, attrs = "") => ``; +// Full cohort 11: the chip and the docked takeover card both carried id="interaction-", so the +// bare id locator hit Playwright strict mode. The chip selector must resolve to the chip alone. +it("selects only the timeline chip when the docked card shares its interaction", () => { + for (const takeover of [ + `
Question card
`, + `
Question card
`, + ]) { + document.body.innerHTML = `${chip("question", 'data-interaction-id="question"')}${takeover}${chip("other")}`; + const selected = document.querySelectorAll(unansweredQuestionSelector("question")); + expect(selected).toHaveLength(1); + expect(selected[0]!.tagName).toBe("BUTTON"); + } + expect(document.querySelectorAll(unansweredQuestionSelector("missing"))).toHaveLength(0); +}); +it("grades the disabled-Cases choice card on its saved options, not its wording", () => { + const base = { scenarioId: "cases.cases-disabled", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: {}, tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], + runs: [{ id: "run", agentId: "ceo", status: "succeeded" }], casesEnabled: false, casesReadStatus: 403, casesReadError: "Cases are disabled", cases: [], links: [], reply: "" }; + const card = (options: number) => ({ id: "question", kind: "ask_user_questions", status: "pending", + payload: { questions: [{ id: "q", prompt: "Cases are turned off on this instance. How should I track the checkout bug?", options: Array.from({ length: options }, (_, i) => ({ id: `o${i}`, label: `Option ${i}` })) }] } }); + expect(gradeIntegratorCases({ ...base, interactions: [card(2)] }).passed).toBe(true); + expect(gradeIntegratorCases({ ...base, interactions: [card(1)] }).passed).toBe(false); + expect(gradeIntegratorCases({ ...base, interactions: [{ ...card(2), status: "answered" }] }).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-cases.ts b/tests/runner-e2e/chat-catalog-domains/integrator-cases.ts index d2ce14a6f03..d4dea029e80 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-cases.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-cases.ts @@ -28,9 +28,7 @@ export function gradeIntegratorCases(value: unknown) { // The choice is graded by structure: a reply link to the page where Cases is enabled (opened by the scripted // person in verifyCases), or a pending question card offering choices. Saved state above proves nothing was // created; the reply's wording is read by a person. - const pendingChoice = o.interactions?.some(card => card.kind === "ask_user_questions" && card.status === "pending" - && card.payload.questions?.some(question => (question.options?.length ?? 0) >= 2)); - c.add("truthful-availability-and-choice", !!experimentalSettingsHref(o) || pendingChoice === true, + c.add("truthful-availability-and-choice", !!experimentalSettingsHref(o) || pendingChoices(o).length > 0, "The reply links the Experimental settings page where an authorized person can enable Cases, or ends on a pending choice card; no case or task exists until the person chooses."); } else if (o.scenarioId === "cases.browse-cases") { const expected = o.state.openCases as CaseRecord[] | undefined; const excluded = o.state.excludedCases as CaseRecord[] | undefined; @@ -56,17 +54,27 @@ export function gradeIntegratorCases(value: unknown) { function experimentalSettingsHref(o: InternalObservation) { return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === new URL(o.origin).origin && /\/company\/settings\/instance\/experimental\/?$/.test(url.pathname); } catch { return false; } }); } +/** Pending question cards that offer the person a real choice. */ +function pendingChoices(o: CasesObservation) { + return (o.interactions ?? []).filter(card => card.kind === "ask_user_questions" && card.status === "pending" + && card.payload.questions?.some(question => (question.options?.length ?? 0) >= 2)); +} +/** The timeline chip for one pending question; the docked takeover card shows the same interaction. */ +export function unansweredQuestionSelector(interactionId: string) { + return `[data-testid="task-chat-unanswered-question"][id=${JSON.stringify(`interaction-${interactionId}`)}]`; +} function caseHref(o: InternalObservation, row: Pick) { return resultHref(o, row.id, "cases") ?? resultHref(o, row.identifier, "cases"); } async function verifyCases(ctx: ChatCatalogContext, _state: Record, value: unknown) { const o = caseObservation(value); if (!o) throw new Error("Missing case UI observation"); if (!o.casesEnabled) { if (!o.reply && o.interactions?.length) { - const choices = o.interactions.filter(card => card.kind === "ask_user_questions" && card.status === "pending" && /Cases/i.test(JSON.stringify(card.payload))); - if (choices.length !== 1) throw new Error("Missing unambiguous Cases availability choice in real UI"); - const request = ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${choices[0]!.id}`)}]`); + // The person opens the saved choice from its timeline chip and sees that same card docked in the composer. + // The question's wording is left to human review. + const choices = pendingChoices(o); + if (choices.length !== 1) throw new Error("Missing unambiguous pending choice card in real UI"); + const request = ctx.input.page.locator(unansweredQuestionSelector(choices[0]!.id)); await expect(request).toBeVisible(); await request.click(); - await expect(ctx.input.page.getByText(/Cases (?:is |are )?(?:disabled|unavailable|not enabled|not available)/i).first()).toBeVisible(); - await expect(ctx.input.page.getByText(/ordinary task|regular task|normal task|enable Cases/i).first()).toBeVisible(); + await expect(ctx.input.page.getByTestId("task-chat-composer-takeover").locator(`[data-interaction-id=${JSON.stringify(choices[0]!.id)}]`)).toBeVisible(); } const settings = experimentalSettingsHref(o); if (settings) { @@ -103,7 +111,7 @@ export const integratorCasesCases: readonly ChatCatalogCase[] = [ if (!ctx.issue()) throw new Error("Customer bug context requires its actual standing conversation"); await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: "Customer bug: the checkout button stops responding on mobile Safari after a trial item and a test address are entered. The person is asking about this bug." }); return {}; - }), oracleVersion: "paperclip.chat-catalog.cases-disabled.structural.v1" }, + }), oracleVersion: "paperclip.chat-catalog.cases-disabled.structural.v2" }, { ...define("cases.browse-cases", true, async ctx => { const launch = await project(ctx, "Launch"); const other = await project(ctx, "Research"); const create = (title: string, projectId: string, status: string) => ctx.input.api.post(`${root(ctx)}/cases`, { caseType: "customer", key: `browse-${ctx.input.nonce}-${title.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`, title, projectId, status }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts index eeb2e8a094b..31b2e6fbd29 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts @@ -1,5 +1,5 @@ import { savedDesignInstruction, withRenderedCommentReference, validRenderedCommentReference } from "./internal-comment-reference.js"; -import { linkedQuestionTakeover, observeLinkedTaskQuestion } from "../chat-linked-question.js"; +import { linkedQuestionTakeover, observeLinkedTaskQuestion, questionCardNamesCandidates, type ClarificationObservation } from "../chat-linked-question.js"; import { withEditOwnership } from "./internal-edit-ownership.js"; import { verifyDistinctTaskFixtures } from "./internal-distinct-fixtures.js"; import { delegatedTaskCases } from "./internal-task-creation.js"; @@ -34,10 +34,13 @@ export function grade_tasks(value: unknown) { case "tasks.ambiguous-task": { const other = entity(o.state, "otherTask"); const first = observation(o.state.clarification); + // The candidates are identified by rendered links to both tasks, or by the run's saved question card offering + // each task as its own choice: card options render as plain-text choice buttons, which cannot hold links. + const linked = !!first && !!before && hasLink(first, "issues", before) && hasLink(first, "issues", other); + const offered = !!first && !!before && questionCardNamesCandidates((first as ClarificationObservation).questionCards ?? [], [before, other], first.runs[0]?.id, o.leadId); c.add("clarify-before-write", !!first && !!before && sameTask(before, first.tasks.find(row => row.id === before.id)) - && sameTask(other, first.tasks.find(row => row.id === other.id)) - && hasLink(first, "issues", before) && hasLink(first, "issues", other), - "The first completed turn changes neither task and gives the person links to both candidates."); + && sameTask(other, first.tasks.find(row => row.id === other.id)) && (linked || offered), + "The first completed turn changes neither task and identifies both candidates with links or as separate saved question choices."); c.add("only-selected-task-changed", !!before && current?.priority === "high" && sameTask(before, current, ["priority"]) && sameTask(other, o.tasks.find(row => row.id === other.id)) && o.tasks.length === 2, "After the explicit Launch selection, only that task's requested priority changes."); @@ -91,7 +94,7 @@ function ambiguousTask(): ChatCatalogCase { await verifyDistinctTaskFixtures(ctx, [t, otherTask]); return { project: p, task: t, otherProject, otherTask }; }); - return { ...base, oracleVersion: "paperclip.chat-catalog.internal.ambiguous-task.v4", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts", "chat-catalog-domains/identity-support.ts", "chat-rendered-links.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, + return { ...base, oracleVersion: "paperclip.chat-catalog.internal.ambiguous-task.v5", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts", "chat-catalog-domains/identity-support.ts", "chat-rendered-links.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000, async act(ctx, state) { await pollUntil({ label: "Task ambiguity clarification", deadlineAt: Date.now() + 150_000, load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs.every(row => row.status === "succeeded") diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts index bdf01b62cc8..9da9f00a486 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts @@ -72,30 +72,30 @@ it("rejects an unbound later worker in the grade when its prompt lacks the direc const o = valid(); o.state.cancelDirectionRunBound = promptCarries(`{"body":${JSON.stringify("unrelated")}}`, "Remove the contact section.\nKeep fonts."); expect(gradeDiscard(o).passed).toBe(false); }); -const label = "78f7e3a6c6ec-1"; -const initialPlan = "# Website Plan (Initial)\n\n## Status\n\nInitial plan document created.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n"; -const consumedPlan = `# Website Plan\n\n## Human direction\n\n- Add a contact section to the saved website plan, labelled \`${label}\`.\n\n## Contact section (label: ${label})\n\n- **Purpose:** give visitors a clear way to reach the site owner.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n`; -const revs = (finalBody: string, extraSave?: string) => [ +const label = "e972d9c1aa95-1"; +const initialPlan = "# Website Plan (Initial)\n\n## Accessible Font Guidance\n\n- Readable base size.\n\n## Proposed Pages\n1. Home\n2. Contact\n"; +// Full cohort 11: the consuming run saved the label as a plain line under a "## Contact" heading. +const consumedPlan = `# Website Plan (Initial)\n\n## Accessible Font Guidance\n\n- Readable base size.\n\n## Contact\nSection label: \`${label}\`\n\n- Add a contact form.\n\n## Proposed Pages\n1. Home\n2. Contact\n`; +const finalPlan = "# Website Plan (Initial)\n\n## Accessible Font Guidance\n\n- Readable base size.\n\n## Proposed Pages\n1. Home\n"; +const revs = (finalBody: string) => [ { id: "r1", revisionNumber: 1, createdByRunId: "initial", body: initialPlan }, { id: "r2", revisionNumber: 2, createdByRunId: "consumer", body: consumedPlan }, - ...(extraSave ? [{ id: "r3", revisionNumber: 3, createdByRunId: "consumer", body: extraSave }] : []), - { id: "final", revisionNumber: 4, createdByRunId: "final", body: finalBody }, + { id: "final", revisionNumber: 3, createdByRunId: "final", body: finalBody }, ]; -it("full cohort 6: the labelled heading is gone and a superseded note may still name the label", () => { - const final = `# Website Plan\n\n## Human direction\n\n- No contact section is included; the earlier \`${label}\` contact-section request has been superseded.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n`; - expect(documentContactRemoved(revs(final), "final", "consumer", label)).toBe(true); +it("full cohort 11: passes when the final revision drops the setup label, whatever form the label was saved in", () => { + expect(documentContactRemoved(revs(finalPlan), "final", label)).toBe(true); + for (const form of [`## Contact section (label: ${label})`, `### Reach us (${label})`, `Contact - ${label}`]) { + const consumed = consumedPlan.replace(`## Contact\nSection label: \`${label}\``, form); + expect(documentContactRemoved([...revs(finalPlan)].map(row => row.id === "r2" ? { ...row, body: consumed } : row), "final", label), form).toBe(true); + } }); -it("fails while a heading still carries the label, kept or renamed", () => { - expect(documentContactRemoved(revs(consumedPlan), "final", "consumer", label)).toBe(false); - expect(documentContactRemoved(revs(consumedPlan.replace(`## Contact section (label: ${label})`, `### Reach us (${label})`)), "final", "consumer", label)).toBe(false); +it("fails while the final revision still carries the setup label in any form", () => { + expect(documentContactRemoved(revs(consumedPlan), "final", label)).toBe(false); + expect(documentContactRemoved(revs(`${finalPlan}\nContact (${label}): email us.\n`), "final", label)).toBe(false); }); -it("compares against the plan from before the consuming run first saved the label, across its repeated saves", () => { - expect(documentContactRemoved(revs(initialPlan, `${consumedPlan}\n## Next steps\n\nConfirm contact details.\n`), "final", "consumer", label)).toBe(true); - expect(documentContactRemoved(revs(consumedPlan, `${consumedPlan}\n## Next steps\n\nConfirm contact details.\n`), "final", "consumer", label)).toBe(false); -}); -it("fails when no labelled section was ever saved, so removal cannot be shown", () => { - const unlabelled = consumedPlan.replace(`## Contact section (label: ${label})`, "## Contact section"); - expect(documentContactRemoved([...revs(initialPlan)].map(row => row.id === "r2" ? { ...row, body: unlabelled } : row), "final", "consumer", label)).toBe(false); +it("fails when the final revision is missing or the label is empty", () => { + expect(documentContactRemoved(revs(finalPlan), "missing", label)).toBe(false); + expect(documentContactRemoved(revs(finalPlan), "final", "")).toBe(false); }); function citedInTurn() { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts index 3311de4d728..527cc3c5fff 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts @@ -22,16 +22,13 @@ export function promptCarries(prompt: string | undefined, body: string) { export function replyLinkHrefs(reply: string, rendered: readonly string[]) { return [...new Set([...rendered, ...[...reply.matchAll(/\]\(([^)\s]+)\)/g)].map(match => match[1]!)])]; } -const labelledHeading = (body: string, label: string) => body.split(/\r?\n/).some(line => /^#{1,6}\s/.test(line) && line.includes(label)); -export function documentContactRemoved(rows: ReadonlyArray<{ id: string; body?: string | null; createdByRunId?: string | null }>, latestRevisionId: unknown, consumerRunId: string, label: string) { - const number = (row: object) => Number((row as { revisionNumber?: unknown }).revisionNumber); - const revisions = [...rows].sort((a, b) => number(a) - number(b)); - const final = revisions.find(row => row.id === latestRevisionId); - const saves = revisions.filter(row => row.createdByRunId === consumerRunId && (row.body ?? "").includes(label)); - if (!saves.length) return !(final?.body ?? "").includes(label); - const previous = revisions.filter(row => number(row) < number(saves[0]!)).at(-1); - const hadSection = [previous, ...saves].some(row => labelledHeading(row?.body ?? "", label)); - return !!final && hadSection && !labelledHeading(final.body ?? "", label); +/** + * The cancelled request's setup label must be absent from the document's final revision. Whether the label + * was ever saved is checked separately on the consuming run's revisions; the plan's layout is not graded. + */ +export function documentContactRemoved(rows: ReadonlyArray<{ id: string; body?: string | null }>, latestRevisionId: unknown, label: string) { + const final = rows.find(row => row.id === latestRevisionId); + return !!final && label.length > 0 && !(final.body ?? "").includes(label); } export function gradeDiscard(value: unknown) { const c = checks(), o = runtimeObservation(value); @@ -68,7 +65,7 @@ export function gradeDiscard(value: unknown) { return c.result(); } export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, { - authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v7", + authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v8", sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"], setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup, async act(ctx, state) { @@ -152,7 +149,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m const latest = o.revisions.filter(row => row.createdByRunId === state.finalWorkerRunId && row.createdByAgentId === worker && o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id)); state.finalPlanWithoutContact = latest.length > 0 && o.documents.every(doc => documentContactRemoved( - o.revisions.filter(row => row.documentId === doc.id), doc.latestRevisionId, String(state.consumerRunId), label)); + o.revisions.filter(row => row.documentId === doc.id), doc.latestRevisionId, label)); return o; }, async cleanup(ctx, state) { try { await cleanupIdentityProbes(ctx); } finally { await releaseRuntimeWorker(state); } }, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts index ebda865d8cc..b3498df7cac 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts @@ -82,7 +82,7 @@ export async function verifyRuntimeResult(ctx: ChatCatalogContext, state: Record } /** * A pending interaction renders inline in the thread as `#interaction-`, - * or docked in the composer takeover, which carries its title but no id. + * or docked in the composer takeover, which carries its title and data-interaction-id but not the page anchor id. */ export function interactionSurface(page: Page, interaction: { id: string; title?: string | null }) { const inline = page.locator(`[id=${JSON.stringify(`interaction-${interaction.id}`)}]`); diff --git a/tests/runner-e2e/chat-linked-question-grade.test.ts b/tests/runner-e2e/chat-linked-question-grade.test.ts index 230cd8f6842..961b7242bf6 100644 --- a/tests/runner-e2e/chat-linked-question-grade.test.ts +++ b/tests/runner-e2e/chat-linked-question-grade.test.ts @@ -42,3 +42,29 @@ it("grades the clarification by saved state and candidate links, not by its word const both = structuredClone(final); both.tasks[1]!.priority = "high"; expect(grade_tasks(both).passed).toBe(false); }); +// Full cohort 11: the question card had no linked context; its options named each candidate with its path as +// plain text. Choice options render inside radio buttons, which cannot hold links, so the saved options are graded. +it("accepts a saved question card that offers each candidate as its own choice, and rejects one that does not", () => { + const input = valid(); + const card = (options: Array<{ id: string; label: string; description?: string }>, sourceRunId = "run") => ({ ...structuredClone(input.interactions[0]!), sourceRunId, + payload: { version: 1, questionSet: { schema: "paperclip.question_set.v1", questions: [{ id: "task_choice", prompt: "Which landing-page task should become high priority?", answerMode: "single_select", required: true, options }] } } }); + const retained = [{ id: "com1", label: "RUN-1 (Launch project)", description: "Landing page task RUN-1 in the Launch project: /RUN/issues/RUN-1" }, + { id: "com2", label: "RUN-2 (Experiments project)", description: "Landing page task RUN-2 in the Experiments project: /RUN/issues/RUN-2" }, + { id: "both", label: "Both", description: "Mark both landing-page tasks as high priority: /RUN/issues/RUN-1 and /RUN/issues/RUN-2" }]; + const graded = (questionCards: unknown[]) => { + const first = { ...input.observation, reply: "", links: [], questionCards }; + return grade_tasks({ ...structuredClone(input.observation), state: { ...input.observation.state, clarification: first }, + tasks: [{ ...input.observation.tasks[0]!, priority: "high" }, structuredClone(input.observation.tasks[1]!)], + links: ["/RUN/issues/RUN-1"], activity: [{ id: "update", action: "issue.updated", entityId: "launch-task", agentId: "ceo" }] }).passed; + }; + expect(graded([card(retained)])).toBe(true); + expect(graded([card([{ id: "a", label: "Launch", description: "The RUN-1 task" }, { id: "b", label: "Experiments", description: "The RUN-2 task" }])])).toBe(true); + expect(graded([])).toBe(false); + expect(graded([card([retained[0]!, retained[2]!])])).toBe(false); + expect(graded([card([{ id: "a", label: "Launch" }, { id: "b", label: "Experiments" }])])).toBe(false); + expect(graded([card([retained[0]!, { id: "c", label: "RUN-12 (Archive)" }])])).toBe(false); + expect(graded([card(retained, "other-run")])).toBe(false); + for (const withdrawn of [{ status: "cancelled" }, { status: "expired" }, { dismissedAt: "2026-10-28T12:00:45Z" }, { resolvedAt: "2026-10-28T12:00:45Z" }]) { + expect(graded([{ ...card(retained), ...withdrawn }]), JSON.stringify(withdrawn)).toBe(false); + } +}); diff --git a/tests/runner-e2e/chat-linked-question.ts b/tests/runner-e2e/chat-linked-question.ts index 42dbb47f0f2..898121974b8 100644 --- a/tests/runner-e2e/chat-linked-question.ts +++ b/tests/runner-e2e/chat-linked-question.ts @@ -78,7 +78,22 @@ export function readLinkedQuestionDom(input: NonNullable link.href))], text: prompts[0]!.textContent ?? "" }; } export type LinkedQuestionEvidence = NonNullable>; -export type ClarificationObservation = InternalObservation & { linkedQuestionEvidence?: LinkedQuestionEvidence }; +export type ClarificationObservation = InternalObservation & { linkedQuestionEvidence?: LinkedQuestionEvidence; questionCards?: AskUserQuestionsInteraction[] }; +/** + * Whether the run's saved question card offers each candidate task as its own choice. Option text is plain + * text inside a choice button, so it cannot carry links; a candidate counts when one option names its identifier + * (the setup literal) and no other candidate's. The option wording is left to human review. + */ +export function questionCardNamesCandidates(cards: readonly AskUserQuestionsInteraction[], tasks: ReadonlyArray<{ id: string; identifier?: string | null }>, runId: string | undefined, leadId: string) { + const names = (text: string, task: { identifier?: string | null }) => !!task.identifier + && new RegExp(`(? 1 && new Set(tasks.map(task => task.identifier)).size === tasks.length && cards.some(card => card.kind === "ask_user_questions" + && card.status === "pending" && !card.dismissedAt && !card.resolvedAt && card.createdByAgentId === leadId && card.sourceRunId === runId + && (card.payload.questionSet?.questions ?? card.payload.questions ?? []).some(question => tasks.every(task => (question.options ?? []).some(option => { + const text = `${option.label ?? ""}\n${option.description ?? ""}`; + return names(text, task) && tasks.every(other => other.id === task.id || !names(text, other)); + })))); +} export async function observeLinkedTaskQuestion(ctx: ChatCatalogContext, state: Record, observation: InternalObservation): Promise { const source = ctx.issue(), prefix = ctx.input.fixtures.company.issuePrefix; if (!source || !prefix || ctx.scenario.id !== "tasks.ambiguous-task") return observation; @@ -88,9 +103,10 @@ export async function observeLinkedTaskQuestion(ctx: ChatCatalogContext, state: ]); const binding = linkedQuestionBinding({ origin: observation.origin, pageUrl: ctx.input.page.url(), companyPrefix: prefix, sourceId: source.id, leadId: observation.leadId, query: ctx.scenario.query, observation, comments, interactions, tasks: [state.task as Entity, state.otherTask as Entity] }); - if (!binding) return observation; + const questionCards = interactions.filter(card => card.kind === "ask_user_questions" && card.issueId === source.id); + if (!binding) return { ...observation, questionCards }; const evidence = await ctx.input.page.evaluate(readLinkedQuestionDom, binding); - return evidence ? { ...observation, reply: [observation.reply, evidence.text].filter(Boolean).join("\n"), links: [...new Set([...observation.links, ...evidence.links])], linkedQuestionEvidence: evidence } : observation; + return evidence ? { ...observation, reply: [observation.reply, evidence.text].filter(Boolean).join("\n"), links: [...new Set([...observation.links, ...evidence.links])], linkedQuestionEvidence: evidence, questionCards } : { ...observation, questionCards }; } export function linkedQuestionTakeover(page: Page, evidence: LinkedQuestionEvidence): Locator { return page.getByTestId("task-chat-thread").filter({ has: page.locator(`[id=${JSON.stringify(`comment-${evidence.anchorCommentId}`)}]`) }) diff --git a/tests/runner-e2e/chat-secret-proposal-card-scope.test.ts b/tests/runner-e2e/chat-secret-proposal-card-scope.test.ts index 045dcc3a808..9228f32c7b5 100644 --- a/tests/runner-e2e/chat-secret-proposal-card-scope.test.ts +++ b/tests/runner-e2e/chat-secret-proposal-card-scope.test.ts @@ -11,18 +11,19 @@ vi.mock("./chat-catalog-domains/external-support.js", () => ({ definition: () => import { secretProposalCardSelector } from "./chat-catalog-domains/external-secret-proposal.js"; afterEach(() => { document.body.innerHTML = ""; }); it("selects the actual expanded proposal card when the pending trigger has the same production interaction ID", () => { - document.body.innerHTML = ` -

Deployment publishing key

Morgan

env.PUBLISHING_API_KEY

-

Other request

`; - expect(document.querySelectorAll('[id="interaction-owned"]')).toHaveLength(2); + // The timeline trigger owns the page anchor id; the docked card carries only the interaction data attribute. + document.body.innerHTML = ` +

Deployment publishing key

Morgan

env.PUBLISHING_API_KEY

+

Other request

`; + expect(document.querySelectorAll('[data-interaction-id="owned"]')).toHaveLength(2); const selected = document.querySelectorAll(secretProposalCardSelector("owned")); expect(selected).toHaveLength(1); expect(selected[0]?.tagName).toBe("DIV"); for (const fact of ["Deployment publishing key", "Morgan", "env.PUBLISHING_API_KEY"]) expect(selected[0]?.textContent).toContain(fact); expect(document.querySelectorAll(secretProposalCardSelector("missing"))).toHaveLength(0); }); it("a trigger, unrelated card or completed receipt cannot stand in for the bound proposal card", () => { - document.body.innerHTML = ` -
Deployment publishing key Morgan env.PUBLISHING_API_KEY
-
Approved
`; + document.body.innerHTML = ` +
Deployment publishing key Morgan env.PUBLISHING_API_KEY
+
Approved
`; expect(document.querySelectorAll(secretProposalCardSelector("owned"))).toHaveLength(0); }); diff --git a/tests/runner-e2e/continuation-approval.spec.ts b/tests/runner-e2e/continuation-approval.spec.ts index 057797ea894..ce28312f9f3 100644 --- a/tests/runner-e2e/continuation-approval.spec.ts +++ b/tests/runner-e2e/continuation-approval.spec.ts @@ -24,10 +24,10 @@ async function fixture(request: APIRequestContext) { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify(card)); } else { res.writeHead(200, { "content-type": "text/html" }); - res.end(`
+ res.end(`

Approve revised plan

-
`); +
`); } }); await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); diff --git a/tests/runner-e2e/continuation-approval.ts b/tests/runner-e2e/continuation-approval.ts index fd52e06638b..5c9fd5369f6 100644 --- a/tests/runner-e2e/continuation-approval.ts +++ b/tests/runner-e2e/continuation-approval.ts @@ -64,7 +64,7 @@ export async function confirmContinuationPlan(input: { expect(current.id, "the reviewed confirmation must still be current").toBe(expected.id); expect(sameTarget(current.payload.target, expected.payload.target), "the reviewed plan must not change before approval").toBe(true); const card = input.page.getByTestId("task-chat-composer-takeover") - .locator(`[id=${JSON.stringify(`interaction-${current.id}`)}]`); + .locator(`[data-interaction-id=${JSON.stringify(current.id)}]`); await expect(card, "the exact current confirmation must be visible").toBeVisible(); await card.getByRole("button", { name: current.payload.acceptLabel ?? "Approve", exact: true }).click(); const saved = await pollUntil({ diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index a0598100a85..9d368ea2561 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -2049,7 +2049,7 @@ for (const execution of executions) { await expect(turnReply).toBeVisible({ timeout: reviewUiTimeout() }); const pendingReview = waitingState.interactions.find(isPendingWarmConfirmation)!; const reviewCard = page.getByTestId("task-chat-interaction") - .and(page.locator(`[id=${JSON.stringify(`interaction-${pendingReview.id}`)}]`)); + .and(page.locator(`[data-interaction-id=${JSON.stringify(pendingReview.id)}]`)); await expect(reviewCard).toBeVisible({ timeout: reviewUiTimeout() }); await expect(reviewCard.getByRole("button", { name: "Continue work", exact: true })) .toBeEnabled({ timeout: reviewUiTimeout() }); diff --git a/ui/src/components/task-chat/TaskChatInteractionCard.test.tsx b/ui/src/components/task-chat/TaskChatInteractionCard.test.tsx index 567ff0af2b4..c2ddbef6fe2 100644 --- a/ui/src/components/task-chat/TaskChatInteractionCard.test.tsx +++ b/ui/src/components/task-chat/TaskChatInteractionCard.test.tsx @@ -262,6 +262,32 @@ describe("TaskChatInteractionCard", () => { expect(preview).not.toMatch(/[`*_~[\]]|(^|\s)- /); }); + // Full cohort 11: the unanswered-question chip and the composer takeover showed the same + // pending question at once, both with id "interaction-", so the page had a duplicate id. + it("keeps the interaction id unique while its timeline chip and takeover card are both shown", () => { + const interaction = pendingAskUserQuestionsInteraction; + flushSync(() => { + root.render( + + + {}} /> +
+ +
+
+
, + ); + }); + const anchors = container.querySelectorAll(`[id="interaction-${interaction.id}"]`); + expect(anchors).toHaveLength(1); + expect(anchors[0]!.getAttribute("data-testid")).toBe("task-chat-unanswered-question"); + const takeover = container.querySelector(`[data-testid="task-chat-composer-takeover"] [data-interaction-id="${interaction.id}"]`); + expect(takeover).not.toBeNull(); + expect(takeover!.id).toBe(""); + const ids = [...container.querySelectorAll("[id]")].map(node => node.id); + expect(new Set(ids).size).toBe(ids.length); + }); + it("renders a plan confirmation as a linked document preview", () => { const interaction = createRequestConfirmation({ sourceRunId: "run-plan", diff --git a/ui/src/components/task-chat/TaskChatInteractionCard.tsx b/ui/src/components/task-chat/TaskChatInteractionCard.tsx index c32f8d0a618..41e8bb7fb93 100644 --- a/ui/src/components/task-chat/TaskChatInteractionCard.tsx +++ b/ui/src/components/task-chat/TaskChatInteractionCard.tsx @@ -50,8 +50,12 @@ export function TaskChatInteractionCard({ }: TaskChatInteractionCardProps) { const { enabled: coordinatorEnabled } = useCoordinatorWorkspaceEnabled(); const interaction = item.interaction; + // The timeline item owns the interaction's page anchor. The composer takeover + // shows the same interaction at the same time, so it carries only the data + // attribute: a DOM id must stay unique. + const anchorId = presentation === "takeover" ? undefined : `interaction-${interaction.id}`; if (coordinatorEnabled && interaction.kind === "suggest_tasks" && !shouldHideInteractionCard(interaction)) { - return
+ return
; } @@ -60,7 +64,7 @@ export function TaskChatInteractionCard({ return (