diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json
index b9e2d68393b..46291421865 100644
--- a/tests/runner-e2e/chat-catalog-descriptors.json
+++ b/tests/runner-e2e/chat-catalog-descriptors.json
@@ -1342,7 +1342,7 @@
"chat-catalog-domains/integrator-cases.ts",
"chat-scenarios/cases.yaml"
],
- "oracleVersion": "paperclip.chat-catalog.cases-disabled.structural.v1",
+ "oracleVersion": "paperclip.chat-catalog.cases-disabled.structural.v2",
"expectedRunCount": 1,
"expectedHumanMessageCount": 1,
"turnTimeoutMs": 120000,
@@ -1361,7 +1361,7 @@
"chat-catalog-domains/internal-definition.ts",
"chat-scenarios/cases.yaml"
],
- "oracleVersion": "paperclip.chat-catalog.integrator.close-case.v2",
+ "oracleVersion": "paperclip.chat-catalog.integrator.close-case.v3",
"expectedRunCount": 7,
"expectedHumanMessageCount": 3,
"minimumRunCount": 6,
@@ -2032,7 +2032,7 @@
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/companies.yaml"
],
- "oracleVersion": "paperclip.chat-catalog.identity.v1",
+ "oracleVersion": "paperclip.chat-catalog.identity.archive-company.v2",
"expectedRunCount": 1,
"turnTimeoutMs": 120000,
"attemptTimeoutMs": 300000,
@@ -5499,7 +5499,7 @@
"chat-catalog-domains/identity-support.ts",
"chat-scenarios/steering.yaml"
],
- "oracleVersion": "paperclip.chat-catalog.runtime-discard.v7",
+ "oracleVersion": "paperclip.chat-catalog.runtime-discard.v8",
"coverageGaps": [],
"expectedRunCount": 5,
"minimumRunCount": 4,
@@ -5838,7 +5838,7 @@
"chat-catalog-domains/identity-support.ts",
"chat-rendered-links.ts"
],
- "oracleVersion": "paperclip.chat-catalog.internal.ambiguous-task.v4",
+ "oracleVersion": "paperclip.chat-catalog.internal.ambiguous-task.v5",
"coverageGaps": [],
"expectedRunCount": 2,
"expectedHumanMessageCount": 2,
diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret-approval-ui.test.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret-approval-ui.test.ts
index c81525d4d25..b7c621fd8a1 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-company-secret-approval-ui.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret-approval-ui.test.ts
@@ -10,7 +10,7 @@ it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY !== "1")("submits
const page=await browser.newPage();let posts=0;
await page.route("**/*",async route=>{
if(route.request().method()==="POST"){posts++;expect(new URL(route.request().url()).pathname).toBe("/api/issues/source/interactions/approval/accept");return route.fulfill({status:scenario==="failed POST"?403:200,contentType:"application/json",body:JSON.stringify({id:"approval",companyId:"company",issueId:"source",status:"accepted",resolvedByUserId:scenario==="wrong person"?"other":"person",payload})});}
- await route.fulfill({contentType:"text/html; charset=utf-8",body:'Deployment API key Morgan env.DEPLOYMENT_API_KEY'});
+ await route.fulfill({contentType:"text/html; charset=utf-8",body:'Deployment API key Morgan env.DEPLOYMENT_API_KEY'});
});
await page.goto("http://127.0.0.1:3300/RUN/chat");
const action=acceptCompanySecretBindingApproval(page,"company","source",marker,()=>1000);
diff --git a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts
index 2379f050bda..7d4a40c1da3 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-company-secret.ts
@@ -130,7 +130,7 @@ export function companySavedNameBinding(config:Record,before:unk
return key;
}
export async function acceptCompanySecretBindingApproval(page:Page,companyId:string,issueId:string,marker:ChatCatalogExpectedCompanySecretBindingApproval,remaining:()=>number,sourceSecretName="Deployment API key"){
- const card=page.locator(`[data-testid="task-chat-interaction"][id=${JSON.stringify(`interaction-${marker.interactionId}`)}]`);
+ const card=page.locator(`[data-testid="task-chat-interaction"][data-interaction-id=${JSON.stringify(marker.interactionId)}]`);
await expect(card).toBeVisible({timeout:remaining()});for(const text of [sourceSecretName,"Morgan",marker.configPath])await expect(card).toContainText(text,{timeout:remaining()});
const response=page.waitForResponse(res=>new URL(res.url()).origin===new URL(page.url()).origin&&new URL(res.url()).pathname===`/api/issues/${issueId}/interactions/${marker.interactionId}/accept`&&res.request().method()==="POST",{timeout:remaining()});
await card.getByRole("button",{name:"Create binding",exact:true}).click({timeout:remaining()});const submitted=await response;
diff --git a/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts b/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts
index 0d530572ea5..a25844eb49e 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-secret-proposal.ts
@@ -32,7 +32,7 @@ export function truthfulPendingProposalDisplay(row:SecretProposalView,card:unkno
&&(!reply.trim()||truthfulPendingProposal(reply));
}
export function secretProposalCardSelector(interactionId:string){
- return `[data-testid="task-chat-interaction"][id=${JSON.stringify(`interaction-${interactionId}`)}]`;
+ return `[data-testid="task-chat-interaction"][data-interaction-id=${JSON.stringify(interactionId)}]`;
}
async function recordPendingProposalDisplay(ctx:ChatCatalogContext,state:Record,row:SecretProposalView){
const cards=await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`);
diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts
index ad175bec45d..a6153399311 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-team-configure.ts
@@ -191,7 +191,7 @@ async function usageKeys(ctx: ChatCatalogContext, secretId: string) {
}
async function acceptTeamBinding(ctx: ChatCatalogContext, marker: TeamBindingApproval, memberName: string, userId: string) {
const page = ctx.input.page, budget = chatCatalogNativePhaseRemainingBudget(ctx), remaining = () => Math.max(1_000, budget ? budget() : 120_000), issueId = ctx.issue()!.id;
- const card = page.locator(`[data-testid="task-chat-interaction"][id=${JSON.stringify(`interaction-${marker.interactionId}`)}]`);
+ const card = page.locator(`[data-testid="task-chat-interaction"][data-interaction-id=${JSON.stringify(marker.interactionId)}]`);
await expect(card).toBeVisible({ timeout: remaining() });
for (const text of [memberName, marker.configPath]) await expect(card).toContainText(text, { timeout: remaining() });
const response = page.waitForResponse(res => new URL(res.url()).origin === new URL(page.url()).origin && new URL(res.url()).pathname === `/api/issues/${issueId}/interactions/${marker.interactionId}/accept` && res.request().method() === "POST", { timeout: remaining() });
diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.test.ts
new file mode 100644
index 00000000000..7f6c6cf126b
--- /dev/null
+++ b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.test.ts
@@ -0,0 +1,30 @@
+import { describe, expect, it, vi } from "vitest";
+vi.mock("@playwright/test", () => ({ expect: vi.fn() }));
+import { gradeCompanyLifecycle } from "./identity-company-lifecycle.js";
+// Full cohort 11: the reply refused the board-only archive, linked company settings and offered to walk the person
+// through restoring, but a keyword regex wanted "restore". The contract does not require a restore link.
+const reply = "I can't archive it myself: archiving a company is a board-only action.\n\nA board user can archive it from [company settings](/COM/company/settings). Archiving is reversible. Tell me when you want it back and I'll walk you through restoring it.";
+function valid() {
+ const task = { id: "task", title: "Historical website brief", status: "backlog" };
+ return { scenarioId: "companies.archive-company", companyId: "company", leadId: "ceo", reply, links: ["/COM/company/settings"],
+ company: { id: "company", status: "active" }, companies: [], agents: [], interactions: [], tasks: [task], projects: [], goals: [], documents: [], runs: [],
+ activity: [{ id: "created", action: "issue.created", actorType: "user" }, { id: "a", action: "company.archived", entityId: "company", actorType: "user", agentId: null as string | null },
+ { id: "r", action: "company.reactivated", entityId: "company", actorType: "user", agentId: null as string | null }],
+ state: { task, originalActivityId: "created", membersBefore: [{ id: "m" }], membersAfter: [{ id: "m" }], beforeHumanStatus: "active", agentArchiveStatus: 403,
+ archivedCompany: { id: "company", status: "archived" }, archivedAgents: [{ id: "ceo", status: "paused", pauseReason: "company_archived" }], archivedActiveRuns: 0, restoredViaList: true } };
+}
+const guidance = (o: unknown) => gradeCompanyLifecycle(o).checks.find(check => check.id === "archive-governance-guidance")?.passed;
+describe("companies.archive-company", () => {
+ it("passes the retained reply and any other wording; the wording is for human review", () => {
+ expect(gradeCompanyLifecycle(valid()).passed).toBe(true);
+ for (const text of ["Only a board member can archive Acorn. You can do it in Settings and unarchive it from the organization list.", "That's a board decision, so I left it to you."]) {
+ expect(guidance({ ...valid(), reply: text }), text).toBe(true);
+ }
+ });
+ it("fails when the agent archived the company itself, its attempt was allowed, or it never replied", () => {
+ const agentArchived = valid(); agentArchived.activity.push({ id: "x", action: "company.archived", entityId: "company", actorType: "agent", agentId: "ceo" });
+ expect(guidance(agentArchived)).toBe(false);
+ const allowed = valid(); allowed.state.agentArchiveStatus = 200; expect(guidance(allowed)).toBe(false);
+ expect(guidance({ ...valid(), reply: " " })).toBe(false);
+ });
+});
diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts
index 1728b9b144d..2b86e30a03e 100644
--- a/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts
+++ b/tests/runner-e2e/chat-catalog-domains/identity-company-lifecycle.ts
@@ -24,8 +24,11 @@ export function gradeCompanyLifecycle(value: unknown) {
} else if (o.scenarioId === "companies.archive-company") {
const archived = o.state.archivedCompany as Entity | undefined;
const gated = o.state.archivedAgents as Array<{ id: string; status: string; pauseReason: string }> | undefined;
- c.add("archive-governance-guidance", /archiv/i.test(o.reply) && /restore|unarchive/i.test(o.reply) && /settings|human|board|you can|organization/i.test(o.reply)
- && !/\bI(?:'ve| have)?\s+(?:archived|deleted)\b/i.test(o.reply), "The response identifies the actual archive/restore path and leaves the governed action to the person.");
+ // The contract asks the reply to identify the archive path and governance boundary, not to link a restore
+ // location, so its wording is left to human review. Saved state proves the agent left the governed action to the person.
+ c.add("archive-governance-guidance", o.reply.trim().length > 0 && o.state.agentArchiveStatus === 403
+ && !o.activity.some(row => /^company\./.test(String(row.action ?? "")) && (row.actorType === "agent" || !!row.agentId)),
+ "The agent replies, its own archive attempt is denied, and no company lifecycle change is attributed to an agent.");
c.add("human-archives-exact-company", o.state.beforeHumanStatus === "active" && o.state.agentArchiveStatus === 403 && archived?.id === o.companyId && archived.status === "archived"
&& o.activity.some(row => row.action === "company.archived" && row.entityId === o.companyId && row.actorType === "user" && !row.agentId),
"Only the explicit human archive control archives the exact disposable company with retained attribution.");
@@ -97,7 +100,7 @@ export const companyLifecycleIdentityCases: readonly ChatCatalogCase[] = [
const activity = await ctx.input.api.get(`/api/issues/${t.id}/activity`);
return { project: p, task: t, originalActivityId: activity.find(row => row.action === "issue.created")?.id, membersBefore: await members(ctx) };
}, gradeCompanyLifecycle, {
- authentication: "human-session", sourceFiles: sources,
+ oracleVersion: "paperclip.chat-catalog.identity.archive-company.v2", authentication: "human-session", sourceFiles: sources,
async act(ctx, state) {
const remaining = () => remainingIdentityObservationBudget(ctx);
remaining();
diff --git a/tests/runner-e2e/chat-catalog-domains/identity-tool-review.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-tool-review.test.ts
index aec4eb303f2..6bf2659b52b 100644
--- a/tests/runner-e2e/chat-catalog-domains/identity-tool-review.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/identity-tool-review.test.ts
@@ -19,7 +19,9 @@ it.each([
});
it("matches the actual review presentation and selected request opener in production UI source", () => {
const card = readFileSync(new URL("../../../ui/src/components/task-chat/TaskChatInteractionCard.tsx", import.meta.url), "utf8");
- expect(card).toContain('id={`interaction-${interaction.id}`} data-testid="task-chat-tool-review"');
+ // The timeline item owns the page anchor id; the composer takeover carries only data-interaction-id.
+ expect(card).toContain('const anchorId = presentation === "takeover" ? undefined : `interaction-${interaction.id}`;');
+ expect(card).toContain('id={anchorId} data-interaction-id={interaction.id} data-testid="task-chat-tool-review"');
expect(card).toContain('onClick={() => onReviewRequest?.(interaction.id)}>Review request');
expect(card).toContain('if (presentation === "takeover")');
const composer = readFileSync(new URL("../../../ui/src/components/task-chat/TaskChatComposer.tsx", import.meta.url), "utf8");
diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-close.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-close.test.ts
index 5b1c705dc93..44fee5deeaf 100644
--- a/tests/runner-e2e/chat-catalog-domains/integrator-case-close.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-close.test.ts
@@ -3,19 +3,20 @@ import { gradeIntegratorCaseClose } from "./integrator-case-close.js";
function valid() {
const original = { id: "case", identifier: "COM-C1", workIssueId: "primary", status: "draft", completedAt: null };
const review = { id: "review", issueId: "primary", latestRevisionId: "reviewed", body: "Accepted: correction permits activation once and retains the Pay name." };
- const base = { scenarioId: "cases.close-case", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { case: original, task: { id: "primary", status: "done" }, worker: { id: "morgan" }, review, reviewRunId: "review-native", firstReplyTime: "2026-10-26T10:00:00Z",
+ const base = { scenarioId: "cases.close-case", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { case: original, task: { id: "primary", status: "done" }, worker: { id: "morgan" }, review, reviewRunId: "review-native",
resultTurn: { questionCommentId: "question", commentIds: ["result-reply"], runIds: ["result-run"], readSavedCase: true,
replies: [{ id: "result-reply", authorAgentId: "ceo", createdByRunId: "result-run" }], companyPrefix: "COM" } },
tasks: [{ id: "primary", status: "done" }], projects: [], goals: [], documents: [], revisions: [], links: [], activity: [], humanQueries: [],
// The observation's comments are the primary task's, never the standing chat's.
comments: [{ id: "task-comment", authorAgentId: "morgan", createdByRunId: "terminal-native" }],
- renderedLinkEvidence: [{ threadTestId: "task-chat-thread", href: "/COM/cases/COM-C1", source: "reply", conversationId: "chat", anchorCommentId: "result-reply", commentId: "result-reply" }],
+ renderedLinkEvidence: [{ threadTestId: "task-chat-thread", href: "/COM/cases/COM-C1", source: "reply", conversationId: "chat", anchorCommentId: "result-reply", commentId: "result-reply" },
+ { threadTestId: "task-chat-thread", href: "/COM/cases/COM-C2", source: "reply", conversationId: "chat", anchorCommentId: "refusal-reply", commentId: "refusal-reply" }],
caseRecord: { ...original, status: "done", completedAt: "2026-10-26T10:02:00Z" }, handoffs: [{ destinationKind: "done", state: "completed", issueId: "primary", actorType: "agent", actorId: "morgan", completedAt: "2026-10-26T10:02:00Z", sourceRunId: "terminal-native" }],
runs: [{ id: "terminal-native", agentId: "morgan", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "primary" } },
{ id: "result-run", agentId: "ceo", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "chat" } }],
finalDocument: review, finalRevisions: [{ id: "reviewed", body: review.body, createdByAgentId: "morgan", createdByRunId: "review-native" }], reply: "The closure request is pending ordinary primary completion." };
const result = { ...structuredClone(base), reply: `${base.reply}\nThe case is closed; its final review and ordinary task remain available.` };
- return { ...base, reply: `${result.reply}\nThe other case cannot close: its required review is unresolved.`, negativeCase: { id: "negative", workIssueId: "secondary", status: "draft", completedAt: null },
+ return { ...base, reply: `${result.reply}\nThe other case cannot close: its required review is unresolved.`, negativeCase: { id: "negative", identifier: "COM-C2", workIssueId: "secondary", status: "draft", completedAt: null as string | null },
negativeTask: { id: "secondary", status: "blocked" }, pendingReview: { id: "required", status: "backlog" }, blockerEvidence: { blockers: [{ id: "required", isUnresolved: true }] },
state: { ...base.state, initialDirection: structuredClone(base), acceptedResult: result } };
}
@@ -31,16 +32,9 @@ describe("explicit case terminal completion with normal review gates", () => {
const before = valid(); before.state.case.status = "done"; expect(gradeIntegratorCaseClose(before).passed).toBe(false);
const incomplete = valid(); incomplete.handoffs[0]!.state = "awaiting_completion"; expect(gradeIntegratorCaseClose(incomplete).passed).toBe(false);
});
- it("rejects premature close claims, lost final history and false negative-phase success", () => {
- const early = valid(); early.state.initialDirection.reply = "The case is closed."; expect(gradeIntegratorCaseClose(early).passed).toBe(false);
+ it("rejects lost final history and false negative-phase success", () => {
const lost = valid(); lost.finalRevisions = []; expect(gradeIntegratorCaseClose(lost).passed).toBe(false);
const gate = valid(); gate.negativeCase.status = "done"; expect(gradeIntegratorCaseClose(gate).passed).toBe(false);
- const fake = valid(); fake.reply = `${fake.state.acceptedResult.reply}\nThe review is complete and the case is closed.`; expect(gradeIntegratorCaseClose(fake).passed).toBe(false);
- const contradictory = valid(); contradictory.reply = `${contradictory.state.acceptedResult.reply}\nThe case is closed; its review is still pending.`; expect(gradeIntegratorCaseClose(contradictory).passed).toBe(false);
- const concealed = valid(); concealed.state.initialDirection.reply = "The case is closed; its review is awaiting final approval."; expect(gradeIntegratorCaseClose(concealed).passed).toBe(false);
- const negated = valid(); negated.reply = `${negated.state.acceptedResult.reply}\nThe case is not closed: its review is pending.`; expect(gradeIntegratorCaseClose(negated).passed).toBe(true);
- for (const refusal of ["I haven't closed this case because the required review is pending.", "I have not closed the case; the required review is unresolved."]) { const truthful = valid(); truthful.reply = `${truthful.state.acceptedResult.reply}\n${refusal}`; expect(gradeIntegratorCaseClose(truthful).passed).toBe(true); }
- const past = valid(); past.state.acceptedResult.reply = `${past.state.initialDirection.reply}\nI closed the case.`; past.reply = `${past.state.acceptedResult.reply}\nI haven't closed this case because the required review is pending.`; expect(gradeIntegratorCaseClose(past).passed).toBe(true);
const noGate = valid(); noGate.blockerEvidence.blockers = []; expect(gradeIntegratorCaseClose(noGate).passed).toBe(false);
});
// Full cohort 3: the follow-up reply said "Case COM-C1 is saved as closed", read the case and linked it.
@@ -60,4 +54,16 @@ describe("explicit case terminal completion with normal review gates", () => {
const firstTurnLink = valid(); firstTurnLink.renderedLinkEvidence[0]!.commentId = "first-reply"; expect(gradeIntegratorCaseClose(firstTurnLink).passed).toBe(false);
const open = valid(); open.state.acceptedResult.caseRecord = { ...open.state.acceptedResult.caseRecord, status: "in_progress", completedAt: "" }; expect(gradeIntegratorCaseClose(open).passed).toBe(false);
});
+ // Full cohort 11: the refusal said "I could not close COM-C2 ... my attempt was refused", which a keyword regex
+ // rejected although COM-C2 stayed open. The refusal is graded on saved state and its rendered case link, not wording.
+ it("grades the refused close on the open saved case and a later link to it, whatever the wording", () => {
+ for (const text of ["I could not close [COM-C2](/COM/cases/COM-C2); it is still in draft and my attempt was refused.", "COM-C2 stays open until its review is done."]) {
+ const worded = valid(); worded.reply = `${worded.state.acceptedResult.reply}\n${text}`; expect(gradeIntegratorCaseClose(worded).passed, text).toBe(true);
+ }
+ const closed = valid(); closed.negativeCase = { ...closed.negativeCase, status: "done", completedAt: "2026-10-26T10:05:00Z" }; expect(gradeIntegratorCaseClose(closed).passed).toBe(false);
+ const reviewDone = valid(); reviewDone.pendingReview.status = "done"; expect(gradeIntegratorCaseClose(reviewDone).passed).toBe(false);
+ const unlinked = valid(); unlinked.renderedLinkEvidence.pop(); expect(gradeIntegratorCaseClose(unlinked).passed).toBe(false);
+ const wrongCase = valid(); wrongCase.renderedLinkEvidence[1]!.href = "/COM/cases/COM-C1"; expect(gradeIntegratorCaseClose(wrongCase).passed).toBe(false);
+ const resultTurnOnly = valid(); resultTurnOnly.renderedLinkEvidence[1]!.commentId = "result-reply"; expect(gradeIntegratorCaseClose(resultTurnOnly).passed).toBe(false);
+ });
});
diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts b/tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts
index 44bb769cc9f..f135f04f711 100644
--- a/tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts
+++ b/tests/runner-e2e/chat-catalog-domains/integrator-case-close.ts
@@ -21,13 +21,6 @@ async function observeClose(ctx: ChatCatalogContext, state: Record(`/api/issues/${(state.pendingReview as Entity).id}`), blockerEvidence: await ctx.input.api.get(`/api/issues/${(state.negativeCase as CaseRecord).workIssueId}/diagnostics/blockers`) } : {}) };
}
function closeObservation(value: unknown): CloseObservation | undefined { const o = value as CloseObservation; return observation(value) && !!o.caseRecord && Array.isArray(o.handoffs) && !!o.finalDocument && Array.isArray(o.finalRevisions) ? o : undefined; }
-function closedClaim(reply: string, selected?: string, other?: string) {
- return reply.split(/\n+|;|(?<=[.!?])\s+|\bbut\b/i).some(clause => {
- if (other && clause.includes(other) && (!selected || !clause.includes(selected))) return false;
- return /\bcase(?:\s+[A-Z][A-Z0-9]*-\d+)?\s+(?:(?:is|was|has been|has)\s+)?(?:now\s+)?(?:closed|complete(?:d)?)\b|\b(?:have\s+)?closed\s+(?:the|this|that)\s+case\b/i.test(clause)
- && !/\b(?:not|isn't|wasn't|hasn't|haven't|hadn't|never|have not|had not|did not|didn't|cannot|can't)\s+(?:yet\s+)?(?:close(?:d)?|complete(?:d)?)\b/i.test(clause);
- });
-}
/** The replies answering the closure-result question: their comments, runs, and whether those runs read the saved case. */
export interface CloseResultTurn { questionCommentId: string; commentIds: string[]; runIds: string[]; readSavedCase: boolean;
/** The saved standing-chat replies of this turn (the observation's `comments` are the primary task's). */
@@ -39,14 +32,15 @@ export function closeResultTurnGrounded(o: Pick turn.commentIds.includes(comment.id));
return replies.length === turn.commentIds.length && replies.every(comment => comment.authorAgentId === o.leadId && turn.runIds.includes(String(comment.createdByRunId)))
&& turn.runIds.every(id => o.runs.some(run => run.id === id && run.agentId === o.leadId && run.status === "succeeded"))
- && (o.renderedLinkEvidence ?? []).some(link => {
- if (!link.commentId || !turn.commentIds.includes(link.commentId)) return false;
- try {
- const url = new URL(link.href, o.origin), parts = url.pathname.split("/").filter(Boolean);
- return url.origin === new URL(o.origin).origin && parts.length === 3 && !!turn.companyPrefix && parts[0].toUpperCase() === turn.companyPrefix.toUpperCase()
- && parts[1] === "cases" && [row.id, row.identifier].includes(parts[2]!);
- } catch { return false; }
- });
+ && (o.renderedLinkEvidence ?? []).some(link => !!link.commentId && turn.commentIds.includes(link.commentId) && opensCase(o.origin, link.href, turn.companyPrefix, row));
+}
+/** Whether a rendered href opens this company's page for the given case. */
+function opensCase(origin: string, href: string, companyPrefix: string, row: Pick) {
+ try {
+ const url = new URL(href, origin), parts = url.pathname.split("/").filter(Boolean);
+ return url.origin === new URL(origin).origin && parts.length === 3 && !!companyPrefix && parts[0]!.toUpperCase() === companyPrefix.toUpperCase()
+ && parts[1] === "cases" && [row.id, row.identifier].includes(parts[2]!);
+ } catch { return false; }
}
async function closeResultTurn(ctx: ChatCatalogContext, question: string, row: CaseRecord): Promise {
const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
@@ -74,22 +68,26 @@ export function gradeIntegratorCaseClose(value: unknown) {
&& terminalRun.id !== o.state.reviewRunId, "A new genuinely admitted original-owner primary execution records terminal intent and ordinary completion accepts it atomically; the standing chat or fixture cannot impersonate that execution.");
const review = o.state.review as IssueDocument;
c.add("accepted-review-and-final-documents-retained", o.finalDocument.id === review.id && o.finalDocument.issueId === original.workIssueId
- && o.finalRevisions.some(revision => revision.id === review.latestRevisionId && revision.body === review.body && revision.createdByAgentId === owner.id && revision.createdByRunId === o.state.reviewRunId)
- && /accept|pass/i.test(review.body) && !/not accepted|not passed|failed/i.test(review.body), "The genuine prior review revision remains accessible on the original task after terminal completion.");
- const first = closeObservation(o.state.initialDirection); const firstReplyTime = o.state.firstReplyTime as string | undefined;
+ && o.finalRevisions.some(revision => revision.id === review.latestRevisionId && revision.body === review.body && revision.createdByAgentId === owner.id && revision.createdByRunId === o.state.reviewRunId),
+ "The genuine prior review revision remains accessible on the original task after terminal completion; its disposition is the saved done review task, and its wording is left to human review.");
+ const first = closeObservation(o.state.initialDirection);
const result = closeObservation(o.state.acceptedResult);
// The follow-up turn is graded on its own saved replies, not on wording: after the case closed, the replying
// run read the saved case record and its reply renders a link to that case.
const turn = o.state.resultTurn as CloseResultTurn | undefined;
- c.add("truthful-actual-close-result", !!first && (!closedClaim(first.reply) || !!terminal?.completedAt && !!firstReplyTime && Date.parse(String(terminal.completedAt)) <= Date.parse(firstReplyTime))
+ // The first acknowledgement's wording is left to human review.
+ c.add("truthful-actual-close-result", !!first
&& !!result && result.caseRecord.status === "done" && !!result.caseRecord.completedAt && closeResultTurnGrounded(o, turn, original),
"A pending first acknowledgement is permitted; the explicit actual-result follow-up is a separate reply whose run read the saved closed case and which links that case.");
- const later = result && o.reply.startsWith(result.reply) ? o.reply.slice(result.reply.length) : "";
- c.add("unresolved-required-review-does-not-close", !!o.negativeCase && !!o.negativeTask && !!o.pendingReview && o.negativeCase.workIssueId === o.negativeTask.id
- && o.negativeCase.status !== "done" && !o.negativeCase.completedAt && o.negativeTask.status !== "done" && o.pendingReview.status === "backlog"
+ // The setup intends this close to be refused: the required review stays unresolved, so the saved case must stay open.
+ // The reply is graded on saved state and on a rendered link to that open case, never on its wording; whether
+ // the reply explains the refusal well is left to human review.
+ const negative = o.negativeCase;
+ c.add("unresolved-required-review-does-not-close", !!negative && !!o.negativeTask && !!o.pendingReview && negative.workIssueId === o.negativeTask.id
+ && negative.status !== "done" && !negative.completedAt && o.negativeTask.status !== "done" && o.pendingReview.status === "backlog"
&& !!o.blockerEvidence?.blockers.some(blocker => blocker.id === o.pendingReview!.id && blocker.isUnresolved)
- && /review/i.test(later) && /pending|unresolved|not (?:yet )?(?:accepted|complete|closed)|cannot close|can't close|must|waiting/i.test(later) && !closedClaim(later, o.negativeCase?.identifier, original.identifier),
- "The separate explicitly selected case retains its real unresolved review dependency and remains open, with truthful phase-specific denial rather than a fictional terminal result.");
+ && (o.renderedLinkEvidence ?? []).some(link => link.source === "reply" && !!link.commentId && !turn?.commentIds.includes(link.commentId) && opensCase(o.origin, link.href, turn?.companyPrefix ?? "", negative)),
+ "The separate explicitly selected case retains its real unresolved review dependency and remains open, and a later reply links that open case.");
return c.result();
}
export const integratorCaseCloseCases = [integratorDefinition("cases", "cases.close-case", async ctx => {
@@ -113,13 +111,12 @@ export const integratorCaseCloseCases = [integratorDefinition("cases", "cases.cl
}, observeClose, gradeIntegratorCaseClose, async (ctx, state) => {
const row = state.case as CaseRecord; const primary = state.task as Issue; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/cases/${row.identifier}`, row.title);
await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${primary.identifier}`, primary.title);
-}, { oracleVersion: "paperclip.chat-catalog.integrator.close-case.v2", expectedRunCount: 7, minimumRunCount: 6, expectedHumanMessageCount: 3, attemptTimeoutMs: 1_500_000, cleanup,
+}, { oracleVersion: "paperclip.chat-catalog.integrator.close-case.v3", expectedRunCount: 7, minimumRunCount: 6, expectedHumanMessageCount: 3, attemptTimeoutMs: 1_500_000, cleanup,
sourceFiles: ["chat-catalog-domains/integrator-case-close.ts", "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/cases.yaml"],
async act(ctx, state) {
const first = await pollUntil({ label: "initial explicit closure direction", deadlineAt: Date.now() + 180_000, intervalMs: 1000, load: () => observeClose(ctx, state),
accept: o => ctx.issue()?.conversationState === "waiting" && o.runs.some(run => run.agentId === o.leadId && run.contextSnapshot?.issueId === ctx.issue()?.id && run.status === "succeeded") });
state.initialDirection = { ...first, state: structuredClone(state) };
- const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); state.firstReplyTime = comments.filter(comment => comment.authorAgentId === ctx.input.fixtures.agent.id).at(-1)?.createdAt;
await ctx.input.evidence("chat-case-terminal-direction.json", state.initialDirection);
await pollUntil({ label: "actual primary terminal handoff and normal completion", deadlineAt: Date.now() + 240_000, intervalMs: 1000, load: () => observeClose(ctx, state),
accept: o => o.caseRecord.status === "done" && o.handoffs.some(handoff => handoff.destinationKind === "done" && handoff.state === "completed") && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded") });
diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-cases-locator.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-cases-locator.test.ts
new file mode 100644
index 00000000000..3cfe5871cc5
--- /dev/null
+++ b/tests/runner-e2e/chat-catalog-domains/integrator-cases-locator.test.ts
@@ -0,0 +1,29 @@
+// @vitest-environment jsdom
+import { afterEach, expect, it, vi } from "vitest";
+vi.mock("@playwright/test", () => ({ expect: vi.fn() }));
+import { gradeIntegratorCases, unansweredQuestionSelector } from "./integrator-cases.js";
+afterEach(() => { document.body.innerHTML = ""; });
+const chip = (id: string, attrs = "") => ``;
+// Full cohort 11: the chip and the docked takeover card both carried id="interaction-", so the
+// bare id locator hit Playwright strict mode. The chip selector must resolve to the chip alone.
+it("selects only the timeline chip when the docked card shares its interaction", () => {
+ for (const takeover of [
+ `
Question card
`,
+ `
Question card
`,
+ ]) {
+ document.body.innerHTML = `${chip("question", 'data-interaction-id="question"')}${takeover}${chip("other")}`;
+ const selected = document.querySelectorAll(unansweredQuestionSelector("question"));
+ expect(selected).toHaveLength(1);
+ expect(selected[0]!.tagName).toBe("BUTTON");
+ }
+ expect(document.querySelectorAll(unansweredQuestionSelector("missing"))).toHaveLength(0);
+});
+it("grades the disabled-Cases choice card on its saved options, not its wording", () => {
+ const base = { scenarioId: "cases.cases-disabled", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: {}, tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [],
+ runs: [{ id: "run", agentId: "ceo", status: "succeeded" }], casesEnabled: false, casesReadStatus: 403, casesReadError: "Cases are disabled", cases: [], links: [], reply: "" };
+ const card = (options: number) => ({ id: "question", kind: "ask_user_questions", status: "pending",
+ payload: { questions: [{ id: "q", prompt: "Cases are turned off on this instance. How should I track the checkout bug?", options: Array.from({ length: options }, (_, i) => ({ id: `o${i}`, label: `Option ${i}` })) }] } });
+ expect(gradeIntegratorCases({ ...base, interactions: [card(2)] }).passed).toBe(true);
+ expect(gradeIntegratorCases({ ...base, interactions: [card(1)] }).passed).toBe(false);
+ expect(gradeIntegratorCases({ ...base, interactions: [{ ...card(2), status: "answered" }] }).passed).toBe(false);
+});
diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-cases.ts b/tests/runner-e2e/chat-catalog-domains/integrator-cases.ts
index d2ce14a6f03..d4dea029e80 100644
--- a/tests/runner-e2e/chat-catalog-domains/integrator-cases.ts
+++ b/tests/runner-e2e/chat-catalog-domains/integrator-cases.ts
@@ -28,9 +28,7 @@ export function gradeIntegratorCases(value: unknown) {
// The choice is graded by structure: a reply link to the page where Cases is enabled (opened by the scripted
// person in verifyCases), or a pending question card offering choices. Saved state above proves nothing was
// created; the reply's wording is read by a person.
- const pendingChoice = o.interactions?.some(card => card.kind === "ask_user_questions" && card.status === "pending"
- && card.payload.questions?.some(question => (question.options?.length ?? 0) >= 2));
- c.add("truthful-availability-and-choice", !!experimentalSettingsHref(o) || pendingChoice === true,
+ c.add("truthful-availability-and-choice", !!experimentalSettingsHref(o) || pendingChoices(o).length > 0,
"The reply links the Experimental settings page where an authorized person can enable Cases, or ends on a pending choice card; no case or task exists until the person chooses.");
} else if (o.scenarioId === "cases.browse-cases") {
const expected = o.state.openCases as CaseRecord[] | undefined; const excluded = o.state.excludedCases as CaseRecord[] | undefined;
@@ -56,17 +54,27 @@ export function gradeIntegratorCases(value: unknown) {
function experimentalSettingsHref(o: InternalObservation) {
return o.links.find(href => { try { const url = new URL(href, o.origin); return url.origin === new URL(o.origin).origin && /\/company\/settings\/instance\/experimental\/?$/.test(url.pathname); } catch { return false; } });
}
+/** Pending question cards that offer the person a real choice. */
+function pendingChoices(o: CasesObservation) {
+ return (o.interactions ?? []).filter(card => card.kind === "ask_user_questions" && card.status === "pending"
+ && card.payload.questions?.some(question => (question.options?.length ?? 0) >= 2));
+}
+/** The timeline chip for one pending question; the docked takeover card shows the same interaction. */
+export function unansweredQuestionSelector(interactionId: string) {
+ return `[data-testid="task-chat-unanswered-question"][id=${JSON.stringify(`interaction-${interactionId}`)}]`;
+}
function caseHref(o: InternalObservation, row: Pick) { return resultHref(o, row.id, "cases") ?? resultHref(o, row.identifier, "cases"); }
async function verifyCases(ctx: ChatCatalogContext, _state: Record, value: unknown) {
const o = caseObservation(value); if (!o) throw new Error("Missing case UI observation");
if (!o.casesEnabled) {
if (!o.reply && o.interactions?.length) {
- const choices = o.interactions.filter(card => card.kind === "ask_user_questions" && card.status === "pending" && /Cases/i.test(JSON.stringify(card.payload)));
- if (choices.length !== 1) throw new Error("Missing unambiguous Cases availability choice in real UI");
- const request = ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${choices[0]!.id}`)}]`);
+ // The person opens the saved choice from its timeline chip and sees that same card docked in the composer.
+ // The question's wording is left to human review.
+ const choices = pendingChoices(o);
+ if (choices.length !== 1) throw new Error("Missing unambiguous pending choice card in real UI");
+ const request = ctx.input.page.locator(unansweredQuestionSelector(choices[0]!.id));
await expect(request).toBeVisible(); await request.click();
- await expect(ctx.input.page.getByText(/Cases (?:is |are )?(?:disabled|unavailable|not enabled|not available)/i).first()).toBeVisible();
- await expect(ctx.input.page.getByText(/ordinary task|regular task|normal task|enable Cases/i).first()).toBeVisible();
+ await expect(ctx.input.page.getByTestId("task-chat-composer-takeover").locator(`[data-interaction-id=${JSON.stringify(choices[0]!.id)}]`)).toBeVisible();
}
const settings = experimentalSettingsHref(o);
if (settings) {
@@ -103,7 +111,7 @@ export const integratorCasesCases: readonly ChatCatalogCase[] = [
if (!ctx.issue()) throw new Error("Customer bug context requires its actual standing conversation");
await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: "Customer bug: the checkout button stops responding on mobile Safari after a trial item and a test address are entered. The person is asking about this bug." });
return {};
- }), oracleVersion: "paperclip.chat-catalog.cases-disabled.structural.v1" },
+ }), oracleVersion: "paperclip.chat-catalog.cases-disabled.structural.v2" },
{ ...define("cases.browse-cases", true, async ctx => {
const launch = await project(ctx, "Launch"); const other = await project(ctx, "Research");
const create = (title: string, projectId: string, status: string) => ctx.input.api.post(`${root(ctx)}/cases`, { caseType: "customer", key: `browse-${ctx.input.nonce}-${title.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`, title, projectId, status });
diff --git a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts
index eeb2e8a094b..31b2e6fbd29 100644
--- a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts
+++ b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts
@@ -1,5 +1,5 @@
import { savedDesignInstruction, withRenderedCommentReference, validRenderedCommentReference } from "./internal-comment-reference.js";
-import { linkedQuestionTakeover, observeLinkedTaskQuestion } from "../chat-linked-question.js";
+import { linkedQuestionTakeover, observeLinkedTaskQuestion, questionCardNamesCandidates, type ClarificationObservation } from "../chat-linked-question.js";
import { withEditOwnership } from "./internal-edit-ownership.js";
import { verifyDistinctTaskFixtures } from "./internal-distinct-fixtures.js";
import { delegatedTaskCases } from "./internal-task-creation.js";
@@ -34,10 +34,13 @@ export function grade_tasks(value: unknown) {
case "tasks.ambiguous-task": {
const other = entity(o.state, "otherTask");
const first = observation(o.state.clarification);
+ // The candidates are identified by rendered links to both tasks, or by the run's saved question card offering
+ // each task as its own choice: card options render as plain-text choice buttons, which cannot hold links.
+ const linked = !!first && !!before && hasLink(first, "issues", before) && hasLink(first, "issues", other);
+ const offered = !!first && !!before && questionCardNamesCandidates((first as ClarificationObservation).questionCards ?? [], [before, other], first.runs[0]?.id, o.leadId);
c.add("clarify-before-write", !!first && !!before && sameTask(before, first.tasks.find(row => row.id === before.id))
- && sameTask(other, first.tasks.find(row => row.id === other.id))
- && hasLink(first, "issues", before) && hasLink(first, "issues", other),
- "The first completed turn changes neither task and gives the person links to both candidates.");
+ && sameTask(other, first.tasks.find(row => row.id === other.id)) && (linked || offered),
+ "The first completed turn changes neither task and identifies both candidates with links or as separate saved question choices.");
c.add("only-selected-task-changed", !!before && current?.priority === "high" && sameTask(before, current, ["priority"])
&& sameTask(other, o.tasks.find(row => row.id === other.id)) && o.tasks.length === 2,
"After the explicit Launch selection, only that task's requested priority changes.");
@@ -91,7 +94,7 @@ function ambiguousTask(): ChatCatalogCase {
await verifyDistinctTaskFixtures(ctx, [t, otherTask]);
return { project: p, task: t, otherProject, otherTask };
});
- return { ...base, oracleVersion: "paperclip.chat-catalog.internal.ambiguous-task.v4", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts", "chat-catalog-domains/identity-support.ts", "chat-rendered-links.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000,
+ return { ...base, oracleVersion: "paperclip.chat-catalog.internal.ambiguous-task.v5", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-distinct-fixtures.ts", "chat-linked-question.ts", "chat-catalog-domains/identity-support.ts", "chat-rendered-links.ts"], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000,
async act(ctx, state) {
await pollUntil({ label: "Task ambiguity clarification", deadlineAt: Date.now() + 150_000,
load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs.every(row => row.status === "succeeded")
diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts
index bdf01b62cc8..9da9f00a486 100644
--- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.test.ts
@@ -72,30 +72,30 @@ it("rejects an unbound later worker in the grade when its prompt lacks the direc
const o = valid(); o.state.cancelDirectionRunBound = promptCarries(`{"body":${JSON.stringify("unrelated")}}`, "Remove the contact section.\nKeep fonts.");
expect(gradeDiscard(o).passed).toBe(false);
});
-const label = "78f7e3a6c6ec-1";
-const initialPlan = "# Website Plan (Initial)\n\n## Status\n\nInitial plan document created.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n";
-const consumedPlan = `# Website Plan\n\n## Human direction\n\n- Add a contact section to the saved website plan, labelled \`${label}\`.\n\n## Contact section (label: ${label})\n\n- **Purpose:** give visitors a clear way to reach the site owner.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n`;
-const revs = (finalBody: string, extraSave?: string) => [
+const label = "e972d9c1aa95-1";
+const initialPlan = "# Website Plan (Initial)\n\n## Accessible Font Guidance\n\n- Readable base size.\n\n## Proposed Pages\n1. Home\n2. Contact\n";
+// Full cohort 11: the consuming run saved the label as a plain line under a "## Contact" heading.
+const consumedPlan = `# Website Plan (Initial)\n\n## Accessible Font Guidance\n\n- Readable base size.\n\n## Contact\nSection label: \`${label}\`\n\n- Add a contact form.\n\n## Proposed Pages\n1. Home\n2. Contact\n`;
+const finalPlan = "# Website Plan (Initial)\n\n## Accessible Font Guidance\n\n- Readable base size.\n\n## Proposed Pages\n1. Home\n";
+const revs = (finalBody: string) => [
{ id: "r1", revisionNumber: 1, createdByRunId: "initial", body: initialPlan },
{ id: "r2", revisionNumber: 2, createdByRunId: "consumer", body: consumedPlan },
- ...(extraSave ? [{ id: "r3", revisionNumber: 3, createdByRunId: "consumer", body: extraSave }] : []),
- { id: "final", revisionNumber: 4, createdByRunId: "final", body: finalBody },
+ { id: "final", revisionNumber: 3, createdByRunId: "final", body: finalBody },
];
-it("full cohort 6: the labelled heading is gone and a superseded note may still name the label", () => {
- const final = `# Website Plan\n\n## Human direction\n\n- No contact section is included; the earlier \`${label}\` contact-section request has been superseded.\n\n## Accessible font guidance (retained)\n\n- Readable base size.\n`;
- expect(documentContactRemoved(revs(final), "final", "consumer", label)).toBe(true);
+it("full cohort 11: passes when the final revision drops the setup label, whatever form the label was saved in", () => {
+ expect(documentContactRemoved(revs(finalPlan), "final", label)).toBe(true);
+ for (const form of [`## Contact section (label: ${label})`, `### Reach us (${label})`, `Contact - ${label}`]) {
+ const consumed = consumedPlan.replace(`## Contact\nSection label: \`${label}\``, form);
+ expect(documentContactRemoved([...revs(finalPlan)].map(row => row.id === "r2" ? { ...row, body: consumed } : row), "final", label), form).toBe(true);
+ }
});
-it("fails while a heading still carries the label, kept or renamed", () => {
- expect(documentContactRemoved(revs(consumedPlan), "final", "consumer", label)).toBe(false);
- expect(documentContactRemoved(revs(consumedPlan.replace(`## Contact section (label: ${label})`, `### Reach us (${label})`)), "final", "consumer", label)).toBe(false);
+it("fails while the final revision still carries the setup label in any form", () => {
+ expect(documentContactRemoved(revs(consumedPlan), "final", label)).toBe(false);
+ expect(documentContactRemoved(revs(`${finalPlan}\nContact (${label}): email us.\n`), "final", label)).toBe(false);
});
-it("compares against the plan from before the consuming run first saved the label, across its repeated saves", () => {
- expect(documentContactRemoved(revs(initialPlan, `${consumedPlan}\n## Next steps\n\nConfirm contact details.\n`), "final", "consumer", label)).toBe(true);
- expect(documentContactRemoved(revs(consumedPlan, `${consumedPlan}\n## Next steps\n\nConfirm contact details.\n`), "final", "consumer", label)).toBe(false);
-});
-it("fails when no labelled section was ever saved, so removal cannot be shown", () => {
- const unlabelled = consumedPlan.replace(`## Contact section (label: ${label})`, "## Contact section");
- expect(documentContactRemoved([...revs(initialPlan)].map(row => row.id === "r2" ? { ...row, body: unlabelled } : row), "final", "consumer", label)).toBe(false);
+it("fails when the final revision is missing or the label is empty", () => {
+ expect(documentContactRemoved(revs(finalPlan), "missing", label)).toBe(false);
+ expect(documentContactRemoved(revs(finalPlan), "final", "")).toBe(false);
});
function citedInTurn() {
diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts
index 3311de4d728..527cc3c5fff 100644
--- a/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts
+++ b/tests/runner-e2e/chat-catalog-domains/runtime-discard.ts
@@ -22,16 +22,13 @@ export function promptCarries(prompt: string | undefined, body: string) {
export function replyLinkHrefs(reply: string, rendered: readonly string[]) {
return [...new Set([...rendered, ...[...reply.matchAll(/\]\(([^)\s]+)\)/g)].map(match => match[1]!)])];
}
-const labelledHeading = (body: string, label: string) => body.split(/\r?\n/).some(line => /^#{1,6}\s/.test(line) && line.includes(label));
-export function documentContactRemoved(rows: ReadonlyArray<{ id: string; body?: string | null; createdByRunId?: string | null }>, latestRevisionId: unknown, consumerRunId: string, label: string) {
- const number = (row: object) => Number((row as { revisionNumber?: unknown }).revisionNumber);
- const revisions = [...rows].sort((a, b) => number(a) - number(b));
- const final = revisions.find(row => row.id === latestRevisionId);
- const saves = revisions.filter(row => row.createdByRunId === consumerRunId && (row.body ?? "").includes(label));
- if (!saves.length) return !(final?.body ?? "").includes(label);
- const previous = revisions.filter(row => number(row) < number(saves[0]!)).at(-1);
- const hadSection = [previous, ...saves].some(row => labelledHeading(row?.body ?? "", label));
- return !!final && hadSection && !labelledHeading(final.body ?? "", label);
+/**
+ * The cancelled request's setup label must be absent from the document's final revision. Whether the label
+ * was ever saved is checked separately on the consuming run's revisions; the plan's layout is not graded.
+ */
+export function documentContactRemoved(rows: ReadonlyArray<{ id: string; body?: string | null }>, latestRevisionId: unknown, label: string) {
+ const final = rows.find(row => row.id === latestRevisionId);
+ return !!final && label.length > 0 && !(final.body ?? "").includes(label);
}
export function gradeDiscard(value: unknown) {
const c = checks(), o = runtimeObservation(value);
@@ -68,7 +65,7 @@ export function gradeDiscard(value: unknown) {
return c.result();
}
export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-message", ctx => humanQueuedRuntime(ctx, contactText), gradeDiscard, {
- authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v7",
+ authentication: "human-session", expectedHumanMessageCount: 2, expectedRunCount: 5, minimumRunCount: 4, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-discard.v8",
sourceFiles: ["chat-catalog-domains/runtime-discard.ts", "chat-catalog-domains/runtime-queue-controls.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-cited-delivery.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/steering.yaml"],
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
async act(ctx, state) {
@@ -152,7 +149,7 @@ export const runtimeDiscardCases = [runtimeDefinition("steering.discard-queued-m
const latest = o.revisions.filter(row => row.createdByRunId === state.finalWorkerRunId && row.createdByAgentId === worker
&& o.documents.some(doc => doc.id === row.documentId && doc.latestRevisionId === row.id));
state.finalPlanWithoutContact = latest.length > 0 && o.documents.every(doc => documentContactRemoved(
- o.revisions.filter(row => row.documentId === doc.id), doc.latestRevisionId, String(state.consumerRunId), label));
+ o.revisions.filter(row => row.documentId === doc.id), doc.latestRevisionId, label));
return o;
},
async cleanup(ctx, state) { try { await cleanupIdentityProbes(ctx); } finally { await releaseRuntimeWorker(state); } },
diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts
index ebda865d8cc..b3498df7cac 100644
--- a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts
+++ b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts
@@ -82,7 +82,7 @@ export async function verifyRuntimeResult(ctx: ChatCatalogContext, state: Record
}
/**
* A pending interaction renders inline in the thread as `#interaction-`,
- * or docked in the composer takeover, which carries its title but no id.
+ * or docked in the composer takeover, which carries its title and data-interaction-id but not the page anchor id.
*/
export function interactionSurface(page: Page, interaction: { id: string; title?: string | null }) {
const inline = page.locator(`[id=${JSON.stringify(`interaction-${interaction.id}`)}]`);
diff --git a/tests/runner-e2e/chat-linked-question-grade.test.ts b/tests/runner-e2e/chat-linked-question-grade.test.ts
index 230cd8f6842..961b7242bf6 100644
--- a/tests/runner-e2e/chat-linked-question-grade.test.ts
+++ b/tests/runner-e2e/chat-linked-question-grade.test.ts
@@ -42,3 +42,29 @@ it("grades the clarification by saved state and candidate links, not by its word
const both = structuredClone(final); both.tasks[1]!.priority = "high";
expect(grade_tasks(both).passed).toBe(false);
});
+// Full cohort 11: the question card had no linked context; its options named each candidate with its path as
+// plain text. Choice options render inside radio buttons, which cannot hold links, so the saved options are graded.
+it("accepts a saved question card that offers each candidate as its own choice, and rejects one that does not", () => {
+ const input = valid();
+ const card = (options: Array<{ id: string; label: string; description?: string }>, sourceRunId = "run") => ({ ...structuredClone(input.interactions[0]!), sourceRunId,
+ payload: { version: 1, questionSet: { schema: "paperclip.question_set.v1", questions: [{ id: "task_choice", prompt: "Which landing-page task should become high priority?", answerMode: "single_select", required: true, options }] } } });
+ const retained = [{ id: "com1", label: "RUN-1 (Launch project)", description: "Landing page task RUN-1 in the Launch project: /RUN/issues/RUN-1" },
+ { id: "com2", label: "RUN-2 (Experiments project)", description: "Landing page task RUN-2 in the Experiments project: /RUN/issues/RUN-2" },
+ { id: "both", label: "Both", description: "Mark both landing-page tasks as high priority: /RUN/issues/RUN-1 and /RUN/issues/RUN-2" }];
+ const graded = (questionCards: unknown[]) => {
+ const first = { ...input.observation, reply: "", links: [], questionCards };
+ return grade_tasks({ ...structuredClone(input.observation), state: { ...input.observation.state, clarification: first },
+ tasks: [{ ...input.observation.tasks[0]!, priority: "high" }, structuredClone(input.observation.tasks[1]!)],
+ links: ["/RUN/issues/RUN-1"], activity: [{ id: "update", action: "issue.updated", entityId: "launch-task", agentId: "ceo" }] }).passed;
+ };
+ expect(graded([card(retained)])).toBe(true);
+ expect(graded([card([{ id: "a", label: "Launch", description: "The RUN-1 task" }, { id: "b", label: "Experiments", description: "The RUN-2 task" }])])).toBe(true);
+ expect(graded([])).toBe(false);
+ expect(graded([card([retained[0]!, retained[2]!])])).toBe(false);
+ expect(graded([card([{ id: "a", label: "Launch" }, { id: "b", label: "Experiments" }])])).toBe(false);
+ expect(graded([card([retained[0]!, { id: "c", label: "RUN-12 (Archive)" }])])).toBe(false);
+ expect(graded([card(retained, "other-run")])).toBe(false);
+ for (const withdrawn of [{ status: "cancelled" }, { status: "expired" }, { dismissedAt: "2026-10-28T12:00:45Z" }, { resolvedAt: "2026-10-28T12:00:45Z" }]) {
+ expect(graded([{ ...card(retained), ...withdrawn }]), JSON.stringify(withdrawn)).toBe(false);
+ }
+});
diff --git a/tests/runner-e2e/chat-linked-question.ts b/tests/runner-e2e/chat-linked-question.ts
index 42dbb47f0f2..898121974b8 100644
--- a/tests/runner-e2e/chat-linked-question.ts
+++ b/tests/runner-e2e/chat-linked-question.ts
@@ -78,7 +78,22 @@ export function readLinkedQuestionDom(input: NonNullable link.href))], text: prompts[0]!.textContent ?? "" };
}
export type LinkedQuestionEvidence = NonNullable>;
-export type ClarificationObservation = InternalObservation & { linkedQuestionEvidence?: LinkedQuestionEvidence };
+export type ClarificationObservation = InternalObservation & { linkedQuestionEvidence?: LinkedQuestionEvidence; questionCards?: AskUserQuestionsInteraction[] };
+/**
+ * Whether the run's saved question card offers each candidate task as its own choice. Option text is plain
+ * text inside a choice button, so it cannot carry links; a candidate counts when one option names its identifier
+ * (the setup literal) and no other candidate's. The option wording is left to human review.
+ */
+export function questionCardNamesCandidates(cards: readonly AskUserQuestionsInteraction[], tasks: ReadonlyArray<{ id: string; identifier?: string | null }>, runId: string | undefined, leadId: string) {
+ const names = (text: string, task: { identifier?: string | null }) => !!task.identifier
+ && new RegExp(`(? 1 && new Set(tasks.map(task => task.identifier)).size === tasks.length && cards.some(card => card.kind === "ask_user_questions"
+ && card.status === "pending" && !card.dismissedAt && !card.resolvedAt && card.createdByAgentId === leadId && card.sourceRunId === runId
+ && (card.payload.questionSet?.questions ?? card.payload.questions ?? []).some(question => tasks.every(task => (question.options ?? []).some(option => {
+ const text = `${option.label ?? ""}\n${option.description ?? ""}`;
+ return names(text, task) && tasks.every(other => other.id === task.id || !names(text, other));
+ }))));
+}
export async function observeLinkedTaskQuestion(ctx: ChatCatalogContext, state: Record, observation: InternalObservation): Promise {
const source = ctx.issue(), prefix = ctx.input.fixtures.company.issuePrefix;
if (!source || !prefix || ctx.scenario.id !== "tasks.ambiguous-task") return observation;
@@ -88,9 +103,10 @@ export async function observeLinkedTaskQuestion(ctx: ChatCatalogContext, state:
]);
const binding = linkedQuestionBinding({ origin: observation.origin, pageUrl: ctx.input.page.url(), companyPrefix: prefix, sourceId: source.id, leadId: observation.leadId,
query: ctx.scenario.query, observation, comments, interactions, tasks: [state.task as Entity, state.otherTask as Entity] });
- if (!binding) return observation;
+ const questionCards = interactions.filter(card => card.kind === "ask_user_questions" && card.issueId === source.id);
+ if (!binding) return { ...observation, questionCards };
const evidence = await ctx.input.page.evaluate(readLinkedQuestionDom, binding);
- return evidence ? { ...observation, reply: [observation.reply, evidence.text].filter(Boolean).join("\n"), links: [...new Set([...observation.links, ...evidence.links])], linkedQuestionEvidence: evidence } : observation;
+ return evidence ? { ...observation, reply: [observation.reply, evidence.text].filter(Boolean).join("\n"), links: [...new Set([...observation.links, ...evidence.links])], linkedQuestionEvidence: evidence, questionCards } : { ...observation, questionCards };
}
export function linkedQuestionTakeover(page: Page, evidence: LinkedQuestionEvidence): Locator {
return page.getByTestId("task-chat-thread").filter({ has: page.locator(`[id=${JSON.stringify(`comment-${evidence.anchorCommentId}`)}]`) })
diff --git a/tests/runner-e2e/chat-secret-proposal-card-scope.test.ts b/tests/runner-e2e/chat-secret-proposal-card-scope.test.ts
index 045dcc3a808..9228f32c7b5 100644
--- a/tests/runner-e2e/chat-secret-proposal-card-scope.test.ts
+++ b/tests/runner-e2e/chat-secret-proposal-card-scope.test.ts
@@ -11,18 +11,19 @@ vi.mock("./chat-catalog-domains/external-support.js", () => ({ definition: () =>
import { secretProposalCardSelector } from "./chat-catalog-domains/external-secret-proposal.js";
afterEach(() => { document.body.innerHTML = ""; });
it("selects the actual expanded proposal card when the pending trigger has the same production interaction ID", () => {
- document.body.innerHTML = `
-
Deployment publishing key
Morgan
env.PUBLISHING_API_KEY
-
Other request
`;
- expect(document.querySelectorAll('[id="interaction-owned"]')).toHaveLength(2);
+ // The timeline trigger owns the page anchor id; the docked card carries only the interaction data attribute.
+ document.body.innerHTML = `
+
Deployment publishing key
Morgan
env.PUBLISHING_API_KEY
+
Other request
`;
+ expect(document.querySelectorAll('[data-interaction-id="owned"]')).toHaveLength(2);
const selected = document.querySelectorAll(secretProposalCardSelector("owned"));
expect(selected).toHaveLength(1); expect(selected[0]?.tagName).toBe("DIV");
for (const fact of ["Deployment publishing key", "Morgan", "env.PUBLISHING_API_KEY"]) expect(selected[0]?.textContent).toContain(fact);
expect(document.querySelectorAll(secretProposalCardSelector("missing"))).toHaveLength(0);
});
it("a trigger, unrelated card or completed receipt cannot stand in for the bound proposal card", () => {
- document.body.innerHTML = `
-
Deployment publishing key Morgan env.PUBLISHING_API_KEY
-
Approved
`;
+ document.body.innerHTML = `
+
Deployment publishing key Morgan env.PUBLISHING_API_KEY