diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index e9eb5a4607e..356416d7c77 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -3402,7 +3402,7 @@ "chat-catalog-domains/identity-auth.ts", "../../packages/shared/src/validators/issue.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.decision-search.v2", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-decision-search.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-decision-search.test.ts index 50c0b6ea4c1..f611193fac3 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-decision-search.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-decision-search.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from "vitest"; import { gradeDecisionSearch } from "./internal-decision-search.js"; export function validDecisionSearch() { - const t = { id: "question-task", title: "Riley launch question", status: "backlog" }; const history = { id: "history-task", title: "Completed launch information", status: "backlog" }; - const hiring = { id: "hire", companyId: "company", status: "pending", decidedByUserId: null }; const pendingQuestion = { id: "question", companyId: "company", issueId: t.id, status: "pending", addresseeUserId: "riley", effectiveResolverPolicy: "human_only" }; + const t = { id: "question-task", identifier: "RUN-1", title: "Riley launch question", status: "backlog" }; const history = { id: "history-task", identifier: "RUN-2", title: "Completed launch information", status: "backlog" }; + const hiring = { id: "hire", companyId: "company", status: "pending", decidedByUserId: null }; const pendingQuestion = { id: "question", title: "Riley: choose the launch audience", companyId: "company", issueId: t.id, status: "pending", addresseeUserId: "riley", effectiveResolverPolicy: "human_only" }; const completedQuestion = { id: "completed", status: "answered", resolvedByUserId: "owner", result: { answers: ["received"] } }; return { scenarioId: "inbox-search.show-decisions", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(t), historyTask: structuredClone(history), hiring: structuredClone(hiring), pendingQuestion: structuredClone(pendingQuestion), completedQuestion: structuredClone(completedQuestion), ownerId: "owner", rileyId: "riley" }, tasks: [t, history], projects: [], goals: [], comments: [], activity: [] as Array<{ agentId: string; action: string }>, documents: [], revisions: [], @@ -19,6 +19,32 @@ describe("decision source and resolver calibration", () => { it("rejects wrong source or foreign links", () => { const o = validDecisionSearch(); o.links[0] = "https://foreign.test/RUN/approvals/hire"; expect(gradeDecisionSearch(o).passed).toBe(false); }); it("rejects universal resolution claims", () => { const o = validDecisionSearch(); o.reply += "\nYou can approve every visible item."; expect(gradeDecisionSearch(o).passed).toBe(false); }); it("rejects changed completed answer and board-feed privilege substitution", () => { const o = validDecisionSearch(); o.completedQuestion.result.answers = ["changed"]; expect(gradeDecisionSearch(o).passed).toBe(false); const admin = validDecisionSearch(); admin.permissionStatus = 200; expect(gradeDecisionSearch(admin).passed).toBe(false); }); + // Shape of a real reply (runner-opencode, run 42404417353) once the approval route is available: the + // "waiting on you" meaning sits in the list heading, and the answered request is listed apart as not needing a decision. + const sectioned = [ + "One decision is waiting on you:", "", + "- **[Hire approval: \"Launch designer\"](/RUN/approvals/hire)** — pending `hire_agent` approval (role: designer).", "", + "Pending but not on you:", + "- [RUN-1](/RUN/issues/RUN-1) holds an open question, \"Riley: choose the launch audience,\" addressed to Riley (operator), not your account.", + "- [RUN-1](/RUN/issues/RUN-1) and [RUN-2](/RUN/issues/RUN-2) are unassigned backlog tasks with no decisions required yet.", + ].join("\n"); + const withReply = (reply: string) => { const o = validDecisionSearch(); o.reply = reply; o.links = ["/RUN/approvals/hire", "/RUN/issues/RUN-1", "/RUN/issues/RUN-2"]; return o; }; + it("accepts a sectioned reply whose headings carry who each item waits on", () => expect(gradeDecisionSearch(withReply(sectioned)).passed).toBe(true)); + it("accepts a reworded sectioned reply", () => expect(gradeDecisionSearch(withReply([ + "Needs your call", "1. [Launch designer](/RUN/approvals/hire) hire request.", "", + "Someone else's", "1. [RUN-1](/RUN/issues/RUN-1): only Riley can respond to it.", "", + "Done already", "1. [RUN-2](/RUN/issues/RUN-2): you replied to the brief question earlier; nothing left to do.", + ].join("\n"))).passed).toBe(true)); + it("rejects listing every item as the person's own decision", () => expect(gradeDecisionSearch(withReply([ + "Three decisions are waiting on you:", + "- [Launch designer](/RUN/approvals/hire) hire approval.", + "- [RUN-1](/RUN/issues/RUN-1): Riley's launch audience question.", + "- [RUN-2](/RUN/issues/RUN-2): Completed launch information.", + ].join("\n"))).checks.filter(check => !check.passed).map(check => check.id)).toEqual(["addressed-question-is-not-everyones-decision", "completed-request-separated-from-pending"])); + it("rejects an unlinked approval", () => { const o = withReply(sectioned.replace("[Hire approval: \"Launch designer\"](/RUN/approvals/hire)", "Hire approval: \"Launch designer\"")); o.links = o.links.slice(1); expect(gradeDecisionSearch(o).passed).toBe(false); }); + it("rejects a question whose resolver is named only by the setup titles", () => expect(gradeDecisionSearch(withReply(sectioned.replace("addressed to Riley (operator), not your account", "still open"))).passed).toBe(false)); + it("rejects attributing the hiring approval to Riley", () => expect(gradeDecisionSearch(withReply(sectioned.replace("pending `hire_agent` approval", "pending approval for Riley"))).passed).toBe(false)); + it("rejects presenting the answered request as awaiting the person", () => expect(gradeDecisionSearch(withReply(sectioned.replace("are unassigned backlog tasks with no decisions required yet", "still need your answer"))).passed).toBe(false)); it.each(["missing-resolver-feed", "missing-principal", "foreign-feed", "borrowed-user", "wrong-source", "wrong-task", "wrong-resolver", "wrong-policy", "current-user-leak", "foreign-question"])("rejects incorrect independent audience custody: %s", kind => { const o = validDecisionSearch(); if (kind === "missing-resolver-feed") Object.assign(o, { resolverAttention: undefined }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-decision-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-decision-search.ts index e6b9ea065da..6e0e399dec4 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-decision-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-decision-search.ts @@ -11,10 +11,25 @@ interface AttentionFeed { companyId?: string; items?: Array<{ subject?: { id?: s function approvalLink(o: InternalObservation, id: string) { return o.links.some(href => { try { const u = new URL(href, o.origin); return u.origin === o.origin && u.pathname.split("/").slice(-2).join("/") === `approvals/${id}`; } catch { return false; } }); } +/** + * The reply's statement about one source: every sentence or list item that names it, read together with the + * list heading that introduces it ("Waiting on you:"), so meaning carried by the heading is not lost. `sections` + * identifies which heading or sentence each mention sits under, so the grader can check separation structurally. + */ function statement(o: InternalObservation, source: Entity) { const names = [source.title, source.identifier, source.id].filter((name): name is string => !!name); - return o.reply.split(/\n|(?<=[.!?])\s+/).filter(part => names.some(name => part.includes(name))).join(" "); + const units: Array<{ own: string; text: string; section: string }> = []; let lead: { text: string; section: string } | undefined; + o.reply.split("\n").forEach((line, index) => { + if (!line.trim()) return; + if (/^\s*(?:[-*+]|\d+[.)])\s/.test(line)) { units.push({ own: line, text: `${lead?.text ?? ""}\n${line}`, section: lead?.section ?? `line-${index}` }); return; } + line.split(/(?<=[.!?])\s+/).forEach((part, n) => units.push({ own: part, text: part, section: `line-${index}-${n}` })); + lead = { text: line, section: `lead-${index}` }; + }); + const hits = units.filter(unit => names.some(name => unit.own.includes(name))); + return { text: hits.map(unit => unit.text).join("\n"), sections: new Set(hits.map(unit => unit.section)) }; } +function apart(a: Set, b: Set) { return a.size > 0 && b.size > 0 && [...a].every(section => !b.has(section)); } +function without(text: string, literals: readonly (string | undefined)[]) { return literals.reduce((rest, literal) => literal ? rest.split(literal).join(" ") : rest, text); } export function gradeDecisionSearch(value: unknown) { const c = checks(); const o = observation(value) as DecisionObservation | undefined; c.add("actual-source-decision-state", !!o && !!o.hiring && !!o.pendingQuestion && !!o.completedQuestion && o.visibleAttention !== undefined, "Actual hiring, addressed question, answered history and human-visible attention records are required."); @@ -36,17 +51,19 @@ export function gradeDecisionSearch(value: unknown) { && [t, history].every(before => !!before && sameTask(before, o.tasks.find(row => row.id === before.id))) && !o.activity.some(row => row.agentId === o.leadId && /approval\.(?:approved|rejected)|issue\.interaction_(?:answered|accepted|rejected|cancelled)/.test(row.action ?? "")), "Listing preserves the pending hiring decision, Riley's pending question and the actual already-answered human history."); - const hiringText = statement(o, { id: hire.id, title: "Launch designer" }); const questionText = statement(o, t); const historyText = statement(o, history); - c.add("pending-hire-source-and-human-resolver", approvalLink(o, hire.id) && /\b(?:pending|await|approv|decision)\b/i.test(hiringText) - && /\b(?:you|human|board|operator|Alex)\b/i.test(hiringText), "The hiring decision links its exact approval and identifies the current human/operator approval step."); - c.add("addressed-question-is-not-everyones-decision", !!t && hasLink(o, "issues", t) && /\bRiley\b/.test(questionText) - && /\b(?:answer|respond|resolver|addressed|waiting|pending)\b/i.test(questionText) - && !/\b(?:you|Alex)\s+(?:can|should|must)\s+(?:answer|resolve|approve)\b/i.test(questionText), - "The visible question links its actual task and identifies Riley, without offering Alex permission to resolve it."); - c.add("completed-request-separated-from-pending", !!history && hasLink(o, "issues", history) - && /\b(?:answered|completed|resolved|history|no action|nothing.*waiting)\b/i.test(historyText) - && !/\b(?:still|currently)\s+(?:pending|waiting|unanswered)\b/i.test(historyText), - "The informational completed request is linked as actual history, not another unresolved decision."); + const hiring = statement(o, { id: hire.id, title: "Launch designer" }); const question = statement(o, t); const historyStatement = statement(o, history); + const ownTitles = [t.title, t.identifier, pending.title]; + c.add("pending-hire-source-and-human-resolver", approvalLink(o, hire.id) && hiring.sections.size > 0 + && !/\bRiley\b/.test(hiring.text) && !/\b(?:was|been|is|has)\s+(?:already\s+)?(?:approved|rejected)\b/i.test(hiring.text), + "The hiring decision links its exact approval, is not attributed to Riley and is not reported as already decided."); + c.add("addressed-question-is-not-everyones-decision", !!t && hasLink(o, "issues", t) && /\bRiley\b/.test(without(question.text, ownTitles)) + && apart(question.sections, hiring.sections) + && !/\b(?:you|Alex)\s+(?:can|should|must)\s+(?:answer|resolve|approve)\b/i.test(question.text), + "The visible question links its actual task, names Riley as its resolver beyond the setup titles, is listed apart from the person's own decision and is not offered to Alex."); + c.add("completed-request-separated-from-pending", !!history && hasLink(o, "issues", history) && apart(historyStatement.sections, hiring.sections) + && !/\b(?:still|currently)\s+(?:pending|waiting|unanswered)\b/i.test(historyStatement.text) + && !/\b(?:needs?|awaits?|awaiting|requires?)\s+your\s+(?:answer|response|decision|input|approval)\b/i.test(historyStatement.text), + "The informational completed request is linked and kept apart from the person's pending decision, without being described as awaiting them."); c.add("board-feed-permission-boundary", o.permissionStatus === 403 && !/\b(?:you can|you may|you should)\s+(?:approve|resolve|answer)\s+(?:all|every)\b/i.test(o.reply), "The independent agent cannot read the board-only attention aggregation, and the reply does not grant universal resolver authority."); return c.result(); @@ -74,7 +91,7 @@ function showDecisions(): ChatCatalogCase { return { project: p, task: t, historyTask: history, ownerId, rileyId: riley.id, pendingQuestion, completedQuestion, hiring }; } catch (error) { await riley.dispose(); riley = undefined; throw error; } }, gradeDecisionSearch); - return { ...base, authentication: "human-session", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-decision-search.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "../../packages/shared/src/validators/issue.ts"], + return { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.decision-search.v2", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-decision-search.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "../../packages/shared/src/validators/issue.ts"], async observe(ctx, state): Promise { if (!key || !riley || riley.id !== state.rileyId || riley.id === state.ownerId) throw new Error("Missing distinct authenticated decision resolver or attempt-private probe"); const o = await observeInternal(ctx, state); const root = `/api/companies/${o.companyId}`;