diff --git a/tests/runner-e2e/chat-catalog-domains/internal-decision-search.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-decision-search.test.ts new file mode 100644 index 00000000000..591b52a4000 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-decision-search.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { gradeDecisionSearch } from "./internal-decision-search.js"; +export function validDecisionSearch() { + const t = { id: "question-task", title: "Riley launch question", status: "backlog" }; const history = { id: "history-task", title: "Completed launch information", status: "backlog" }; + const hiring = { id: "hire", status: "pending", decidedByUserId: null }; const pendingQuestion = { id: "question", status: "pending", addresseeUserId: "riley" }; + const completedQuestion = { id: "completed", status: "answered", resolvedByUserId: "owner", result: { answers: ["received"] } }; + return { scenarioId: "inbox-search.show-decisions", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(t), historyTask: structuredClone(history), hiring: structuredClone(hiring), pendingQuestion: structuredClone(pendingQuestion), completedQuestion: structuredClone(completedQuestion), ownerId: "owner", rileyId: "riley" }, + tasks: [t, history], projects: [], goals: [], comments: [], activity: [] as Array<{ agentId: string; action: string }>, documents: [], revisions: [], + reply: "Launch designer: pending hiring approval for you as the human operator.\nRiley launch question: waiting for Riley to answer; it is addressed to Riley.\nCompleted launch information: answered history, no action remains.", + links: ["/RUN/approvals/hire", "/RUN/issues/question-task", "/RUN/issues/history-task"], humanQueries: [], runs: [], hiring, pendingQuestion, completedQuestion, + visibleAttention: { items: [{ subject: { id: "hire" } }, { subject: { id: "question" }, resolverAudience: { addresseeUserId: "riley", effectiveResolverPolicy: "human_only" } }] }, permissionStatus: 403 }; +} +describe("decision source and resolver calibration", () => { + it("accepts pending hiring, Riley's question and answered history without acting", () => expect(gradeDecisionSearch(validDecisionSearch()).passed).toBe(true)); + it("rejects approving while listing", () => { const o = validDecisionSearch(); o.hiring.status = "approved"; expect(gradeDecisionSearch(o).passed).toBe(false); }); + it("rejects calling history unresolved", () => { const o = validDecisionSearch(); o.reply = o.reply.replace("answered history, no action remains", "still pending"); expect(gradeDecisionSearch(o).passed).toBe(false); }); + it("rejects offering Alex Riley's resolver authority", () => { const o = validDecisionSearch(); o.reply = o.reply.replace("waiting for Riley to answer; it is addressed to Riley", "Riley asked it; you can answer now"); expect(gradeDecisionSearch(o).passed).toBe(false); }); + it("rejects wrong source or foreign links", () => { const o = validDecisionSearch(); o.links[0] = "https://foreign.test/RUN/approvals/hire"; expect(gradeDecisionSearch(o).passed).toBe(false); }); + it("rejects universal resolution claims", () => { const o = validDecisionSearch(); o.reply += "\nYou can approve every visible item."; expect(gradeDecisionSearch(o).passed).toBe(false); }); + it("rejects changed completed answer and board-feed privilege substitution", () => { const o = validDecisionSearch(); o.completedQuestion.result.answers = ["changed"]; expect(gradeDecisionSearch(o).passed).toBe(false); const admin = validDecisionSearch(); admin.permissionStatus = 200; expect(gradeDecisionSearch(admin).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-decision-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-decision-search.ts new file mode 100644 index 00000000000..c0990954bf9 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-decision-search.ts @@ -0,0 +1,98 @@ +import { expect } from "@playwright/test"; +import { createIssueThreadInteractionSchema } from "../../../packages/shared/src/validators/issue.js"; +import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import { makeDefinition } from "./internal-definition.js"; +import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js"; +import { mainPerson } from "./internal-privacy.js"; +import { checks, entity, hasLink, observation, observeInternal, project, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js"; +interface Request extends Entity { addresseeUserId?: string; resolvedByUserId?: string; result?: unknown; payload?: unknown; decidedByUserId?: string | null } +interface DecisionObservation extends InternalObservation { hiring: Request; pendingQuestion?: Request; completedQuestion?: Request; visibleAttention: unknown; permissionStatus: number } +function approvalLink(o: InternalObservation, id: string) { + return o.links.some(href => { try { const u = new URL(href, o.origin); return u.origin === o.origin && u.pathname.split("/").slice(-2).join("/") === `approvals/${id}`; } catch { return false; } }); +} +function statement(o: InternalObservation, source: Entity) { + const names = [source.title, source.identifier, source.id].filter((name): name is string => !!name); + return o.reply.split(/\n|(?<=[.!?])\s+/).filter(part => names.some(name => part.includes(name))).join(" "); +} +export function gradeDecisionSearch(value: unknown) { + const c = checks(); const o = observation(value) as DecisionObservation | undefined; + c.add("actual-source-decision-state", !!o && !!o.hiring && !!o.pendingQuestion && !!o.completedQuestion && o.visibleAttention !== undefined, "Actual hiring, addressed question, answered history and human-visible attention records are required."); + if (!o || !o.hiring || !o.pendingQuestion || !o.completedQuestion) return c.result(); + const feed = o.visibleAttention as { items?: Array<{ subject?: { id?: string }; resolverAudience?: { addresseeUserId?: string | null; effectiveResolverPolicy?: string } }> }; + const hire = o.state.hiring as Request; const pending = o.state.pendingQuestion as Request; const done = o.state.completedQuestion as Request; + const t = o.state.task as Entity; const history = o.state.historyTask as Entity; + c.add("visible-sources-and-real-resolver", Array.isArray(feed?.items) && feed.items.some(row => row.subject?.id === hire?.id) + && feed.items.some(row => row.subject?.id === pending?.id && row.resolverAudience?.addresseeUserId === o.state.rileyId && row.resolverAudience?.effectiveResolverPolicy === "human_only") + && o.state.rileyId !== o.state.ownerId, "The real human-visible queue contains both pending sources and the server-evaluated audience names the other human as question resolver."); + c.add("list-does-not-resolve", o.hiring.id === hire?.id && o.hiring.status === "pending" && !o.hiring.decidedByUserId && JSON.stringify(o.hiring.payload) === JSON.stringify(hire.payload) + && o.pendingQuestion.id === pending?.id && o.pendingQuestion.status === "pending" && !o.pendingQuestion.result && JSON.stringify(o.pendingQuestion.payload) === JSON.stringify(pending.payload) + && o.pendingQuestion.addresseeUserId === o.state.rileyId && o.completedQuestion.id === done?.id && o.completedQuestion.status === "answered" + && o.completedQuestion.resolvedByUserId === o.state.ownerId && JSON.stringify(o.completedQuestion.result) === JSON.stringify(done.result) + && [t, history].every(before => !!before && sameTask(before, o.tasks.find(row => row.id === before.id))) + && !o.activity.some(row => row.agentId === o.leadId && /approval\.(?:approved|rejected)|issue\.interaction_(?:answered|accepted|rejected|cancelled)/.test(row.action ?? "")), + "Listing preserves the pending hiring decision, Riley's pending question and the actual already-answered human history."); + const hiringText = statement(o, { id: hire.id, title: "Launch designer" }); const questionText = statement(o, t); const historyText = statement(o, history); + c.add("pending-hire-source-and-human-resolver", approvalLink(o, hire.id) && /\b(?:pending|await|approv|decision)\b/i.test(hiringText) + && /\b(?:you|human|board|operator|Alex)\b/i.test(hiringText), "The hiring decision links its exact approval and identifies the current human/operator approval step."); + c.add("addressed-question-is-not-everyones-decision", !!t && hasLink(o, "issues", t) && /\bRiley\b/.test(questionText) + && /\b(?:answer|respond|resolver|addressed|waiting|pending)\b/i.test(questionText) + && !/\b(?:you|Alex)\s+(?:can|should|must)\s+(?:answer|resolve|approve)\b/i.test(questionText), + "The visible question links its actual task and identifies Riley, without offering Alex permission to resolve it."); + c.add("completed-request-separated-from-pending", !!history && hasLink(o, "issues", history) + && /\b(?:answered|completed|resolved|history|no action|nothing.*waiting)\b/i.test(historyText) + && !/\b(?:still|currently)\s+(?:pending|waiting|unanswered)\b/i.test(historyText), + "The informational completed request is linked as actual history, not another unresolved decision."); + c.add("board-feed-permission-boundary", o.permissionStatus === 403 && !/\b(?:you can|you may|you should)\s+(?:approve|resolve|answer)\s+(?:all|every)\b/i.test(o.reply), + "The independent agent cannot read the board-only attention aggregation, and the reply does not grant universal resolver authority."); + return c.result(); +} +function showDecisions(): ChatCatalogCase { + let riley: Awaited> | undefined; let key: { id: string; token: string } | undefined; + const base = makeDefinition("inbox-search.show-decisions", async ctx => { + const ownerId = await mainPerson(ctx); const browser = ctx.input.page.context().browser(); + if (!browser || !ctx.input.privateSecrets) throw new Error("Decision audiences require an independent signed-in human and private credential redaction"); + riley = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Riley" }); + try { + await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, riley, ctx.input.privateSecrets, "operator"); + const p = await project(ctx); const t = await task(ctx, "Riley launch question", p.id); + const history = await task(ctx, "Completed launch information", p.id); + const question = (title: string, addresseeUserId: string) => createIssueThreadInteractionSchema.parse({ kind: "ask_user_questions", title, resolverPolicy: "human_only", addresseeUserId, continuationPolicy: "none", + payload: { version: 1, supersedeOnUserComment: false, questionSet: { schema: "paperclip.question_set.v1", questions: [{ id: "launch-input", prompt: title, answerMode: "text", required: true }] } } }); + const pendingQuestion = await ctx.input.api.post(`/api/issues/${t.id}/interactions`, question("Riley: choose the launch audience", riley.id)); + const informational = await ctx.input.api.post(`/api/issues/${history.id}/interactions`, question("Confirm receipt of the launch brief", ownerId)); + const completedQuestion = await ctx.input.api.post(`/api/issues/${history.id}/interactions/${informational.id}/respond`, { answers: [{ questionId: "launch-input", optionIds: [], otherText: "Received the launch brief; no further decision is needed." }] }); + const hiring = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/approvals`, { type: "hire_agent", requestedByAgentId: ctx.input.fixtures.agent.id, + payload: { name: "Launch designer", role: "designer", title: "Launch designer", capabilities: "Create launch design proposals", adapterType: "process", adapterConfig: { command: "node", args: ["-e", "process.exit(0)"] } } }); + if (hiring.status !== "pending" || pendingQuestion.status !== "pending" || completedQuestion.status !== "answered") throw new Error("Decision prerequisites did not persist their genuine pending/history states"); + if (!ctx.input.privateSecrets) throw new Error("Decision scope probe requires attempt-private credential redaction"); + key = await ctx.input.api.postSensitive(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `decision-scope-${ctx.input.nonce}` }); ctx.input.privateSecrets.push(key!.token); + return { project: p, task: t, historyTask: history, ownerId, rileyId: riley.id, pendingQuestion, completedQuestion, hiring }; + } catch (error) { await riley.dispose(); riley = undefined; throw error; } + }, gradeDecisionSearch); + return { ...base, authentication: "human-session", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-decision-search.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "../../packages/shared/src/validators/issue.ts"], + async observe(ctx, state): Promise { + if (!key) throw new Error("Missing attempt-private decision probe"); + const o = await observeInternal(ctx, state); const root = `/api/companies/${o.companyId}`; + const [hiring, pendingRows, completedRows, visibleAttention] = await Promise.all([ + ctx.input.api.get(`/api/approvals/${entity(state, "hiring").id}`), ctx.input.api.get(`/api/issues/${entity(state, "task").id}/interactions`), + ctx.input.api.get(`/api/issues/${entity(state, "historyTask").id}/interactions`), ctx.input.api.get(`${root}/attention?all=true&includeDismissed=true`), + ]); + const url = new URL(`${root}/attention?all=true`, ctx.input.api.baseURL); + const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}` } }); await response.body?.cancel(); + return { ...o, hiring, pendingQuestion: pendingRows.find(row => row.id === entity(state, "pendingQuestion").id), completedQuestion: completedRows.find(row => row.id === entity(state, "completedQuestion").id), visibleAttention, permissionStatus: response.status }; + }, + async verifyResult(ctx, state, value) { + const o = value as DecisionObservation; await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); + try { + for (const row of [{ kind: "approvals", record: entity(state, "hiring"), title: "Launch designer" }, { kind: "issues", record: entity(state, "historyTask"), title: entity(state, "historyTask").title! }]) { + const href = o.links.find(link => { try { const u = new URL(link, o.origin); return u.origin === o.origin && u.pathname.includes(`/${row.kind}/`) && [row.record.id, row.record.identifier].some(id => id && u.pathname.split("/").includes(id)); } catch { return false; } }); + if (!href) throw new Error(`Missing exact linked ${row.kind} result`); + await ctx.input.page.goto(new URL(href, o.origin).href, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByText(row.title, { exact: true }).first()).toBeVisible(); + await ctx.input.evidence(`chat-catalog-decision-${row.kind}-opened.json`, { id: row.record.id, url: ctx.input.page.url(), visible: true }); + } + } finally { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } + }, + async cleanup(ctx) { try { if (key) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true }); key = undefined; } } finally { await riley?.dispose(); riley = undefined; } }, + }; +} +export const decisionSearchCases: readonly ChatCatalogCase[] = [showDecisions()]; diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index 49d78990441..a5748b9a5d1 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -15,6 +15,15 @@ function event(action: string, entityId: string, details?: Record