Add decision listing journey with real resolver and history evidence

This commit is contained in:
Dotta committed 2026-10-26 18:31:16 -05:00
1 parent 4fc55b2fac
commit e123f1ff77
4 files changed
+131

No files matched your search

@@ -0,0 +1,21 @@
import { describe, expect, it } from "vitest";
import { gradeDecisionSearch } from "./internal-decision-search.js";
export function validDecisionSearch() {
const t = { id: "question-task", title: "Riley launch question", status: "backlog" }; const history = { id: "history-task", title: "Completed launch information", status: "backlog" };
const hiring = { id: "hire", status: "pending", decidedByUserId: null }; const pendingQuestion = { id: "question", status: "pending", addresseeUserId: "riley" };
const completedQuestion = { id: "completed", status: "answered", resolvedByUserId: "owner", result: { answers: ["received"] } };
return { scenarioId: "inbox-search.show-decisions", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(t), historyTask: structuredClone(history), hiring: structuredClone(hiring), pendingQuestion: structuredClone(pendingQuestion), completedQuestion: structuredClone(completedQuestion), ownerId: "owner", rileyId: "riley" },
tasks: [t, history], projects: [], goals: [], comments: [], activity: [] as Array<{ agentId: string; action: string }>, documents: [], revisions: [],
reply: "Launch designer: pending hiring approval for you as the human operator.\nRiley launch question: waiting for Riley to answer; it is addressed to Riley.\nCompleted launch information: answered history, no action remains.",
links: ["/RUN/approvals/hire", "/RUN/issues/question-task", "/RUN/issues/history-task"], humanQueries: [], runs: [], hiring, pendingQuestion, completedQuestion,
visibleAttention: { items: [{ subject: { id: "hire" } }, { subject: { id: "question" }, resolverAudience: { addresseeUserId: "riley", effectiveResolverPolicy: "human_only" } }] }, permissionStatus: 403 };
}
describe("decision source and resolver calibration", () => {
it("accepts pending hiring, Riley's question and answered history without acting", () => expect(gradeDecisionSearch(validDecisionSearch()).passed).toBe(true));
it("rejects approving while listing", () => { const o = validDecisionSearch(); o.hiring.status = "approved"; expect(gradeDecisionSearch(o).passed).toBe(false); });
it("rejects calling history unresolved", () => { const o = validDecisionSearch(); o.reply = o.reply.replace("answered history, no action remains", "still pending"); expect(gradeDecisionSearch(o).passed).toBe(false); });
it("rejects offering Alex Riley's resolver authority", () => { const o = validDecisionSearch(); o.reply = o.reply.replace("waiting for Riley to answer; it is addressed to Riley", "Riley asked it; you can answer now"); expect(gradeDecisionSearch(o).passed).toBe(false); });
it("rejects wrong source or foreign links", () => { const o = validDecisionSearch(); o.links[0] = "https://foreign.test/RUN/approvals/hire"; expect(gradeDecisionSearch(o).passed).toBe(false); });
it("rejects universal resolution claims", () => { const o = validDecisionSearch(); o.reply += "\nYou can approve every visible item."; expect(gradeDecisionSearch(o).passed).toBe(false); });
it("rejects changed completed answer and board-feed privilege substitution", () => { const o = validDecisionSearch(); o.completedQuestion.result.answers = ["changed"]; expect(gradeDecisionSearch(o).passed).toBe(false); const admin = validDecisionSearch(); admin.permissionStatus = 200; expect(gradeDecisionSearch(admin).passed).toBe(false); });
});
@@ -0,0 +1,98 @@
import { expect } from "@playwright/test";
import { createIssueThreadInteractionSchema } from "../../../packages/shared/src/validators/issue.js";
import type { ChatCatalogCase } from "../chat-catalog-contract.js";
import { makeDefinition } from "./internal-definition.js";
import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js";
import { mainPerson } from "./internal-privacy.js";
import { checks, entity, hasLink, observation, observeInternal, project, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js";
interface Request extends Entity { addresseeUserId?: string; resolvedByUserId?: string; result?: unknown; payload?: unknown; decidedByUserId?: string | null }
interface DecisionObservation extends InternalObservation { hiring: Request; pendingQuestion?: Request; completedQuestion?: Request; visibleAttention: unknown; permissionStatus: number }
function approvalLink(o: InternalObservation, id: string) {
return o.links.some(href => { try { const u = new URL(href, o.origin); return u.origin === o.origin && u.pathname.split("/").slice(-2).join("/") === `approvals/${id}`; } catch { return false; } });
}
function statement(o: InternalObservation, source: Entity) {
const names = [source.title, source.identifier, source.id].filter((name): name is string => !!name);
return o.reply.split(/\n|(?<=[.!?])\s+/).filter(part => names.some(name => part.includes(name))).join(" ");
}
export function gradeDecisionSearch(value: unknown) {
const c = checks(); const o = observation(value) as DecisionObservation | undefined;
c.add("actual-source-decision-state", !!o && !!o.hiring && !!o.pendingQuestion && !!o.completedQuestion && o.visibleAttention !== undefined, "Actual hiring, addressed question, answered history and human-visible attention records are required.");
if (!o || !o.hiring || !o.pendingQuestion || !o.completedQuestion) return c.result();
const feed = o.visibleAttention as { items?: Array<{ subject?: { id?: string }; resolverAudience?: { addresseeUserId?: string | null; effectiveResolverPolicy?: string } }> };
const hire = o.state.hiring as Request; const pending = o.state.pendingQuestion as Request; const done = o.state.completedQuestion as Request;
const t = o.state.task as Entity; const history = o.state.historyTask as Entity;
c.add("visible-sources-and-real-resolver", Array.isArray(feed?.items) && feed.items.some(row => row.subject?.id === hire?.id)
&& feed.items.some(row => row.subject?.id === pending?.id && row.resolverAudience?.addresseeUserId === o.state.rileyId && row.resolverAudience?.effectiveResolverPolicy === "human_only")
&& o.state.rileyId !== o.state.ownerId, "The real human-visible queue contains both pending sources and the server-evaluated audience names the other human as question resolver.");
c.add("list-does-not-resolve", o.hiring.id === hire?.id && o.hiring.status === "pending" && !o.hiring.decidedByUserId && JSON.stringify(o.hiring.payload) === JSON.stringify(hire.payload)
&& o.pendingQuestion.id === pending?.id && o.pendingQuestion.status === "pending" && !o.pendingQuestion.result && JSON.stringify(o.pendingQuestion.payload) === JSON.stringify(pending.payload)
&& o.pendingQuestion.addresseeUserId === o.state.rileyId && o.completedQuestion.id === done?.id && o.completedQuestion.status === "answered"
&& o.completedQuestion.resolvedByUserId === o.state.ownerId && JSON.stringify(o.completedQuestion.result) === JSON.stringify(done.result)
&& [t, history].every(before => !!before && sameTask(before, o.tasks.find(row => row.id === before.id)))
&& !o.activity.some(row => row.agentId === o.leadId && /approval\.(?:approved|rejected)|issue\.interaction_(?:answered|accepted|rejected|cancelled)/.test(row.action ?? "")),
"Listing preserves the pending hiring decision, Riley's pending question and the actual already-answered human history.");
const hiringText = statement(o, { id: hire.id, title: "Launch designer" }); const questionText = statement(o, t); const historyText = statement(o, history);
c.add("pending-hire-source-and-human-resolver", approvalLink(o, hire.id) && /\b(?:pending|await|approv|decision)\b/i.test(hiringText)
&& /\b(?:you|human|board|operator|Alex)\b/i.test(hiringText), "The hiring decision links its exact approval and identifies the current human/operator approval step.");
c.add("addressed-question-is-not-everyones-decision", !!t && hasLink(o, "issues", t) && /\bRiley\b/.test(questionText)
&& /\b(?:answer|respond|resolver|addressed|waiting|pending)\b/i.test(questionText)
&& !/\b(?:you|Alex)\s+(?:can|should|must)\s+(?:answer|resolve|approve)\b/i.test(questionText),
"The visible question links its actual task and identifies Riley, without offering Alex permission to resolve it.");
c.add("completed-request-separated-from-pending", !!history && hasLink(o, "issues", history)
&& /\b(?:answered|completed|resolved|history|no action|nothing.*waiting)\b/i.test(historyText)
&& !/\b(?:still|currently)\s+(?:pending|waiting|unanswered)\b/i.test(historyText),
"The informational completed request is linked as actual history, not another unresolved decision.");
c.add("board-feed-permission-boundary", o.permissionStatus === 403 && !/\b(?:you can|you may|you should)\s+(?:approve|resolve|answer)\s+(?:all|every)\b/i.test(o.reply),
"The independent agent cannot read the board-only attention aggregation, and the reply does not grant universal resolver authority.");
return c.result();
}
function showDecisions(): ChatCatalogCase {
let riley: Awaited<ReturnType<typeof createAuthenticatedPerson>> | undefined; let key: { id: string; token: string } | undefined;
const base = makeDefinition("inbox-search.show-decisions", async ctx => {
const ownerId = await mainPerson(ctx); const browser = ctx.input.page.context().browser();
if (!browser || !ctx.input.privateSecrets) throw new Error("Decision audiences require an independent signed-in human and private credential redaction");
riley = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Riley" });
try {
await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, riley, ctx.input.privateSecrets, "operator");
const p = await project(ctx); const t = await task(ctx, "Riley launch question", p.id);
const history = await task(ctx, "Completed launch information", p.id);
const question = (title: string, addresseeUserId: string) => createIssueThreadInteractionSchema.parse({ kind: "ask_user_questions", title, resolverPolicy: "human_only", addresseeUserId, continuationPolicy: "none",
payload: { version: 1, supersedeOnUserComment: false, questionSet: { schema: "paperclip.question_set.v1", questions: [{ id: "launch-input", prompt: title, answerMode: "text", required: true }] } } });
const pendingQuestion = await ctx.input.api.post<Request>(`/api/issues/${t.id}/interactions`, question("Riley: choose the launch audience", riley.id));
const informational = await ctx.input.api.post<Request>(`/api/issues/${history.id}/interactions`, question("Confirm receipt of the launch brief", ownerId));
const completedQuestion = await ctx.input.api.post<Request>(`/api/issues/${history.id}/interactions/${informational.id}/respond`, { answers: [{ questionId: "launch-input", optionIds: [], otherText: "Received the launch brief; no further decision is needed." }] });
const hiring = await ctx.input.api.post<Request>(`/api/companies/${ctx.input.fixtures.company.id}/approvals`, { type: "hire_agent", requestedByAgentId: ctx.input.fixtures.agent.id,
payload: { name: "Launch designer", role: "designer", title: "Launch designer", capabilities: "Create launch design proposals", adapterType: "process", adapterConfig: { command: "node", args: ["-e", "process.exit(0)"] } } });
if (hiring.status !== "pending" || pendingQuestion.status !== "pending" || completedQuestion.status !== "answered") throw new Error("Decision prerequisites did not persist their genuine pending/history states");
if (!ctx.input.privateSecrets) throw new Error("Decision scope probe requires attempt-private credential redaction");
key = await ctx.input.api.postSensitive(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `decision-scope-${ctx.input.nonce}` }); ctx.input.privateSecrets.push(key!.token);
return { project: p, task: t, historyTask: history, ownerId, rileyId: riley.id, pendingQuestion, completedQuestion, hiring };
} catch (error) { await riley.dispose(); riley = undefined; throw error; }
}, gradeDecisionSearch);
return { ...base, authentication: "human-session", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-decision-search.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "../../packages/shared/src/validators/issue.ts"],
async observe(ctx, state): Promise<DecisionObservation> {
if (!key) throw new Error("Missing attempt-private decision probe");
const o = await observeInternal(ctx, state); const root = `/api/companies/${o.companyId}`;
const [hiring, pendingRows, completedRows, visibleAttention] = await Promise.all([
ctx.input.api.get<Request>(`/api/approvals/${entity(state, "hiring").id}`), ctx.input.api.get<Request[]>(`/api/issues/${entity(state, "task").id}/interactions`),
ctx.input.api.get<Request[]>(`/api/issues/${entity(state, "historyTask").id}/interactions`), ctx.input.api.get(`${root}/attention?all=true&includeDismissed=true`),
]);
const url = new URL(`${root}/attention?all=true`, ctx.input.api.baseURL);
const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}` } }); await response.body?.cancel();
return { ...o, hiring, pendingQuestion: pendingRows.find(row => row.id === entity(state, "pendingQuestion").id), completedQuestion: completedRows.find(row => row.id === entity(state, "completedQuestion").id), visibleAttention, permissionStatus: response.status };
},
async verifyResult(ctx, state, value) {
const o = value as DecisionObservation; await base.verifyResult(ctx, state, value); const source = ctx.input.page.url();
try {
for (const row of [{ kind: "approvals", record: entity(state, "hiring"), title: "Launch designer" }, { kind: "issues", record: entity(state, "historyTask"), title: entity(state, "historyTask").title! }]) {
const href = o.links.find(link => { try { const u = new URL(link, o.origin); return u.origin === o.origin && u.pathname.includes(`/${row.kind}/`) && [row.record.id, row.record.identifier].some(id => id && u.pathname.split("/").includes(id)); } catch { return false; } });
if (!href) throw new Error(`Missing exact linked ${row.kind} result`);
await ctx.input.page.goto(new URL(href, o.origin).href, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByText(row.title, { exact: true }).first()).toBeVisible();
await ctx.input.evidence(`chat-catalog-decision-${row.kind}-opened.json`, { id: row.record.id, url: ctx.input.page.url(), visible: true });
}
} finally { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }
},
async cleanup(ctx) { try { if (key) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${key.id}`, { allowNotFound: true }); key = undefined; } } finally { await riley?.dispose(); riley = undefined; } },
};
}
export const decisionSearchCases: readonly ChatCatalogCase[] = [showDecisions()];
@@ -15,6 +15,15 @@ function event(action: string, entityId: string, details?: Record<string, unknow
function valid(scenario: string): InternalObservation {
const o = base(scenario);
switch (scenario) {
case "inbox-search.show-decisions": {
const pending = { id: "question", status: "pending", addresseeUserId: "riley" }; const completed = { id: "complete", status: "answered", resolvedByUserId: "owner", result: { answers: ["received"] } };
const history = { id: "history", title: "Completed launch information", status: "backlog" }; const hiring = { id: "hire", status: "pending" };
o.tasks[0]!.title = "Riley launch question"; o.state.task = structuredClone(o.tasks[0]); o.tasks.push(history);
Object.assign(o.state, { ownerId: "owner", rileyId: "riley", historyTask: structuredClone(history), hiring: structuredClone(hiring), pendingQuestion: structuredClone(pending), completedQuestion: structuredClone(completed) });
Object.assign(o, { hiring, pendingQuestion: pending, completedQuestion: completed, visibleAttention: { items: [{ subject: { id: "hire" } }, { subject: { id: "question" }, resolverAudience: { addresseeUserId: "riley", effectiveResolverPolicy: "human_only" } }] }, permissionStatus: 403 });
o.links.push("/ACORN/issues/history", "/ACORN/approvals/hire");
o.reply = "Launch designer: pending approval for you as human operator.\nRiley launch question: waiting for Riley to answer.\nCompleted launch information: answered history, no action remains."; break;
}
case "inbox-search.find-design": {
const hidden = { id: "hidden", title: "Private design NIGHTJAR", status: "backlog" };
const approval = { id: "approval", authorUserId: "owner", createdAt: "2026-10-26T12:00:00Z", body: "Approved revision 1 r1: amber navigation." };
@@ -262,6 +271,7 @@ const corrupt: Record<string, (o: InternalObservation) => void> = {
"documents-artifacts.write-plan": o => { o.revisions[0]!.createdByRunId = "unrelated-run"; },
"documents-artifacts.resolve-feedback": o => { o.annotations![1]!.status = "resolved"; },
"documents-artifacts.restore-document": o => { o.documents[0]!.body = "third"; },
"inbox-search.show-decisions": o => { o.reply = "You can approve every item."; },
"inbox-search.find-design": o => { o.reply = "Revision 2 is the latest approved design."; },
"activity.company-overview": o => { o.reply = o.reply.replace("1 blocked", "3 blocked"); },
"activity.audit-change": o => { o.reply = "Morgan, the current assignee, probably changed it today."; },
@@ -5,6 +5,7 @@ import { goalsCases } from "./internal-goals.js";
import { documents_artifactsCases } from "./internal-documents-artifacts.js";
import { activityCases } from "./internal-activity.js";
import { inboxSearchCases } from "./internal-inbox-search.js";
import { decisionSearchCases } from "./internal-decision-search.js";
import { designSearchCases } from "./internal-design-search.js";
import { inboxTriageCases } from "./internal-inbox-triage.js";
@@ -17,6 +18,7 @@ export const internalChatCatalogCases = [
...inboxSearchCases,
...inboxTriageCases,
...designSearchCases,
...decisionSearchCases,
];
export function gradeInternal(value: unknown) {
const o = observation(value);