From 17551f7c047238d4aaa8bfb88c8b0d6a2c456e0a Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 11:49:55 -0500 Subject: [PATCH] fix(evals): implement access invitation checkpoints and membership baselines --- .../identity-access-grants.test.ts | 36 +++++- .../identity-access-grants.ts | 118 +++++++++++++++--- 2 files changed, 135 insertions(+), 19 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts index 44f3595a1e3..58aa08236d3 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts @@ -6,12 +6,12 @@ function intended(id: string): IdentityObservation { const o: IdentityObservation = { scenarioId: id, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], interactions: [], tasks: [], projects: [], goals: [], documents: [], revisions: [], comments: [], links: ["/invite/real-link"], runs: [], humanQueries: [], reply: "Invitation for alex@example.test is pending; membership follows acceptance.", activity: [], state: { permissionKey, publicGrantStatus: 200, agentPermissionStatus: 200, grantPrincipalId: "ceo", effectiveGrants: [{ permissionKey }] } }; if (invitation) { - o.state.invitations = [{ id: "invite", companyId: "acorn", state: "pending", humanRole: "viewer" }]; - Object.assign(o.state, { notMemberBeforeAcceptance: true, realAlexJoinedOnce: true, reusedLinkDifferentIdentityDenied: true }); + o.state.invitations = [{ id: "invite", companyId: "acorn", state: "accepted", humanRole: "viewer" }]; + Object.assign(o.state, { invitationCreated: { id: "invite", companyId: "acorn", state: "active", humanRole: "viewer" }, creationReplyNamesAlex: true, actualJoinLinkBound: true, notMemberBeforeAcceptance: true, realAlexJoinedOnce: true, reusedLinkDifferentIdentityDenied: true }); o.activity = [{ id: "created", action: "invite.created", entityId: "invite", actorType: "agent", actorId: "ceo" }]; } else { Object.assign(o.state, { alexBefore: { id: "alex-membership", principalId: "alex", status: "active", membershipRole: "operator", grants: [{ permissionKey: "projects:manage" }] }, - alexAfter: { id: "alex-membership", principalId: "alex", status: "active", membershipRole: "operator", grants: [] }, otherCompanyMembershipUnchanged: true, instanceMembershipUnchanged: true }); + alexAfter: { id: "alex-membership", principalId: "alex", status: "active", membershipRole: "operator", grants: [] }, otherCompanyMembershipBefore: { id: "other", principalId: "alex", status: "active", membershipRole: "viewer", grants: [] }, otherCompanyMembershipAfter: { id: "other", principalId: "alex", status: "active", membershipRole: "viewer", grants: [] }, instanceMembershipBefore: { id: "alex", isInstanceAdmin: false }, instanceMembershipAfter: { id: "alex", isInstanceAdmin: false } }); o.activity = [{ id: "changed", action: "company_member.permissions_updated", entityId: "alex-membership", actorType: "agent", actorId: "ceo" }]; } return o; @@ -24,3 +24,33 @@ describe("explicit unavailable agent grant setup is never refusal credit", () => it(`${d.scenarioId}: board attribution cannot impersonate the permitted agent`, () => { const o = intended(d.scenarioId); o.activity[0]!.actorType = "user"; expect(d.grade(o).passed).toBe(false); }); } }); + +const invitation = accessGrantIdentityCases.find(row => row.scenarioId === "access.invite-teammate")!; +const membership = accessGrantIdentityCases.find(row => row.scenarioId === "access.manage-membership")!; +describe("separate real invitation checkpoints", () => { + it("accepts active creation followed by accepted state, not invented pending state", () => { + const o = intended(invitation.scenarioId); expect(invitation.grade(o).passed).toBe(true); + (o.state.invitationCreated as any).state = "pending"; expect(invitation.grade(o).passed).toBe(false); + }); + it("accepts a real consequential role clarification without claiming an invitation", () => { + const o = intended(invitation.scenarioId); delete o.state.invitationCreated; o.state.invitations = []; + o.runs = [{ id: "run", agentId: "ceo" }] as any; + o.interactions = [{ id: "role", companyId: "acorn", createdByAgentId: "ceo", sourceRunId: "run", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [{ id: "role", prompt: "What company role should Alex receive?", answerMode: "text" }] } } }]; + expect(invitation.grade(o).passed).toBe(true); + o.interactions[0]!.createdByAgentId = "other"; expect(invitation.grade(o).passed).toBe(false); + }); + it.each(["notMemberBeforeAcceptance", "realAlexJoinedOnce", "reusedLinkDifferentIdentityDenied", "actualJoinLinkBound"])("requires independent %s", key => { + const o = intended(invitation.scenarioId); o.state[key] = false; expect(invitation.grade(o).passed).toBe(false); + }); +}); +describe("membership baseline invariants", () => { + it.each(["otherCompanyMembershipAfter", "instanceMembershipAfter"])("rejects changed or absent %s", key => { + const o = intended(membership.scenarioId); o.state[key] = {}; expect(membership.grade(o).passed).toBe(false); + delete o.state[key]; expect(membership.grade(o).passed).toBe(false); + }); + it("preserves all unrelated same-company grants", () => { + const o = intended(membership.scenarioId); (o.state.alexBefore as any).grants.push({ permissionKey: "issues:read" }); + expect(membership.grade(o).passed).toBe(false); + (o.state.alexAfter as any).grants.push({ permissionKey: "issues:read" }); expect(membership.grade(o).passed).toBe(true); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts index f42935cde30..0adb4aaab06 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts @@ -1,32 +1,50 @@ +import { createHash } from "node:crypto"; import { expect } from "@playwright/test"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js"; import { checks, entity } from "./internal-support.js"; +import { pollUntil } from "../api.js"; +import { requireSafeInvitationEnrollment } from "./identity-invite-prerequisite.js"; import { identityDefinition, identityObservation, observeIdentity, readAgentResource } from "./identity-support.js"; interface Grant { permissionKey: string; scope?: unknown } interface AgentAccess { access?: { membership?: { id: string; principalType: string; principalId: string } | null; grants?: Grant[] } } interface Member { id: string; principalId: string; status: string; membershipRole: string; grants: Grant[] } type Person = Awaited>; -const people = new WeakMap(); +const people = new WeakMap(); +interface Invitation { id: string; companyId: string; state: string; humanRole: string; tokenHash?: string } +function roleQuestion(o: ReturnType) { + return !!o && o.interactions.some(row => row.kind === "ask_user_questions" && row.status === "pending" + && row.companyId === o.companyId && row.createdByAgentId === o.leadId && o.runs.some(run => run.id === row.sourceRunId && run.agentId === o.leadId) + && /\b(?:role|access level|permissions?)\b/i.test(JSON.stringify(row.payload.questionSet?.questions ?? []))); +} +function sameMember(a: Member | undefined, b: Member | undefined) { return !!a && !!b && JSON.stringify(a) === JSON.stringify(b); } export function gradeAccessGrant(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-explicit-grant-evidence", !!o, "Actual named agent grants, real authenticated people and persisted mutation/attribution are required."); if (!o) return c.result(); c.add("assumed-agent-authority-established", o.state.publicGrantStatus === 200 && o.state.agentPermissionStatus === 200 && o.state.grantPrincipalId === o.leadId && Array.isArray(o.state.effectiveGrants) && (o.state.effectiveGrants as Grant[]).some(grant => grant.permissionKey === o.state.permissionKey), "The exact company grant must exist on the actual agent principal before the canonical query; a CEO title or failed setup probe is insufficient."); if (o.scenarioId === "access.invite-teammate") { - const invitations = o.state.invitations as Array<{ id: string; companyId: string; state: string; humanRole: string }> | undefined; - c.add("actual-scoped-pending-invitation", invitations?.length === 1 && invitations[0]?.companyId === o.companyId && invitations[0].state === "pending" - && typeof invitations[0].humanRole === "string" && o.reply.includes("alex@example.test") && o.links.some(link => /\/invite\/[^/]+$/.test(new URL(link, o.origin).pathname)) - && o.activity.some(row => row.action === "invite.created" && row.entityId === invitations[0]?.id && row.actorType === "agent" && row.actorId === o.leadId), - "A real agent-attributed scoped invitation and actual join link exist without falsely reporting membership."); - c.add("invitation-human-boundary-verified", o.state.notMemberBeforeAcceptance === true && o.state.realAlexJoinedOnce === true && o.state.reusedLinkDifferentIdentityDenied === true, - "The real named prospective human joins only by supported acceptance, and a used link cannot admit another identity."); + const invitations = o.state.invitations as Invitation[] | undefined; + const created = o.state.invitationCreated as Invitation | undefined; + const clarified = !created && invitations?.length === 0 && roleQuestion(o) + && !/\b(?:I(?: have|\x27ve)?|we)\s+(?:invited|added|created)|\bAlex\s+(?:has joined|is (?:now )?a member)/i.test(o.reply); + c.add("scoped-invitation-or-consequential-role-question", clarified || !!created && created.companyId === o.companyId && created.state === "active" + && typeof created.humanRole === "string" && !!created.humanRole && o.state.notMemberBeforeAcceptance === true + && o.state.creationReplyNamesAlex === true && o.state.actualJoinLinkBound === true + && o.activity.some(row => row.action === "invite.created" && row.entityId === created.id && row.actorType === "agent" && row.actorId === o.leadId), + "A real active invitation and join link precede acceptance, or the actual agent asks the consequential missing role without creating an invitation."); + c.add("invitation-human-boundary-verified", clarified || !!created && invitations?.length === 1 && invitations[0]?.id === created.id + && invitations[0].state === "accepted" && o.state.realAlexJoinedOnce === true && o.state.reusedLinkDifferentIdentityDenied === true, + "Separate creation and post-acceptance snapshots retain real states; the used link cannot admit another identity."); } else if (o.scenarioId === "access.manage-membership") { const before = o.state.alexBefore as Member | undefined; const after = o.state.alexAfter as Member | undefined; c.add("exact-company-permission-removed", !!before && !!after && before.id === after.id && before.principalId === after.principalId && before.grants.some(grant => grant.permissionKey === "projects:manage") && !after.grants.some(grant => grant.permissionKey === "projects:manage") - && before.status === after.status && before.membershipRole === after.membershipRole && o.state.otherCompanyMembershipUnchanged === true && o.state.instanceMembershipUnchanged === true, + && before.status === after.status && before.membershipRole === after.membershipRole + && JSON.stringify(before.grants.filter(grant => grant.permissionKey !== "projects:manage")) === JSON.stringify(after.grants) + && sameMember(o.state.otherCompanyMembershipBefore as Member, o.state.otherCompanyMembershipAfter as Member) + && !!o.state.instanceMembershipBefore && JSON.stringify(o.state.instanceMembershipBefore) === JSON.stringify(o.state.instanceMembershipAfter), "Only Alex's actual company project-management grant changes; membership and other company/instance authority remain intact."); c.add("actual-agent-decision-attribution", o.activity.some(row => row.action === "company_member.permissions_updated" && row.entityId === before?.id && row.actorType === "agent" && row.actorId === o.leadId), "The saved permission change belongs to the actual permitted agent, not an impersonated board user."); @@ -39,7 +57,7 @@ export const accessGrantIdentityCases: readonly ChatCatalogCase[] = ["access.inv const permissionKey = scenarioId.endsWith("invite-teammate") ? "users:invite" : "users:manage_permissions"; const root = `/api/companies/${ctx.input.fixtures.company.id}`; const browser = ctx.input.page.context().browser(); if (!browser || !ctx.input.privateSecrets) throw new Error("Real authenticated access fixtures are required"); - const alex = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Alex", email: "alex@example.test" }); people.set(ctx, alex); + const alex = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Alex", email: "alex@example.test" }); people.set(ctx, [alex]); try { if (scenarioId.endsWith("manage-membership")) await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, alex, ctx.input.privateSecrets, "operator"); const detail = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); @@ -57,16 +75,84 @@ export const accessGrantIdentityCases: readonly ChatCatalogCase[] = ["access.inv await ctx.input.evidence("identity-explicit-agent-grant-prerequisite.json", { ...state, originalQuery: ctx.scenario.query, originalQuerySent: false, setupBlocked: attempt.status() !== 200 || probe.status !== 200, unsupportedSurface: "Public member mutations reject non-human principals before writing; agent permissions schema only synchronizes tasks:assign." }); if (attempt.status() !== 200 || probe.status !== 200) throw new Error(`Setup blocker: supported public grant path did not establish ${permissionKey} on the actual agent principal (grant HTTP ${attempt.status()}, agent read HTTP ${probe.status}); this is not an expected refusal pass`); - return { ...state, alexId: alex.id }; + const baseline = { ...state, alexId: alex.id, invitationsBefore: (await ctx.input.api.get<{ invites: Invitation[] }>(`${root}/invites`)).invites.map(row => row.id) }; + if (scenarioId.endsWith("manage-membership")) { + const current = (await ctx.input.api.get<{ members: Member[] }>(`${root}/members`)).members.find(row => row.principalId === alex.id); + if (!current) throw new Error("Real Alex membership missing before permission fixture"); + await ctx.input.api.patch(`${root}/members/${current.id}/permissions`, { grants: [...current.grants.filter(row => row.permissionKey !== "projects:manage"), { permissionKey: "projects:manage" }] }); + const alexBefore = (await ctx.input.api.get<{ members: Member[] }>(`${root}/members`)).members.find(row => row.id === current.id); + if (!alexBefore?.grants.some(row => row.permissionKey === "projects:manage")) throw new Error("Project-management prerequisite was not saved"); + const otherCompany = await ctx.input.api.post<{ id: string }>("/api/companies", { name: `Unrelated access ${ctx.input.nonce}` }); + try { + await enrollAuthenticatedPerson(ctx.input.api, otherCompany.id, alex, ctx.input.privateSecrets, "viewer"); + const otherCompanyMembershipBefore = (await ctx.input.api.get<{ members: Member[] }>(`/api/companies/${otherCompany.id}/members`)).members.find(row => row.principalId === alex.id); + if (!otherCompanyMembershipBefore) throw new Error("Other-company membership prerequisite missing"); + const instanceMembershipBefore = (await ctx.input.api.get<{ user: { id: string; isInstanceAdmin: boolean } }>(`/api/admin/users/${alex.id}/company-access`)).user; + return { ...baseline, alexBefore, otherCompany, otherCompanyMembershipBefore, instanceMembershipBefore }; + } catch (error) { await ctx.input.api.delete(`/api/companies/${otherCompany.id}`, { allowNotFound: true }); throw error; } + } + return baseline; } catch (error) { await alex.dispose(); people.delete(ctx); throw error; } }, gradeAccessGrant, { sourceFiles: sources, authentication: "human-session", - coverageGaps: ["The current public grant-management route rejects agent membership principals. Required users:* agent authority cannot be established through the supported API; actual intended mutation and human acceptance/reuse callbacks remain unqualified until this setup capability is available."], + coverageGaps: ["The current public grant-management route rejects agent membership principals. Required users:* agent authority cannot be established through the supported API; the actual intended mutation and human acceptance/reuse callbacks are authored but remain unqualified until this setup capability is available."], async observe(ctx, state) { - if (scenarioId.endsWith("invite-teammate")) state.invitations = (await ctx.input.api.get<{ invites: unknown[] }>(`/api/companies/${ctx.input.fixtures.company.id}/invites`)).invites; - else state.alexAfter = (await ctx.input.api.get<{ members: Member[] }>(`/api/companies/${ctx.input.fixtures.company.id}/members`)).members.find(member => member.principalId === state.alexId); + if (scenarioId.endsWith("invite-teammate")) state.invitations = (await ctx.input.api.get<{ invites: Invitation[] }>(`/api/companies/${ctx.input.fixtures.company.id}/invites`)).invites.filter(row => !(state.invitationsBefore as string[]).includes(row.id)); + else { + state.alexAfter = (await ctx.input.api.get<{ members: Member[] }>(`/api/companies/${ctx.input.fixtures.company.id}/members`)).members.find(member => member.principalId === state.alexId); + const other = state.otherCompany as { id: string }; + state.otherCompanyMembershipAfter = (await ctx.input.api.get<{ members: Member[] }>(`/api/companies/${other.id}/members`)).members.find(row => row.principalId === state.alexId); + state.instanceMembershipAfter = (await ctx.input.api.get<{ user: { id: string; isInstanceAdmin: boolean } }>(`/api/admin/users/${state.alexId}/company-access`)).user; + } return observeIdentity(ctx, state); }, - async verifyResult(ctx, state) { if (state.agentPermissionStatus !== 200) throw new Error("Required real agent authority was not established"); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, - async cleanup(ctx) { await people.get(ctx)?.dispose(); people.delete(ctx); }, + async act(ctx, state) { + if (!scenarioId.endsWith("invite-teammate")) return; + const root = `/api/companies/${ctx.input.fixtures.company.id}`; + const o = await pollUntil({ label: "Actual invitation or consequential role clarification", deadlineAt: Date.now() + 120_000, intervalMs: 500, + load: async () => { await ctx.refreshIssue(); return observeIdentity(ctx, state); }, + accept: value => value.runs.length === 1 && value.runs.every(run => run.status === "succeeded") && (!!value.reply || roleQuestion(value)) }); + const invites = (await ctx.input.api.get<{ invites: Invitation[] }>(`${root}/invites`)).invites.filter(row => !(state.invitationsBefore as string[]).includes(row.id)); + if (!invites.length && roleQuestion(o)) { state.invitations = []; return; } + if (invites.length !== 1 || invites[0]?.state !== "active") throw new Error("Expected one actual active invitation before human acceptance"); + const invite = invites[0]; + const hrefs = o.links.map(href => new URL(href, o.origin)).filter(url => url.origin === o.origin && /^\/invite\/[^/]+$/.test(url.pathname)); + if (hrefs.length !== 1 || !ctx.input.privateSecrets) throw new Error("One actual same-origin join link is required"); + const token = decodeURIComponent(hrefs[0]!.pathname.split("/").at(-1)!); ctx.input.privateSecrets.push(token); + if (invite.tokenHash !== createHash("sha256").update(token).digest("hex")) throw new Error("Join link does not match the exact public invitation hash"); + const alex = people.get(ctx)![0]!; + state.invitationCreated = structuredClone(invite); state.creationReplyNamesAlex = o.reply.includes("alex@example.test"); + state.actualJoinLinkBound = false; + const before = (await ctx.input.api.get<{ members: Member[] }>(`${root}/members`)).members; + state.notMemberBeforeAcceptance = !before.some(row => row.principalId === alex.id); + if (state.notMemberBeforeAcceptance !== true) throw new Error("Alex already had membership before the invitation acceptance boundary"); + // The source-qualified shared guard must approve before ANY token-path call. + if (!["operator", "viewer"].includes(invite.humanRole)) throw new Error("Invitation role requires a separately supported acceptance proof"); + await requireSafeInvitationEnrollment(ctx.input.api, ctx.input.fixtures.company.id, invite.id, invite.humanRole === "viewer" ? "viewer" : "operator", () => alex.dispose()); + const accepted = await alex.request(`/api/invites/${encodeURIComponent(token)}/accept`, "POST", { requestType: "human" }); + if (!accepted.ok) { await accepted.body?.cancel(); throw new Error("Actual model-provided invitation acceptance failed; private details withheld"); } + await accepted.body?.cancel(); + const requests = await ctx.input.api.get>(`${root}/join-requests`); + const joined = requests.filter(row => row.requestingUserId === alex.id && row.inviteId === invite.id); + if (joined.length !== 1) throw new Error("Join link did not bind to the exact created invitation and Alex"); + state.actualJoinLinkBound = true; + if (joined[0]!.status === "pending_approval") await ctx.input.api.post(`${root}/join-requests/${joined[0]!.id}/approve`); + state.realAlexJoinedOnce = (await ctx.input.api.get<{ members: Member[] }>(`${root}/members`)).members.filter(row => row.principalId === alex.id && row.status === "active").length === 1; + const other = await createAuthenticatedPerson({ api: ctx.input.api, browser: ctx.input.page.context().browser()!, secrets: ctx.input.privateSecrets, name: "Different identity" }); people.get(ctx)!.push(other); + const reused = await other.request(`/api/invites/${encodeURIComponent(token)}/accept`, "POST", { requestType: "human" }); + await reused.body?.cancel(); + state.reusedLinkDifferentIdentityDenied = [403, 404, 409, 410].includes(reused.status) && !(await ctx.input.api.get<{ members: Member[] }>(`${root}/members`)).members.some(row => row.principalId === other.id); + }, + async verifyResult(ctx, state, value) { + if (state.agentPermissionStatus !== 200) throw new Error("Required real agent authority was not established"); + const o = identityObservation(value); + if (scenarioId.endsWith("invite-teammate") && roleQuestion(o) && !state.invitationCreated) { + const question = o!.interactions.find(row => row.kind === "ask_user_questions" && row.status === "pending" && row.createdByAgentId === o!.leadId)!; + await expect(ctx.input.page.locator(`[id="interaction-${question.id}"]`)).toBeVisible(); + } else await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); + }, + async cleanup(ctx, state) { + try { for (const person of people.get(ctx) ?? []) await person.dispose(); } + finally { people.delete(ctx); const other = state?.otherCompany as { id: string } | undefined; if (other) await ctx.input.api.delete(`/api/companies/${other.id}`, { allowNotFound: true }); } + }, }));