From 36dc8027e9b3a2c18e195c4073d9de4570ffeacc Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 12:08:37 -0500 Subject: [PATCH] fix(evals): bind role clarification and reject current skill denial --- .../identity-access-grants.test.ts | 20 +++++++++++++++++++ .../identity-access-grants.ts | 12 ++++++++++- .../integrator-skill-enable.test.ts | 20 +++++++++++++++++++ .../integrator-skill-enable.ts | 20 ++++++++++++++----- 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts index 0ad45c50232..a57c6565bec 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts @@ -54,3 +54,23 @@ describe("membership baseline invariants", () => { (o.state.alexAfter as any).grants.push({ permissionKey: "issues:read" }); expect(membership.grade(o).passed).toBe(true); }); }); + +function clarifiedInvite(prompt: string) { + const o = intended(invitation.scenarioId); delete o.state.invitationCreated; o.state.invitations = []; o.state.alexMembershipAfter = null; + o.runs = [{ id: "run", agentId: "ceo" }] as any; + o.interactions = [{ id: "role", companyId: "acorn", createdByAgentId: "ceo", sourceRunId: "run", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [{ id: "role", prompt, answerMode: "text" }] } } }]; + return o; +} +it.each([ + "What company role should Morgan receive?", + "What company role should morgan receive?", + "Which access level for Riley?", + "Viewer or operator for Morgan?", + "Which role should alex@other.test receive?", +])("rejects a role question explicitly about a different invitee: %s", prompt => expect(invitation.grade(clarifiedInvite(prompt)).passed).toBe(false)); +it.each([ + "What company role should Alex receive?", + "What company role should they receive?", + "Which access level should the new teammate have?", + "Viewer or operator for Alex?", +])("accepts target-bound or contextual role clarification: %s", prompt => expect(invitation.grade(clarifiedInvite(prompt)).passed).toBe(true)); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts index 2ee6700f0c7..a6775020553 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts @@ -13,10 +13,20 @@ interface Member { id: string; principalId: string; status: string; membershipRo type Person = Awaited>; const people = new WeakMap(); interface Invitation { id: string; companyId: string; state: string; humanRole: string; tokenHash?: string } +/** Bind a role question to the canonical invitee. Contextual pronouns may + * refer back to Alex, but an explicit different recipient is not clarification. */ +function inviteeRolePrompt(prompt: string) { + if (!/\b(?:role|access level|permissions?|viewer|operator|admin|owner)\b/i.test(prompt)) return false; + const emails = prompt.match(/[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}/g) ?? []; + if (emails.some(email => email.toLowerCase() !== "alex@example.test")) return false; + const recipients = [...prompt.matchAll(/\b(?:should|does|will)\s+([\w-]+)\s+(?:receive|have|need|get|be|join)\b|\b(?:role|access(?: level)?|permissions?|viewer|operator|admin|owner)\s+(?:for|to)\s+([\w-]+)/gi)] + .map(match => (match[1] ?? match[2]!).toLowerCase()); + return recipients.every(name => ["alex", "they", "he", "she", "them", "him", "her", "the", "this", "that", "our"].includes(name)); +} function roleQuestion(o: ReturnType) { return !!o && o.interactions.some(row => row.kind === "ask_user_questions" && row.status === "pending" && row.companyId === o.companyId && row.createdByAgentId === o.leadId && o.runs.some(run => run.id === row.sourceRunId && run.agentId === o.leadId) - && /\b(?:role|access level|permissions?)\b/i.test(row.payload.questionSet?.questions.map(question => question.prompt).join(" ") ?? "")); + && row.payload.questionSet?.questions.some(question => inviteeRolePrompt(question.prompt))); } function sameMember(a: Member | undefined, b: Member | undefined) { return !!a && !!b && JSON.stringify(a) === JSON.stringify(b); } export function gradeAccessGrant(value: unknown) { diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.test.ts index 358527a2cda..4a6f8ef8be9 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.test.ts @@ -56,3 +56,23 @@ it.each(["companyId", "companySkillId", "id"])("rejects a foreign or unselected const o = valid(); (o.versions[0] as Record)[field] = "wrong"; expect(gradeIntegratorSkillEnable(o).passed).toBe(false); }); + +it.each([ + "Morgan cannot use accessibility review, but the installed skill is available.", + "Morgan can't use accessibility review. The skill is installed and available.", + "Morgan is not allowed to use it, but the assignment is saved.", + "Morgan still cannot use it; they are assigned the installed skill.", + "Morgan cannot currently use it, but they are assigned the skill.", + "Morgan now has no access to it, but the installed skill is available.", + "Earlier Morgan could not use it, and Morgan still cannot use it. The skill is available.", +])("rejects contradictory current use claim: %s", reply => { + const o = valid(); o.reply = reply; expect(gradeIntegratorSkillEnable(o).passed).toBe(false); +}); +it.each([ + "Previously Morgan could not use the skill, but Morgan can now use it on future work.", + "Morgan could not use it before. It is now assigned to them.", + "Morgan was not assigned the skill earlier; they are now configured to use it.", + "The installed skill is ready for Morgan. No policy changes were needed.", +])("accepts natural current assignment and historical negation: %s", reply => { + const o = valid(); o.reply = reply; expect(gradeIntegratorSkillEnable(o).passed).toBe(true); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.ts index dd8656ac3d9..131945d74df 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.ts @@ -8,7 +8,7 @@ import { pollUntil } from "../api.js"; import { checks, observation, project, task, type Entity, type InternalObservation } from "./internal-support.js"; import { fixtureWorker } from "./internal-definition.js"; import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js"; -import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js"; +import { integratorDefinition, inspectResult, observeInternal, groundedStatement } from "./integrator-support.js"; interface EnableObservation extends InternalObservation { skill: CompanySkillDetail; assignment: AgentSkillSnapshot; effectivePolicy: unknown; versions: CompanySkillVersion[] } const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`; async function observeEnable(ctx: ChatCatalogContext, state: Record): Promise { @@ -18,6 +18,18 @@ async function observeEnable(ctx: ChatCatalogContext, state: Record clause.trim()).filter(Boolean); + const current = clauses.filter(clause => !/\b(?:previously|formerly|earlier|before|used to|could not|couldn't|was unable|was not|wasn't|had not|hadn't)\b/i.test(clause) + || /\b(?:now|currently|still|remains?|continues?)\b/i.test(clause)); + const denial = /\b(?:cannot|can't|unable to|not able to|must not|may not|not permitted to|not allowed to|does not|doesn't)\s+(?:(?:now|currently|still)\s+)?(?:use|access|receive|have)|\b(?:not|isn't|is not|aren't|are not)\s+(?:now\s+)?(?:assigned|enabled|configured|available|ready)|\b(?:has|have)\s+no\s+(?:access|permission)|\bcan no longer\s+(?:use|access)|\b(?:denied|unavailable|unassigned|disabled)\b/i; + return groundedStatement(current.join(". "), /\b(?:Morgan|they|he|she|them|him|her)\b/i, + /\b(?:enabled|assigned|configured|added|available|ready|can (?:now )?use|will (?:receive|use)|now has)\b/i, denial) + && /\bMorgan\b/i.test(reply) + && !/\b(?:all agents|every agent|changed (?:the )?policy|overrode (?:the )?policy|installed (?:the )?skill again)\b/i.test(reply); +} export function gradeIntegratorSkillEnable(value: unknown) { const c = checks(); const o = enableObservation(value); c.add("real-assignment-policy-and-future-admission", !!o, "Requires actual permitted assignment, unchanged installed revision/policy and a real subsequent native admission."); if (!o) return c.result(); const original = o.state.skill as CompanySkillDetail; const worker = o.state.worker as Entity; @@ -40,10 +52,8 @@ export function gradeIntegratorSkillEnable(value: unknown) { c.add("future-execution-receives-intended-version", future?.runtimeMode === "native" && future.status === "succeeded" && exactBytes && (!original.currentVersionId || selectedId === original.currentVersionId), "A separately admitted actual Morgan task receives the selected immutable skill revision in its saved native skill selection snapshot."); - c.add("truthful-assignment-not-global-policy", /Morgan/i.test(o.reply) - && /\b(?:enabled|assigned|configured|added|available|ready|can (?:now )?use|will (?:receive|use)|now has)\b/i.test(o.reply) - && !/\b(?:not assigned|not enabled|not installed|all agents|every agent|changed (?:the )?policy|overrode (?:the )?policy|installed (?:the )?skill again)\b/i.test(o.reply), - "Natural assignment wording is accepted; the independent saved installation, policy and runtime bytes establish the distinctions without prescribed nouns."); + c.add("truthful-assignment-not-global-policy", truthfulAssignmentReply(o.reply), + "Current target assignment must be affirmed without a contradictory current-use denial; historical lack of access is not a current denial."); return c.result(); } /** This is a public prerequisite experiment, never a substitute for the