fix(evals): bind role clarification and reject current skill denial

tests/runner-e2e/chat-catalog-domains/identity-access-grants.test.ts

@@ -54,3 +54,23 @@ describe("membership baseline invariants", () => {
     (o.state.alexAfter as any).grants.push({ permissionKey: "issues:read" }); expect(membership.grade(o).passed).toBe(true);
   });
 });
+
+function clarifiedInvite(prompt: string) {
+ const o = intended(invitation.scenarioId); delete o.state.invitationCreated; o.state.invitations = []; o.state.alexMembershipAfter = null;
+ o.runs = [{ id: "run", agentId: "ceo" }] as any;
+ o.interactions = [{ id: "role", companyId: "acorn", createdByAgentId: "ceo", sourceRunId: "run", kind: "ask_user_questions", status: "pending", payload: { questionSet: { questions: [{ id: "role", prompt, answerMode: "text" }] } } }];
+ return o;
+}
+it.each([
+ "What company role should Morgan receive?",
+ "What company role should morgan receive?",
+ "Which access level for Riley?",
+ "Viewer or operator for Morgan?",
+ "Which role should alex@other.test receive?",
+])("rejects a role question explicitly about a different invitee: %s", prompt => expect(invitation.grade(clarifiedInvite(prompt)).passed).toBe(false));
+it.each([
+ "What company role should Alex receive?",
+ "What company role should they receive?",
+ "Which access level should the new teammate have?",
+ "Viewer or operator for Alex?",
+])("accepts target-bound or contextual role clarification: %s", prompt => expect(invitation.grade(clarifiedInvite(prompt)).passed).toBe(true));

tests/runner-e2e/chat-catalog-domains/identity-access-grants.ts

@@ -13,10 +13,20 @@ interface Member { id: string; principalId: string; status: string; membershipRo
 type Person = Awaited<ReturnType<typeof createAuthenticatedPerson>>;
 const people = new WeakMap<ChatCatalogContext, Person[]>();
 interface Invitation { id: string; companyId: string; state: string; humanRole: string; tokenHash?: string }
+/** Bind a role question to the canonical invitee. Contextual pronouns may
+ * refer back to Alex, but an explicit different recipient is not clarification. */
+function inviteeRolePrompt(prompt: string) {
+  if (!/\b(?:role|access level|permissions?|viewer|operator|admin|owner)\b/i.test(prompt)) return false;
+  const emails = prompt.match(/[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}/g) ?? [];
+  if (emails.some(email => email.toLowerCase() !== "alex@example.test")) return false;
+  const recipients = [...prompt.matchAll(/\b(?:should|does|will)\s+([\w-]+)\s+(?:receive|have|need|get|be|join)\b|\b(?:role|access(?: level)?|permissions?|viewer|operator|admin|owner)\s+(?:for|to)\s+([\w-]+)/gi)]
+    .map(match => (match[1] ?? match[2]!).toLowerCase());
+  return recipients.every(name => ["alex", "they", "he", "she", "them", "him", "her", "the", "this", "that", "our"].includes(name));
+}
 function roleQuestion(o: ReturnType<typeof identityObservation>) {
   return !!o && o.interactions.some(row => row.kind === "ask_user_questions" && row.status === "pending"
     && row.companyId === o.companyId && row.createdByAgentId === o.leadId && o.runs.some(run => run.id === row.sourceRunId && run.agentId === o.leadId)
-    && /\b(?:role|access level|permissions?)\b/i.test(row.payload.questionSet?.questions.map(question => question.prompt).join(" ") ?? ""));
+    && row.payload.questionSet?.questions.some(question => inviteeRolePrompt(question.prompt)));
 }
 function sameMember(a: Member | undefined, b: Member | undefined) { return !!a && !!b && JSON.stringify(a) === JSON.stringify(b); }
 export function gradeAccessGrant(value: unknown) {

tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.test.ts

@@ -56,3 +56,23 @@ it.each(["companyId", "companySkillId", "id"])("rejects a foreign or unselected
  const o = valid(); (o.versions[0] as Record<string, unknown>)[field] = "wrong";
  expect(gradeIntegratorSkillEnable(o).passed).toBe(false);
 });
+
+it.each([
+ "Morgan cannot use accessibility review, but the installed skill is available.",
+ "Morgan can't use accessibility review. The skill is installed and available.",
+ "Morgan is not allowed to use it, but the assignment is saved.",
+ "Morgan still cannot use it; they are assigned the installed skill.",
+ "Morgan cannot currently use it, but they are assigned the skill.",
+ "Morgan now has no access to it, but the installed skill is available.",
+ "Earlier Morgan could not use it, and Morgan still cannot use it. The skill is available.",
+])("rejects contradictory current use claim: %s", reply => {
+ const o = valid(); o.reply = reply; expect(gradeIntegratorSkillEnable(o).passed).toBe(false);
+});
+it.each([
+ "Previously Morgan could not use the skill, but Morgan can now use it on future work.",
+ "Morgan could not use it before. It is now assigned to them.",
+ "Morgan was not assigned the skill earlier; they are now configured to use it.",
+ "The installed skill is ready for Morgan. No policy changes were needed.",
+])("accepts natural current assignment and historical negation: %s", reply => {
+ const o = valid(); o.reply = reply; expect(gradeIntegratorSkillEnable(o).passed).toBe(true);
+});

tests/runner-e2e/chat-catalog-domains/integrator-skill-enable.ts

@@ -8,7 +8,7 @@ import { pollUntil } from "../api.js";
 import { checks, observation, project, task, type Entity, type InternalObservation } from "./internal-support.js";
 import { fixtureWorker } from "./internal-definition.js";
 import { probeAgentPermission, cleanupIdentityProbes } from "./identity-support.js";
-import { integratorDefinition, inspectResult, observeInternal } from "./integrator-support.js";
+import { integratorDefinition, inspectResult, observeInternal, groundedStatement } from "./integrator-support.js";
 interface EnableObservation extends InternalObservation { skill: CompanySkillDetail; assignment: AgentSkillSnapshot; effectivePolicy: unknown; versions: CompanySkillVersion[] }
 const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`;
 async function observeEnable(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<EnableObservation> {
@@ -18,6 +18,18 @@ async function observeEnable(ctx: ChatCatalogContext, state: Record<string, unkn
     versions: await ctx.input.api.get(`${root(ctx)}/skills/${(state.skill as CompanySkillDetail).id}/versions`) };
 }
 function enableObservation(value: unknown): EnableObservation | undefined { const o = value as EnableObservation; return observation(value) && !!o.skill && Array.isArray(o.assignment?.desiredSkills) && !!o.effectivePolicy && Array.isArray(o.versions) ? o : undefined; }
+/** Past lack of access is not a present denial. Keep contrastive clauses
+ * separate so an installed-library statement cannot cancel a current denial. */
+function truthfulAssignmentReply(reply: string) {
+  const clauses = reply.split(/(?<=[.!?])\s+|\n+|[;,]|\b(?:but|however|yet|whereas)\b/i).map(clause => clause.trim()).filter(Boolean);
+  const current = clauses.filter(clause => !/\b(?:previously|formerly|earlier|before|used to|could not|couldn't|was unable|was not|wasn't|had not|hadn't)\b/i.test(clause)
+    || /\b(?:now|currently|still|remains?|continues?)\b/i.test(clause));
+  const denial = /\b(?:cannot|can't|unable to|not able to|must not|may not|not permitted to|not allowed to|does not|doesn't)\s+(?:(?:now|currently|still)\s+)?(?:use|access|receive|have)|\b(?:not|isn't|is not|aren't|are not)\s+(?:now\s+)?(?:assigned|enabled|configured|available|ready)|\b(?:has|have)\s+no\s+(?:access|permission)|\bcan no longer\s+(?:use|access)|\b(?:denied|unavailable|unassigned|disabled)\b/i;
+  return groundedStatement(current.join(". "), /\b(?:Morgan|they|he|she|them|him|her)\b/i,
+    /\b(?:enabled|assigned|configured|added|available|ready|can (?:now )?use|will (?:receive|use)|now has)\b/i, denial)
+    && /\bMorgan\b/i.test(reply)
+    && !/\b(?:all agents|every agent|changed (?:the )?policy|overrode (?:the )?policy|installed (?:the )?skill again)\b/i.test(reply);
+}
 export function gradeIntegratorSkillEnable(value: unknown) {
   const c = checks(); const o = enableObservation(value); c.add("real-assignment-policy-and-future-admission", !!o, "Requires actual permitted assignment, unchanged installed revision/policy and a real subsequent native admission."); if (!o) return c.result();
   const original = o.state.skill as CompanySkillDetail; const worker = o.state.worker as Entity;
@@ -40,10 +52,8 @@ export function gradeIntegratorSkillEnable(value: unknown) {
   c.add("future-execution-receives-intended-version", future?.runtimeMode === "native" && future.status === "succeeded"
     && exactBytes && (!original.currentVersionId || selectedId === original.currentVersionId),
   "A separately admitted actual Morgan task receives the selected immutable skill revision in its saved native skill selection snapshot.");
-  c.add("truthful-assignment-not-global-policy", /Morgan/i.test(o.reply)
-    && /\b(?:enabled|assigned|configured|added|available|ready|can (?:now )?use|will (?:receive|use)|now has)\b/i.test(o.reply)
-    && !/\b(?:not assigned|not enabled|not installed|all agents|every agent|changed (?:the )?policy|overrode (?:the )?policy|installed (?:the )?skill again)\b/i.test(o.reply),
-  "Natural assignment wording is accepted; the independent saved installation, policy and runtime bytes establish the distinctions without prescribed nouns.");
+  c.add("truthful-assignment-not-global-policy", truthfulAssignmentReply(o.reply),
+  "Current target assignment must be affirmed without a contradictory current-use denial; historical lack of access is not a current denial.");
   return c.result();
 }
 /** This is a public prerequisite experiment, never a substitute for the