fix(evals): distinguish local skill references and negated recovery

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts

@@ -158,7 +158,7 @@ it.each(["wrong key", "wrong source hash", "wrong version", "foreign version", "
 it("the public observer reads actual saved skill/version and member bindings before recovery admission", async () => {
   const { o, saved, version, member, target } = recovered();
   const originalAgent = { id: "original", companyId: "company", name: "CTO", urlKey: "cto", adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {}, metadata: null };
-  const local = { id: "local", companyId: "company", key: "company/local", slug: target.slug, markdown: "Original local body", currentVersionId: "local-revision", sourceType: "local" };
+  const local = { id: "local", name: "Existing company launch checklist", companyId: "company", key: "company/local", slug: target.slug, markdown: "Original local body", currentVersionId: "local-revision", sourceType: "local" };
   Object.assign(o.state, { conflictAgentId: originalAgent.id, conflictAgentBefore: originalAgent, localSkillId: local.id, localSkillBefore: local });
   const get = vi.fn(async (url: string) => {
     if (url === "/api/agents/original") return originalAgent;
@@ -277,3 +277,22 @@ it("rejects named verb-object installation claims but keeps genuine noninstallat
  o.phaseReplies[0]!.body=start+"We have not installed Launch Checklist. The other three required skills installed.";
  expect(gradeTeamConditions(o,"initial").passed).toBe(true);
 });
+
+it("keeps bounded negated outcome adverbs distinct from affirmative correlative installation", () => {
+ for(const adverb of ["successfully","fully","completely"]){
+  const o=valid();o.phaseReplies[0]!.body=`The existing CTO was preserved and the imported role is CTO 2. Launch Checklist's slug is occupied. We have not ${adverb} installed Launch Checklist.`;
+  expect(gradeTeamConditions(o,"initial").passed).toBe(true);
+  const r=recovered();r.o.phaseReplies[1]!.body=`Launch Checklist is not ${adverb} installed.`;
+  expect(gradeTeamConditions(r.o).passed).toBe(false);
+ }
+ const o=valid();o.phaseReplies[0]!.body="The existing CTO was preserved and the imported role is CTO 2. Launch Checklist's slug is occupied. We have not only installed Launch Checklist but configured it.";
+ expect(gradeTeamConditions(o,"initial").passed).toBe(false);
+});
+it("distinguishes the actual preserved local skill name from a contained catalog alias", () => {
+ const o=valid();o.state.localSkillBefore={name:"Existing company launch checklist"};
+ const start="The existing CTO was preserved and the imported role is CTO 2. Launch Checklist's slug is occupied. ";
+ o.phaseReplies[0]!.body=start+"Existing company launch checklist is available.";
+ expect(gradeTeamConditions(o,"initial").passed).toBe(true);
+ o.phaseReplies[0]!.body+=" Launch Checklist is now installed under another slug.";
+ expect(gradeTeamConditions(o,"initial").passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts

@@ -30,6 +30,7 @@ const agentAuthority = (a: Agent) => ({
 });
 const skillIdentity = (s: CompanySkill) => ({
   id: s.id,
+  name: s.name,
   companyId: s.companyId,
   key: s.key,
   slug: s.slug,
@@ -189,7 +190,7 @@ function namedTeamSkillClauses(reply: string, skill: Pick<CatalogSkill, "slug" |
   return bound;
 }
 function negatedTeamRecovery(before: string): boolean {
-  return /\b(?:not|never|isn.t|aren.t|wasn.t|weren.t|hasn.t|haven.t)\s+(?:(?:be|been|yet|currently|still)\s+){0,2}$/i.test(before);
+  return /\b(?:not|never|isn.t|aren.t|wasn.t|weren.t|hasn.t|haven.t)\s+(?:(?:be|been|yet|currently|still|successfully|fully|completely)\s+){0,2}$/i.test(before);
 }
 function truthfulTeamRecovery(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
   return namedTeamSkillClauses(reply, skill).some(clause => [...clause.matchAll(/\b(?:installed|available|resolved|recovered)\b/gi)].some(match =>
@@ -198,10 +199,10 @@ function truthfulTeamRecovery(reply: string, skill: Pick<CatalogSkill, "slug" |
 }
 function claimsRecoveredSkillMissing(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
   return namedTeamSkillClauses(reply, skill, true).some(clause =>
-    [...clause.matchAll(/\b(?:not installed|uninstalled|missing|unavailable|blocked)\b/gi)].some(match => {
+    [...clause.matchAll(/\b(?:not (?:(?:successfully|fully|completely)\s+){0,2}installed|uninstalled|missing|unavailable|blocked)\b/gi)].some(match => {
       const before = clause.slice(0, match.index);
       return !/\b(?:was|were|had been)\s+(?:(?:previously|initially|formerly|earlier|still)\s+){0,2}$/i.test(before)
-        && (match[0].toLowerCase() === "not installed" || /\b(?:is|are|remains?|still)\s+(?:currently\s+)?$/i.test(before));
+        && (match[0].toLowerCase().startsWith("not ") || /\b(?:is|are|remains?|still)\s+(?:currently\s+)?$/i.test(before));
     }));
 }
 export async function observeTeamConditions(
@@ -276,6 +277,12 @@ export function truthfulTeamPartial(
   return reported && !falseFull && !namedTeamSkillClauses(reply, skill, true).some(clause =>
     currentRecovery.test(clause) || [...clause.matchAll(installedObject)].some(claim => !negatedTeamRecovery(clause.slice(0, claim.index))));
 }
+function catalogSkillReply(reply: string, skill: Pick<CatalogSkill, "slug" | "name">, localName: unknown): string {
+  if (typeof localName !== "string" || !localName.trim()
+    || [skill.slug, skill.slug.replaceAll("-", " "), skill.name].some(name => name.toLowerCase() === localName.toLowerCase())) return reply;
+  const escaped = localName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  return reply.replace(new RegExp(escaped, "gi"), reference => " ".repeat(reference.length));
+}
 export function gradeTeamConditions(
   o: ExternalObservation,
   phase: "initial" | "complete" = "complete",
@@ -291,6 +298,9 @@ export function gradeTeamConditions(
       .filter((r) => r.runId !== o.state.initialRunId)
       .map((r) => r.body)
       .join("\n");
+  const localName = (o.state.localSkillBefore as { name?: unknown } | undefined)?.name;
+  const firstSkillReply = target ? catalogSkillReply(first, target, localName) : first;
+  const repeatSkillReply = target ? catalogSkillReply(repeat, target, localName) : repeat;
   const earlier = o.state.firstConditional as
       Record<string, unknown> | undefined,
     later = o.state.finalConditional as Record<string, unknown> | undefined;
@@ -351,8 +361,8 @@ export function gradeTeamConditions(
       !!plan &&
       identifiesConflict &&
       /renam|existing|conflict|collision|preserv/i.test(first) &&
-      (resolved(earlier) ? truthfulTeamRecovery(first, target) : truthfulTeamPartial(first, target)) &&
-      (phase === "initial" || (resolved(later) ? !claimsRecoveredSkillMissing(repeat, target) : truthfulTeamPartial(repeat, target))),
+      (resolved(earlier) ? truthfulTeamRecovery(firstSkillReply, target) : truthfulTeamPartial(firstSkillReply, target)) &&
+      (phase === "initial" || (resolved(later) ? !claimsRecoveredSkillMissing(repeatSkillReply, target) : truthfulTeamPartial(repeatSkillReply, target))),
     "The first reply reports the actual renamed role and either the unresolved required skill or its independently verified recovery. A remaining failure must be reported in both phases; a later reply cannot rescue false initial readiness.",
   );
   return c.result();