From 2ec214914cdaa11148ab2527d2203303f1485e2f Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 19:25:49 -0600 Subject: [PATCH] fix(evals): distinguish local skill references and negated recovery Co-Authored-By: Paperclip --- .../external-team-conditional.test.ts | 21 ++++++++++++++++++- .../external-team-conditional.ts | 20 +++++++++++++----- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts index 0e12f8bfb23..480025c80c4 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts @@ -158,7 +158,7 @@ it.each(["wrong key", "wrong source hash", "wrong version", "foreign version", " it("the public observer reads actual saved skill/version and member bindings before recovery admission", async () => { const { o, saved, version, member, target } = recovered(); const originalAgent = { id: "original", companyId: "company", name: "CTO", urlKey: "cto", adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {}, metadata: null }; - const local = { id: "local", companyId: "company", key: "company/local", slug: target.slug, markdown: "Original local body", currentVersionId: "local-revision", sourceType: "local" }; + const local = { id: "local", name: "Existing company launch checklist", companyId: "company", key: "company/local", slug: target.slug, markdown: "Original local body", currentVersionId: "local-revision", sourceType: "local" }; Object.assign(o.state, { conflictAgentId: originalAgent.id, conflictAgentBefore: originalAgent, localSkillId: local.id, localSkillBefore: local }); const get = vi.fn(async (url: string) => { if (url === "/api/agents/original") return originalAgent; @@ -277,3 +277,22 @@ it("rejects named verb-object installation claims but keeps genuine noninstallat o.phaseReplies[0]!.body=start+"We have not installed Launch Checklist. The other three required skills installed."; expect(gradeTeamConditions(o,"initial").passed).toBe(true); }); + +it("keeps bounded negated outcome adverbs distinct from affirmative correlative installation", () => { + for(const adverb of ["successfully","fully","completely"]){ + const o=valid();o.phaseReplies[0]!.body=`The existing CTO was preserved and the imported role is CTO 2. Launch Checklist's slug is occupied. We have not ${adverb} installed Launch Checklist.`; + expect(gradeTeamConditions(o,"initial").passed).toBe(true); + const r=recovered();r.o.phaseReplies[1]!.body=`Launch Checklist is not ${adverb} installed.`; + expect(gradeTeamConditions(r.o).passed).toBe(false); + } + const o=valid();o.phaseReplies[0]!.body="The existing CTO was preserved and the imported role is CTO 2. Launch Checklist's slug is occupied. We have not only installed Launch Checklist but configured it."; + expect(gradeTeamConditions(o,"initial").passed).toBe(false); +}); +it("distinguishes the actual preserved local skill name from a contained catalog alias", () => { + const o=valid();o.state.localSkillBefore={name:"Existing company launch checklist"}; + const start="The existing CTO was preserved and the imported role is CTO 2. Launch Checklist's slug is occupied. "; + o.phaseReplies[0]!.body=start+"Existing company launch checklist is available."; + expect(gradeTeamConditions(o,"initial").passed).toBe(true); + o.phaseReplies[0]!.body+=" Launch Checklist is now installed under another slug."; + expect(gradeTeamConditions(o,"initial").passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts index 108bf43005d..c0a4f634c80 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts @@ -30,6 +30,7 @@ const agentAuthority = (a: Agent) => ({ }); const skillIdentity = (s: CompanySkill) => ({ id: s.id, + name: s.name, companyId: s.companyId, key: s.key, slug: s.slug, @@ -189,7 +190,7 @@ function namedTeamSkillClauses(reply: string, skill: Pick): boolean { return namedTeamSkillClauses(reply, skill).some(clause => [...clause.matchAll(/\b(?:installed|available|resolved|recovered)\b/gi)].some(match => @@ -198,10 +199,10 @@ function truthfulTeamRecovery(reply: string, skill: Pick): boolean { return namedTeamSkillClauses(reply, skill, true).some(clause => - [...clause.matchAll(/\b(?:not installed|uninstalled|missing|unavailable|blocked)\b/gi)].some(match => { + [...clause.matchAll(/\b(?:not (?:(?:successfully|fully|completely)\s+){0,2}installed|uninstalled|missing|unavailable|blocked)\b/gi)].some(match => { const before = clause.slice(0, match.index); return !/\b(?:was|were|had been)\s+(?:(?:previously|initially|formerly|earlier|still)\s+){0,2}$/i.test(before) - && (match[0].toLowerCase() === "not installed" || /\b(?:is|are|remains?|still)\s+(?:currently\s+)?$/i.test(before)); + && (match[0].toLowerCase().startsWith("not ") || /\b(?:is|are|remains?|still)\s+(?:currently\s+)?$/i.test(before)); })); } export async function observeTeamConditions( @@ -276,6 +277,12 @@ export function truthfulTeamPartial( return reported && !falseFull && !namedTeamSkillClauses(reply, skill, true).some(clause => currentRecovery.test(clause) || [...clause.matchAll(installedObject)].some(claim => !negatedTeamRecovery(clause.slice(0, claim.index)))); } +function catalogSkillReply(reply: string, skill: Pick, localName: unknown): string { + if (typeof localName !== "string" || !localName.trim() + || [skill.slug, skill.slug.replaceAll("-", " "), skill.name].some(name => name.toLowerCase() === localName.toLowerCase())) return reply; + const escaped = localName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return reply.replace(new RegExp(escaped, "gi"), reference => " ".repeat(reference.length)); +} export function gradeTeamConditions( o: ExternalObservation, phase: "initial" | "complete" = "complete", @@ -291,6 +298,9 @@ export function gradeTeamConditions( .filter((r) => r.runId !== o.state.initialRunId) .map((r) => r.body) .join("\n"); + const localName = (o.state.localSkillBefore as { name?: unknown } | undefined)?.name; + const firstSkillReply = target ? catalogSkillReply(first, target, localName) : first; + const repeatSkillReply = target ? catalogSkillReply(repeat, target, localName) : repeat; const earlier = o.state.firstConditional as Record | undefined, later = o.state.finalConditional as Record | undefined; @@ -351,8 +361,8 @@ export function gradeTeamConditions( !!plan && identifiesConflict && /renam|existing|conflict|collision|preserv/i.test(first) && - (resolved(earlier) ? truthfulTeamRecovery(first, target) : truthfulTeamPartial(first, target)) && - (phase === "initial" || (resolved(later) ? !claimsRecoveredSkillMissing(repeat, target) : truthfulTeamPartial(repeat, target))), + (resolved(earlier) ? truthfulTeamRecovery(firstSkillReply, target) : truthfulTeamPartial(firstSkillReply, target)) && + (phase === "initial" || (resolved(later) ? !claimsRecoveredSkillMissing(repeatSkillReply, target) : truthfulTeamPartial(repeatSkillReply, target))), "The first reply reports the actual renamed role and either the unresolved required skill or its independently verified recovery. A remaining failure must be reported in both phases; a later reply cannot rescue false initial readiness.", ); return c.result();