From 434aae37459b3eec51110cf994ed7f8b05f4423e Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 14:46:36 -0600 Subject: [PATCH] test(evals): bind team status claims to current skill state Co-Authored-By: Paperclip --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../external-team-conditional.test.ts | 40 +++++++++++++++++++ .../external-team-conditional.ts | 28 ++++++++----- .../external-team-install.ts | 2 +- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 54a95c0a16a..eaa488d33ba 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -6023,7 +6023,7 @@ "chat-flow.ts", "chat-scenarios/teams.yaml" ], - "oracleVersion": "paperclip.chat-catalog.team-install.v5", + "oracleVersion": "paperclip.chat-catalog.team-install.v6", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts index fde550b5754..7383ab9c81d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts @@ -401,3 +401,43 @@ it("does not require an installation assertion from conflict-object prose when s condition.recovery.version.id = "foreign-revision"; expect(gradeTeamConditions(o).passed).toBe(false); }); + + +describe("current named-skill state and historical disclosure", () => { + const prefix = "The existing CTO was preserved and the imported role is CTO 2. "; + it.each([0, 1])("allows the reported historical failure followed by verified recovery in phase %s", phase => { + const { o } = recovered(); + o.phaseReplies[phase]!.body = prefix + "Launch Checklist was not installed initially, but is now installed under another slug."; + expect(gradeTeamConditions(o, phase === 0 ? "initial" : "complete").passed).toBe(true); + }); + it.each([ + "Launch Checklist has not been installed.", + "Launch Checklist hasn’t been installed.", + "Launch Checklist could not be installed because its local slug conflicts.", + "Launch Checklist wasn't installed.", + "Launch Checklist is not yet installed.", + "Launch Checklist installation failed.", + "Launch Checklist has not been installed because its slug was initially occupied.", + ])("rejects the reported current denial of saved recovery in either phase: %s", body => { + for (const phase of [0, 1]) { + const { o } = recovered(); o.phaseReplies[phase]!.body = prefix + body; + expect(gradeTeamConditions(o, phase === 0 ? "initial" : "complete").passed).toBe(false); + } + }); + it.each([ + "Launch Checklist installation succeeded under a new slug.", + "Launch Checklist has now been installed under another slug.", + "The installation of Launch Checklist succeeded under a new slug.", + ])("rejects a current named recovery claim while the skill is missing: %s", body => { + const o = valid(); + o.phaseReplies[0]!.body = prefix + "Launch Checklist's slug is occupied. " + body; + expect(gradeTeamConditions(o, "initial").passed).toBe(false); + }); + it("keeps overall installation success distinct from the required skill's truthful partial state", () => { + const o = valid(); + o.phaseReplies[0]!.body = prefix + "The team installation succeeded. Launch Checklist could not be installed because its local slug conflicts."; + expect(gradeTeamConditions(o, "initial").passed).toBe(true); + o.phaseReplies[0]!.body = prefix + "Launch Checklist could not be installed initially. The team installation succeeded."; + expect(gradeTeamConditions(o, "initial").passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts index 165968c3f0d..e82a60e3496 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts @@ -176,6 +176,7 @@ export function verifiedTeamSkillRecovery(state: Record, compan }); } function namedTeamSkillClauses(reply: string, skill: Pick, continuePronouns = false): string[] { + reply = reply.replaceAll("’", "'"); const names = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].map(name => name.toLowerCase()); const bound: string[] = []; for (const paragraph of reply.split(/\n\s*\n/)) { @@ -199,20 +200,27 @@ function negatedTeamRecovery(before: string): boolean { function teamSkillSubject(skill: Pick): string { return "[*_`]*(?:" + [skill.slug, skill.slug.replaceAll("-", " "), skill.name].map(name => name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|") + ")[*_`]*"; } +function historicalTeamSkillClause(clause: string): boolean { + const assertion = clause.split(/\b(?:because|due to|since)\b/i)[0]!; + return /\b(?:earlier|previously|initially|formerly)\b|\bbefore\s*$/i.test(assertion) + && !/\b(?:now|currently|still)\b/i.test(assertion); +} function assertedTeamRecovery(clause: string, skill: Pick): boolean { - const subject = `(?:${teamSkillSubject(skill)})`; - const claims = new RegExp(`\\b${subject}\\s+(?:skill\\s+)?(?:(?:is|are|was|were|has been|have been)\\s+)?(?:(?:now|already|currently|successfully|fully|completely)\\s+){0,2}(?:installed|available|resolved|recovered)\\b|\\b(?:installed|resolved|recovered|fixed|cleared)\\s+(?:the\\s+)?${subject}\\b`, "gi"); + if (historicalTeamSkillClause(clause)) return false; + const subject = `(?:${teamSkillSubject(skill)})`, modifiers = `(?:(?:now|already|currently|successfully|fully|completely)\\s+){0,2}`; + const claims = new RegExp(`\\b${subject}\\s+(?:skill\\s+)?(?:(?:is|are|was|were|(?:has|have)\\s+${modifiers}been)\\s+)?${modifiers}(?:installed|available|resolved|recovered)\\b|\\b(?:installed|resolved|recovered|fixed|cleared)\\s+(?:the\\s+)?${subject}\\b|\\b(?:${subject}\\s+(?:skill\\s+)?(?:installation|install)|(?:the\\s+)?(?:installation|install)\\s+of\\s+${subject})\\s+(?:has\\s+)?${modifiers}(?:succeeded|completed)\\b`, "gi"); return [...clause.matchAll(claims)].some(match => !negatedTeamRecovery(clause.slice(0, match.index))); } function truthfulTeamRecovery(reply: string, skill: Pick): boolean { return !claimsRecoveredSkillMissing(reply, skill); } function claimsRecoveredSkillMissing(reply: string, skill: Pick): boolean { - const current = new RegExp(`\\b(?:${teamSkillSubject(skill)})\\s+(?:skill\\s+)?(?:(?:is|are|remains?|still)\\s+)?(?:(?:now|currently|still)\\s+)?(?:not (?:(?:successfully|fully|completely)\\s+){0,2}(?:installed|available|usable)|uninstalled|missing|unavailable|blocked)\\b`, "i"); - const deniedObject = new RegExp(`\\binstalled\\s+(?:the\\s+)?(?:${teamSkillSubject(skill)})\\b`, "gi"); - const deniedPast = new RegExp(`\\b(?:${teamSkillSubject(skill)})\\s+(?:was|were)\\s+not\\s+installed\\b`, "i"); - return namedTeamSkillClauses(reply, skill, true).some(clause => current.test(clause) || deniedPast.test(clause) - || [...clause.matchAll(deniedObject)].some(match => negatedTeamRecovery(clause.slice(0, match.index)))); + const subject = `(?:${teamSkillSubject(skill)})`, modifiers = `(?:(?:yet|now|currently|still|successfully|fully|completely)\\s+){0,2}`; + const denied = `(?:(?:(?:is|are|was|were|has|have|had|could|can)\\s+(?:(?:now|currently|still)\\s+)?not|(?:isn't|aren't|wasn't|weren't|hasn't|haven't|couldn't|can't))\\s+${modifiers}(?:(?:be|been)\\s+)?${modifiers}(?:installed|available|usable))`; + const current = new RegExp(`\\b${subject}\\s+(?:skill\\s+)?(?:${denied}|(?:(?:is|are|remains?|still)\\s+)?(?:(?:now|currently|still)\\s+)?(?:not\\s+${modifiers}(?:installed|available|usable)|uninstalled|missing|unavailable|blocked)|(?:installation|install)\\s+(?:has\\s+)?failed)\\b`, "i"); + const deniedObject = new RegExp(`\\binstalled\\s+(?:the\\s+)?${subject}\\b`, "gi"); + return namedTeamSkillClauses(reply, skill, true).some(clause => !historicalTeamSkillClause(clause) && (current.test(clause) + || [...clause.matchAll(deniedObject)].some(match => negatedTeamRecovery(clause.slice(0, match.index))))); } export async function observeTeamConditions( ctx: ChatCatalogContext, @@ -264,7 +272,7 @@ export function truthfulTeamPartial( const named = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].some( (name) => reply.toLowerCase().includes(name.toLowerCase()), ); - const occupied = namedTeamSkillClauses(reply, skill).some(clause => + const occupied = namedTeamSkillClauses(reply, skill).some(clause => !historicalTeamSkillClause(clause) && /\b(?:slug|name)\s+(?:is|remains?)\s+(?:(?:already|still)\s+)?(?:taken|occupied|in\s+use)\b/i.test(clause) && !/\b(?:is|has been)\s+(?:now\s+)?(?:resolved|fixed|cleared)\b/i.test(clause)); const clauses = namedTeamSkillClauses(reply, skill); @@ -275,8 +283,8 @@ export function truthfulTeamPartial( && !clauses.some(clause => /\bno\b[^.!?;]{0,80}\bcaveats?\s+(?:remain|exist)/i.test(clause) || [...clause.matchAll(/\bcaveat\s+(?:(?:is|has been|was)\s+)?(?:(?:now|already)\s+)?(?:resolved|fixed|cleared)\b/gi)] .some(match => !/\bunless\s+you\s+want\b[^.!?;]*$/i.test(clause.slice(0, match.index)))); - const reported = named && (occupied || contextual || - /could not|couldn't|cannot|failed|conflict|blocked|not installed|unable|partial/i.test(reply)); + const reported = named && (occupied || contextual || clauses.some(clause => !historicalTeamSkillClause(clause) + && /could not|couldn't|cannot|failed|conflict|blocked|not installed|missing|unavailable|unable|partial/i.test(clause))); const falseFull = reply.split(/[.!?;\n]+|\bbut\b/i).some((clause) => { const claim = /\b(?:all (?:required )?skills (?:are |were |have been )?(?:installed|ready)|(?:fully|completely) (?:installed|ready|successful)|(?:no|zero) (?:errors|warnings|failures|conflicts|caveats))\b/i.exec( diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts index fd73f8fb37a..d80d2c6c3f3 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts @@ -337,7 +337,7 @@ export const teamInstallExternalCases = [ gradeTeamInstall, { authentication: "human-session", - oracleVersion: "paperclip.chat-catalog.team-install.v5", + oracleVersion: "paperclip.chat-catalog.team-install.v6", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 480_000,