From 88c4f357186dc308e2fe185b4a60ad22c8c3ef18 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 6 Nov 2026 00:17:10 -0600 Subject: [PATCH] test(evals): recognize current unresolved team skill conflicts Accept truthful exact-skill unresolved installation reports while rejecting unrelated or denied conflicts and independent current recovery claims. Keep saved state, actor, phase, links and original bounds strict. Co-Authored-By: Paperclip --- tests/runner-e2e/CHAT-FEATURES.md | 9 +++++++++ tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../external-team-conditional.test.ts | 17 +++++++++++++++++ .../external-team-conditional.ts | 12 ++++++++++-- .../external-team-install.ts | 2 +- 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 63335a56168..e29e8c61fda 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -489,3 +489,12 @@ Menu evidence labels the parameter as `expectedDefinitionId`; the observed UI pr is the current-company route and visible definition key. Private entry retains the independent exact-ID/current-owner check. Delayed flyout and own-entry visibility are awaited within the same passed navigation deadline. + +Team install v8 accepts a current, exact-skill unresolved installation or slug-conflict +report when the saved required skill is still missing. Independent current recovery +claims and all-skills-installed claims still fail. Result 5171 remains originally +FAILED; a local diagnostic changed only its truthful repeat phrase from "remains +unresolved" to the already supported "remains blocked" and passed the owning grade. +This is an assertion correction, with no new native pass or runtime-migration credit. +The derived runtime fixture, protected local skill, actor/phase/resource/link checks, +original query, two native/human turns and 480000ms attempt are unchanged. diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index d29741a87b0..e6487857eab 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -6045,7 +6045,7 @@ "chat-flow.ts", "chat-scenarios/teams.yaml" ], - "oracleVersion": "paperclip.chat-catalog.team-install.v7", + "oracleVersion": "paperclip.chat-catalog.team-install.v8", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts index efcd615386c..60455bbd498 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts @@ -489,3 +489,20 @@ it("does not inherit history for a coordinated past assertion with a current mar expect(gradeTeamConditions(o, phase === 0 ? "initial" : "complete").passed).toBe(false); } }); + +it("accepts a current exact-skill unresolved installation disclosure without inventing recovery", () => { + const target = { slug: "doc-maintenance", name: "doc-maintenance" }; + const actual = "The existing CTO was preserved; `doc-maintenance` remains unresolved due to an existing company skill slug."; + expect(truthfulTeamPartial(actual, target, true)).toBe(true); + const o = valid(); + o.state.blockedSkill = target; + o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. The doc-maintenance skill was not installed because the local slug conflicts."; + o.phaseReplies[1]!.body = actual; + expect(gradeTeamConditions(o).passed).toBe(true); + for (const contradiction of [ + "A security review remains unresolved. The team is installed.", + "No unresolved doc-maintenance conflicts remain.", + "All required skills are installed.", + actual + " The doc-maintenance skill is now installed.", + ]) expect(truthfulTeamPartial(contradiction, target, true)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts index 5d7ab46ccce..28bc7b486f8 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts @@ -294,7 +294,12 @@ export function truthfulTeamPartial( && !clauses.some(clause => /\bno\b[^.!?;]{0,80}\bcaveats?\s+(?:remain|exist)/i.test(clause) || [...clause.matchAll(/\bcaveat\s+(?:(?:is|has been|was)\s+)?(?:(?:now|already)\s+)?(?:resolved|fixed|cleared)\b/gi)] .some(match => !/\bunless\s+you\s+want\b[^.!?;]*$/i.test(clause.slice(0, match.index)))); - const reported = named && (occupied || contextual || clauses.some(clause => !historicalTeamSkillClause(clause, skill) + const unresolved = clauses.some(clause => { + const claim = new RegExp(`\\b${teamSkillSubject(skill)}\\s+(?:skill\\s+)?(?:is|remains?)\\s+(?:still\\s+)?unresolved\\b`, "i").exec(clause); + return !!claim && !historicalTeamSkillClause(clause, skill) + && !negatedTeamRecovery(clause.slice(0, claim.index)) && /\b(?:installation|install|conflict|collision|slug)\b/i.test(clause); + }); + const reported = named && (occupied || contextual || unresolved || clauses.some(clause => !historicalTeamSkillClause(clause, skill) && /could not|couldn't|cannot|failed|conflict|blocked|not installed|missing|unavailable|unable|partial/i.test(clause))); const falseFull = reply.split(/[.!?;\n]+|\bbut\b/i).some((clause) => { const claim = @@ -308,7 +313,10 @@ export function truthfulTeamPartial( ) ); }); - return reported && !falseFull && !namedTeamSkillClauses(reply, skill, true).some(clause => assertedTeamRecovery(clause, skill)); + const deniesUnresolvedConflict = clauses.some(clause => !historicalTeamSkillClause(clause, skill) + && new RegExp(`\\bno\\s+unresolved\\s+${teamSkillSubject(skill)}\\s+(?:skill\\s+)?conflicts?\\b`, "i").test(clause)); + return reported && !falseFull && !deniesUnresolvedConflict + && !namedTeamSkillClauses(reply, skill, true).some(clause => assertedTeamRecovery(clause, skill)); } function catalogSkillReply(reply: string, skill: Pick, localName: unknown): string { if (typeof localName !== "string" || !localName.trim() diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts index 00d3a7427f5..17e399a9f51 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts @@ -346,7 +346,7 @@ export const teamInstallExternalCases = [ gradeTeamInstall, { authentication: "human-session", - oracleVersion: "paperclip.chat-catalog.team-install.v7", + oracleVersion: "paperclip.chat-catalog.team-install.v8", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 480_000,