fix(evals): admit verified team skill conflict recovery

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/CHAT-FEATURES.md

@@ -249,6 +249,17 @@ version. The receipt identifies the actual editor surface. The original case
 clock, query, and one-human/one-native limits remain unchanged. Older failed
 attempts are retained; component controls do not establish a native pass.
 
+### Team install conflict recovery
+
+`teams.install-team` v2 accepts either an honestly reported missing required
+skill or a supported catalog recovery that preserves the original local skill
+and agent. Recovery requires the accepted catalog key and source hash, exact
+current immutable-version bytes, and that key assigned to the intended imported
+members. It does not accept merely installing an unattached skill. The original
+repeat query, resource identity reconciliation, final rendered-link inspection,
+two-human/two-native limits, and case clock remain unchanged. Failed earlier
+attempts remain failed; this correction does not establish a native pass.
+
 ### Six prerequisite probes are not completed journeys
 
 The following six definitions retain explicit coverage gaps and must not be

tests/runner-e2e/chat-catalog-descriptors.json

@@ -6015,7 +6015,7 @@
       "chat-flow.ts",
       "chat-scenarios/teams.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.external.v1",
+    "oracleVersion": "paperclip.chat-catalog.team-install.v2",
     "expectedRunCount": 2,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts

@@ -1,6 +1,8 @@
-import { describe, it, expect } from "vitest";
+import { createHash } from "node:crypto";
+import { describe, it, expect, vi } from "vitest";
 import {
   gradeTeamConditions,
+  observeTeamConditions,
   truthfulTeamPartial,
 } from "./external-team-conditional.js";
 import type { ExternalObservation } from "./external-support.js";
@@ -119,3 +121,71 @@ describe("real team post-import conditional controls", () => {
       expect(truthfulTeamPartial(text, skill)).toBe(false);
   });
 });
+
+function recovered() {
+  const o = valid(); o.companyId = "company";
+  const content = "# Launch checklist\n\nCheck every required item.";
+  const target = { id: "catalog", key: "catalog/launch-checklist", slug: skill.slug, name: skill.name, contentHash: "sha256:catalog",
+    files: [{ path: "SKILL.md", kind: "skill", sha256: createHash("sha256").update(content).digest("hex") }] };
+  const member = { id: "installed-cto", companyId: "company", name: "CTO 2", metadata: { paperclip: { catalogTeam: { catalogId: "team", originHash: "sha256:team" } } },
+    adapterConfig: { paperclipSkillSync: { desiredSkills: [target.key] } } };
+  const saved = { id: "saved", companyId: "company", key: target.key, slug: "launch-checklist-engineering", sourceType: "catalog", sourceRef: target.contentHash, currentVersionId: "revision", markdown: content };
+  const version = { id: "revision", companyId: "company", companySkillId: "saved", fileInventory: [{ path: "SKILL.md", kind: "skill", content }] };
+  Object.assign(o.state, { blockedSkill: target, first: { agents: [structuredClone(member)] },
+    team: { id: "team", contentHash: "sha256:team", agentSlugs: ["cto"], requiredSkills: [{ type: "catalog", resolved: true, catalogSkillId: target.id, catalogSkillKey: target.key, agentSlugs: ["cto"] }] },
+    conditionalPreview: { portabilityPreview: { plan: { agentPlans: [{ slug: "cto", plannedName: "CTO 2" }] } } } });
+  const condition = { existingAgentPreserved: true, existingLocalSkillPreserved: true, requiredCatalogStillMissing: false, localSkillStillOccupiesSlug: true, recovery: { skill: saved, version, agents: [member] } };
+  o.state.firstConditional = condition; o.state.finalConditional = structuredClone(condition);
+  o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist was installed under a separate slug; the original local skill was preserved.";
+  o.phaseReplies[1]!.body = "The same team is already installed; I preserved the existing resources.";
+  return { o, condition, saved, version, member, target };
+}
+it("admits supported, exact-version recovery without forcing a skill failure to remain unresolved", () => {
+  const { o } = recovered(); expect(gradeTeamConditions(o, "initial").passed).toBe(true); expect(gradeTeamConditions(o).passed).toBe(true);
+});
+it.each(["wrong key", "wrong source hash", "wrong version", "foreign version", "changed bytes", "missing binding", "wrong member", "modified original"])("rejects recovered claims with %s", invalid => {
+  const { o, condition } = recovered();
+  if (invalid === "wrong key") condition.recovery.skill.key = "unrelated";
+  if (invalid === "wrong source hash") condition.recovery.skill.sourceRef = "sha256:other";
+  if (invalid === "wrong version") condition.recovery.version.id = "other";
+  if (invalid === "foreign version") condition.recovery.version.companyId = "foreign";
+  if (invalid === "changed bytes") condition.recovery.version.fileInventory[0]!.content = "Unrelated body";
+  if (invalid === "missing binding") condition.recovery.agents[0]!.adapterConfig.paperclipSkillSync.desiredSkills = [];
+  if (invalid === "wrong member") condition.recovery.agents[0]!.id = "unrelated-agent";
+  if (invalid === "modified original") condition.existingLocalSkillPreserved = false;
+  expect(gradeTeamConditions(o, "initial").passed).toBe(false);
+});
+it("the public observer reads actual saved skill/version and member bindings before recovery admission", async () => {
+  const { o, saved, version, member, target } = recovered();
+  const originalAgent = { id: "original", companyId: "company", name: "CTO", urlKey: "cto", adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {}, metadata: null };
+  const local = { id: "local", companyId: "company", key: "company/local", slug: target.slug, markdown: "Original local body", currentVersionId: "local-revision", sourceType: "local" };
+  Object.assign(o.state, { conflictAgentId: originalAgent.id, conflictAgentBefore: originalAgent, localSkillId: local.id, localSkillBefore: local });
+  const get = vi.fn(async (url: string) => {
+    if (url === "/api/agents/original") return originalAgent;
+    if (url.endsWith("/skills/local")) return local;
+    if (url.endsWith("/skills")) return [local, saved];
+    if (url.endsWith("/skills/saved")) return saved;
+    if (url.endsWith("/versions/revision")) return version;
+    if (url.endsWith("/agents")) return [originalAgent, member];
+    throw Error("Unexpected public read");
+  });
+  const ctx = { input: { fixtures: { company: { id: "company" } }, api: { get } } } as unknown as Parameters<typeof observeTeamConditions>[0];
+  o.state.firstConditional = await observeTeamConditions(ctx, o.state);
+  expect(gradeTeamConditions(o, "initial").passed).toBe(true);
+  expect(get).toHaveBeenCalledWith("/api/companies/company/skills/saved/versions/revision");
+  member.adapterConfig.paperclipSkillSync.desiredSkills = [];
+  o.state.firstConditional = await observeTeamConditions(ctx, o.state);
+  expect(gradeTeamConditions(o, "initial").passed).toBe(false);
+});
+
+it("requires the selected catalog version for a pinned binding and rechecks bindings after repeat", () => {
+  const { o, condition } = recovered();
+  const selections = condition.recovery.agents[0]!.adapterConfig.paperclipSkillSync as { desiredSkills: unknown[] };
+  selections.desiredSkills = [{ key: condition.recovery.skill.key, versionId: condition.recovery.version.id }];
+  expect(gradeTeamConditions(o, "initial").passed).toBe(true);
+  selections.desiredSkills = [{ key: condition.recovery.skill.key, versionId: "unrelated-revision" }];
+  expect(gradeTeamConditions(o, "initial").passed).toBe(false);
+  selections.desiredSkills = [condition.recovery.skill.key];
+  (o.state.finalConditional as typeof condition).recovery.agents[0]!.adapterConfig.paperclipSkillSync.desiredSkills = [];
+  expect(gradeTeamConditions(o).passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts

@@ -1,9 +1,12 @@
+import { createHash } from "node:crypto";
 import { isDeepStrictEqual } from "node:util";
 import type { Agent } from "../../../packages/shared/src/types/agent.js";
 import type {
   CatalogSkill,
   CompanySkill,
   CompanySkillListItem,
+  CompanySkillDetail,
+  CompanySkillVersion,
 } from "../../../packages/shared/src/types/company-skill.js";
 import type {
   CatalogTeam,
@@ -126,6 +129,56 @@ export async function prepareTeamConditions(
     conditionalPreview: conflict,
   };
 }
+interface TeamSkillRecovery {
+  skill: CompanySkillDetail;
+  version: CompanySkillVersion;
+  agents: Agent[];
+}
+export function verifiedTeamSkillRecovery(state: Record<string, unknown>, companyId: string, value: unknown): boolean {
+  const recovery = value as TeamSkillRecovery | undefined, target = state.blockedSkill as CatalogSkill;
+  const team = state.team as CatalogTeam, preview = state.conditionalPreview as CatalogTeamImportPreviewResult;
+  const first = state.first as { agents: Agent[] } | undefined;
+  if (!recovery?.skill || !recovery.version || !Array.isArray(recovery.agents) || !target?.key || !target.contentHash
+    || !team?.id || !preview?.portabilityPreview?.plan || !first?.agents) return false;
+  const { skill, version, agents } = recovery;
+  if (!skill.id || skill.companyId !== companyId || skill.key !== target.key || skill.sourceType !== "catalog" || skill.sourceRef !== target.contentHash
+    || !skill.currentVersionId || version.id !== skill.currentVersionId || version.companyId !== companyId || version.companySkillId !== skill.id
+    || !Array.isArray(version.fileInventory) || !target.files?.length || version.fileInventory.length !== target.files.length
+    || new Set(version.fileInventory.map(file => file.path)).size !== target.files.length
+    || version.fileInventory.find(file => file.path === "SKILL.md")?.content !== skill.markdown
+    || !target.files.every(expected => {
+      const file = version.fileInventory.find(file => file.path === expected.path);
+      return file && file.kind === expected.kind && typeof file.content === "string"
+        && createHash("sha256").update(Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8")).digest("hex") === expected.sha256;
+    })) return false;
+  const required = team.requiredSkills.find(row => row.type === "catalog" && row.resolved && row.catalogSkillId === target.id && row.catalogSkillKey === target.key);
+  if (!required?.agentSlugs.length) return false;
+  const plans = preview.portabilityPreview.plan.agentPlans;
+  const members = (rows: Agent[]) => rows.filter(agent => {
+    const provenance = (agent.metadata as { paperclip?: { catalogTeam?: { catalogId?: string; originHash?: string } } } | null)?.paperclip?.catalogTeam;
+    return agent.companyId === companyId && provenance?.catalogId === team.id && provenance.originHash === team.contentHash;
+  });
+  const initialMembers = members(first.agents), currentMembers = members(agents);
+  if (initialMembers.length !== team.agentSlugs.length || currentMembers.length !== initialMembers.length
+    || new Set(currentMembers.map(agent => agent.id)).size !== currentMembers.length) return false;
+  return required.agentSlugs.every(slug => {
+    const planned = plans.filter(plan => plan.slug === slug);
+    if (planned.length !== 1) return false;
+    const initial = initialMembers.filter(agent => agent.name === planned[0]!.plannedName);
+    if (initial.length !== 1) return false;
+    const current = currentMembers.find(agent => agent.id === initial[0]!.id);
+    const sync = current?.adapterConfig?.paperclipSkillSync as { desiredSkills?: unknown[] } | undefined;
+    const selection = sync?.desiredSkills?.find(value => typeof value === "string" ? value.trim() === target.key
+      : !!value && typeof value === "object" && "key" in value && value.key === target.key);
+    return typeof selection === "string" || !!selection && typeof selection === "object" && "versionId" in selection
+      && (selection.versionId === null || selection.versionId === version.id);
+  });
+}
+function truthfulTeamRecovery(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
+  return [skill.slug, skill.slug.replaceAll("-", " "), skill.name].some(name => reply.toLowerCase().includes(name.toLowerCase()))
+    && /\b(?:installed|available|resolved|recovered)\b/i.test(reply)
+    && !/\b(?:not installed|still (?:blocked|missing|unavailable)|remains (?:blocked|missing|unavailable))\b/i.test(reply);
+}
 export async function observeTeamConditions(
   ctx: ChatCatalogContext,
   state: Record<string, unknown>,
@@ -138,6 +191,18 @@ export async function observeTeamConditions(
     ctx.input.api.get<CompanySkillListItem[]>(`${root(ctx)}/skills`),
   ]);
   const target = state.blockedSkill as CatalogSkill;
+  const installed = skills.filter(row => row.key === target.key && row.sourceType === "catalog");
+  let recovery: TeamSkillRecovery | undefined;
+  if (installed.length === 1) {
+    const detail = await ctx.input.api.get<CompanySkillDetail>(`${root(ctx)}/skills/${encodeURIComponent(installed[0]!.id)}`);
+    if (detail.id === installed[0]!.id && detail.currentVersionId) {
+      const [version, agents] = await Promise.all([
+        ctx.input.api.get<CompanySkillVersion>(`${root(ctx)}/skills/${encodeURIComponent(detail.id)}/versions/${encodeURIComponent(detail.currentVersionId)}`),
+        ctx.input.api.get<Agent[]>(`${root(ctx)}/agents`),
+      ]);
+      recovery = { skill: detail, version, agents };
+    }
+  }
   return {
     existingAgentPreserved: isDeepStrictEqual(
       agentAuthority(agent),
@@ -147,6 +212,7 @@ export async function observeTeamConditions(
       skillIdentity(skill),
       state.localSkillBefore,
     ),
+    recovery,
     requiredCatalogStillMissing: !skills.some(
       (s) => s.key === target.key && s.sourceType === "catalog",
     ),
@@ -227,15 +293,16 @@ export function gradeTeamConditions(
                 clause,
               ),
           )));
+  const resolved = (condition: Record<string, unknown> | undefined) => condition?.requiredCatalogStillMissing === false
+    && verifiedTeamSkillRecovery(o.state, o.companyId, condition.recovery);
+  const accounted = (condition: Record<string, unknown> | undefined) => condition?.localSkillStillOccupiesSlug === true
+    && (condition.requiredCatalogStillMissing === true || resolved(condition));
   const events = o.activity as Array<Entity & { runId?: string | null }>;
   c.add(
     "actual-post-import-partial-warning",
     !!target &&
-      earlier?.requiredCatalogStillMissing === true &&
-      earlier.localSkillStillOccupiesSlug === true &&
-      (phase === "initial" ||
-        (later?.requiredCatalogStillMissing === true &&
-          later.localSkillStillOccupiesSlug === true)) &&
+      accounted(earlier) &&
+      (phase === "initial" || accounted(later)) &&
       events.some(
         (e) =>
           e.action === "company.team_catalog_installed" &&
@@ -246,7 +313,7 @@ export function gradeTeamConditions(
           Number(e.details?.warningCount) > 0 &&
           Number(e.details?.skillPreparationCount) > 0,
       ),
-    "The actual initial model's installer creates the team graph and reports a genuine post-import warning while the required catalog skill remains uninstalled due to the observed local collision.",
+    "The actual initial model's installer creates the team graph and reports a genuine post-import warning with the required skill either still blocked or independently recovered through its exact catalog version and intended agent bindings.",
   );
   c.add(
     "phase-specific-honest-conflict-partial-replies",
@@ -254,9 +321,9 @@ export function gradeTeamConditions(
       !!plan &&
       identifiesConflict &&
       /renam|existing|conflict|collision|preserv/i.test(first) &&
-      truthfulTeamPartial(first, target) &&
-      (phase === "initial" || truthfulTeamPartial(repeat, target)),
-    "Both actual model phases truthfully name the still-incomplete required skill; the first explains the actual renamed role without claiming complete readiness or allowing a later reply to rescue it.",
+      (resolved(earlier) ? truthfulTeamRecovery(first, target) : truthfulTeamPartial(first, target)) &&
+      (phase === "initial" || (resolved(later) || truthfulTeamPartial(repeat, target))),
+    "The first reply reports the actual renamed role and either the unresolved required skill or its independently verified recovery. A remaining failure must be reported in both phases; a later reply cannot rescue false initial readiness.",
   );
   return c.result();
 }

tests/runner-e2e/chat-catalog-domains/external-team-install.ts

@@ -337,6 +337,7 @@ export const teamInstallExternalCases = [
     gradeTeamInstall,
     {
       authentication: "human-session",
+      oracleVersion: "paperclip.chat-catalog.team-install.v2",
       expectedRunCount: 2,
       expectedHumanMessageCount: 2,
       attemptTimeoutMs: 480_000,
@@ -381,11 +382,11 @@ export const teamInstallExternalCases = [
         state.initialRunId = (await ctx.allRuns())[0]?.id;
         state.firstConditional = await observeTeamConditions(ctx, state);
         const firstObservation = await observeExternal(ctx, state);
-        // A later correct repeat cannot conceal an unreported initial partial.
+        // A later correct repeat cannot conceal an unaccounted initial outcome.
         const firstGrade = gradeTeamConditions(firstObservation, "initial");
         if (!firstGrade.passed)
           throw new Error(
-            "Actual initial installer did not truthfully preserve/report its genuine conflict and partial outcome before repeat",
+            "Actual initial installer did not truthfully account for its genuine conflict or verified recovery before repeat",
           );
         await ctx.input.evidence("chat-catalog-team-initial-conditional.json", {
           initialRunId: state.initialRunId,