From 6ada37107578eeb3a50972da628716f5a2209e4f Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 16:46:42 -0600 Subject: [PATCH] fix(evals): admit verified team skill conflict recovery Co-Authored-By: Paperclip --- tests/runner-e2e/CHAT-FEATURES.md | 11 +++ .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../external-team-conditional.test.ts | 72 +++++++++++++++- .../external-team-conditional.ts | 85 +++++++++++++++++-- .../external-team-install.ts | 5 +- 5 files changed, 162 insertions(+), 13 deletions(-) diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 73efea89367..e8479259b68 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -239,6 +239,17 @@ Domain descriptors declare real setup/actions/independent grading and exact boun Only `steering.preserve-draft` and `tasks.attachment-brief` may currently declare one `queryAttachments` receipt. The canonical typed query remains unchanged; the production composer appends its exact attachment reference. Before and after send, the harness independently reads the public same-source receipt and actual bytes, verifies SHA-256 and human attribution, and binds it to the real canonical comment. Extra markdown, text changes, foreign receipts and hidden followups fail. Attachment fingerprints pin both production composer source hashes as data, without importing the UI graph. +### Team install conflict recovery + +`teams.install-team` v2 accepts either an honestly reported missing required +skill or a supported catalog recovery that preserves the original local skill +and agent. Recovery requires the accepted catalog key and source hash, exact +current immutable-version bytes, and that key assigned to the intended imported +members. It does not accept merely installing an unattached skill. The original +repeat query, resource identity reconciliation, final rendered-link inspection, +two-human/two-native limits, and case clock remain unchanged. Failed earlier +attempts remain failed; this correction does not establish a native pass. + ### Six prerequisite probes are not completed journeys The following six definitions retain explicit coverage gaps and must not be diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index d14ae9b3a2f..9786e3626b1 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -6013,7 +6013,7 @@ "chat-flow.ts", "chat-scenarios/teams.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v1", + "oracleVersion": "paperclip.chat-catalog.team-install.v2", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts index fa25fbdb654..af125dd9118 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts @@ -1,6 +1,8 @@ -import { describe, it, expect } from "vitest"; +import { createHash } from "node:crypto"; +import { describe, it, expect, vi } from "vitest"; import { gradeTeamConditions, + observeTeamConditions, truthfulTeamPartial, } from "./external-team-conditional.js"; import type { ExternalObservation } from "./external-support.js"; @@ -119,3 +121,71 @@ describe("real team post-import conditional controls", () => { expect(truthfulTeamPartial(text, skill)).toBe(false); }); }); + +function recovered() { + const o = valid(); o.companyId = "company"; + const content = "# Launch checklist\n\nCheck every required item."; + const target = { id: "catalog", key: "catalog/launch-checklist", slug: skill.slug, name: skill.name, contentHash: "sha256:catalog", + files: [{ path: "SKILL.md", kind: "skill", sha256: createHash("sha256").update(content).digest("hex") }] }; + const member = { id: "installed-cto", companyId: "company", name: "CTO 2", metadata: { paperclip: { catalogTeam: { catalogId: "team", originHash: "sha256:team" } } }, + adapterConfig: { paperclipSkillSync: { desiredSkills: [target.key] } } }; + const saved = { id: "saved", companyId: "company", key: target.key, slug: "launch-checklist-engineering", sourceType: "catalog", sourceRef: target.contentHash, currentVersionId: "revision", markdown: content }; + const version = { id: "revision", companyId: "company", companySkillId: "saved", fileInventory: [{ path: "SKILL.md", kind: "skill", content }] }; + Object.assign(o.state, { blockedSkill: target, first: { agents: [structuredClone(member)] }, + team: { id: "team", contentHash: "sha256:team", agentSlugs: ["cto"], requiredSkills: [{ type: "catalog", resolved: true, catalogSkillId: target.id, catalogSkillKey: target.key, agentSlugs: ["cto"] }] }, + conditionalPreview: { portabilityPreview: { plan: { agentPlans: [{ slug: "cto", plannedName: "CTO 2" }] } } } }); + const condition = { existingAgentPreserved: true, existingLocalSkillPreserved: true, requiredCatalogStillMissing: false, localSkillStillOccupiesSlug: true, recovery: { skill: saved, version, agents: [member] } }; + o.state.firstConditional = condition; o.state.finalConditional = structuredClone(condition); + o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist was installed under a separate slug; the original local skill was preserved."; + o.phaseReplies[1]!.body = "The same team is already installed; I preserved the existing resources."; + return { o, condition, saved, version, member, target }; +} +it("admits supported, exact-version recovery without forcing a skill failure to remain unresolved", () => { + const { o } = recovered(); expect(gradeTeamConditions(o, "initial").passed).toBe(true); expect(gradeTeamConditions(o).passed).toBe(true); +}); +it.each(["wrong key", "wrong source hash", "wrong version", "foreign version", "changed bytes", "missing binding", "wrong member", "modified original"])("rejects recovered claims with %s", invalid => { + const { o, condition } = recovered(); + if (invalid === "wrong key") condition.recovery.skill.key = "unrelated"; + if (invalid === "wrong source hash") condition.recovery.skill.sourceRef = "sha256:other"; + if (invalid === "wrong version") condition.recovery.version.id = "other"; + if (invalid === "foreign version") condition.recovery.version.companyId = "foreign"; + if (invalid === "changed bytes") condition.recovery.version.fileInventory[0]!.content = "Unrelated body"; + if (invalid === "missing binding") condition.recovery.agents[0]!.adapterConfig.paperclipSkillSync.desiredSkills = []; + if (invalid === "wrong member") condition.recovery.agents[0]!.id = "unrelated-agent"; + if (invalid === "modified original") condition.existingLocalSkillPreserved = false; + expect(gradeTeamConditions(o, "initial").passed).toBe(false); +}); +it("the public observer reads actual saved skill/version and member bindings before recovery admission", async () => { + const { o, saved, version, member, target } = recovered(); + const originalAgent = { id: "original", companyId: "company", name: "CTO", urlKey: "cto", adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {}, metadata: null }; + const local = { id: "local", companyId: "company", key: "company/local", slug: target.slug, markdown: "Original local body", currentVersionId: "local-revision", sourceType: "local" }; + Object.assign(o.state, { conflictAgentId: originalAgent.id, conflictAgentBefore: originalAgent, localSkillId: local.id, localSkillBefore: local }); + const get = vi.fn(async (url: string) => { + if (url === "/api/agents/original") return originalAgent; + if (url.endsWith("/skills/local")) return local; + if (url.endsWith("/skills")) return [local, saved]; + if (url.endsWith("/skills/saved")) return saved; + if (url.endsWith("/versions/revision")) return version; + if (url.endsWith("/agents")) return [originalAgent, member]; + throw Error("Unexpected public read"); + }); + const ctx = { input: { fixtures: { company: { id: "company" } }, api: { get } } } as unknown as Parameters[0]; + o.state.firstConditional = await observeTeamConditions(ctx, o.state); + expect(gradeTeamConditions(o, "initial").passed).toBe(true); + expect(get).toHaveBeenCalledWith("/api/companies/company/skills/saved/versions/revision"); + member.adapterConfig.paperclipSkillSync.desiredSkills = []; + o.state.firstConditional = await observeTeamConditions(ctx, o.state); + expect(gradeTeamConditions(o, "initial").passed).toBe(false); +}); + +it("requires the selected catalog version for a pinned binding and rechecks bindings after repeat", () => { + const { o, condition } = recovered(); + const selections = condition.recovery.agents[0]!.adapterConfig.paperclipSkillSync as { desiredSkills: unknown[] }; + selections.desiredSkills = [{ key: condition.recovery.skill.key, versionId: condition.recovery.version.id }]; + expect(gradeTeamConditions(o, "initial").passed).toBe(true); + selections.desiredSkills = [{ key: condition.recovery.skill.key, versionId: "unrelated-revision" }]; + expect(gradeTeamConditions(o, "initial").passed).toBe(false); + selections.desiredSkills = [condition.recovery.skill.key]; + (o.state.finalConditional as typeof condition).recovery.agents[0]!.adapterConfig.paperclipSkillSync.desiredSkills = []; + expect(gradeTeamConditions(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts index 510a881a8a5..93579071f4c 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts @@ -1,9 +1,12 @@ +import { createHash } from "node:crypto"; import { isDeepStrictEqual } from "node:util"; import type { Agent } from "../../../packages/shared/src/types/agent.js"; import type { CatalogSkill, CompanySkill, CompanySkillListItem, + CompanySkillDetail, + CompanySkillVersion, } from "../../../packages/shared/src/types/company-skill.js"; import type { CatalogTeam, @@ -126,6 +129,56 @@ export async function prepareTeamConditions( conditionalPreview: conflict, }; } +interface TeamSkillRecovery { + skill: CompanySkillDetail; + version: CompanySkillVersion; + agents: Agent[]; +} +export function verifiedTeamSkillRecovery(state: Record, companyId: string, value: unknown): boolean { + const recovery = value as TeamSkillRecovery | undefined, target = state.blockedSkill as CatalogSkill; + const team = state.team as CatalogTeam, preview = state.conditionalPreview as CatalogTeamImportPreviewResult; + const first = state.first as { agents: Agent[] } | undefined; + if (!recovery?.skill || !recovery.version || !Array.isArray(recovery.agents) || !target?.key || !target.contentHash + || !team?.id || !preview?.portabilityPreview?.plan || !first?.agents) return false; + const { skill, version, agents } = recovery; + if (!skill.id || skill.companyId !== companyId || skill.key !== target.key || skill.sourceType !== "catalog" || skill.sourceRef !== target.contentHash + || !skill.currentVersionId || version.id !== skill.currentVersionId || version.companyId !== companyId || version.companySkillId !== skill.id + || !Array.isArray(version.fileInventory) || !target.files?.length || version.fileInventory.length !== target.files.length + || new Set(version.fileInventory.map(file => file.path)).size !== target.files.length + || version.fileInventory.find(file => file.path === "SKILL.md")?.content !== skill.markdown + || !target.files.every(expected => { + const file = version.fileInventory.find(file => file.path === expected.path); + return file && file.kind === expected.kind && typeof file.content === "string" + && createHash("sha256").update(Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8")).digest("hex") === expected.sha256; + })) return false; + const required = team.requiredSkills.find(row => row.type === "catalog" && row.resolved && row.catalogSkillId === target.id && row.catalogSkillKey === target.key); + if (!required?.agentSlugs.length) return false; + const plans = preview.portabilityPreview.plan.agentPlans; + const members = (rows: Agent[]) => rows.filter(agent => { + const provenance = (agent.metadata as { paperclip?: { catalogTeam?: { catalogId?: string; originHash?: string } } } | null)?.paperclip?.catalogTeam; + return agent.companyId === companyId && provenance?.catalogId === team.id && provenance.originHash === team.contentHash; + }); + const initialMembers = members(first.agents), currentMembers = members(agents); + if (initialMembers.length !== team.agentSlugs.length || currentMembers.length !== initialMembers.length + || new Set(currentMembers.map(agent => agent.id)).size !== currentMembers.length) return false; + return required.agentSlugs.every(slug => { + const planned = plans.filter(plan => plan.slug === slug); + if (planned.length !== 1) return false; + const initial = initialMembers.filter(agent => agent.name === planned[0]!.plannedName); + if (initial.length !== 1) return false; + const current = currentMembers.find(agent => agent.id === initial[0]!.id); + const sync = current?.adapterConfig?.paperclipSkillSync as { desiredSkills?: unknown[] } | undefined; + const selection = sync?.desiredSkills?.find(value => typeof value === "string" ? value.trim() === target.key + : !!value && typeof value === "object" && "key" in value && value.key === target.key); + return typeof selection === "string" || !!selection && typeof selection === "object" && "versionId" in selection + && (selection.versionId === null || selection.versionId === version.id); + }); +} +function truthfulTeamRecovery(reply: string, skill: Pick): boolean { + return [skill.slug, skill.slug.replaceAll("-", " "), skill.name].some(name => reply.toLowerCase().includes(name.toLowerCase())) + && /\b(?:installed|available|resolved|recovered)\b/i.test(reply) + && !/\b(?:not installed|still (?:blocked|missing|unavailable)|remains (?:blocked|missing|unavailable))\b/i.test(reply); +} export async function observeTeamConditions( ctx: ChatCatalogContext, state: Record, @@ -138,6 +191,18 @@ export async function observeTeamConditions( ctx.input.api.get(`${root(ctx)}/skills`), ]); const target = state.blockedSkill as CatalogSkill; + const installed = skills.filter(row => row.key === target.key && row.sourceType === "catalog"); + let recovery: TeamSkillRecovery | undefined; + if (installed.length === 1) { + const detail = await ctx.input.api.get(`${root(ctx)}/skills/${encodeURIComponent(installed[0]!.id)}`); + if (detail.id === installed[0]!.id && detail.currentVersionId) { + const [version, agents] = await Promise.all([ + ctx.input.api.get(`${root(ctx)}/skills/${encodeURIComponent(detail.id)}/versions/${encodeURIComponent(detail.currentVersionId)}`), + ctx.input.api.get(`${root(ctx)}/agents`), + ]); + recovery = { skill: detail, version, agents }; + } + } return { existingAgentPreserved: isDeepStrictEqual( agentAuthority(agent), @@ -147,6 +212,7 @@ export async function observeTeamConditions( skillIdentity(skill), state.localSkillBefore, ), + recovery, requiredCatalogStillMissing: !skills.some( (s) => s.key === target.key && s.sourceType === "catalog", ), @@ -227,15 +293,16 @@ export function gradeTeamConditions( clause, ), ))); + const resolved = (condition: Record | undefined) => condition?.requiredCatalogStillMissing === false + && verifiedTeamSkillRecovery(o.state, o.companyId, condition.recovery); + const accounted = (condition: Record | undefined) => condition?.localSkillStillOccupiesSlug === true + && (condition.requiredCatalogStillMissing === true || resolved(condition)); const events = o.activity as Array; c.add( "actual-post-import-partial-warning", !!target && - earlier?.requiredCatalogStillMissing === true && - earlier.localSkillStillOccupiesSlug === true && - (phase === "initial" || - (later?.requiredCatalogStillMissing === true && - later.localSkillStillOccupiesSlug === true)) && + accounted(earlier) && + (phase === "initial" || accounted(later)) && events.some( (e) => e.action === "company.team_catalog_installed" && @@ -246,7 +313,7 @@ export function gradeTeamConditions( Number(e.details?.warningCount) > 0 && Number(e.details?.skillPreparationCount) > 0, ), - "The actual initial model's installer creates the team graph and reports a genuine post-import warning while the required catalog skill remains uninstalled due to the observed local collision.", + "The actual initial model's installer creates the team graph and reports a genuine post-import warning with the required skill either still blocked or independently recovered through its exact catalog version and intended agent bindings.", ); c.add( "phase-specific-honest-conflict-partial-replies", @@ -254,9 +321,9 @@ export function gradeTeamConditions( !!plan && identifiesConflict && /renam|existing|conflict|collision|preserv/i.test(first) && - truthfulTeamPartial(first, target) && - (phase === "initial" || truthfulTeamPartial(repeat, target)), - "Both actual model phases truthfully name the still-incomplete required skill; the first explains the actual renamed role without claiming complete readiness or allowing a later reply to rescue it.", + (resolved(earlier) ? truthfulTeamRecovery(first, target) : truthfulTeamPartial(first, target)) && + (phase === "initial" || (resolved(later) || truthfulTeamPartial(repeat, target))), + "The first reply reports the actual renamed role and either the unresolved required skill or its independently verified recovery. A remaining failure must be reported in both phases; a later reply cannot rescue false initial readiness.", ); return c.result(); } diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts index 866b7742b5c..8324c90ab7f 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts @@ -337,6 +337,7 @@ export const teamInstallExternalCases = [ gradeTeamInstall, { authentication: "human-session", + oracleVersion: "paperclip.chat-catalog.team-install.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 480_000, @@ -381,11 +382,11 @@ export const teamInstallExternalCases = [ state.initialRunId = (await ctx.allRuns())[0]?.id; state.firstConditional = await observeTeamConditions(ctx, state); const firstObservation = await observeExternal(ctx, state); - // A later correct repeat cannot conceal an unreported initial partial. + // A later correct repeat cannot conceal an unaccounted initial outcome. const firstGrade = gradeTeamConditions(firstObservation, "initial"); if (!firstGrade.passed) throw new Error( - "Actual initial installer did not truthfully preserve/report its genuine conflict and partial outcome before repeat", + "Actual initial installer did not truthfully account for its genuine conflict or verified recovery before repeat", ); await ctx.input.evidence("chat-catalog-team-initial-conditional.json", { initialRunId: state.initialRunId,