diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 15240d235a7..58a70fc4225 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -249,6 +249,17 @@ version. The receipt identifies the actual editor surface. The original case clock, query, and one-human/one-native limits remain unchanged. Older failed attempts are retained; component controls do not establish a native pass. +### Team install conflict recovery + +`teams.install-team` v2 accepts either an honestly reported missing required +skill or a supported catalog recovery that preserves the original local skill +and agent. Recovery requires the accepted catalog key and source hash, exact +current immutable-version bytes, and that key assigned to the intended imported +members. It does not accept merely installing an unattached skill. The original +repeat query, resource identity reconciliation, final rendered-link inspection, +two-human/two-native limits, and case clock remain unchanged. Failed earlier +attempts remain failed; this correction does not establish a native pass. + ### Six prerequisite probes are not completed journeys The following six definitions retain explicit coverage gaps and must not be diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 57bcc9c9cae..466429a09cf 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -6015,7 +6015,7 @@ "chat-flow.ts", "chat-scenarios/teams.yaml" ], - "oracleVersion": "paperclip.chat-catalog.external.v1", + "oracleVersion": "paperclip.chat-catalog.team-install.v2", "expectedRunCount": 2, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts index fa25fbdb654..af125dd9118 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts @@ -1,6 +1,8 @@ -import { describe, it, expect } from "vitest"; +import { createHash } from "node:crypto"; +import { describe, it, expect, vi } from "vitest"; import { gradeTeamConditions, + observeTeamConditions, truthfulTeamPartial, } from "./external-team-conditional.js"; import type { ExternalObservation } from "./external-support.js"; @@ -119,3 +121,71 @@ describe("real team post-import conditional controls", () => { expect(truthfulTeamPartial(text, skill)).toBe(false); }); }); + +function recovered() { + const o = valid(); o.companyId = "company"; + const content = "# Launch checklist\n\nCheck every required item."; + const target = { id: "catalog", key: "catalog/launch-checklist", slug: skill.slug, name: skill.name, contentHash: "sha256:catalog", + files: [{ path: "SKILL.md", kind: "skill", sha256: createHash("sha256").update(content).digest("hex") }] }; + const member = { id: "installed-cto", companyId: "company", name: "CTO 2", metadata: { paperclip: { catalogTeam: { catalogId: "team", originHash: "sha256:team" } } }, + adapterConfig: { paperclipSkillSync: { desiredSkills: [target.key] } } }; + const saved = { id: "saved", companyId: "company", key: target.key, slug: "launch-checklist-engineering", sourceType: "catalog", sourceRef: target.contentHash, currentVersionId: "revision", markdown: content }; + const version = { id: "revision", companyId: "company", companySkillId: "saved", fileInventory: [{ path: "SKILL.md", kind: "skill", content }] }; + Object.assign(o.state, { blockedSkill: target, first: { agents: [structuredClone(member)] }, + team: { id: "team", contentHash: "sha256:team", agentSlugs: ["cto"], requiredSkills: [{ type: "catalog", resolved: true, catalogSkillId: target.id, catalogSkillKey: target.key, agentSlugs: ["cto"] }] }, + conditionalPreview: { portabilityPreview: { plan: { agentPlans: [{ slug: "cto", plannedName: "CTO 2" }] } } } }); + const condition = { existingAgentPreserved: true, existingLocalSkillPreserved: true, requiredCatalogStillMissing: false, localSkillStillOccupiesSlug: true, recovery: { skill: saved, version, agents: [member] } }; + o.state.firstConditional = condition; o.state.finalConditional = structuredClone(condition); + o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist was installed under a separate slug; the original local skill was preserved."; + o.phaseReplies[1]!.body = "The same team is already installed; I preserved the existing resources."; + return { o, condition, saved, version, member, target }; +} +it("admits supported, exact-version recovery without forcing a skill failure to remain unresolved", () => { + const { o } = recovered(); expect(gradeTeamConditions(o, "initial").passed).toBe(true); expect(gradeTeamConditions(o).passed).toBe(true); +}); +it.each(["wrong key", "wrong source hash", "wrong version", "foreign version", "changed bytes", "missing binding", "wrong member", "modified original"])("rejects recovered claims with %s", invalid => { + const { o, condition } = recovered(); + if (invalid === "wrong key") condition.recovery.skill.key = "unrelated"; + if (invalid === "wrong source hash") condition.recovery.skill.sourceRef = "sha256:other"; + if (invalid === "wrong version") condition.recovery.version.id = "other"; + if (invalid === "foreign version") condition.recovery.version.companyId = "foreign"; + if (invalid === "changed bytes") condition.recovery.version.fileInventory[0]!.content = "Unrelated body"; + if (invalid === "missing binding") condition.recovery.agents[0]!.adapterConfig.paperclipSkillSync.desiredSkills = []; + if (invalid === "wrong member") condition.recovery.agents[0]!.id = "unrelated-agent"; + if (invalid === "modified original") condition.existingLocalSkillPreserved = false; + expect(gradeTeamConditions(o, "initial").passed).toBe(false); +}); +it("the public observer reads actual saved skill/version and member bindings before recovery admission", async () => { + const { o, saved, version, member, target } = recovered(); + const originalAgent = { id: "original", companyId: "company", name: "CTO", urlKey: "cto", adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: {}, permissions: {}, metadata: null }; + const local = { id: "local", companyId: "company", key: "company/local", slug: target.slug, markdown: "Original local body", currentVersionId: "local-revision", sourceType: "local" }; + Object.assign(o.state, { conflictAgentId: originalAgent.id, conflictAgentBefore: originalAgent, localSkillId: local.id, localSkillBefore: local }); + const get = vi.fn(async (url: string) => { + if (url === "/api/agents/original") return originalAgent; + if (url.endsWith("/skills/local")) return local; + if (url.endsWith("/skills")) return [local, saved]; + if (url.endsWith("/skills/saved")) return saved; + if (url.endsWith("/versions/revision")) return version; + if (url.endsWith("/agents")) return [originalAgent, member]; + throw Error("Unexpected public read"); + }); + const ctx = { input: { fixtures: { company: { id: "company" } }, api: { get } } } as unknown as Parameters[0]; + o.state.firstConditional = await observeTeamConditions(ctx, o.state); + expect(gradeTeamConditions(o, "initial").passed).toBe(true); + expect(get).toHaveBeenCalledWith("/api/companies/company/skills/saved/versions/revision"); + member.adapterConfig.paperclipSkillSync.desiredSkills = []; + o.state.firstConditional = await observeTeamConditions(ctx, o.state); + expect(gradeTeamConditions(o, "initial").passed).toBe(false); +}); + +it("requires the selected catalog version for a pinned binding and rechecks bindings after repeat", () => { + const { o, condition } = recovered(); + const selections = condition.recovery.agents[0]!.adapterConfig.paperclipSkillSync as { desiredSkills: unknown[] }; + selections.desiredSkills = [{ key: condition.recovery.skill.key, versionId: condition.recovery.version.id }]; + expect(gradeTeamConditions(o, "initial").passed).toBe(true); + selections.desiredSkills = [{ key: condition.recovery.skill.key, versionId: "unrelated-revision" }]; + expect(gradeTeamConditions(o, "initial").passed).toBe(false); + selections.desiredSkills = [condition.recovery.skill.key]; + (o.state.finalConditional as typeof condition).recovery.agents[0]!.adapterConfig.paperclipSkillSync.desiredSkills = []; + expect(gradeTeamConditions(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts index 510a881a8a5..93579071f4c 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts @@ -1,9 +1,12 @@ +import { createHash } from "node:crypto"; import { isDeepStrictEqual } from "node:util"; import type { Agent } from "../../../packages/shared/src/types/agent.js"; import type { CatalogSkill, CompanySkill, CompanySkillListItem, + CompanySkillDetail, + CompanySkillVersion, } from "../../../packages/shared/src/types/company-skill.js"; import type { CatalogTeam, @@ -126,6 +129,56 @@ export async function prepareTeamConditions( conditionalPreview: conflict, }; } +interface TeamSkillRecovery { + skill: CompanySkillDetail; + version: CompanySkillVersion; + agents: Agent[]; +} +export function verifiedTeamSkillRecovery(state: Record, companyId: string, value: unknown): boolean { + const recovery = value as TeamSkillRecovery | undefined, target = state.blockedSkill as CatalogSkill; + const team = state.team as CatalogTeam, preview = state.conditionalPreview as CatalogTeamImportPreviewResult; + const first = state.first as { agents: Agent[] } | undefined; + if (!recovery?.skill || !recovery.version || !Array.isArray(recovery.agents) || !target?.key || !target.contentHash + || !team?.id || !preview?.portabilityPreview?.plan || !first?.agents) return false; + const { skill, version, agents } = recovery; + if (!skill.id || skill.companyId !== companyId || skill.key !== target.key || skill.sourceType !== "catalog" || skill.sourceRef !== target.contentHash + || !skill.currentVersionId || version.id !== skill.currentVersionId || version.companyId !== companyId || version.companySkillId !== skill.id + || !Array.isArray(version.fileInventory) || !target.files?.length || version.fileInventory.length !== target.files.length + || new Set(version.fileInventory.map(file => file.path)).size !== target.files.length + || version.fileInventory.find(file => file.path === "SKILL.md")?.content !== skill.markdown + || !target.files.every(expected => { + const file = version.fileInventory.find(file => file.path === expected.path); + return file && file.kind === expected.kind && typeof file.content === "string" + && createHash("sha256").update(Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8")).digest("hex") === expected.sha256; + })) return false; + const required = team.requiredSkills.find(row => row.type === "catalog" && row.resolved && row.catalogSkillId === target.id && row.catalogSkillKey === target.key); + if (!required?.agentSlugs.length) return false; + const plans = preview.portabilityPreview.plan.agentPlans; + const members = (rows: Agent[]) => rows.filter(agent => { + const provenance = (agent.metadata as { paperclip?: { catalogTeam?: { catalogId?: string; originHash?: string } } } | null)?.paperclip?.catalogTeam; + return agent.companyId === companyId && provenance?.catalogId === team.id && provenance.originHash === team.contentHash; + }); + const initialMembers = members(first.agents), currentMembers = members(agents); + if (initialMembers.length !== team.agentSlugs.length || currentMembers.length !== initialMembers.length + || new Set(currentMembers.map(agent => agent.id)).size !== currentMembers.length) return false; + return required.agentSlugs.every(slug => { + const planned = plans.filter(plan => plan.slug === slug); + if (planned.length !== 1) return false; + const initial = initialMembers.filter(agent => agent.name === planned[0]!.plannedName); + if (initial.length !== 1) return false; + const current = currentMembers.find(agent => agent.id === initial[0]!.id); + const sync = current?.adapterConfig?.paperclipSkillSync as { desiredSkills?: unknown[] } | undefined; + const selection = sync?.desiredSkills?.find(value => typeof value === "string" ? value.trim() === target.key + : !!value && typeof value === "object" && "key" in value && value.key === target.key); + return typeof selection === "string" || !!selection && typeof selection === "object" && "versionId" in selection + && (selection.versionId === null || selection.versionId === version.id); + }); +} +function truthfulTeamRecovery(reply: string, skill: Pick): boolean { + return [skill.slug, skill.slug.replaceAll("-", " "), skill.name].some(name => reply.toLowerCase().includes(name.toLowerCase())) + && /\b(?:installed|available|resolved|recovered)\b/i.test(reply) + && !/\b(?:not installed|still (?:blocked|missing|unavailable)|remains (?:blocked|missing|unavailable))\b/i.test(reply); +} export async function observeTeamConditions( ctx: ChatCatalogContext, state: Record, @@ -138,6 +191,18 @@ export async function observeTeamConditions( ctx.input.api.get(`${root(ctx)}/skills`), ]); const target = state.blockedSkill as CatalogSkill; + const installed = skills.filter(row => row.key === target.key && row.sourceType === "catalog"); + let recovery: TeamSkillRecovery | undefined; + if (installed.length === 1) { + const detail = await ctx.input.api.get(`${root(ctx)}/skills/${encodeURIComponent(installed[0]!.id)}`); + if (detail.id === installed[0]!.id && detail.currentVersionId) { + const [version, agents] = await Promise.all([ + ctx.input.api.get(`${root(ctx)}/skills/${encodeURIComponent(detail.id)}/versions/${encodeURIComponent(detail.currentVersionId)}`), + ctx.input.api.get(`${root(ctx)}/agents`), + ]); + recovery = { skill: detail, version, agents }; + } + } return { existingAgentPreserved: isDeepStrictEqual( agentAuthority(agent), @@ -147,6 +212,7 @@ export async function observeTeamConditions( skillIdentity(skill), state.localSkillBefore, ), + recovery, requiredCatalogStillMissing: !skills.some( (s) => s.key === target.key && s.sourceType === "catalog", ), @@ -227,15 +293,16 @@ export function gradeTeamConditions( clause, ), ))); + const resolved = (condition: Record | undefined) => condition?.requiredCatalogStillMissing === false + && verifiedTeamSkillRecovery(o.state, o.companyId, condition.recovery); + const accounted = (condition: Record | undefined) => condition?.localSkillStillOccupiesSlug === true + && (condition.requiredCatalogStillMissing === true || resolved(condition)); const events = o.activity as Array; c.add( "actual-post-import-partial-warning", !!target && - earlier?.requiredCatalogStillMissing === true && - earlier.localSkillStillOccupiesSlug === true && - (phase === "initial" || - (later?.requiredCatalogStillMissing === true && - later.localSkillStillOccupiesSlug === true)) && + accounted(earlier) && + (phase === "initial" || accounted(later)) && events.some( (e) => e.action === "company.team_catalog_installed" && @@ -246,7 +313,7 @@ export function gradeTeamConditions( Number(e.details?.warningCount) > 0 && Number(e.details?.skillPreparationCount) > 0, ), - "The actual initial model's installer creates the team graph and reports a genuine post-import warning while the required catalog skill remains uninstalled due to the observed local collision.", + "The actual initial model's installer creates the team graph and reports a genuine post-import warning with the required skill either still blocked or independently recovered through its exact catalog version and intended agent bindings.", ); c.add( "phase-specific-honest-conflict-partial-replies", @@ -254,9 +321,9 @@ export function gradeTeamConditions( !!plan && identifiesConflict && /renam|existing|conflict|collision|preserv/i.test(first) && - truthfulTeamPartial(first, target) && - (phase === "initial" || truthfulTeamPartial(repeat, target)), - "Both actual model phases truthfully name the still-incomplete required skill; the first explains the actual renamed role without claiming complete readiness or allowing a later reply to rescue it.", + (resolved(earlier) ? truthfulTeamRecovery(first, target) : truthfulTeamPartial(first, target)) && + (phase === "initial" || (resolved(later) || truthfulTeamPartial(repeat, target))), + "The first reply reports the actual renamed role and either the unresolved required skill or its independently verified recovery. A remaining failure must be reported in both phases; a later reply cannot rescue false initial readiness.", ); return c.result(); } diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts index 866b7742b5c..8324c90ab7f 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-install.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-install.ts @@ -337,6 +337,7 @@ export const teamInstallExternalCases = [ gradeTeamInstall, { authentication: "human-session", + oracleVersion: "paperclip.chat-catalog.team-install.v2", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 480_000, @@ -381,11 +382,11 @@ export const teamInstallExternalCases = [ state.initialRunId = (await ctx.allRuns())[0]?.id; state.firstConditional = await observeTeamConditions(ctx, state); const firstObservation = await observeExternal(ctx, state); - // A later correct repeat cannot conceal an unreported initial partial. + // A later correct repeat cannot conceal an unaccounted initial outcome. const firstGrade = gradeTeamConditions(firstObservation, "initial"); if (!firstGrade.passed) throw new Error( - "Actual initial installer did not truthfully preserve/report its genuine conflict and partial outcome before repeat", + "Actual initial installer did not truthfully account for its genuine conflict or verified recovery before repeat", ); await ctx.input.evidence("chat-catalog-team-initial-conditional.json", { initialRunId: state.initialRunId,