Files
PaperClipAI/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts
T

263 lines
10 KiB
TypeScript

import { isDeepStrictEqual } from "node:util";
import type { Agent } from "../../../packages/shared/src/types/agent.js";
import type {
CatalogSkill,
CompanySkill,
CompanySkillListItem,
} from "../../../packages/shared/src/types/company-skill.js";
import type {
CatalogTeam,
CatalogTeamImportPreviewResult,
} from "../../../packages/shared/src/types/teams-catalog.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import { RunnerApiHttpError } from "../api.js";
import { identityWorker } from "./identity-support.js";
import { checks, root, type ExternalObservation } from "./external-support.js";
import type { Entity } from "./internal-support.js";
const agentAuthority = (a: Agent) => ({
id: a.id,
companyId: a.companyId,
name: a.name,
urlKey: a.urlKey,
adapterType: a.adapterType,
adapterConfig: a.adapterConfig,
runtimeConfig: a.runtimeConfig,
permissions: a.permissions,
metadata: a.metadata,
});
const skillIdentity = (s: CompanySkill) => ({
id: s.id,
companyId: s.companyId,
key: s.key,
slug: s.slug,
markdown: s.markdown,
currentVersionId: s.currentVersionId,
sourceType: s.sourceType,
});
/** Genuine post-import failure: the catalog installer imports the team graph,
* then installFromCatalog rejects this separately owned local skill's occupied
* slug. No test flag, permission denial, missing account or catalog mutation. */
export async function prepareTeamConditions(
ctx: ChatCatalogContext,
team: CatalogTeam,
) {
const preview = await ctx.input.api.post<CatalogTeamImportPreviewResult>(
`${root(ctx)}/teams/catalog/${encodeURIComponent(team.id)}/preview`,
{ targetManagerAgentId: ctx.input.fixtures.agent.id },
);
const rootAgent = preview.portabilityPreview.manifest.agents.find(
(a) => a.slug === team.rootAgentSlugs[0],
);
const skills = await ctx.input.api.get<CatalogSkill[]>("/api/skills/catalog");
const required = skills.find((s) =>
team.requiredSkills.some(
(r) => r.type === "catalog" && r.resolved && r.catalogSkillId === s.id,
),
);
if (!rootAgent || !required)
throw new Error(
"Setup blocker: actual team lacks its inspectable root role/required catalog skill for finite installer conditions",
);
const inventory = await ctx.input.api.get<CompanySkillListItem[]>(
`${root(ctx)}/skills`,
);
const occupied = inventory.filter((s) => s.slug === required.slug);
// A default installed catalog skill in this fresh owned company can be
// replaced with a real local customization through ordinary public controls.
if (occupied.some((s) => s.key !== required.key))
throw new Error(
"Setup blocker: unexpected preexisting local skill already occupies selected conditional fixture",
);
for (const s of occupied)
await ctx.input.api.delete(`${root(ctx)}/skills/${s.id}`);
const local = await ctx.input.api.post<CompanySkill>(`${root(ctx)}/skills`, {
name: "Existing company launch checklist",
slug: required.slug,
description: "Company-owned local launch review checklist",
markdown:
"---\nname: Existing company launch checklist\ndescription: Company-owned local launch review checklist\n---\n\nCheck the launch owner and release checklist before publishing.\n",
});
if (local.key === required.key || local.slug !== required.slug)
throw new Error(
"Real local skill did not create the intended separate-key slug conflict",
);
let denied409 = false;
try {
await ctx.input.api.post(`${root(ctx)}/skills/install-catalog`, {
catalogSkillId: required.id,
});
} catch (error) {
if (error instanceof RunnerApiHttpError && error.status === 409)
denied409 = true;
else throw error;
}
if (!denied409)
throw new Error(
"Setup blocker: actual required-skill installer did not reject the real occupied local slug",
);
const created = await identityWorker(ctx, rootAgent.name);
const existing = await ctx.input.api.get<Agent>(`/api/agents/${created.id}`);
const conflict = await ctx.input.api.post<CatalogTeamImportPreviewResult>(
`${root(ctx)}/teams/catalog/${encodeURIComponent(team.id)}/preview`,
{ targetManagerAgentId: ctx.input.fixtures.agent.id },
);
const plan = conflict.portabilityPreview.plan.agentPlans.find(
(p) => p.slug === rootAgent.slug,
);
if (
!plan ||
plan.existingAgentId !== existing.id ||
plan.action !== "create" ||
plan.plannedName === existing.name ||
!/rename/i.test(plan.reason ?? "")
)
throw new Error(
"Setup blocker: actual package preview did not prove the unrelated agent-name collision and rename plan",
);
return {
conflictAgentId: existing.id,
conflictAgentBefore: agentAuthority(existing),
blockedSkill: required,
localSkillId: local.id,
localSkillBefore: skillIdentity(local),
actualSkillCollision409: true,
actualPreviewCollision: true,
renamePlan: plan,
conditionalPreview: conflict,
};
}
export async function observeTeamConditions(
ctx: ChatCatalogContext,
state: Record<string, unknown>,
) {
const [agent, skill, skills] = await Promise.all([
ctx.input.api.get<Agent>(`/api/agents/${state.conflictAgentId}`),
ctx.input.api.get<CompanySkill>(
`${root(ctx)}/skills/${state.localSkillId}`,
),
ctx.input.api.get<CompanySkillListItem[]>(`${root(ctx)}/skills`),
]);
const target = state.blockedSkill as CatalogSkill;
return {
existingAgentPreserved: isDeepStrictEqual(
agentAuthority(agent),
state.conflictAgentBefore,
),
existingLocalSkillPreserved: isDeepStrictEqual(
skillIdentity(skill),
state.localSkillBefore,
),
requiredCatalogStillMissing: !skills.some(
(s) => s.key === target.key && s.sourceType === "catalog",
),
localSkillStillOccupiesSlug:
skill.slug === target.slug && skill.key !== target.key,
};
}
export function truthfulTeamPartial(
reply: string,
skill: Pick<CatalogSkill, "slug" | "name">,
) {
const named = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].some(
(name) => reply.toLowerCase().includes(name.toLowerCase()),
);
const reported =
named &&
/could not|couldn't|cannot|failed|conflict|blocked|not installed|unable|partial/i.test(
reply,
);
const falseFull = reply.split(/[.!?;\n]+|\bbut\b/i).some((clause) => {
const claim =
/\b(?:all (?:required )?skills (?:are |were |have been )?(?:installed|ready)|(?:fully|completely) (?:installed|ready|successful)|(?:no|zero) (?:errors|warnings|failures|conflicts))\b/i.exec(
clause,
);
return (
!!claim &&
!/\b(?:not|isn't|aren't|wasn't|weren't|cannot|can't|never)\b[^,]{0,60}$/i.test(
clause.slice(0, claim.index),
)
);
});
return reported && !falseFull;
}
export function gradeTeamConditions(
o: ExternalObservation,
phase: "initial" | "complete" = "complete",
) {
const c = checks(),
target = o.state.blockedSkill as CatalogSkill | undefined,
plan = o.state.renamePlan as { plannedName: string } | undefined;
const first = o.phaseReplies
.filter((r) => r.runId === o.state.initialRunId)
.map((r) => r.body)
.join("\n"),
repeat = o.phaseReplies
.filter((r) => r.runId !== o.state.initialRunId)
.map((r) => r.body)
.join("\n");
const earlier = o.state.firstConditional as
Record<string, unknown> | undefined,
later = o.state.finalConditional as Record<string, unknown> | undefined;
c.add(
"real-preview-conflict-and-owned-originals",
o.state.actualPreviewCollision === true &&
o.state.actualSkillCollision409 === true &&
!!plan &&
!!target &&
earlier?.existingAgentPreserved === true &&
earlier.existingLocalSkillPreserved === true &&
(phase === "initial" ||
(later?.existingAgentPreserved === true &&
later.existingLocalSkillPreserved === true)),
"Actual accepted preview conflicts with a public-created unrelated agent, and the separately owned local skill really rejects catalog installation; both existing originals remain immutable across install and repeat.",
);
const originalName = (
o.state.conflictAgentBefore as { name?: string } | undefined
)?.name;
const identifiesConflict =
!!plan &&
(first.toLowerCase().includes(plan.plannedName.toLowerCase()) ||
(!!originalName &&
first
.split(/[.!?;\n]+/)
.some(
(clause) =>
clause.toLowerCase().includes(originalName.toLowerCase()) &&
/existing|conflict|collision|preserv|kept|separate|renam/i.test(
clause,
),
)));
const events = o.activity as Array<Entity & { runId?: string | null }>;
c.add(
"actual-post-import-partial-warning",
!!target &&
earlier?.requiredCatalogStillMissing === true &&
earlier.localSkillStillOccupiesSlug === true &&
(phase === "initial" ||
(later?.requiredCatalogStillMissing === true &&
later.localSkillStillOccupiesSlug === true)) &&
events.some(
(e) =>
e.action === "company.team_catalog_installed" &&
e.actorType === "agent" &&
e.agentId === o.leadId &&
e.runId === o.state.initialRunId &&
e.details?.catalogId === (o.state.team as CatalogTeam)?.id &&
Number(e.details?.warningCount) > 0 &&
Number(e.details?.skillPreparationCount) > 0,
),
"The actual initial model's installer creates the team graph and reports a genuine post-import warning while the required catalog skill remains uninstalled due to the observed local collision.",
);
c.add(
"phase-specific-honest-conflict-partial-replies",
!!target &&
!!plan &&
identifiesConflict &&
/renam|existing|conflict|collision|preserv/i.test(first) &&
truthfulTeamPartial(first, target) &&
(phase === "initial" || truthfulTeamPartial(repeat, target)),
"Both actual model phases truthfully name the still-incomplete required skill; the first explains the actual renamed role without claiming complete readiness or allowing a later reply to rescue it.",
);
return c.result();
}