test(evals): install vetted skills and preserve editable forks

tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from "vitest";
+import { gradeIntegratorSkillImports, integratorSkillImportCases, preservesMobileChecks } from "./integrator-skill-imports.js";
+import { sourceSkillMarkdown, mobileChecks } from "./integrator-skill-source.js";
+function valid(id: string) {
+  const url = "http://127.0.0.1:12345/accessibility-review/SKILL.md";
+  const upstream = { id: "upstream", companyId: "company", name: "accessibility-review", sourceLocator: url, sourceType: "url", editable: false, markdown: sourceSkillMarkdown, currentVersionId: "original", usedByAgents: [] };
+  const version = { id: "original", fileInventory: [{ path: "SKILL.md", content: sourceSkillMarkdown }] };
+  const local = { ...upstream, id: "local", name: "Accessibility local notes", sourceType: "local_path", editable: true, forkedFromSkillId: "upstream", forkedFromCompanyId: "company", markdown: `${sourceSkillMarkdown}\nKeep ORCHID customer escalation notes unchanged.`, currentVersionId: "notes" };
+  const fork = { ...local, markdown: `${local.markdown}\n${mobileChecks.join("\n")}`, currentVersionId: "edited" };
+  const find = id === "skills.find-and-install";
+  return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { sourceURL: url, importAllowed: true, upstream: structuredClone(upstream), localFork: structuredClone(local), upstreamVersions: [structuredClone(version)] },
+    tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], links: [], runs: [{ id: "run", agentId: "ceo", status: "succeeded" }],
+    activity: find ? [{ id: "import", action: "company.skills_imported", agentId: "ceo", details: { source: url, importedCount: 1 } }] : [{ id: "edit", action: "company.skill_file_updated", agentId: "ceo", entityId: "local" }],
+    sourceRequests: [{ method: "GET", path: "/accessibility-review/SKILL.md", at: 1, sha256: "digest" }],
+    skills: find ? [upstream] : [upstream, fork], skillVersions: find ? { upstream: [version] } : { upstream: [version], local: [{ id: "edited", authorAgentId: "ceo", fileInventory: [{ path: "SKILL.md", content: fork.markdown }] }] },
+    reply: find ? "Installed accessibility-review from the vetted source. Assignment is a separate step." : "Use Accessibility local notes for future assignments. The upstream original is unchanged and remains read-only." };
+}
+describe("owned vetted source and editable fork controls", () => {
+  for (const definition of integratorSkillImportCases) it(`${definition.scenarioId}: saved source/version evidence and missing inputs`, () => {
+    expect(gradeIntegratorSkillImports(valid(definition.scenarioId)).passed).toBe(true);
+    expect(gradeIntegratorSkillImports(undefined).passed).toBe(false);
+    expect(gradeIntegratorSkillImports({ ...valid(definition.scenarioId), sourceRequests: [] }).passed).toBe(false);
+  });
+  it("rejects a substituted source, fetch-without-install and fictional board approval", () => {
+    const o = valid("skills.find-and-install");
+    expect(gradeIntegratorSkillImports({ ...o, activity: [] }).passed).toBe(false);
+    const replaced = structuredClone(o); replaced.skills[0]!.sourceLocator = "http://other.test/SKILL.md";
+    expect(gradeIntegratorSkillImports(replaced).passed).toBe(false);
+    expect(gradeIntegratorSkillImports({ ...o, reply: "Board approval is required to install accessibility-review." }).passed).toBe(false);
+    const wrongBytes = structuredClone(o); wrongBytes.skills[0]!.markdown = "A suggested skill.";
+    expect(gradeIntegratorSkillImports(wrongBytes).passed).toBe(false);
+  });
+  it("rejects upstream edits, lost unrelated local notes, missing substantive mobile checks and false in-place success", () => {
+    const o = valid("skills.fork-read-only"); const upstreamChanged = structuredClone(o); upstreamChanged.skills[0]!.markdown += "Modified upstream";
+    expect(gradeIntegratorSkillImports(upstreamChanged).passed).toBe(false);
+    const lostNotes = structuredClone(o); lostNotes.skills[1]!.markdown = `${sourceSkillMarkdown}\n${mobileChecks.join("\n")}`;
+    expect(gradeIntegratorSkillImports(lostNotes).passed).toBe(false);
+    const wrongTouch = structuredClone(o); wrongTouch.skills[1]!.markdown = wrongTouch.skills[1]!.markdown.replace("44 by 44", "24 by 24");
+    expect(gradeIntegratorSkillImports(wrongTouch).passed).toBe(false);
+    expect(gradeIntegratorSkillImports({ ...o, reply: "I edited the upstream. Use Accessibility local notes; the original is unchanged." }).passed).toBe(false);
+    expect(preservesMobileChecks("Check touch targets at least 44x44 pixels.\nCheck both portrait and landscape orientation.\nReview mobile Safari checkout.")).toBe(true);
+    expect(preservesMobileChecks("Ignore touch targets at least 44x44 pixels.\nCheck both portrait and landscape orientation.\nReview mobile Safari checkout.")).toBe(false);
+  });
+});

tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.ts

@@ -0,0 +1,82 @@
+import type { CompanySkillDetail, CompanySkillListItem, CompanySkillVersion } from "../../../packages/shared/src/types/company-skill.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import { checks, observation, project, type Entity, type InternalObservation } from "./internal-support.js";
+import { groundedStatement, integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js";
+import { preservesWebsiteChecklist } from "./integrator-skills.js";
+import { sourceSkillMarkdown, mobileChecks, openVettedSkillSource, skillSourceRequests, closeVettedSkillSource, importReadOnlySkill, editableLocalNotes } from "./integrator-skill-source.js";
+interface ImportObservation extends InternalObservation {
+  skills: CompanySkillDetail[]; skillVersions: Record<string, CompanySkillVersion[]>;
+  sourceRequests: ReturnType<typeof skillSourceRequests>;
+}
+const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}/skills`;
+async function observeImports(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<ImportObservation> {
+  const base = await observeInternal(ctx, state); const api = ctx.input.api;
+  const rows = await api.get<CompanySkillListItem[]>(root(ctx));
+  const skills = await Promise.all(rows.map(row => api.get<CompanySkillDetail>(`${root(ctx)}/${row.id}`)));
+  const skillVersions = Object.fromEntries(await Promise.all(rows.map(async row => [row.id, await api.get<CompanySkillVersion[]>(`${root(ctx)}/${row.id}/versions`)])));
+  return { ...base, skills, skillVersions, sourceRequests: skillSourceRequests(ctx) };
+}
+function importObservation(value: unknown): ImportObservation | undefined {
+  const o = value as ImportObservation;
+  return observation(value) && Array.isArray(o.skills) && !!o.skillVersions && Array.isArray(o.sourceRequests) ? o : undefined;
+}
+export function preservesMobileChecks(markdown: string) {
+  const lines = markdown.split(/\n+|(?<=[.!?])\s+/).filter(text => !/\b(?:skip|omit|ignore|do not|don't|optional)\b/i.test(text));
+  return lines.some(text => /touch/i.test(text) && /44\s*(?:by|x|×)\s*44/i.test(text) && /at least|minimum|>=|≥/.test(text))
+    && lines.some(text => /portrait/i.test(text) && /landscape/i.test(text))
+    && lines.some(text => /mobile Safari/i.test(text) && /checkout/i.test(text));
+}
+function changedFork(o: ImportObservation) {
+  const upstream = o.state.upstream as CompanySkillDetail | undefined;
+  return o.skills.find(row => row.forkedFromSkillId === upstream?.id && row.editable && preservesMobileChecks(row.markdown)
+    && o.activity.some(event => event.entityId === row.id && event.agentId === o.leadId && ["company.skill_forked", "company.skill_file_updated", "company.skill_created"].includes(event.action ?? "")));
+}
+export function gradeIntegratorSkillImports(value: unknown) {
+  const c = checks(); const o = importObservation(value);
+  c.add("actual-source-library-and-version-evidence", !!o, "Requires the real vetted source request log, public company library detail and revision history.");
+  if (!o) return c.result();
+  c.add("permitted-source-prerequisite", o.state.importAllowed === true && o.sourceRequests.length >= 1, "Effective skill policy permits ordinary import, and the owned synthetic source was genuinely read.");
+  if (o.scenarioId === "skills.find-and-install") {
+    const installed = o.skills.filter(row => row.sourceLocator === o.state.sourceURL); const skill = installed[0];
+    c.add("one-intended-source-installation", installed.length === 1 && !!skill && skill.companyId === o.companyId && skill.sourceType === "url" && skill.markdown === sourceSkillMarkdown
+      && !!skill.currentVersionId && o.skillVersions[skill.id]?.some(row => row.id === skill.currentVersionId && row.fileInventory.some(file => file.path === "SKILL.md" && file.content === sourceSkillMarkdown)), "One intended skill retains the genuine source URL and exact fetched original revision bytes.");
+    c.add("agent-imported-approved-source", o.activity.some(event => event.action === "company.skills_imported" && event.agentId === o.leadId && event.details?.source === o.state.sourceURL
+      && event.details?.importedCount === 1), "The responding agent imports the actual authorized source through the supported company route.");
+    c.add("truthful-installation-and-usage", !!skill && o.reply.includes(skill.name) && skill.usedByAgents.length === 0
+      && !/\b(?:board|human|admin(?:istrator)?)\b[^.!?\n]{0,40}\b(?:approval (?:is )?required|must approve|need(?:s)? to approve)\b/i.test(o.reply)
+      && !/\b(?:enabled|assigned|activated)\b[^.!?\n]{0,30}\b(?:all|every)\s+agents?\b/i.test(o.reply), "The response identifies the saved skill without a fictional approval gate or global usage assignment.");
+  } else if (o.scenarioId === "skills.fork-read-only") {
+    const before = o.state.upstream as CompanySkillDetail | undefined; const original = o.skills.find(row => row.id === before?.id); const fork = changedFork(o);
+    c.add("upstream-remains-read-only-and-identical", !!before && !!original && original.editable === false && original.sourceType === "url" && original.sourceLocator === before.sourceLocator
+      && original.currentVersionId === before.currentVersionId && original.markdown === before.markdown
+      && JSON.stringify(o.skillVersions[original.id]) === JSON.stringify(o.state.upstreamVersions), "The real protected upstream package and full revision history remain unchanged.");
+    c.add("editable-provenance-preserving-mobile-fork", !!fork && !!before && fork.id !== before.id && fork.forkedFromSkillId === before.id && fork.forkedFromCompanyId === o.companyId
+      && preservesWebsiteChecklist(fork.markdown) && preservesMobileChecks(fork.markdown)
+      && o.skillVersions[fork.id]?.some(version => version.id === fork.currentVersionId && version.authorAgentId === o.leadId && version.fileInventory.some(file => file.path === "SKILL.md" && file.content === fork.markdown)), "A saved editable fork retains upstream provenance, the original substantive checklist and all actual mobile additions.");
+    const localBefore = o.state.localFork as CompanySkillDetail | undefined; const localNow = o.skills.find(row => row.id === localBefore?.id);
+    c.add("unrelated-local-notes-preserved", !!localBefore && !!localNow && localNow.markdown.includes("ORCHID customer escalation notes unchanged")
+      && (localNow.id === fork?.id || localNow.currentVersionId === localBefore.currentVersionId && localNow.markdown === localBefore.markdown), "Existing local notes survive whether the agent edits the existing copy or selects a separate fork.");
+    c.add("correct-copy-explained", !!fork && o.reply.includes(fork.name) && groundedStatement(o.reply, /\b(?:upstream|original|source)\b/i, /\b(?:unchanged|read.only|protected|not (?:changed|modified|edited))\b/i,
+      /\b(?:edited|updated|modified|changed)\s+(?:the )?(?:upstream|original source)\b/i), "Chat distinguishes the editable copy for future use from the unmodified upstream.");
+  } else c.add("owned-import-oracle", false, "No independent import/fork oracle exists for this scenario.");
+  return c.result();
+}
+function define(id: "skills.find-and-install" | "skills.fork-read-only") {
+  return integratorDefinition("skills", id, async ctx => {
+    const source = await openVettedSkillSource(ctx);
+    try {
+      const decision = await ctx.input.api.post<{ allowed: boolean }>(`/api/companies/${ctx.input.fixtures.company.id}/skill-policy/evaluate`, { principal: { agentId: ctx.input.fixtures.agent.id }, action: "skills.import", resource: { sourceType: "external_package", sourceLocator: source.url } });
+      if (!decision.allowed) throw new Error("Synthetic source fixture does not have the canonical permitted skill policy");
+      const p = await project(ctx, "Launch", { description: `Our vetted accessibility review source is ${source.url}. It is a synthetic Markdown-only fixture. Our mobile review checklist is: ${mobileChecks.join(" ")}` });
+      if (id === "skills.find-and-install") return { project: p, sourceURL: source.url, importAllowed: true };
+      const upstream = await importReadOnlySkill(ctx, source.url); const localFork = await editableLocalNotes(ctx, upstream);
+      return { project: p, sourceURL: source.url, importAllowed: true, upstream, localFork, upstreamVersions: await ctx.input.api.get<CompanySkillVersion[]>(`${root(ctx)}/${upstream.id}/versions`) };
+    } catch (error) { await closeVettedSkillSource(ctx); throw error; }
+  }, observeImports, gradeIntegratorSkillImports, async (ctx, _state, value) => {
+    const o = importObservation(value); const skill = o && (id === "skills.find-and-install" ? o.skills.find(row => row.sourceLocator === o.state.sourceURL) : changedFork(o));
+    if (!o || !skill) throw new Error("No actual saved skill copy for browser inspection");
+    await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`, skill.name, resultHref(o, skill.id, "studio"));
+  }, { sourceFiles: ["chat-catalog-domains/integrator-skill-imports.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skills.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"],
+    async cleanup(ctx) { await closeVettedSkillSource(ctx); } });
+}
+export const integratorSkillImportCases = [define("skills.find-and-install"), define("skills.fork-read-only")];

tests/runner-e2e/chat-catalog-domains/integrator-skill-source.ts

@@ -0,0 +1,40 @@
+import { createServer } from "node:http";
+import { createHash } from "node:crypto";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { CompanySkill, CompanySkillDetail, CompanySkillImportResult } from "../../../packages/shared/src/types/company-skill.js";
+import { websiteChecklist } from "./integrator-skills.js";
+export const sourceSkillMarkdown = `---\nname: accessibility-review\ndescription: Review a synthetic website for accessibility and report evidence.\n---\n\n# Website accessibility review\n\n${websiteChecklist.map(item => `- ${item}`).join("\n")}\n\nFor each check, record the page and observed evidence; report failures without claiming every page was tested.\n`;
+export const mobileChecks = ["Check touch targets are at least 44 by 44 pixels.", "Repeat the review in both portrait and landscape orientation.", "Verify the checkout interaction on mobile Safari."];
+const sources = new WeakMap<ChatCatalogContext, Awaited<ReturnType<typeof openSource>>>();
+async function openSource() {
+  const requests: Array<{ method: string; path: string; at: number; sha256: string }> = [];
+  const server = createServer((request, response) => {
+    if (request.method !== "GET" || request.url !== "/accessibility-review/SKILL.md" || requests.length >= 100) { response.writeHead(404).end(); return; }
+    requests.push({ method: request.method, path: request.url, at: Date.now(), sha256: createHash("sha256").update(sourceSkillMarkdown).digest("hex") });
+    response.writeHead(200, { "content-type": "text/markdown; charset=utf-8", "cache-control": "no-store" }); response.end(sourceSkillMarkdown);
+  });
+  await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => resolve()); });
+  const address = server.address(); if (!address || typeof address === "string") throw new Error("Owned skill source did not bind loopback");
+  return { url: `http://127.0.0.1:${address.port}/accessibility-review/SKILL.md`, requests,
+    async close() { server.closeAllConnections(); await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve())); } };
+}
+export async function openVettedSkillSource(ctx: ChatCatalogContext) {
+  if (sources.has(ctx)) throw new Error("Only one owned synthetic skill source is allowed per attempt");
+  const source = await openSource(); sources.set(ctx, source); return source;
+}
+export function skillSourceRequests(ctx: ChatCatalogContext) { return sources.get(ctx)?.requests.map(row => ({ ...row })) ?? []; }
+export async function closeVettedSkillSource(ctx: ChatCatalogContext) { const source = sources.get(ctx); if (source) { await source.close(); sources.delete(ctx); } }
+export async function importReadOnlySkill(ctx: ChatCatalogContext, source: string): Promise<CompanySkillDetail> {
+  const root = `/api/companies/${ctx.input.fixtures.company.id}/skills`;
+  const imported = await ctx.input.api.post<CompanySkillImportResult>(`${root}/import`, { source });
+  if (imported.imported.length !== 1) throw new Error("Owned source import must produce exactly one real company skill");
+  const skill = await ctx.input.api.get<CompanySkillDetail>(`${root}/${imported.imported[0]!.id}`);
+  if (skill.sourceType !== "url" || skill.sourceLocator !== source || skill.editable || skill.markdown !== sourceSkillMarkdown) throw new Error("Public import did not establish the actual read-only upstream skill");
+  return skill;
+}
+export async function editableLocalNotes(ctx: ChatCatalogContext, upstream: CompanySkillDetail) {
+  const root = `/api/companies/${ctx.input.fixtures.company.id}/skills`;
+  const result = await ctx.input.api.post<{ skill: CompanySkill }>(`${root}/${upstream.id}/fork`, { name: "Accessibility local notes", slug: "accessibility-local-notes" });
+  await ctx.input.api.patch(`${root}/${result.skill.id}/files`, { path: "SKILL.md", content: `${result.skill.markdown}\n\n## Local review notes\nKeep ORCHID customer escalation notes unchanged.\n`, expectedVersionId: result.skill.currentVersionId });
+  return ctx.input.api.get<CompanySkillDetail>(`${root}/${result.skill.id}`);
+}