diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.test.ts new file mode 100644 index 00000000000..fb482a50946 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { gradeIntegratorSkillImports, integratorSkillImportCases, preservesMobileChecks } from "./integrator-skill-imports.js"; +import { sourceSkillMarkdown, mobileChecks } from "./integrator-skill-source.js"; +function valid(id: string) { + const url = "http://127.0.0.1:12345/accessibility-review/SKILL.md"; + const upstream = { id: "upstream", companyId: "company", name: "accessibility-review", sourceLocator: url, sourceType: "url", editable: false, markdown: sourceSkillMarkdown, currentVersionId: "original", usedByAgents: [] }; + const version = { id: "original", fileInventory: [{ path: "SKILL.md", content: sourceSkillMarkdown }] }; + const local = { ...upstream, id: "local", name: "Accessibility local notes", sourceType: "local_path", editable: true, forkedFromSkillId: "upstream", forkedFromCompanyId: "company", markdown: `${sourceSkillMarkdown}\nKeep ORCHID customer escalation notes unchanged.`, currentVersionId: "notes" }; + const fork = { ...local, markdown: `${local.markdown}\n${mobileChecks.join("\n")}`, currentVersionId: "edited" }; + const find = id === "skills.find-and-install"; + return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { sourceURL: url, importAllowed: true, upstream: structuredClone(upstream), localFork: structuredClone(local), upstreamVersions: [structuredClone(version)] }, + tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], links: [], runs: [{ id: "run", agentId: "ceo", status: "succeeded" }], + activity: find ? [{ id: "import", action: "company.skills_imported", agentId: "ceo", details: { source: url, importedCount: 1 } }] : [{ id: "edit", action: "company.skill_file_updated", agentId: "ceo", entityId: "local" }], + sourceRequests: [{ method: "GET", path: "/accessibility-review/SKILL.md", at: 1, sha256: "digest" }], + skills: find ? [upstream] : [upstream, fork], skillVersions: find ? { upstream: [version] } : { upstream: [version], local: [{ id: "edited", authorAgentId: "ceo", fileInventory: [{ path: "SKILL.md", content: fork.markdown }] }] }, + reply: find ? "Installed accessibility-review from the vetted source. Assignment is a separate step." : "Use Accessibility local notes for future assignments. The upstream original is unchanged and remains read-only." }; +} +describe("owned vetted source and editable fork controls", () => { + for (const definition of integratorSkillImportCases) it(`${definition.scenarioId}: saved source/version evidence and missing inputs`, () => { + expect(gradeIntegratorSkillImports(valid(definition.scenarioId)).passed).toBe(true); + expect(gradeIntegratorSkillImports(undefined).passed).toBe(false); + expect(gradeIntegratorSkillImports({ ...valid(definition.scenarioId), sourceRequests: [] }).passed).toBe(false); + }); + it("rejects a substituted source, fetch-without-install and fictional board approval", () => { + const o = valid("skills.find-and-install"); + expect(gradeIntegratorSkillImports({ ...o, activity: [] }).passed).toBe(false); + const replaced = structuredClone(o); replaced.skills[0]!.sourceLocator = "http://other.test/SKILL.md"; + expect(gradeIntegratorSkillImports(replaced).passed).toBe(false); + expect(gradeIntegratorSkillImports({ ...o, reply: "Board approval is required to install accessibility-review." }).passed).toBe(false); + const wrongBytes = structuredClone(o); wrongBytes.skills[0]!.markdown = "A suggested skill."; + expect(gradeIntegratorSkillImports(wrongBytes).passed).toBe(false); + }); + it("rejects upstream edits, lost unrelated local notes, missing substantive mobile checks and false in-place success", () => { + const o = valid("skills.fork-read-only"); const upstreamChanged = structuredClone(o); upstreamChanged.skills[0]!.markdown += "Modified upstream"; + expect(gradeIntegratorSkillImports(upstreamChanged).passed).toBe(false); + const lostNotes = structuredClone(o); lostNotes.skills[1]!.markdown = `${sourceSkillMarkdown}\n${mobileChecks.join("\n")}`; + expect(gradeIntegratorSkillImports(lostNotes).passed).toBe(false); + const wrongTouch = structuredClone(o); wrongTouch.skills[1]!.markdown = wrongTouch.skills[1]!.markdown.replace("44 by 44", "24 by 24"); + expect(gradeIntegratorSkillImports(wrongTouch).passed).toBe(false); + expect(gradeIntegratorSkillImports({ ...o, reply: "I edited the upstream. Use Accessibility local notes; the original is unchanged." }).passed).toBe(false); + expect(preservesMobileChecks("Check touch targets at least 44x44 pixels.\nCheck both portrait and landscape orientation.\nReview mobile Safari checkout.")).toBe(true); + expect(preservesMobileChecks("Ignore touch targets at least 44x44 pixels.\nCheck both portrait and landscape orientation.\nReview mobile Safari checkout.")).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.ts new file mode 100644 index 00000000000..afd1ecc77f2 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-imports.ts @@ -0,0 +1,82 @@ +import type { CompanySkillDetail, CompanySkillListItem, CompanySkillVersion } from "../../../packages/shared/src/types/company-skill.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { checks, observation, project, type Entity, type InternalObservation } from "./internal-support.js"; +import { groundedStatement, integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; +import { preservesWebsiteChecklist } from "./integrator-skills.js"; +import { sourceSkillMarkdown, mobileChecks, openVettedSkillSource, skillSourceRequests, closeVettedSkillSource, importReadOnlySkill, editableLocalNotes } from "./integrator-skill-source.js"; +interface ImportObservation extends InternalObservation { + skills: CompanySkillDetail[]; skillVersions: Record; + sourceRequests: ReturnType; +} +const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}/skills`; +async function observeImports(ctx: ChatCatalogContext, state: Record): Promise { + const base = await observeInternal(ctx, state); const api = ctx.input.api; + const rows = await api.get(root(ctx)); + const skills = await Promise.all(rows.map(row => api.get(`${root(ctx)}/${row.id}`))); + const skillVersions = Object.fromEntries(await Promise.all(rows.map(async row => [row.id, await api.get(`${root(ctx)}/${row.id}/versions`)]))); + return { ...base, skills, skillVersions, sourceRequests: skillSourceRequests(ctx) }; +} +function importObservation(value: unknown): ImportObservation | undefined { + const o = value as ImportObservation; + return observation(value) && Array.isArray(o.skills) && !!o.skillVersions && Array.isArray(o.sourceRequests) ? o : undefined; +} +export function preservesMobileChecks(markdown: string) { + const lines = markdown.split(/\n+|(?<=[.!?])\s+/).filter(text => !/\b(?:skip|omit|ignore|do not|don't|optional)\b/i.test(text)); + return lines.some(text => /touch/i.test(text) && /44\s*(?:by|x|×)\s*44/i.test(text) && /at least|minimum|>=|≥/.test(text)) + && lines.some(text => /portrait/i.test(text) && /landscape/i.test(text)) + && lines.some(text => /mobile Safari/i.test(text) && /checkout/i.test(text)); +} +function changedFork(o: ImportObservation) { + const upstream = o.state.upstream as CompanySkillDetail | undefined; + return o.skills.find(row => row.forkedFromSkillId === upstream?.id && row.editable && preservesMobileChecks(row.markdown) + && o.activity.some(event => event.entityId === row.id && event.agentId === o.leadId && ["company.skill_forked", "company.skill_file_updated", "company.skill_created"].includes(event.action ?? ""))); +} +export function gradeIntegratorSkillImports(value: unknown) { + const c = checks(); const o = importObservation(value); + c.add("actual-source-library-and-version-evidence", !!o, "Requires the real vetted source request log, public company library detail and revision history."); + if (!o) return c.result(); + c.add("permitted-source-prerequisite", o.state.importAllowed === true && o.sourceRequests.length >= 1, "Effective skill policy permits ordinary import, and the owned synthetic source was genuinely read."); + if (o.scenarioId === "skills.find-and-install") { + const installed = o.skills.filter(row => row.sourceLocator === o.state.sourceURL); const skill = installed[0]; + c.add("one-intended-source-installation", installed.length === 1 && !!skill && skill.companyId === o.companyId && skill.sourceType === "url" && skill.markdown === sourceSkillMarkdown + && !!skill.currentVersionId && o.skillVersions[skill.id]?.some(row => row.id === skill.currentVersionId && row.fileInventory.some(file => file.path === "SKILL.md" && file.content === sourceSkillMarkdown)), "One intended skill retains the genuine source URL and exact fetched original revision bytes."); + c.add("agent-imported-approved-source", o.activity.some(event => event.action === "company.skills_imported" && event.agentId === o.leadId && event.details?.source === o.state.sourceURL + && event.details?.importedCount === 1), "The responding agent imports the actual authorized source through the supported company route."); + c.add("truthful-installation-and-usage", !!skill && o.reply.includes(skill.name) && skill.usedByAgents.length === 0 + && !/\b(?:board|human|admin(?:istrator)?)\b[^.!?\n]{0,40}\b(?:approval (?:is )?required|must approve|need(?:s)? to approve)\b/i.test(o.reply) + && !/\b(?:enabled|assigned|activated)\b[^.!?\n]{0,30}\b(?:all|every)\s+agents?\b/i.test(o.reply), "The response identifies the saved skill without a fictional approval gate or global usage assignment."); + } else if (o.scenarioId === "skills.fork-read-only") { + const before = o.state.upstream as CompanySkillDetail | undefined; const original = o.skills.find(row => row.id === before?.id); const fork = changedFork(o); + c.add("upstream-remains-read-only-and-identical", !!before && !!original && original.editable === false && original.sourceType === "url" && original.sourceLocator === before.sourceLocator + && original.currentVersionId === before.currentVersionId && original.markdown === before.markdown + && JSON.stringify(o.skillVersions[original.id]) === JSON.stringify(o.state.upstreamVersions), "The real protected upstream package and full revision history remain unchanged."); + c.add("editable-provenance-preserving-mobile-fork", !!fork && !!before && fork.id !== before.id && fork.forkedFromSkillId === before.id && fork.forkedFromCompanyId === o.companyId + && preservesWebsiteChecklist(fork.markdown) && preservesMobileChecks(fork.markdown) + && o.skillVersions[fork.id]?.some(version => version.id === fork.currentVersionId && version.authorAgentId === o.leadId && version.fileInventory.some(file => file.path === "SKILL.md" && file.content === fork.markdown)), "A saved editable fork retains upstream provenance, the original substantive checklist and all actual mobile additions."); + const localBefore = o.state.localFork as CompanySkillDetail | undefined; const localNow = o.skills.find(row => row.id === localBefore?.id); + c.add("unrelated-local-notes-preserved", !!localBefore && !!localNow && localNow.markdown.includes("ORCHID customer escalation notes unchanged") + && (localNow.id === fork?.id || localNow.currentVersionId === localBefore.currentVersionId && localNow.markdown === localBefore.markdown), "Existing local notes survive whether the agent edits the existing copy or selects a separate fork."); + c.add("correct-copy-explained", !!fork && o.reply.includes(fork.name) && groundedStatement(o.reply, /\b(?:upstream|original|source)\b/i, /\b(?:unchanged|read.only|protected|not (?:changed|modified|edited))\b/i, + /\b(?:edited|updated|modified|changed)\s+(?:the )?(?:upstream|original source)\b/i), "Chat distinguishes the editable copy for future use from the unmodified upstream."); + } else c.add("owned-import-oracle", false, "No independent import/fork oracle exists for this scenario."); + return c.result(); +} +function define(id: "skills.find-and-install" | "skills.fork-read-only") { + return integratorDefinition("skills", id, async ctx => { + const source = await openVettedSkillSource(ctx); + try { + const decision = await ctx.input.api.post<{ allowed: boolean }>(`/api/companies/${ctx.input.fixtures.company.id}/skill-policy/evaluate`, { principal: { agentId: ctx.input.fixtures.agent.id }, action: "skills.import", resource: { sourceType: "external_package", sourceLocator: source.url } }); + if (!decision.allowed) throw new Error("Synthetic source fixture does not have the canonical permitted skill policy"); + const p = await project(ctx, "Launch", { description: `Our vetted accessibility review source is ${source.url}. It is a synthetic Markdown-only fixture. Our mobile review checklist is: ${mobileChecks.join(" ")}` }); + if (id === "skills.find-and-install") return { project: p, sourceURL: source.url, importAllowed: true }; + const upstream = await importReadOnlySkill(ctx, source.url); const localFork = await editableLocalNotes(ctx, upstream); + return { project: p, sourceURL: source.url, importAllowed: true, upstream, localFork, upstreamVersions: await ctx.input.api.get(`${root(ctx)}/${upstream.id}/versions`) }; + } catch (error) { await closeVettedSkillSource(ctx); throw error; } + }, observeImports, gradeIntegratorSkillImports, async (ctx, _state, value) => { + const o = importObservation(value); const skill = o && (id === "skills.find-and-install" ? o.skills.find(row => row.sourceLocator === o.state.sourceURL) : changedFork(o)); + if (!o || !skill) throw new Error("No actual saved skill copy for browser inspection"); + await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`, skill.name, resultHref(o, skill.id, "studio")); + }, { sourceFiles: ["chat-catalog-domains/integrator-skill-imports.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-skills.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"], + async cleanup(ctx) { await closeVettedSkillSource(ctx); } }); +} +export const integratorSkillImportCases = [define("skills.find-and-install"), define("skills.fork-read-only")]; diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-source.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-source.ts new file mode 100644 index 00000000000..73f19e02396 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-source.ts @@ -0,0 +1,40 @@ +import { createServer } from "node:http"; +import { createHash } from "node:crypto"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { CompanySkill, CompanySkillDetail, CompanySkillImportResult } from "../../../packages/shared/src/types/company-skill.js"; +import { websiteChecklist } from "./integrator-skills.js"; +export const sourceSkillMarkdown = `---\nname: accessibility-review\ndescription: Review a synthetic website for accessibility and report evidence.\n---\n\n# Website accessibility review\n\n${websiteChecklist.map(item => `- ${item}`).join("\n")}\n\nFor each check, record the page and observed evidence; report failures without claiming every page was tested.\n`; +export const mobileChecks = ["Check touch targets are at least 44 by 44 pixels.", "Repeat the review in both portrait and landscape orientation.", "Verify the checkout interaction on mobile Safari."]; +const sources = new WeakMap>>(); +async function openSource() { + const requests: Array<{ method: string; path: string; at: number; sha256: string }> = []; + const server = createServer((request, response) => { + if (request.method !== "GET" || request.url !== "/accessibility-review/SKILL.md" || requests.length >= 100) { response.writeHead(404).end(); return; } + requests.push({ method: request.method, path: request.url, at: Date.now(), sha256: createHash("sha256").update(sourceSkillMarkdown).digest("hex") }); + response.writeHead(200, { "content-type": "text/markdown; charset=utf-8", "cache-control": "no-store" }); response.end(sourceSkillMarkdown); + }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => resolve()); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Owned skill source did not bind loopback"); + return { url: `http://127.0.0.1:${address.port}/accessibility-review/SKILL.md`, requests, + async close() { server.closeAllConnections(); await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); } }; +} +export async function openVettedSkillSource(ctx: ChatCatalogContext) { + if (sources.has(ctx)) throw new Error("Only one owned synthetic skill source is allowed per attempt"); + const source = await openSource(); sources.set(ctx, source); return source; +} +export function skillSourceRequests(ctx: ChatCatalogContext) { return sources.get(ctx)?.requests.map(row => ({ ...row })) ?? []; } +export async function closeVettedSkillSource(ctx: ChatCatalogContext) { const source = sources.get(ctx); if (source) { await source.close(); sources.delete(ctx); } } +export async function importReadOnlySkill(ctx: ChatCatalogContext, source: string): Promise { + const root = `/api/companies/${ctx.input.fixtures.company.id}/skills`; + const imported = await ctx.input.api.post(`${root}/import`, { source }); + if (imported.imported.length !== 1) throw new Error("Owned source import must produce exactly one real company skill"); + const skill = await ctx.input.api.get(`${root}/${imported.imported[0]!.id}`); + if (skill.sourceType !== "url" || skill.sourceLocator !== source || skill.editable || skill.markdown !== sourceSkillMarkdown) throw new Error("Public import did not establish the actual read-only upstream skill"); + return skill; +} +export async function editableLocalNotes(ctx: ChatCatalogContext, upstream: CompanySkillDetail) { + const root = `/api/companies/${ctx.input.fixtures.company.id}/skills`; + const result = await ctx.input.api.post<{ skill: CompanySkill }>(`${root}/${upstream.id}/fork`, { name: "Accessibility local notes", slug: "accessibility-local-notes" }); + await ctx.input.api.patch(`${root}/${result.skill.id}/files`, { path: "SKILL.md", content: `${result.skill.markdown}\n\n## Local review notes\nKeep ORCHID customer escalation notes unchanged.\n`, expectedVersionId: result.skill.currentVersionId }); + return ctx.input.api.get(`${root}/${result.skill.id}`); +}