From 30c57ebd5f62cf6553ad5dc82c37cd8d9f0dabdf Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 11 Jun 2026 16:16:19 +0000 Subject: [PATCH] Preserve runtime skill version pins --- .../__tests__/heartbeat-project-env.test.ts | 25 ++ .../heartbeat-runtime-skills.test.ts | 225 ++++++++++++++++++ server/src/services/company-skills.ts | 12 +- server/src/services/heartbeat.ts | 11 +- 4 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 server/src/__tests__/heartbeat-runtime-skills.test.ts diff --git a/server/src/__tests__/heartbeat-project-env.test.ts b/server/src/__tests__/heartbeat-project-env.test.ts index 1f62aa76197..2fde8594c64 100644 --- a/server/src/__tests__/heartbeat-project-env.test.ts +++ b/server/src/__tests__/heartbeat-project-env.test.ts @@ -333,4 +333,29 @@ describe("applyRunScopedMentionedSkillKeys", () => { }, }); }); + + it("preserves existing version pins when adding mentioned skills", () => { + const originalConfig = { + command: "codex", + paperclipSkillSync: { + desiredSkills: [ + { key: "company/company-1/release-changelog", versionId: "version-1" }, + ], + }, + }; + + const updatedConfig = applyRunScopedMentionedSkillKeys(originalConfig, [ + "company/company-1/security-review", + ]); + + expect(updatedConfig).toEqual({ + command: "codex", + paperclipSkillSync: { + desiredSkills: [ + { key: "company/company-1/release-changelog", versionId: "version-1" }, + { key: "company/company-1/security-review", versionId: null }, + ], + }, + }); + }); }); diff --git a/server/src/__tests__/heartbeat-runtime-skills.test.ts b/server/src/__tests__/heartbeat-runtime-skills.test.ts new file mode 100644 index 00000000000..2a2512da96a --- /dev/null +++ b/server/src/__tests__/heartbeat-runtime-skills.test.ts @@ -0,0 +1,225 @@ +import { randomUUID } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import { promises as fs } from "node:fs"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, companySkills, createDb } from "@paperclipai/db"; +import type { PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { companySkillService } from "../services/company-skills.ts"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { registerServerAdapter, unregisterServerAdapter } from "../adapters/index.ts"; + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +const TEST_ADAPTER_TYPE = "runtime_skill_capture"; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres heartbeat runtime skill tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +async function waitForRunToFinish( + heartbeat: ReturnType, + runId: string, + timeoutMs = 5_000, +) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const run = await heartbeat.getRun(runId); + if (run && !["queued", "running"].includes(run.status)) return run; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return await heartbeat.getRun(runId); +} + +describeEmbeddedPostgres("heartbeat runtime skill version pins", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + let oldPaperclipHome: string | undefined; + let paperclipHome: string | null = null; + const capturedRuns: Array<{ agentId: string; skills: PaperclipSkillEntry[] }> = []; + const cleanupDirs = new Set(); + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("heartbeat-runtime-skills-"); + db = createDb(tempDb.connectionString); + oldPaperclipHome = process.env.PAPERCLIP_HOME; + paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-skills-home-")); + process.env.PAPERCLIP_HOME = paperclipHome; + registerServerAdapter({ + type: TEST_ADAPTER_TYPE, + execute: async (ctx) => { + capturedRuns.push({ + agentId: ctx.agent.id, + skills: (ctx.config.paperclipRuntimeSkills ?? []) as PaperclipSkillEntry[], + }); + return { + exitCode: 0, + signal: null, + timedOut: false, + label: "Captured runtime skills", + }; + }, + testEnvironment: async () => ({ + adapterType: TEST_ADAPTER_TYPE, + status: "pass", + checks: [], + testedAt: new Date().toISOString(), + }), + }); + }, 20_000); + + afterEach(async () => { + capturedRuns.length = 0; + await db.execute(sql.raw(` + TRUNCATE TABLE + "activity_log", + "environment_leases", + "environments", + "heartbeat_run_events", + "heartbeat_runs", + "agent_wakeup_requests", + "agent_runtime_state", + "company_skill_versions", + "company_skills", + "agents", + "companies" + RESTART IDENTITY CASCADE + `)); + await Promise.all(Array.from(cleanupDirs, (dir) => fs.rm(dir, { recursive: true, force: true }))); + cleanupDirs.clear(); + }); + + afterAll(async () => { + unregisterServerAdapter(TEST_ADAPTER_TYPE); + if (oldPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = oldPaperclipHome; + if (paperclipHome) { + await fs.rm(paperclipHome, { recursive: true, force: true }); + } + await tempDb?.cleanup(); + }); + + it("materializes different pinned skill versions for different agents at runtime", async () => { + const companyId = randomUUID(); + const skillId = randomUUID(); + const firstAgentId = randomUUID(); + const secondAgentId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + const skillKey = `company/${companyId}/runtime-coach`; + const skillDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-versioned-runtime-skill-")); + cleanupDirs.add(skillDir); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + }); + await fs.writeFile(path.join(skillDir, "SKILL.md"), "# Runtime Coach\n\nVersion one.\n", "utf8"); + await db.insert(companySkills).values({ + id: skillId, + companyId, + key: skillKey, + slug: "runtime-coach", + name: "Runtime Coach", + description: null, + markdown: "# Runtime Coach\n\nVersion one.\n", + sourceType: "local_path", + sourceLocator: skillDir, + trustLevel: "markdown_only", + compatibility: "compatible", + fileInventory: [{ path: "SKILL.md", kind: "skill" }], + metadata: { sourceKind: "local_path" }, + }); + + const skills = companySkillService(db); + const versionOne = await skills.createVersion( + companyId, + skillId, + { label: "v1" }, + { type: "user", userId: "board" }, + ); + await fs.writeFile(path.join(skillDir, "SKILL.md"), "# Runtime Coach\n\nVersion two.\n", "utf8"); + await db + .update(companySkills) + .set({ markdown: "# Runtime Coach\n\nVersion two.\n", updatedAt: new Date() }) + .where(eq(companySkills.id, skillId)); + const versionTwo = await skills.createVersion( + companyId, + skillId, + { label: "v2" }, + { type: "user", userId: "board" }, + ); + + await db.insert(agents).values([ + { + id: firstAgentId, + companyId, + name: "Pinned V1", + role: "engineer", + status: "idle", + adapterType: TEST_ADAPTER_TYPE, + adapterConfig: { + paperclipSkillSync: { + desiredSkills: [{ key: skillKey, versionId: versionOne.id }], + }, + }, + runtimeConfig: {}, + permissions: {}, + }, + { + id: secondAgentId, + companyId, + name: "Pinned V2", + role: "engineer", + status: "idle", + adapterType: TEST_ADAPTER_TYPE, + adapterConfig: { + paperclipSkillSync: { + desiredSkills: [{ key: skillKey, versionId: versionTwo.id }], + }, + }, + runtimeConfig: {}, + permissions: {}, + }, + ]); + + const heartbeat = heartbeatService(db); + const firstRun = await heartbeat.invoke(firstAgentId, "on_demand", {}, "manual"); + expect(firstRun).not.toBeNull(); + expect((await waitForRunToFinish(heartbeat, firstRun!.id))?.status).toBe("succeeded"); + + const secondRun = await heartbeat.invoke(secondAgentId, "on_demand", {}, "manual"); + expect(secondRun).not.toBeNull(); + expect((await waitForRunToFinish(heartbeat, secondRun!.id))?.status).toBe("succeeded"); + + const firstSkill = capturedRuns.find((run) => run.agentId === firstAgentId)?.skills + .find((entry) => entry.key === skillKey); + const secondSkill = capturedRuns.find((run) => run.agentId === secondAgentId)?.skills + .find((entry) => entry.key === skillKey); + + expect(firstSkill).toMatchObject({ + key: skillKey, + versionId: versionOne.id, + currentVersionId: versionTwo.id, + sourceStatus: "available", + }); + expect(secondSkill).toMatchObject({ + key: skillKey, + versionId: versionTwo.id, + currentVersionId: versionTwo.id, + sourceStatus: "available", + }); + await expect(fs.readFile(path.join(firstSkill!.source, "SKILL.md"), "utf8")) + .resolves.toContain("Version one."); + await expect(fs.readFile(path.join(secondSkill!.source, "SKILL.md"), "utf8")) + .resolves.toContain("Version two."); + }); +}); diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index 47e2ed3b7fe..3df874749df 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -1711,9 +1711,15 @@ async function resolveRequestedSkillEntriesOrThrow( const match = resolveSkillReference(skills, selection.key); if (match.skill) { - await assertVersionMatchesSkill(db, companyId, match.skill.id, selection.versionId); - if (!resolved.has(match.skill.key)) { - resolved.set(match.skill.key, { key: match.skill.key, versionId: selection.versionId }); + const skill = skills.find((candidate) => candidate.id === match.skill?.id); + if (!skill) { + missing.add(selection.key); + continue; + } + const selectedVersionId = selection.versionId ?? skill.currentVersionId ?? null; + await assertVersionMatchesSkill(db, companyId, skill.id, selectedVersionId); + if (!resolved.has(skill.key)) { + resolved.set(skill.key, { key: skill.key, versionId: selectedVersionId }); } continue; } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index deec2ab691a..a248441a22f 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -555,11 +555,15 @@ export function applyRunScopedMentionedSkillKeys( const existingPreference = readPaperclipSkillSyncPreference(config); return writePaperclipSkillSyncPreference(config, [ - ...existingPreference.desiredSkills, + ...existingPreference.desiredSkillEntries, ...normalizedSkillKeys, ]); } +function skillVersionSelectionMap(entries: Array<{ key: string; versionId: string | null }>) { + return new Map(entries.map((entry) => [entry.key, entry.versionId] as const)); +} + export function computeBoundedTransientHeartbeatRetrySchedule( attempt: number, now = new Date(), @@ -8264,7 +8268,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) resolvedConfig, runScopedMentionedSkillKeys, ); - const runtimeSkillEntries = await companySkills.listRuntimeSkillEntries(agent.companyId); + const runtimeSkillPreference = readPaperclipSkillSyncPreference(effectiveResolvedConfig); + const runtimeSkillEntries = await companySkills.listRuntimeSkillEntries(agent.companyId, { + versionSelections: skillVersionSelectionMap(runtimeSkillPreference.desiredSkillEntries), + }); let runtimeConfig = { ...effectiveResolvedConfig, paperclipRuntimeSkills: runtimeSkillEntries,