diff --git a/doc/skill-package-storage.md b/doc/skill-package-storage.md index 68dbe427614..958c82d8ef2 100644 --- a/doc/skill-package-storage.md +++ b/doc/skill-package-storage.md @@ -189,8 +189,15 @@ package connection to avoid a one-connection pool deadlock. No upload happens on unchanged warm preparation path. Uploads use unique attempt keys. An incomplete attempt has a durable claim with a -ten-minute expiry. Retries after expiry use fresh keys; an ambiguous database result -never triggers deletion of a possibly successful upload. A database trigger records +ten-minute crash expiry. When an attempt fails or is cancelled, it expires only its +own unpublished claim so a retry can use fresh keys immediately. This cleanup has +a one-second client deadline and a transaction-local statement timeout. Cleanup +queued behind a busy pool skips its write after that deadline. If the database is +unavailable, the original crash expiry remains. Concurrent preparation on another +connection or replica waits for the receipt or an expired claim within its existing +three-minute request deadline. These reads stop when the request settles or is cancelled; +they add no idle background polling. An ambiguous database result never triggers +deletion of a possibly successful upload. A database trigger records retired keys in `skill_package_deletions`, including company cascades. Promotion of an upload to a published receipt does not enqueue its objects for deletion. diff --git a/server/src/__tests__/skill-package-store.test.ts b/server/src/__tests__/skill-package-store.test.ts index 672ece8d9db..cd1bb86f700 100644 --- a/server/src/__tests__/skill-package-store.test.ts +++ b/server/src/__tests__/skill-package-store.test.ts @@ -892,7 +892,7 @@ describe.skipIf(!support.supported)("external skill packages", () => { } finally { clearTimeout(timeout); release(); await Promise.allSettled(pending); } }); - it("retries expired upload claims with fresh keys and only deletes the abandoned attempt", async () => { + it("retries failed upload claims immediately with fresh keys and only deletes the abandoned attempt", async () => { const companyId = await company(); const store = skillPackageStore(db, storage); const put = vi.spyOn(storage, "putFile").mockRejectedValueOnce(new Error("connection lost")); @@ -900,8 +900,7 @@ describe.skipIf(!support.supported)("external skill packages", () => { put.mockRestore(); const [pending] = await db.select().from(snapshots).where(eq(snapshots.companyId, companyId)); expect(pending!.storage).toBeNull(); - await expect(store.prepare(companyId, files)).rejects.toThrow("being saved"); - await db.update(snapshots).set({ upload: { ...pending!.upload!, expiresAt: new Date(0).toISOString() } }).where(eq(snapshots.id, pending!.id)); + expect(new Date(pending!.upload!.expiresAt).getTime()).toBeLessThanOrEqual(Date.now()); const id = await store.prepare(companyId, files); expect(id).toBe(pending!.id); const receipt = (await store.descriptor(companyId, id)).storage!; diff --git a/server/src/__tests__/skill-package-upload.test.ts b/server/src/__tests__/skill-package-upload.test.ts new file mode 100644 index 00000000000..f141515ffb1 --- /dev/null +++ b/server/src/__tests__/skill-package-upload.test.ts @@ -0,0 +1,271 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { companies, companySkillRepositorySnapshots as snapshots, createDb, skillPackageDeletions } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import * as storageModule from "../storage/index.js"; +import { createStorageService } from "../storage/service.js"; +import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; +import { companySkillService } from "../services/company-skills.js"; +import { skillPackageStore } from "../services/skill-package-store.js"; +import { skillPackageManifest } from "../services/skill-package-files.js"; + +const support = await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("skill package upload recovery", () => { + let database: Awaited>; + let db: ReturnType; + let otherDb: ReturnType; + let home: string; + let storage: storageModule.StorageService; + const previousHome = process.env.PAPERCLIP_HOME; + const files = [{ path: "SKILL.md", kind: "skill" as const, content: "---\nname: recovery\ndescription: Test recovery\n---\nTest instructions\n" }]; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("skill-package-upload-"); + db = createDb(database.connectionString, { maxConnections: 1 }); + otherDb = createDb(database.connectionString, { maxConnections: 1 }); + home = await fs.mkdtemp(path.join(os.tmpdir(), "skill-package-upload-")); + process.env.PAPERCLIP_HOME = home; + storage = createStorageService(createLocalDiskStorageProvider(path.join(home, "objects"))); + }); + afterEach(() => vi.restoreAllMocks()); + afterAll(async () => { + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = previousHome; + await Promise.all([db?.$client.end({ timeout: 1 }), otherDb?.$client.end({ timeout: 1 })]); + await database?.cleanup(); + if (home) await fs.rm(home, { recursive: true, force: true }); + }); + async function company() { + const id = randomUUID(); + await db.insert(companies).values({ id, name: "Upload recovery", issuePrefix: `P${id.slice(0, 6)}` }); + return id; + } + async function pendingUpload(companyId: string, expiresAt: Date) { + const token = randomUUID(); + const [row] = await db.insert(snapshots).values({ companyId, contentHash: skillPackageManifest(files).contentHash, + upload: { token, legacy: false, provider: storage.provider, archiveKey: `${companyId}/${token}/files.tar.gz`, + manifestKey: `${companyId}/${token}/manifest.json`, expiresAt: expiresAt.toISOString() } }).returning(); + return row!; + } + + it.each([false, true])("joins an upload on another database client and recovers a failed owner: %s", async failed => { + const companyId = await company(); + let release!: () => void; + let entered!: () => void; + let contended!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const uploading = new Promise(resolve => { entered = resolve; }); + const contention = new Promise(resolve => { contended = resolve; }); + const put = storage.putFile.bind(storage); + const puts = vi.spyOn(storage, "putFile").mockImplementationOnce(async input => { + entered(); + await blocked; + if (failed) throw new Error("upload disconnected"); + return put(input); + }); + const debug = otherDb.$client.options.debug; + otherDb.$client.options.debug = (_connection, query) => { + if (query.includes('update "company_skill_repository_snapshots" set "upload"')) contended(); + }; + const owner = skillPackageStore(db, storage).prepare(companyId, files).catch(error => error as Error); + let joined: Promise | undefined; + const cancellation = new AbortController(); + const timeout = setTimeout(() => cancellation.abort(new Error("Upload join did not settle")), 5_000); + try { + await uploading; + joined = skillPackageStore(otherDb, storage).prepare(companyId, files, false, cancellation.signal).catch(error => error as Error); + await contention; + release(); + const [first, second] = await Promise.all([owner, joined]); + expect(second).toEqual(expect.any(String)); + if (failed) expect(first).toEqual(new Error("upload disconnected")); + else expect(second).toBe(first); + expect(puts).toHaveBeenCalledTimes(failed ? 3 : 2); + const rows = await db.select().from(snapshots).where(eq(snapshots.companyId, companyId)); + expect(rows).toHaveLength(1); + expect(rows[0]!.id).toBe(second); + expect(rows[0]!.upload).toBeNull(); + expect(await skillPackageStore(db, storage).verifyDurable(companyId, rows[0]!.id)).toBe(true); + const retired = await db.select().from(skillPackageDeletions).where(eq(skillPackageDeletions.companyId, companyId)); + expect(retired).toHaveLength(failed ? 1 : 0); + if (failed) expect(retired[0]!.archiveKey).not.toBe(rows[0]!.storage!.archiveKey); + } finally { + clearTimeout(timeout); + cancellation.abort(); + release(); + await Promise.allSettled([owner, joined]); + otherDb.$client.options.debug = debug; + } + }); + + it("retries the first agent's bundled skill initialization immediately after a storage failure", async () => { + const companyId = await company(); + vi.spyOn(storageModule, "getStorageService").mockReturnValue(storage); + vi.spyOn(storage, "putFile").mockRejectedValueOnce(new Error("upload disconnected")); + const skills = companySkillService(db); + const requested = ["paperclipai/paperclip/paperclip-create-agent", "paperclipai/paperclip/first-task"]; + await expect(skills.resolveRequestedSkillEntries(companyId, requested)).rejects.toThrow("upload disconnected"); + const result = await skills.resolveRequestedSkillEntries(companyId, requested); + expect(result.unresolved).toEqual([]); + expect(result.resolved.map(entry => entry.key)).toEqual(requested); + }); + + it("reclaims an expired crash lease with fresh keys", async () => { + const companyId = await company(); + const pending = await pendingUpload(companyId, new Date(0)); + const id = await skillPackageStore(db, storage).prepare(companyId, files); + expect(id).toBe(pending.id); + const [saved] = await db.select().from(snapshots).where(eq(snapshots.id, id)); + expect(saved!.upload).toBeNull(); + expect(saved!.storage!.archiveKey).not.toBe(pending.upload!.archiveKey); + const [retired] = await db.select().from(skillPackageDeletions).where(eq(skillPackageDeletions.companyId, companyId)); + expect(retired!.archiveKey).toBe(pending.upload!.archiveKey); + }); + + it("cancels a waiting caller without expiring another owner's active lease", async () => { + const companyId = await company(); + const pending = await pendingUpload(companyId, new Date("2099-01-01")); + const puts = vi.spyOn(storage, "putFile"); + const cancellation = new AbortController(); + let contended!: () => void; + const contention = new Promise(resolve => { contended = resolve; }); + const debug = db.$client.options.debug; + db.$client.options.debug = (_connection, query) => { + if (query.includes('update "company_skill_repository_snapshots" set "upload"')) contended(); + }; + const preparing = skillPackageStore(db, storage).prepare(companyId, files, false, cancellation.signal).catch(error => error); + try { + await contention; + cancellation.abort(new Error("Caller left onboarding")); + expect(await preparing).toEqual(new Error("Caller left onboarding")); + const [saved] = await db.select().from(snapshots).where(eq(snapshots.id, pending.id)); + expect(saved!.upload).toEqual(pending.upload); + expect(puts).not.toHaveBeenCalled(); + } finally { cancellation.abort(); await preparing; db.$client.options.debug = debug; } + }); + + it.each([false, true])("preserves a possibly published receipt after a lost commit reply: committed=%s", async committed => { + const companyId = await company(); + let transactionCount = 0; + const ambiguousDb = new Proxy(db, { + get(target, property, receiver) { + if (property === "transaction") return async (...args: Parameters) => { + const publishing = ++transactionCount === 2; + const result = await target.transaction(async tx => { + const value = await args[0](tx); + if (publishing && !committed) throw new Error("Lost commit reply"); + return value; + }, args[1]); + if (publishing) throw new Error("Lost commit reply"); + return result; + }; + return Reflect.get(target, property, receiver); + }, + }); + const puts = vi.spyOn(storage, "putFile"); + await expect(skillPackageStore(ambiguousDb, storage).prepare(companyId, files)).rejects.toThrow("Lost commit reply"); + const [beforeRetry] = await db.select().from(snapshots).where(eq(snapshots.companyId, companyId)); + expect(Boolean(beforeRetry!.storage)).toBe(committed); + const store = skillPackageStore(db, storage); + expect(await store.prepare(companyId, files)).toBe(beforeRetry!.id); + expect(puts).toHaveBeenCalledTimes(committed ? 2 : 4); + expect(await store.verifyDurable(companyId, beforeRetry!.id)).toBe(true); + const retired = await db.select().from(skillPackageDeletions).where(eq(skillPackageDeletions.companyId, companyId)); + expect(retired).toHaveLength(committed ? 0 : 1); + }); + + it("does not expire a newer owner's claim after the old upload fails", async () => { + const companyId = await company(); + const replacement = { token: randomUUID(), legacy: false, provider: storage.provider, archiveKey: `${companyId}/replacement/files.tar.gz`, + manifestKey: `${companyId}/replacement/manifest.json`, expiresAt: new Date("2099-01-01").toISOString() }; + vi.spyOn(storage, "putFile").mockImplementationOnce(async () => { + await db.update(snapshots).set({ upload: replacement }).where(eq(snapshots.companyId, companyId)); + throw new Error("Old owner disconnected"); + }); + await expect(skillPackageStore(db, storage).prepare(companyId, files)).rejects.toThrow("Old owner disconnected"); + const [saved] = await db.select().from(snapshots).where(eq(snapshots.companyId, companyId)); + expect(saved!.upload).toEqual(replacement); + }); + + it.each([false, true])("bounds failed-upload cleanup behind a busy pool and rejects late writes: cancellation=%s", async cancelled => { + const companyId = await company(); + const failure = new Error(cancelled ? "Upload cancelled" : "Upload failed"); + const cancellation = new AbortController(); + let release!: () => void; + let entered!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const occupied = new Promise(resolve => { entered = resolve; }); + let holding: Promise | undefined; + vi.spyOn(storage, "putFile").mockImplementationOnce(async () => { + holding = db.transaction(async tx => { + await tx.execute(sql`select 1`); + entered(); + await blocked; + }); + await occupied; + if (cancelled) cancellation.abort(failure); + throw failure; + }); + const preparing = skillPackageStore(db, storage).prepare(companyId, files, false, cancellation.signal).catch(error => error); + let timer: ReturnType | undefined; + let pending: typeof snapshots.$inferSelect | undefined; + try { + await occupied; + [pending] = await otherDb.select().from(snapshots).where(eq(snapshots.companyId, companyId)); + const result = await Promise.race([preparing, new Promise(resolve => { + timer = setTimeout(() => resolve(new Error("Cleanup held the caller behind the pool")), 3_000); + })]); + expect(result).toBe(failure); + } finally { + clearTimeout(timer); + release(); + await holding; + await preparing; + } + await db.execute(sql`select 1`); + const [saved] = await db.select().from(snapshots).where(eq(snapshots.companyId, companyId)); + expect(saved!.upload).toEqual(pending!.upload); + }); + + it("cancels blocked cleanup SQL so the pool remains usable while the row is locked", async () => { + const companyId = await company(); + const failure = new Error("Upload failed before locked cleanup"); + let release!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + let holding: Promise | undefined; + let pending: typeof snapshots.$inferSelect | undefined; + vi.spyOn(storage, "putFile").mockImplementationOnce(async () => { + let entered!: () => void; + const locked = new Promise(resolve => { entered = resolve; }); + holding = otherDb.transaction(async tx => { + [pending] = await tx.select().from(snapshots).where(eq(snapshots.companyId, companyId)).for("update"); + entered(); + await blocked; + }); + await locked; + throw failure; + }); + let timer: ReturnType | undefined; + const preparing = skillPackageStore(db, storage).prepare(companyId, files).catch(error => error); + try { + const checked = (async () => { + expect(await preparing).toBe(failure); + await db.execute(sql`select 1`); + const [saved] = await db.select().from(snapshots).where(eq(snapshots.companyId, companyId)); + expect(saved!.upload).toEqual(pending!.upload); + return true; + })(); + expect(await Promise.race([checked, new Promise(resolve => { + timer = setTimeout(() => resolve(false), 3_000); + })])).toBe(true); + } finally { + clearTimeout(timer); + release(); + await holding; + await preparing; + } + }); +}); diff --git a/server/src/services/skill-package-store.ts b/server/src/services/skill-package-store.ts index 627e3ebe2ba..11c14f3765b 100644 --- a/server/src/services/skill-package-store.ts +++ b/server/src/services/skill-package-store.ts @@ -6,6 +6,7 @@ import { randomUUID } from "node:crypto"; import { constants, createReadStream, createWriteStream, promises as fs } from "node:fs"; import path from "node:path"; import { pipeline } from "node:stream/promises"; +import { setTimeout as delay } from "node:timers/promises"; import { and, eq, getTableColumns, inArray, sql } from "drizzle-orm"; import { companySkillRepositorySnapshots as snapshots, companySkillVersions, companySkillSources, withDedicatedDbConnection, type Db, type SkillPackageFileMapping, type SkillPackageStorageReceipt, type SkillPackageRestoreRecipe } from "@paperclipai/db"; import type { CompanySkillVersionFileInventoryEntry } from "@paperclipai/shared"; @@ -446,6 +447,24 @@ export function skillPackageStore(db: Db, suppliedStorage?: StorageService, cach return result; }); } + async function expireFailedUpload(companyId: string, id: string, token: string) { + const deadline = performance.now() + 1_000; + let timer: ReturnType | undefined; + try { + await Promise.race([ + db.transaction(async tx => { + if (performance.now() >= deadline) return; + const remainingMs = Math.max(1, Math.floor(deadline - performance.now())); + await tx.execute(sql`select set_config('statement_timeout', ${`${remainingMs}ms`}, true)`); + if (performance.now() >= deadline) return; + await tx.update(snapshots).set({ upload: sql`jsonb_set(${snapshots.upload}, '{expiresAt}', to_jsonb(now()))` }) + .where(and(scope(companyId, id), sql`${snapshots.storage} is null`, sql`${snapshots.upload}->>'token' = ${token}`)); + }), + new Promise(resolve => { timer = setTimeout(resolve, 1_000); }), + ]); + } catch { + } finally { clearTimeout(timer); } + } async function prepare(companyId: string, files: CompanySkillVersionFileInventoryEntry[], migrateLegacy = false, cancellation?: AbortSignal, restore?: SkillPackageRestoreRecipe): Promise { cancellation?.throwIfAborted(); const manifest = skillPackageManifest(files); @@ -461,7 +480,8 @@ export function skillPackageStore(db: Db, suppliedStorage?: StorageService, cach migrationAt: null, maintenanceAt: new Date(Date.now() + 24 * 60 * 60_000) }).onConflictDoNothing().returning(descriptorColumns); }); let existing = created ?? (await db.select(descriptorColumns).from(snapshots).where(and(eq(snapshots.companyId, companyId), eq(snapshots.contentHash, manifest.contentHash))))[0]!; - if (!created) { + while (!created) { + signal.throwIfAborted(); if (existing.storage || (!existing.upload && !migrateLegacy && !existing.restorable)) { // Content deduplication must not attach a recipe with different kind or // encoding metadata to an already published manifest. @@ -476,8 +496,12 @@ export function skillPackageStore(db: Db, suppliedStorage?: StorageService, cach const [claimed] = await db.update(snapshots).set({ upload }).where(and(scope(companyId, existing.id), sql`${snapshots.storage} is null`, sql`(${snapshots.upload} is null or (${snapshots.upload}->>'expiresAt')::timestamptz <= now())`)).returning(descriptorColumns); - if (!claimed) throw conflict("This skill package is being saved. Retry after its upload settles."); - existing = claimed; + if (claimed) { existing = claimed; break; } + try { await delay(250, undefined, { signal }); } + catch (error) { signal.throwIfAborted(); throw error; } + const [current] = await db.select(descriptorColumns).from(snapshots).where(scope(companyId, existing.id)); + if (!current) throw notFound("Skill package was removed during preparation"); + existing = current; } return withPackageIo(async () => { const root = skillPackagePaths(companyId, existing.id).parent; @@ -517,11 +541,12 @@ export function skillPackageStore(db: Db, suppliedStorage?: StorageService, cach if (!saved) throw conflict("Skill package upload ownership changed"); return saved.id; } finally { - // Failed/ambiguous uploads retain their durable claim. Maintenance reconciles - // exact owned keys; never delete a successful object on an uncertain SQL result. await fs.rm(staging, { recursive: true, force: true }); } - }, signal); + }, signal).catch(async error => { + await expireFailedUpload(companyId, existing.id, token); + throw error; + }); }); { const [retained] = await db.update(snapshots).set({ maintenanceAt: new Date(Date.now() + 24 * 60 * 60_000),