diff --git a/server/src/__tests__/ephemeral-migration.test.ts b/server/src/__tests__/ephemeral-migration.test.ts index 69733b6eb8d..a3ef74bd1d2 100644 --- a/server/src/__tests__/ephemeral-migration.test.ts +++ b/server/src/__tests__/ephemeral-migration.test.ts @@ -10,8 +10,11 @@ import { randomBytes, randomUUID } from "node:crypto"; import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; +import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "../services/remote-workspace-snapshot-storage.js"; +import { restoreRemoteWorkspaceArchive } from "../services/remote-workspace-snapshot-archive.js"; import { migrateEphemeralServerData } from "../maintenance/ephemeral-migration-data.js"; import { agentFileStore } from "../services/agent-file-store.js"; import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js"; @@ -35,6 +38,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll home = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-migration-")); await s3.send(new CreateBucketCommand({ Bucket: bucket })); vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", process.env.PAPERCLIP_TEST_S3_ENDPOINT!); @@ -43,6 +47,10 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll }); afterEach(async () => { await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(workspaceOperations).where(eq(workspaceOperations.companyId, companyId)); + await db.delete(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.companyId, companyId)); + await db.delete(issues).where(eq(issues.companyId, companyId)); + await db.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId)); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); await db.delete(agents).where(eq(agents.companyId, companyId)); await db.delete(companies).where(eq(companies.id, companyId)); @@ -90,6 +98,33 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll await s3.send(new PutObjectCommand({ Bucket: bucket, Key: file.objectKey, Body: "corrupt" })); await expect(verifyAgentFileObjects(db, getStorageService(), AbortSignal.timeout(30_000))).rejects.toThrow(/agent_file_object_(size|digest)_mismatch/); }, 120_000); + it("migrates overlapping task folders through maintenance and restores both from S3 after volume loss", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32)); + const parent = path.join(home, "project"), child = path.join(parent, ".paperclip/worktrees/task"); + await fs.mkdir(child, { recursive: true }); + const fixtures = [{ cwd: parent, file: "project.txt", bytes: "project work" }, { cwd: child, file: "task.txt", bytes: "nested task work" }]; + const ids: string[] = []; + for (const fixture of fixtures) { + await fs.writeFile(path.join(fixture.cwd, fixture.file), fixture.bytes); + const [issue] = await db.insert(issues).values({ companyId, title: fixture.file }).returning(); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issue!.id, + name: fixture.file, mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: fixture.cwd }).returning(); + ids.push(workspace!.id); + await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issue!.id)); + } + expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 2 } }); + await fs.rm(home, { recursive: true }); home = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-successor-")); + vi.stubEnv("PAPERCLIP_HOME", home); + for (const [index, workspaceId] of ids.entries()) { + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); + const authority = readRemoteWorkspaceAuthority(workspace!.metadata)!; + const reference = await getWorkspaceSnapshot(db, { companyId, workspaceId, snapshotId: authority.snapshotId! }); + const archivePath = path.join(home, `${index}.pcws`), targetDir = path.join(home, `restored-${index}`); + await loadWorkspaceSnapshotArchive({ storage: getStorageService(), reference: reference!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, fixtures[index]!.file), "utf8")).toBe(fixtures[index]!.bytes); + } + }, 120_000); const entry = fileURLToPath(new URL("../../dist-runtime/ephemeral-migration.mjs", import.meta.url)); it.skipIf(!existsSync(entry))("runs the actual bundled maintenance entry with plain Node and exits cleanly", async () => { await agent();