test(workspaces): restore migrated trees from shared object storage

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/ephemeral-migration.test.ts

@@ -10,8 +10,11 @@ import { randomBytes, randomUUID } from "node:crypto";
 import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
 import { eq } from "drizzle-orm";
-import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db";
+import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db";
 import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
+import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "../services/remote-workspace-snapshot-storage.js";
+import { restoreRemoteWorkspaceArchive } from "../services/remote-workspace-snapshot-archive.js";
 import { migrateEphemeralServerData } from "../maintenance/ephemeral-migration-data.js";
 import { agentFileStore } from "../services/agent-file-store.js";
 import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js";
@@ -35,6 +38,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll
     home = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-migration-"));
     await s3.send(new CreateBucketCommand({ Bucket: bucket }));
     vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true");
+    vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
     vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3");
     vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1");
     vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", process.env.PAPERCLIP_TEST_S3_ENDPOINT!);
@@ -43,6 +47,10 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll
   });
   afterEach(async () => {
     await db.delete(activityLog).where(eq(activityLog.companyId, companyId));
+    await db.delete(workspaceOperations).where(eq(workspaceOperations.companyId, companyId));
+    await db.delete(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.companyId, companyId));
+    await db.delete(issues).where(eq(issues.companyId, companyId));
+    await db.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId));
     await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
     await db.delete(agents).where(eq(agents.companyId, companyId));
     await db.delete(companies).where(eq(companies.id, companyId));
@@ -90,6 +98,33 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll
     await s3.send(new PutObjectCommand({ Bucket: bucket, Key: file.objectKey, Body: "corrupt" }));
     await expect(verifyAgentFileObjects(db, getStorageService(), AbortSignal.timeout(30_000))).rejects.toThrow(/agent_file_object_(size|digest)_mismatch/);
   }, 120_000);
+  it("migrates overlapping task folders through maintenance and restores both from S3 after volume loss", async () => {
+    vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32));
+    const parent = path.join(home, "project"), child = path.join(parent, ".paperclip/worktrees/task");
+    await fs.mkdir(child, { recursive: true });
+    const fixtures = [{ cwd: parent, file: "project.txt", bytes: "project work" }, { cwd: child, file: "task.txt", bytes: "nested task work" }];
+    const ids: string[] = [];
+    for (const fixture of fixtures) {
+      await fs.writeFile(path.join(fixture.cwd, fixture.file), fixture.bytes);
+      const [issue] = await db.insert(issues).values({ companyId, title: fixture.file }).returning();
+      const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issue!.id,
+        name: fixture.file, mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: fixture.cwd }).returning();
+      ids.push(workspace!.id);
+      await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issue!.id));
+    }
+    expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 2 } });
+    await fs.rm(home, { recursive: true }); home = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-successor-"));
+    vi.stubEnv("PAPERCLIP_HOME", home);
+    for (const [index, workspaceId] of ids.entries()) {
+      const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
+      const authority = readRemoteWorkspaceAuthority(workspace!.metadata)!;
+      const reference = await getWorkspaceSnapshot(db, { companyId, workspaceId, snapshotId: authority.snapshotId! });
+      const archivePath = path.join(home, `${index}.pcws`), targetDir = path.join(home, `restored-${index}`);
+      await loadWorkspaceSnapshotArchive({ storage: getStorageService(), reference: reference!, targetPath: archivePath });
+      await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest });
+      expect(await fs.readFile(path.join(targetDir, fixtures[index]!.file), "utf8")).toBe(fixtures[index]!.bytes);
+    }
+  }, 120_000);
   const entry = fileURLToPath(new URL("../../dist-runtime/ephemeral-migration.mjs", import.meta.url));
   it.skipIf(!existsSync(entry))("runs the actual bundled maintenance entry with plain Node and exits cleanly", async () => {
     await agent();