PaperClipAI/server/src/__tests__/ephemeral-migration.test.ts

import fs from "node:fs/promises";
import { existsSync } from "node:fs";
import { spawn } from "node:child_process";
import { createServer } from "node:net";
import { fileURLToPath } from "node:url";
import { setTimeout as delay } from "node:timers/promises";
import os from "node:os";
import path from "node:path";
import { randomBytes, randomUUID } from "node:crypto";
import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { activityLog, agentWakeupRequests, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "../services/remote-workspace-snapshot-storage.js";
import { restoreRemoteWorkspaceArchive } from "../services/remote-workspace-snapshot-archive.js";
import { migrateEphemeralServerData } from "../maintenance/ephemeral-migration-data.js";
import { agentFileStore } from "../services/agent-file-store.js";
import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js";
import { materializeInstructionBytes } from "../services/agent-instruction-files.js";
import { verifyAgentFileObjects } from "../services/agent-file-maintenance.js";
import { getStorageService } from "../storage/index.js";
import { resolveDecisionSigningSecret } from "../services/decision-signing.js";
import { getRunLogStore } from "../services/run-log-store.js";
import { companySkillService } from "../services/company-skills.js";
import { skillPackageStore } from "../services/skill-package-store.js";
import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js";
import { migrateLocalSkillPackageObjects } from "../services/skill-package-storage-migration.js";
import { companySkills, companySkillRepositorySnapshots } from "@paperclipai/db";

// Explicit real-dependency qualification. Never contacts off-host storage in
// the default test path; use a disposable S3-compatible service and PostgreSQL.
describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enrollment with PostgreSQL and S3", () => {
  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
  let db: ReturnType<typeof createDb>, s3: S3Client;
  let home: string, companyId: string, bucket: string;
  beforeAll(async () => {
    database = await startEmbeddedPostgresTestDatabase("ephemeral-migration-"); db = createDb(database.connectionString);
    s3 = new S3Client({ endpoint: process.env.PAPERCLIP_TEST_S3_ENDPOINT, region: "us-east-1", forcePathStyle: true });
  }, 90_000);
  beforeEach(async () => {
    companyId = randomUUID(); bucket = `enrollment-${randomUUID()}`;
    home = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-migration-"));
    await s3.send(new CreateBucketCommand({ Bucket: bucket }));
    vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true");
    vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
    vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3");
    vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1");
    vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", process.env.PAPERCLIP_TEST_S3_ENDPOINT!);
    vi.stubEnv("PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "true");
    await db.insert(companies).values({ id: companyId, name: "Enrollment fixture", issuePrefix: "MIG" });
  });
  afterEach(async () => {
    await db.delete(activityLog).where(eq(activityLog.companyId, companyId));
    await db.delete(workspaceOperations).where(eq(workspaceOperations.companyId, companyId));
    await db.delete(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.companyId, companyId));
    await db.delete(issues).where(eq(issues.companyId, companyId));
    await db.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId));
    await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
    await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
    await db.delete(agents).where(eq(agents.companyId, companyId));
    await db.delete(companySkills).where(eq(companySkills.companyId, companyId));
    await db.delete(companySkillRepositorySnapshots).where(eq(companySkillRepositorySnapshots.companyId, companyId));
    await db.delete(companies).where(eq(companies.id, companyId));
    let token: string | undefined;
    do {
      const page = await s3.send(new ListObjectsV2Command({ Bucket: bucket, ContinuationToken: token }));
      if (page.Contents?.length) await s3.send(new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: page.Contents.map(row => ({ Key: row.Key! })) } }));
      token = page.NextContinuationToken;
    } while (token);
    await s3.send(new DeleteBucketCommand({ Bucket: bucket }));
    await fs.rm(home, { recursive: true, force: true }); vi.unstubAllEnvs();
  });
  afterAll(async () => {
    if (db) { await closeFileStorageDb(db, 1); await db.$client.end({ timeout: 1 }); }
    await database?.cleanup(); s3?.destroy();
  });
  const migrate = () => migrateEphemeralServerData(db, { signal: AbortSignal.timeout(120_000), phase: () => {} });
  async function agent(index = 0) {
    const row = { id: randomUUID(), companyId, name: `Migrated ${index}`, adapterConfig: {} };
    const root = resolveManagedInstructionsRoot(row);
    await materializeInstructionBytes(root, "AGENTS.md", Buffer.from(`Original ${index}`));
    await db.insert(agents).values({ ...row, adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root } });
    return { ...row, root };
  }
  it("migrates every page, verifies object bytes, and reads after deleting the entire predecessor filesystem", async () => {
    const rows = [];
    for (let index = 0; index < 26; index++) rows.push(await agent(index));
    const binary = Buffer.alloc(2 * 1024 * 1024, 0x81);
    await materializeInstructionBytes(rows[0]!.root, "nested/data.bin", binary);
    await materializeInstructionBytes(rows[0]!.root, "empty.txt", Buffer.alloc(0));
    expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { agentFiles: 26, verifiedAgentObjects: 28 } });
    await fs.rm(home, { recursive: true, force: true });
    home = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-successor-")); vi.stubEnv("PAPERCLIP_HOME", home);
    vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "false");
    // The service's normal board actor uses instance-operator authority.
    const actor = { type: "board" as const, source: "local_implicit" as const, userId: "local-board" };
    const store = agentFileStore(db);
    for (let index = 0; index < rows.length; index++) {
      expect(await store.read(companyId, rows[index]!.id, "AGENTS.md", actor)).toEqual(Buffer.from(`Original ${index}`));
    }
    expect(await store.read(companyId, rows[0]!.id, "nested/data.bin", actor)).toEqual(binary);
    const [head] = await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, rows[0]!.id));
    const file = head!.entries.find(entry => entry.kind === "file" && entry.objectKey !== null)!;
    if (file.kind !== "file" || !file.objectKey) throw new Error("Missing object");
    await s3.send(new PutObjectCommand({ Bucket: bucket, Key: file.objectKey, Body: "corrupt" }));
    await expect(verifyAgentFileObjects(db, getStorageService(), AbortSignal.timeout(30_000))).rejects.toThrow(/agent_file_object_(size|digest)_mismatch/);
  }, 120_000);
  it("preserves original skill objects and identity when destination readback fails", async () => {
    vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "local_disk");
    const skills = await companySkillService(db).list(companyId);
    const local = getStorageService(), store = skillPackageStore(db, local);
    const before = await Promise.all(skills.map(async skill => {
      const version = await store.version(companyId, skill.id, skill.currentVersionId!);
      return { skillId: skill.id, version: version!, files: await store.versionFiles(version!) };
    }));
    expect(before.length).toBeGreaterThan(0);
    vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3");
    const remote = getStorageService();
    await expect(migrateLocalSkillPackageObjects(db, { source: local, destination: { ...remote,
      getObject: async () => { throw new Error("Synthetic destination readback failed"); } }, writersStopped: true,
      signal: AbortSignal.timeout(30_000) })).rejects.toThrow("Synthetic destination readback failed");
    for (const point of before) {
      expect((await store.descriptor(companyId, point.version.packageSnapshotId!)).storage?.provider).toBe("local_disk");
      expect(await store.versionFiles(point.version)).toEqual(point.files);
    }
  }, 120_000);
  it("verifies bundled skill migration after removal of the old server home", async () => {
    vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "local_disk");
    const skills = await companySkillService(db).list(companyId), local = skillPackageStore(db);
    const before = await Promise.all(skills.map(async skill => {
      const version = await local.version(companyId, skill.id, skill.currentVersionId!);
      return { skillId: skill.id, version: version!, files: await local.versionFiles(version!) };
    }));
    vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3");
    const result = await migrate();
    expect(result.eligibility.eligible).toBe(true);
    expect(result.counts.skillObjects).toBe(before.length);
    await fs.rm(home, { recursive: true, force: true });
    const remote = skillPackageStore(db);
    for (const point of before) {
      const version = await remote.version(companyId, point.skillId, point.version.id);
      expect(version?.packageSnapshotId).toBe(point.version.packageSnapshotId);
      expect(await remote.versionFiles(version!)).toEqual(point.files);
    }
    expect((await skillInstallationMaintenance(db).verifyVersions({ companyId, limit: 50 })).results
      .every(row => ["remote_storage", "database_release_snapshot"].includes(row.status))).toBe(true);
  }, 120_000);
  it("migrates overlapping task folders through maintenance and restores both from S3 after volume loss", async () => {
    vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32));
    const parent = path.join(home, "project"), child = path.join(parent, ".paperclip/worktrees/task");
    await fs.mkdir(child, { recursive: true });
    const fixtures = [{ cwd: parent, file: "project.txt", bytes: "project work" }, { cwd: child, file: "task.txt", bytes: "nested task work" }];
    const ids: string[] = [], queued = [];
    const queuedAgent = await agent();
    for (const fixture of fixtures) {
      await fs.writeFile(path.join(fixture.cwd, fixture.file), fixture.bytes);
      const [issue] = await db.insert(issues).values({ companyId, title: fixture.file }).returning();
      const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issue!.id,
        name: fixture.file, mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: fixture.cwd }).returning();
      ids.push(workspace!.id);
      await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issue!.id));
      const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId: queuedAgent.id,
        source: "on_demand", payload: { issueId: issue!.id, text: "Continue my saved work" } }).returning();
      const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: queuedAgent.id,
        issueId: issue!.id, wakeupRequestId: wake!.id, status: "queued", contextSnapshot: { issueId: issue!.id,
          executionWorkspaceId: workspace!.id } }).returning();
      queued.push({ run: run!, wake: wake! });
    }
    expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 2 } });
    await fs.rm(home, { recursive: true }); home = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-successor-"));
    vi.stubEnv("PAPERCLIP_HOME", home);
    for (const { run, wake } of queued) {
      expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run);
      expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]).toEqual(wake);
    }
    for (const [index, workspaceId] of ids.entries()) {
      const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
      const authority = readRemoteWorkspaceAuthority(workspace!.metadata)!;
      const reference = await getWorkspaceSnapshot(db, { companyId, workspaceId, snapshotId: authority.snapshotId! });
      const archivePath = path.join(home, `${index}.pcws`), targetDir = path.join(home, `restored-${index}`);
      await loadWorkspaceSnapshotArchive({ storage: getStorageService(), reference: reference!, targetPath: archivePath });
      await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest });
      expect(await fs.readFile(path.join(targetDir, fixtures[index]!.file), "utf8")).toBe(fixtures[index]!.bytes);
    }
  }, 120_000);
  const entry = fileURLToPath(new URL("../../dist-runtime/ephemeral-migration.mjs", import.meta.url));
  it.skipIf(!existsSync(entry))("runs the actual bundled maintenance entry with plain Node and exits cleanly", async () => {
    const row = await agent(), runId = randomUUID();
    const logRef = `${companyId}/${row.id}/${runId}.ndjson`, logRoot = path.join(home, "old-run-logs");
    const history = '{"stream":"stdout","chunk":"unfinished historical output"}\n{"partial":';
    vi.stubEnv("RUN_LOG_BASE_PATH", logRoot);
    await fs.mkdir(path.dirname(path.join(logRoot, logRef)), { recursive: true });
    await fs.writeFile(path.join(logRoot, logRef), history);
    await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId: row.id, status: "interrupted",
      runtimeMode: "native", logStore: "local_file", logRef });
    vi.stubEnv("PAPERCLIP_DECISION_SIGNING_SECRET", "");
    const [queued] = await db.insert(heartbeatRuns).values({ companyId, agentId: row.id, status: "queued",
      contextSnapshot: { input: "Resume this turn after maintenance" } }).returning();
    const original = resolveDecisionSigningSecret();
    const listener = createServer();
    await new Promise<void>(resolve => listener.listen(0, "127.0.0.1", resolve));
    const address = listener.address();
    if (!address || typeof address === "string") throw new Error("Missing listener");
    await new Promise<void>(resolve => listener.close(() => resolve()));
    const id = randomUUID(), token = randomBytes(32).toString("base64url");
    const stamp = JSON.parse(await fs.readFile(new URL("../../dist/build-info.json", import.meta.url), "utf8"));
    const child = spawn(process.execPath, [entry], { env: { ...process.env, DATABASE_URL: database.connectionString,
      PAPERCLIP_DATABASE_LISTEN_URL: database.connectionString, PAPERCLIP_SECRETS_MASTER_KEY: randomBytes(32).toString("hex"),
      PAPERCLIP_SHARED_UI_ASSETS: "true", PAPERCLIP_EPHEMERAL_MIGRATION: "true", PAPERCLIP_EPHEMERAL_MIGRATION_ID: id,
      PAPERCLIP_EPHEMERAL_MIGRATION_TOKEN: token, PORT: String(address.port) }, stdio: ["ignore", "pipe", "pipe"] });
    let diagnostic = "";
    child.stderr.on("data", chunk => { diagnostic = (diagnostic + String(chunk)).slice(-8_192); });
    child.stdout.resume();
    const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => {
      child.once("error", reject); child.once("exit", (code, signal) => resolve({ code, signal }));
    });
    const origin = `http://127.0.0.1:${address.port}`;
    const headers = { authorization: `Bearer ${token}` };
    try {
      let status: Record<string, unknown> | undefined;
      for (const until = Date.now() + 45_000; Date.now() < until;) {
        if (child.exitCode !== null) throw new Error(`Maintenance exited: ${diagnostic}`);
        status = await fetch(`${origin}/status`, { headers, signal: AbortSignal.timeout(1_000) }).then(r => r.json() as Promise<Record<string, unknown>>).catch(() => undefined);
        if (status?.state === "ready" || status?.state === "blocked") break;
        await delay(100);
      }
      expect(status).toMatchObject({ id, commit: stamp.commit, state: "ready", eligibility: { eligible: true } });
      expect(JSON.stringify(status)).not.toContain(original);
      expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued!.id)))[0]).toEqual(queued);
      const exported = await fetch(`${origin}/export`, { method: "POST", headers }).then(r => r.json());
      expect(exported).toMatchObject({ id, commit: stamp.commit, decisionSigningSecret: original });
      const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
      expect(run).toMatchObject({ status: "interrupted", logStore: "s3_archive", logBytes: null, logSha256: null });
      await fs.rm(logRoot, { recursive: true });
      expect((await getRunLogStore().read({ store: "s3_archive", logRef: run!.logRef! })).content).toBe(history);
      expect((await fetch(`${origin}/api/companies`, { headers })).status).toBe(404);
      child.kill("SIGTERM");
      expect(await Promise.race([exited, delay(12_000).then(() => { throw new Error("Maintenance did not exit gracefully"); })])).toEqual({ code: 0, signal: null });
    } finally {
      if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
      await exited;
    }
  }, 60_000);

  it("refuses active executions and missing predecessor bytes without publishing an empty head", async () => {
    const row = await agent();
    const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: row.id, status: "running" }).returning();
    await expect(migrate()).rejects.toThrow("active_execution_requires_drain");
    await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, run!.id));
    await fs.rm(home, { recursive: true, force: true });
    await expect(migrate()).rejects.toThrow();
    expect(await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, row.id))).toEqual([]);
  });

  it("retains unsaved terminal-run working copies until their owner records settlement", async () => {
    const row = await agent();
    const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: row.id, status: "succeeded" }).returning();
    const privateRoot = path.join(home, "private-copy");
    await materializeInstructionBytes(privateRoot, "AGENTS.md", Buffer.from("Unsaved work"));
    await db.insert(agentInstructionWorkingCopies).values({ runId: run!.id, companyId, agentId: row.id,
      responsibleUserId: "fixture", entryFile: "AGENTS.md", baseHash: "fixture", localRoot: privateRoot,
      executionRoot: privateRoot, location: "local", state: "pending_collection" });
    await expect(migrate()).rejects.toThrow("local_instruction_copies_require_settlement");
    expect(await fs.readFile(path.join(privateRoot, "AGENTS.md"), "utf8")).toBe("Unsaved work");
    expect(await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, row.id))).toEqual([]);
    // Simulate only the owning service's completed collection receipt. The
    // enrollment coordinator must not fabricate that save or erase the copy.
    await materializeInstructionBytes(row.root, "AGENTS.md", Buffer.from("Unsaved work"));
    await db.update(agentInstructionWorkingCopies).set({ state: "saved", processStoppedAt: new Date() })
      .where(eq(agentInstructionWorkingCopies.runId, run!.id));
    expect((await migrate()).eligibility.eligible).toBe(true);
  });

  it("expires every page of local traces through their owner before final eligibility", async () => {
    const row = await agent();
    const runs = await db.insert(heartbeatRuns).values(Array.from({ length: 101 }, () => ({
      companyId, agentId: row.id, status: "succeeded",
    }))).returning();
    await db.insert(providerTraceRecords).values(runs.map(run => ({ companyId, runId: run.id, provider: "codex",
      traceRef: `${randomUUID()}.ndjson`, status: "complete", requestedBy: "fixture", expiresAt: new Date(0) })));
    expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { expiredTraces: 101 } });
    expect((await db.select().from(providerTraceRecords).where(eq(providerTraceRecords.companyId, companyId)))
      .every(trace => trace.status === "expired" && trace.deletedAt !== null)).toBe(true);
  }, 120_000);

  it("accepts an unavailable directory only after its owner confirms stop and no-save cleanup", async () => {
    const row = await agent();
    const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: row.id, status: "succeeded" }).returning();
    await db.insert(agentInstructionWorkingCopies).values({ runId: run!.id, companyId, agentId: row.id,
      responsibleUserId: "fixture", entryFile: "AGENTS.md", baseHash: "fixture", localRoot: path.join(home, "rejected-copy"),
      executionRoot: path.join(home, "rejected-copy"), location: "local", state: "unavailable",
      errorCode: "AGENT_FILES_LIMIT_EXCEEDED", receipt: { schema: "paperclip.agent-files.v1", cleanupPending: true } });
    await expect(migrate()).rejects.toThrow("local_instruction_copies_require_settlement");
    await db.update(agentInstructionWorkingCopies).set({ receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
      .where(eq(agentInstructionWorkingCopies.runId, run!.id));
    await expect(migrate()).rejects.toThrow("local_instruction_copies_require_settlement");
    await db.update(agentInstructionWorkingCopies).set({ processStoppedAt: new Date() })
      .where(eq(agentInstructionWorkingCopies.runId, run!.id));
    expect((await migrate()).eligibility.eligible).toBe(true);
  });
});