diff --git a/server/src/__tests__/ephemeral-server-startup-run-state.test.ts b/server/src/__tests__/ephemeral-server-startup-run-state.test.ts new file mode 100644 index 00000000000..e6d22a1d028 --- /dev/null +++ b/server/src/__tests__/ephemeral-server-startup-run-state.test.ts @@ -0,0 +1,62 @@ +import { afterAll, afterEach, beforeAll, expect, it, vi } from "vitest"; +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { agents, companies, createDb } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { assertEphemeralServerData } from "../services/ephemeral-server-contract.js"; +import { hasLocalNativeHarnessObligations } from "../services/native-runtime/native-harness-backup-migration.js"; +import { volumeFreeControllerStateOwned } from "../services/native-runtime/volume-free-controller-state.js"; + +vi.mock("../storage/index.js", () => ({ getStorageService: () => ({ provider: "s3" }) })); +vi.mock("../services/native-runtime/native-harness-backup-migration.js", () => ({ hasLocalNativeHarnessObligations: vi.fn() })); +vi.mock("../services/native-runtime/volume-free-controller-state.js", async importOriginal => ({ + ...await importOriginal(), + volumeFreeControllerStateOwned: vi.fn(), +})); + +let database: Awaited>; +let db: ReturnType; +beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("ephemeral-startup-run-state-"); db = createDb(database.connectionString); }, 90_000); +afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); +afterEach(() => { vi.unstubAllEnvs(); vi.mocked(hasLocalNativeHarnessObligations).mockReset(); vi.mocked(volumeFreeControllerStateOwned).mockReset(); }); + +// Staging, 2026-11-11: a volume-free server died mid-run, leaving leases whose +// harness backup never reached durable storage. Every later start refused with +// this blocker, so the workspace stayed down until someone repaired it by hand. +it("starts a volume-free server over harness backups a volume-free server left unsaved", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.mocked(hasLocalNativeHarnessObligations).mockResolvedValue(true); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(true); + await expect(assertEphemeralServerData(db)).resolves.toBeUndefined(); +}); + +it("still refuses configuration that would write new data to the local disk, with the owner marker present", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.mocked(hasLocalNativeHarnessObligations).mockResolvedValue(true); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(true); + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Startup gate", issuePrefix: "SGAT" }); + await db.insert(agents).values({ companyId, name: "Legacy files" }); + try { + const refusal = assertEphemeralServerData(db); + await expect(refusal).rejects.toThrow("agent_files_require_migration"); + await expect(refusal).rejects.not.toThrow("native_harness_backups_require_migration"); + } finally { + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + } +}); + +it("still refuses unsaved harness backups on a filesystem with no volume-free owner marker", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.mocked(hasLocalNativeHarnessObligations).mockResolvedValue(true); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(false); + await expect(assertEphemeralServerData(db)).rejects.toThrow("native_harness_backups_require_migration"); +}); + +it("does not consult the owner marker when harness backups are settled", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.mocked(hasLocalNativeHarnessObligations).mockResolvedValue(false); + await expect(assertEphemeralServerData(db)).resolves.toBeUndefined(); + expect(volumeFreeControllerStateOwned).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/ephemeral-server-contract.ts b/server/src/services/ephemeral-server-contract.ts index 0dfd5498192..6274346f788 100644 --- a/server/src/services/ephemeral-server-contract.ts +++ b/server/src/services/ephemeral-server-contract.ts @@ -11,6 +11,8 @@ import { hasLocalRuntimeSkillObligations } from "./native-runtime/runtime-skill- import { localWorkspaceDependencyWhere } from "./local-workspace-dependencies.js"; import { ephemeralServerEnabled } from "./ephemeral-server-mode.js"; +import { volumeFreeControllerStateOwned } from "./native-runtime/volume-free-controller-state.js"; +import { logger } from "../middleware/logger.js"; export { ephemeralServerEnabled }; @@ -40,11 +42,36 @@ export function assertEphemeralServerConfig(config: Pick = new Set([ + "native_harness_backups_require_migration", + "local_instruction_copies_require_settlement", +]); + /** A readiness gate, not a periodic scan or an implicit migration. Existing * volume data is adopted by the owning maintenance commands before removal. */ export async function assertEphemeralServerData(db: Db) { if (!ephemeralServerEnabled()) return; - const { blockers, controller } = await readEphemeralServerDataEligibility(db, { purpose: "startup" }); + const eligibility = await readEphemeralServerDataEligibility(db, { purpose: "startup" }); + const { controller } = eligibility; + let blockers = eligibility.blockers; + // Run state that never reached durable storage needs migration only while + // it can still be on a volume. Once this filesystem carries the volume-free + // owner marker there is no volume: the state was written by a volume-free + // server, this one or a predecessor that stopped mid-run, and refusing to + // start cannot bring it back. It would only keep every restart from + // starting again. Restart recovery owns those runs; start and say so. + // Configuration that would write new data to the local disk stays refused. + const interrupted = blockers.filter(blocker => interruptedRunBlockers.has(blocker)); + if (interrupted.length && await volumeFreeControllerStateOwned()) { + blockers = blockers.filter(blocker => !interruptedRunBlockers.has(blocker)); + logger.warn({ blockers: interrupted }, + "Volume-free server is starting over run state that never reached durable storage; restart recovery owns those runs"); + } if (!blockers.length) return; const detail = (blocker: string) => blocker === "native_controllers_require_migration" && controller ? `${blocker} (run ${controller.runId} on task ${controller.issueId}: controller state at ${controller.source} is not proven by the database, and this filesystem has no volume-free owner marker, so it may be the only copy of that run's progress; adopt it with the native controller migration before starting a volume-free server)`