diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 59b19543503..afc36029247 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -5168,6 +5168,26 @@ count is kept on the run's context so a resume claim does not reset it, and a Stop, a lost lease or a moved task leave it to normal settlement. A busy lock met while collecting files after a turn is not deferred this way. +### Runner controller state on a restarted volume-free server + +A volume-free server keeps Runner controller files under +`runtime/paperclip-runner/durable-sessions/` while it runs. Its durable record is +the Postgres checkpoint, so those files are disposable. Early in startup, before +any controller can write, a volume-free server claims an empty (or absent) state +directory by writing `.volume-free-owner.json` there. A later start that finds +this marker treats every file in the directory as its own, and the data +eligibility gate does not count it as legacy volume data; normal restart +recovery reconciles it as it would after a crash of a volume-backed server. A +server started without `PAPERCLIP_EPHEMERAL_SERVER` removes the marker, because +from then on it may write files that exist only on that disk. + +A volume-free server that finds files it did not claim (a mounted legacy volume, +or a directory written before this marker existed) logs a warning and keeps the +existing protection: if the native controller check finds state the database does +not prove, startup refuses and the error names the run, the task and the file, +because that file may be the only copy of the run's progress. Adopt it with the +native controller migration first. + ## Cloud startup configuration Cloud stores nonsecret startup choices in a private PostgreSQL singleton. diff --git a/server/src/index.ts b/server/src/index.ts index 37a0c42d20e..a9974828abb 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -64,6 +64,7 @@ import { caseHandoffDeliveryService } from "./services/case-handoff-delivery.js" import { registerScheduledWork } from "./services/scheduled-work.js"; import type { createDeliveryWorkCoordinator } from "./services/delivery-work-coordinator.js"; import { assertEphemeralServerConfig, assertEphemeralServerData } from "./services/ephemeral-server-contract.js"; +import { claimVolumeFreeControllerState } from "./services/native-runtime/volume-free-controller-state.js"; import { deploymentNotifications } from "./services/deployment-notifications.js"; import { registerTerminalWorkspaceWork, TERMINAL_WORKSPACE_RETRY_MS } from "./services/terminal-workspace-work.js"; import { isIdleTaskDrainActive, readTaskDrain, beginIdleTrackedWork, trackIdleWork, registerTaskDrainExpiryWork } from "./services/task-admission.js"; @@ -754,6 +755,10 @@ async function startServerWithDatabaseTeardown( await initializeCloudRuntimeConfig(db as any); let config = "deploymentMode" in startupConfig ? startupConfig : loadConfig(); const deploymentListenUrl = assertEphemeralServerConfig(config); + const runnerStateClaim = await claimVolumeFreeControllerState(deploymentListenUrl !== null); + if (runnerStateClaim === "unclaimed") { + logger.warn("Volume-free server found runner controller state it did not write; the data eligibility gate decides whether it blocks startup"); + } // Cloud storage depends on the canonical DB snapshot. Validate it before // signing keys or other configuration consumers can mutate local state. if (isCloudManagedInstance()) ensureDecisionSigningSecret(); diff --git a/server/src/services/ephemeral-server-contract.ts b/server/src/services/ephemeral-server-contract.ts index f68ad9bc816..0dfd5498192 100644 --- a/server/src/services/ephemeral-server-contract.ts +++ b/server/src/services/ephemeral-server-contract.ts @@ -5,7 +5,7 @@ import { isValidLocalEncryptedMasterKey } from "../secrets/local-encrypted-provi import { agentInstructionsBundleMode } from "./agent-instructions.js"; import { skillInstallationMaintenance } from "./skill-installation-maintenance.js"; import { hasLocalNativeHarnessObligations } from "./native-runtime/native-harness-backup-migration.js"; -import { hasLocalNativeControllerObligations } from "./native-runtime/native-controller-migration.js"; +import { findLocalNativeControllerObligation } from "./native-runtime/native-controller-migration.js"; import { hasUnsettledLocalInstructionCopies } from "./agent-instruction-copy-preservation.js"; import { hasLocalRuntimeSkillObligations } from "./native-runtime/runtime-skill-preservation.js"; import { localWorkspaceDependencyWhere } from "./local-workspace-dependencies.js"; @@ -44,8 +44,12 @@ export function assertEphemeralServerConfig(config: Pick blocker === "native_controllers_require_migration" && controller + ? `${blocker} (run ${controller.runId} on task ${controller.issueId}: controller state at ${controller.source} is not proven by the database, and this filesystem has no volume-free owner marker, so it may be the only copy of that run's progress; adopt it with the native controller migration before starting a volume-free server)` + : blocker; + throw new Error(`Volume-free server data is not eligible: ${blockers.map(detail).join(", ")}`); } export async function readEphemeralServerDataEligibility(db: Db, options: { purpose?: "migration" | "startup" } = {}) { @@ -71,7 +75,8 @@ export async function readEphemeralServerDataEligibility(db: Db, options: { purp )).limit(1); if (localTrace) blockers.push("provider_traces_require_migration"); if (await hasLocalNativeHarnessObligations(db)) blockers.push("native_harness_backups_require_migration"); - if (await hasLocalNativeControllerObligations(db)) blockers.push("native_controllers_require_migration"); + const controller = await findLocalNativeControllerObligation(db); + if (controller) blockers.push("native_controllers_require_migration"); if (await hasLocalRuntimeSkillObligations(db)) blockers.push("runtime_skill_assets_require_migration"); const skills = skillInstallationMaintenance(db); let after: string | undefined; @@ -81,5 +86,5 @@ export async function readEphemeralServerDataEligibility(db: Db, options: { purp if (page.inspected < 50 || !page.after) break; after = page.after; } - return { eligible: blockers.length === 0, blockers }; + return { eligible: blockers.length === 0, blockers, controller }; } diff --git a/server/src/services/native-runtime/native-controller-migration.integration.test.ts b/server/src/services/native-runtime/native-controller-migration.integration.test.ts index 62d1ff52e78..bebd6db2bf9 100644 --- a/server/src/services/native-runtime/native-controller-migration.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-migration.integration.test.ts @@ -24,8 +24,10 @@ import { createLocalDiskStorageProvider } from "../../storage/local-disk-provide import { createStorageService } from "../../storage/service.js"; import { legacyNativeControllerArchive } from "./native-controller-archive.js"; import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; +import { claimVolumeFreeControllerState } from "./volume-free-controller-state.js"; +import { assertEphemeralServerData } from "../ephemeral-server-contract.js"; -describeEmbeddedPostgres.each(["archive_online", "archive_online_owned", "archive_online_changed", "archive_online_commit", "archive_online_active", "archive_online_foreign", +describeEmbeddedPostgres.each(["volume_free_owner", "archive_online", "archive_online_owned", "archive_online_changed", "archive_online_commit", "archive_online_active", "archive_online_foreign", "active_warm", "source_scan", "active_durable", "active_durable_cold", "online_success", "online_owned", "online_same_source", "online_busy", "online_changed", "online_evidence", "import", "archive", "success", "batch", "durable_failed", "durable_inline_failed", "stale_confirmed", "stale_foreign", "stale_changed", "stale_queued", "stale_overlap", "stale_success", "stale_busy", "stale_durable", "stale_durable_absent", "stale_durable_runner", "stale_durable_session", "stale_durable_snapshot", @@ -94,6 +96,21 @@ describeEmbeddedPostgres.each(["archive_online", "archive_online_owned", "archiv taskKey: f.issueId, lastRunId: f.runId }).returning(); return { ...f, companyId, agentId, paperclipHome, runtimeRoot, paths, file, source, profile, session, queuedId, identity, snapshot }; } + if (scenario === "volume_free_owner") it("starts a volume-free server over controller state it wrote itself and keeps unmarked state blocked", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + expect(await claimVolumeFreeControllerState(true)).toBe("claimed"); + const f = await fixture(0, true, true); + await f.db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, f.runId)); + expect(JSON.parse(await readFile(f.file, "utf8")).schema).toBe("paperclip.runner.durable.control-plane-state.v2"); + expect(await claimVolumeFreeControllerState(true)).toBe("owned"); + expect(await hasLocalNativeControllerObligations(f.db)).toBe(false); + expect(await claimVolumeFreeControllerState(false)).toBe("released"); + expect(await hasLocalNativeControllerObligations(f.db)).toBe(true); + expect(await claimVolumeFreeControllerState(true)).toBe("unclaimed"); + expect(await hasLocalNativeControllerObligations(f.db)).toBe(true); + await expect(assertEphemeralServerData(f.db)).rejects.toThrow(`native_controllers_require_migration (run ${f.runId}`); + expect(await readFile(f.file, "utf8")).toBe(f.source); + }); if (scenario === "active_warm") it("preserves an active warm rotation only with its exact older committed authority", async () => { const f = await fixture(0, true, true), controller = await currentNativeControllerIdentity(); const binding = { companyId: f.companyId, issueId: f.issueId, runId: f.runId }; diff --git a/server/src/services/native-runtime/native-controller-migration.ts b/server/src/services/native-runtime/native-controller-migration.ts index aeb22c5e517..14f0b3b1420 100644 --- a/server/src/services/native-runtime/native-controller-migration.ts +++ b/server/src/services/native-runtime/native-controller-migration.ts @@ -20,6 +20,7 @@ import { archiveFailedLegacyNativeController, hasArchivedLegacyNativeController import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js"; import { archiveSettledFailedNativeController } from "./native-controller-archive-source.js"; import { nativeControllerSourceDigest } from "./native-controller-source-digest.js"; +import { isVolumeFreeControllerStateOwnerEntry, volumeFreeControllerStateOwned } from "./volume-free-controller-state.js"; type Selected = { id: string; companyId: string; agentId: string; issueId: string; sessionId: string; createdAt: Date; lastRunId: string }; @@ -257,10 +258,15 @@ async function preservedActiveCheckpoint(db: Db, selected: Selected, } export async function hasLocalNativeControllerObligations(db: Db, storage?: StorageService | null): Promise { - const roots = await readdir(nativeHarnessStateBase()).catch((error: NodeJS.ErrnoException) => { + return await findLocalNativeControllerObligation(db, storage) !== null; +} + +export async function findLocalNativeControllerObligation(db: Db, storage?: StorageService | null) { + const base = nativeHarnessStateBase(); + const roots = await readdir(base).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return [] as string[]; throw error; }); - if (!roots.length) return false; + if (!roots.some(name => !isVolumeFreeControllerStateOwnerEntry(name)) || await volumeFreeControllerStateOwned(base)) return null; for await (const { selected, source, databaseRecoveryPreserved } of retainedSources(db, undefined, 8, true)) { const binding = { companyId: selected.companyId, issueId: selected.issueId, runId: source.id }; if (databaseRecoveryPreserved) continue; @@ -280,9 +286,10 @@ export async function hasLocalNativeControllerObligations(db: Db, storage?: Stor ? await readPrivateSource(paths.supervisorPath, 4 * 1024 * 1024) : undefined; if (controller !== undefined && archiveStorage && await hasArchivedLegacyNativeController(db, { ...binding, storage: archiveStorage, controller, supervisor })) continue; } - if (present || await hasNativeRunnerCheckpoint(db, binding)) return true; + if (present) return { runId: source.id, issueId: selected.issueId, source: paths.controlFile }; + if (await hasNativeRunnerCheckpoint(db, binding)) return { runId: source.id, issueId: selected.issueId, source: "database_checkpoint" }; } - return false; + return null; } function hasMatchingControllerRecovery(selected: Selected, diff --git a/server/src/services/native-runtime/volume-free-controller-state.test.ts b/server/src/services/native-runtime/volume-free-controller-state.test.ts new file mode 100644 index 00000000000..bb1077269aa --- /dev/null +++ b/server/src/services/native-runtime/volume-free-controller-state.test.ts @@ -0,0 +1,83 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { link, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { claimVolumeFreeControllerState, volumeFreeControllerStateOwned } from "./volume-free-controller-state.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function base() { + const root = await mkdtemp(join(tmpdir(), "volume-free-owner-")); + roots.push(root); + return join(root, "durable-sessions"); +} + +it("claims only an empty state directory and releases the claim outside volume-free mode", async () => { + const dir = await base(); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("claimed"); + expect((await stat(dir)).mode & 0o777).toBe(0o700); + expect(JSON.parse(await readFile(join(dir, ".volume-free-owner.json"), "utf8")).schema).toBe("paperclip.volume-free-controller-state.v1"); + await mkdir(join(dir, "a".repeat(64))); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("owned"); + expect(await claimVolumeFreeControllerState(false, dir)).toBe("released"); + expect(await claimVolumeFreeControllerState(false, dir)).toBe("absent"); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("unclaimed"); + expect(await volumeFreeControllerStateOwned(dir)).toBe(false); +}); + +it("lets only one of several concurrent starts claim the directory", async () => { + const dir = await base(); + const results = await Promise.all(Array.from({ length: 8 }, () => claimVolumeFreeControllerState(true, dir))); + expect(results.filter(result => result === "claimed")).toHaveLength(1); + expect(results.every(result => ["claimed", "owned", "unclaimed"].includes(result))).toBe(true); + expect(await volumeFreeControllerStateOwned(dir)).toBe(true); + expect((await readdir(dir)).filter(name => name !== ".volume-free-owner.json")).toEqual([]); +}); + +it("recovers a claim interrupted between publishing the marker and removing its temporary name", async () => { + const dir = await base(); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("claimed"); + await link(join(dir, ".volume-free-owner.json"), join(dir, ".volume-free-owner.json.interrupted")); + expect(await volumeFreeControllerStateOwned(dir)).toBe(false); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("owned"); + expect(await readdir(dir)).toEqual([".volume-free-owner.json"]); + await rm(join(dir, ".volume-free-owner.json")); + await writeFile(join(dir, ".volume-free-owner.json.stale"), "partial"); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("claimed"); + expect(await readdir(dir)).toEqual([".volume-free-owner.json"]); +}); + +it("leaves a live overlapping start's claim in place and removes one whose writer has exited", async () => { + const dir = await base(); + await mkdir(dir, { recursive: true }); + const exited = spawnSync(process.execPath, ["-e", "process.exit(0)"]).pid; + const live = join(dir, `.volume-free-owner.json.${process.pid}.overlapping-start`); + const abandoned = join(dir, `.volume-free-owner.json.${exited}.interrupted-start`); + await writeFile(live, "in progress"); + await writeFile(abandoned, "partial"); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("claimed"); + expect((await readdir(dir)).sort()).toEqual([".volume-free-owner.json", `.volume-free-owner.json.${process.pid}.overlapping-start`]); + expect(await readFile(live, "utf8")).toBe("in progress"); + await rm(live); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("owned"); +}); + +it("ignores a marker with the wrong schema", async () => { + const dir = await base(); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, ".volume-free-owner.json"), JSON.stringify({ schema: "other" })); + expect(await volumeFreeControllerStateOwned(dir)).toBe(false); +}); + +it.each(["directory", "fifo"] as const)("never blocks or throws on a %s at the marker path", async kind => { + const dir = await base(); + await mkdir(dir, { recursive: true }); + const marker = join(dir, ".volume-free-owner.json"); + if (kind === "directory") await mkdir(marker); + else execFileSync("mkfifo", [marker]); + expect(await volumeFreeControllerStateOwned(dir)).toBe(false); + expect(await claimVolumeFreeControllerState(true, dir)).toBe("unclaimed"); + expect(await claimVolumeFreeControllerState(false, dir)).toBe("absent"); + expect((await stat(marker)).isFile()).toBe(false); +}); diff --git a/server/src/services/native-runtime/volume-free-controller-state.ts b/server/src/services/native-runtime/volume-free-controller-state.ts new file mode 100644 index 00000000000..8781261b6fd --- /dev/null +++ b/server/src/services/native-runtime/volume-free-controller-state.ts @@ -0,0 +1,94 @@ +import { constants } from "node:fs"; +import { link, lstat, mkdir, open, readdir, rm } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js"; + +const ownerFile = ".volume-free-owner.json"; +const ownerSchema = "paperclip.volume-free-controller-state.v1"; + +export type VolumeFreeControllerStateClaim = "owned" | "claimed" | "unclaimed" | "released" | "absent"; + +export function isVolumeFreeControllerStateOwnerEntry(name: string) { + return name === ownerFile || name.startsWith(`${ownerFile}.`); +} + +async function regularOwnerFile(path: string) { + const entry = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (["ENOENT", "ENOTDIR"].includes(error.code ?? "")) return undefined; + throw error; + }); + return Boolean(entry?.isFile()); +} + +export async function volumeFreeControllerStateOwned(base = nativeHarnessStateBase()): Promise { + const path = join(base, ownerFile); + if (!await regularOwnerFile(path)) return false; + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK).catch((error: NodeJS.ErrnoException) => { + if (["ENOENT", "ENOTDIR", "ELOOP"].includes(error.code ?? "")) return undefined; + throw error; + }); + if (!handle) return false; + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.nlink !== 1 || stat.size > 4096) return false; + const marker = JSON.parse(await handle.readFile("utf8")) as unknown; + return Boolean(marker && typeof marker === "object" && (marker as Record).schema === ownerSchema); + } catch { return false; } + finally { await handle.close(); } +} + +function claimWriterAlive(name: string) { + const pid = Number(name.slice(ownerFile.length + 1).split(".")[0]); + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + try { process.kill(pid, 0); return true; } + catch (error) { return (error as NodeJS.ErrnoException).code === "EPERM"; } +} + +async function removeAbandonedClaims(base: string, listed: string[]) { + const marker = await lstat(join(base, ownerFile)).catch(() => undefined); + for (const name of listed.filter(name => name.startsWith(`${ownerFile}.`))) { + const entry = await lstat(join(base, name)).catch(() => undefined); + if (!entry?.isFile()) continue; + const published = marker?.isFile() && marker.ino === entry.ino && marker.dev === entry.dev; + if (published || !claimWriterAlive(name)) await rm(join(base, name), { force: true }); + } +} + +export async function claimVolumeFreeControllerState(volumeFree: boolean, base = nativeHarnessStateBase()): Promise { + if (!volumeFree) { + const owned = await volumeFreeControllerStateOwned(base); + if (await regularOwnerFile(join(base, ownerFile))) await rm(join(base, ownerFile), { force: true }); + return owned ? "released" : "absent"; + } + for (let attempt = 0; attempt < 3; attempt++) { + const listed = await readdir(base).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] as string[]; + throw error; + }); + await removeAbandonedClaims(base, listed); + if (await volumeFreeControllerStateOwned(base)) return "owned"; + const entries = listed.filter(name => !name.startsWith(`${ownerFile}.`)); + if (entries.some(name => !isVolumeFreeControllerStateOwnerEntry(name))) return "unclaimed"; + if (entries.includes(ownerFile) && !await regularOwnerFile(join(base, ownerFile))) return "unclaimed"; + await mkdir(base, { recursive: true, mode: 0o700 }); + const temporary = join(base, `${ownerFile}.${process.pid}.${randomUUID()}`); + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { + await handle.writeFile(JSON.stringify({ schema: ownerSchema, claimedAt: new Date().toISOString(), pid: process.pid })); + await handle.sync(); + } finally { await handle.close(); } + try { + await link(temporary, join(base, ownerFile)); + } catch (error) { + await rm(temporary, { force: true }); + if (!["EEXIST", "ENOENT"].includes((error as NodeJS.ErrnoException).code ?? "")) throw error; + continue; + } + await rm(temporary, { force: true }); + const directory = await open(base, constants.O_RDONLY); + try { await directory.sync(); } finally { await directory.close(); } + return "claimed"; + } + return await volumeFreeControllerStateOwned(base) ? "owned" : "unclaimed"; +}