mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-20 18:26:40 +01:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Hosted Paperclip servers can move from a server with a volume to a volume-free server, which allows deploys without downtime > - Before the move, Core checks that the volume-free configuration is complete > - That check requires `PAPERCLIP_DATABASE_LISTEN_URL` on the running volume server, but only the volume-free replacement uses it, and Cloud supplies it when it configures that replacement > - Many older servers never had the setting, so they cannot move online even when all their data is ready > - This pull request stops an unset listener from blocking, and keeps a listener that points at another database as a blocker > - The benefit is that older servers can move online ## Linked Issues or Issue Description **What existing behavior does this improve?** The online move refuses a volume server that has no `PAPERCLIP_DATABASE_LISTEN_URL`. **Subsystem affected** Server: ephemeral enrollment readiness (`server/src/services/ephemeral-enrollment.ts`). **Current behavior** The eligibility read and preparation report `volume_free_database_listener_required` for every server that has no listener setting. **Proposed behavior** An unset listener does not block. A listener that does not match the runtime database still blocks with the same name. **Reason and benefit** The replacement server receives the listener from Cloud. On staging, this check alone blocks 18 servers whose data is otherwise ready. **Breaking changes** None. The startup contract for a running volume-free server is unchanged. ## What Changed - `services/ephemeral-enrollment.ts`: when the only configuration failure is the listener, and the listener is unset, the readiness check passes. The listener check is the last check in `assertEphemeralServerConfig`, so all earlier settings were already valid. - `services/ephemeral-enrollment.test.ts`: tests for an unset listener (eligible, and preparation succeeds) and a mismatched listener (still blocked). ## Verification - `npx vitest run src/services/ephemeral-enrollment.test.ts src/routes/ephemeral-enrollment.test.ts`: 29/29 pass. ## Risks - Low. Cloud's online arming passes `PAPERCLIP_DATABASE_LISTEN_URL` with the replacement's variables. If the replacement did not get the setting, the existing startup contract would still refuse to boot it, and the move would not complete. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`) in Claude Code, with tool use (shell, file edits, browser). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
159 lines
10 KiB
TypeScript
159 lines
10 KiB
TypeScript
import { and, eq, isNotNull, isNull, sql } from "drizzle-orm";
|
|
import { agentFileRoots, agents, executionWorkspaces, heartbeatRuns, providerTraceRecords, type Db } from "@paperclipai/db";
|
|
import type { Config } from "../config.js";
|
|
import { isValidLocalEncryptedMasterKey } from "../secrets/local-encrypted-provider.js";
|
|
import { agentInstructionsBundleMode } from "./agent-instructions.js";
|
|
import { skillInstallationMaintenance } from "./skill-installation-maintenance.js";
|
|
import { hasLocalNativeHarnessObligations } from "./native-runtime/native-harness-backup-migration.js";
|
|
import { findLocalNativeControllerObligation } from "./native-runtime/native-controller-migration.js";
|
|
import { hasUnsettledLocalInstructionCopies } from "./agent-instruction-copy-preservation.js";
|
|
import { hasLocalRuntimeSkillObligations, preserveLocalRuntimeSkillObligations } from "./native-runtime/runtime-skill-preservation.js";
|
|
import { localWorkspaceDependencyWhere } from "./local-workspace-dependencies.js";
|
|
|
|
import { ephemeralServerEnabled } from "./ephemeral-server-mode.js";
|
|
import { volumeFreeControllerStateOwned } from "./native-runtime/volume-free-controller-state.js";
|
|
import { logger } from "../middleware/logger.js";
|
|
|
|
export { ephemeralServerEnabled };
|
|
|
|
/** Validate before consumers can generate a replacement local identity. Error
|
|
* messages name settings only, never credential values or connection URLs. */
|
|
export function assertEphemeralServerConfig(config: Pick<Config, "databaseUrl" | "storageProvider">, env = process.env,
|
|
options: { allowMissingListener?: boolean } = {}): string | null {
|
|
if (env.PAPERCLIP_EPHEMERAL_SERVER !== "true") return null;
|
|
const refuse = (reason: string): never => { throw new Error(`Volume-free server configuration: ${reason}`); };
|
|
if (config.storageProvider !== "s3" || env.PAPERCLIP_DURABLE_AGENT_FILES !== "true" || env.PAPERCLIP_SHARED_UI_ASSETS !== "true") {
|
|
refuse("shared S3 storage, durable agent files and shared UI assets are required");
|
|
}
|
|
if (!env.PAPERCLIP_SECRETS_MASTER_KEY || !isValidLocalEncryptedMasterKey(env.PAPERCLIP_SECRETS_MASTER_KEY)) {
|
|
refuse("a stable PAPERCLIP_SECRETS_MASTER_KEY is required");
|
|
}
|
|
if ((env.PAPERCLIP_DECISION_SIGNING_SECRET?.trim().length ?? 0) < 32) {
|
|
refuse("a stable PAPERCLIP_DECISION_SIGNING_SECRET of at least 32 characters is required; preserve the existing decision key during migration");
|
|
}
|
|
let runtime: URL, listener: URL;
|
|
const waiveListener = options.allowMissingListener === true && !env.PAPERCLIP_DATABASE_LISTEN_URL?.trim();
|
|
const runtimeRequired = waiveListener ? "an external postgres DATABASE_URL is required"
|
|
: "external DATABASE_URL and PAPERCLIP_DATABASE_LISTEN_URL are required";
|
|
try { runtime = new URL(config.databaseUrl ?? ""); }
|
|
catch { return refuse(runtimeRequired); }
|
|
if (waiveListener) {
|
|
if (!["postgres:", "postgresql:"].includes(runtime.protocol)) refuse(runtimeRequired);
|
|
return null;
|
|
}
|
|
try { listener = new URL(env.PAPERCLIP_DATABASE_LISTEN_URL ?? ""); }
|
|
catch { return refuse("external DATABASE_URL and PAPERCLIP_DATABASE_LISTEN_URL are required"); }
|
|
if (![runtime, listener].every(url => ["postgres:", "postgresql:"].includes(url.protocol))
|
|
|| runtime.username !== listener.username || runtime.password !== listener.password || runtime.pathname !== listener.pathname
|
|
|| runtime.hostname.replace(/-pooler(?=\.)/, "") !== listener.hostname
|
|
|| /pooler|pgbouncer/i.test(listener.hostname)) {
|
|
refuse("PAPERCLIP_DATABASE_LISTEN_URL must be a direct connection to the same database using the runtime role");
|
|
}
|
|
return listener.href;
|
|
}
|
|
|
|
/** Findings that describe work in progress when a server stopped, not data
|
|
* waiting on a volume to be migrated. Every volume-free server that stops
|
|
* during a run leaves some of them behind. Runtime skill assets are not in
|
|
* this set: the server first copies the bundles still on this filesystem
|
|
* (settleRuntimeSkillObligations), and waives only the ones not here. */
|
|
const interruptedRunBlockers: ReadonlySet<string> = new Set([
|
|
"native_harness_backups_require_migration",
|
|
"local_instruction_copies_require_settlement",
|
|
]);
|
|
|
|
export const RUNTIME_SKILL_ASSETS_BLOCKER = "runtime_skill_assets_require_migration";
|
|
|
|
/** Runtime skill bundles admitted before Core saved them at admission
|
|
* (Nov 2026) can still be retained by old runs. Copy the ones this filesystem
|
|
* holds into durable storage now. A bundle whose files are not here cannot be
|
|
* saved by refusing to start: on a filesystem with the volume-free owner
|
|
* marker there is no volume to read it from, and a volume-free stack never
|
|
* gets its volume back. Refusing would keep the stack down on every restart
|
|
* (staging, 2026-11-18: twelve days of failed deploys, then an outage). */
|
|
async function settleRuntimeSkillObligations(db: Db, owned: () => Promise<boolean>) {
|
|
let outcome: Awaited<ReturnType<typeof preserveLocalRuntimeSkillObligations>>;
|
|
try { outcome = await preserveLocalRuntimeSkillObligations(db); }
|
|
catch (error) {
|
|
logger.warn({ err: error }, "Volume-free server could not save retained runtime skill bundles to durable storage");
|
|
return false;
|
|
}
|
|
if (outcome.bundles) logger.info({ bundles: outcome.bundles }, "Volume-free server saved retained runtime skill bundles to durable storage");
|
|
// The copy pass visited every obligation, so its result is the answer.
|
|
if (!outcome.unsaved.length) return true;
|
|
if (!await owned()) return false;
|
|
logger.warn({ blockers: [RUNTIME_SKILL_ASSETS_BLOCKER], runs: outcome.unsaved.length },
|
|
"Volume-free server is starting over runtime skill bundles that never reached durable storage; restart recovery owns those runs");
|
|
return true;
|
|
}
|
|
|
|
/** A readiness gate, not a periodic scan. Existing volume data is adopted by
|
|
* the owning maintenance commands before removal; the one copy it makes
|
|
* itself is of retained runtime skill bundles still on this filesystem. */
|
|
export async function assertEphemeralServerData(db: Db) {
|
|
if (!ephemeralServerEnabled()) return;
|
|
const eligibility = await readEphemeralServerDataEligibility(db, { purpose: "startup" });
|
|
const { controller } = eligibility;
|
|
let blockers = eligibility.blockers;
|
|
let ownedMarker: Promise<boolean> | undefined;
|
|
const owned = () => ownedMarker ??= volumeFreeControllerStateOwned();
|
|
if (blockers.includes(RUNTIME_SKILL_ASSETS_BLOCKER) && await settleRuntimeSkillObligations(db, owned)) {
|
|
blockers = blockers.filter(blocker => blocker !== RUNTIME_SKILL_ASSETS_BLOCKER);
|
|
}
|
|
// Run state that never reached durable storage needs migration only while
|
|
// it can still be on a volume. Once this filesystem carries the volume-free
|
|
// owner marker there is no volume: the state was written by a volume-free
|
|
// server, this one or a predecessor that stopped mid-run, and refusing to
|
|
// start cannot bring it back. It would only keep every restart from
|
|
// starting again. Restart recovery owns those runs; start and say so.
|
|
// Configuration that would write new data to the local disk stays refused.
|
|
const interrupted = blockers.filter(blocker => interruptedRunBlockers.has(blocker));
|
|
if (interrupted.length && await owned()) {
|
|
blockers = blockers.filter(blocker => !interruptedRunBlockers.has(blocker));
|
|
logger.warn({ blockers: interrupted },
|
|
"Volume-free server is starting over run state that never reached durable storage; restart recovery owns those runs");
|
|
}
|
|
if (!blockers.length) return;
|
|
const detail = (blocker: string) => blocker === "native_controllers_require_migration" && controller
|
|
? `${blocker} (run ${controller.runId} on task ${controller.issueId}: controller state at ${controller.source} is not proven by the database, and this filesystem has no volume-free owner marker, so it may be the only copy of that run's progress; adopt it with the native controller migration before starting a volume-free server)`
|
|
: blocker;
|
|
throw new Error(`Volume-free server data is not eligible: ${blockers.map(detail).join(", ")}`);
|
|
}
|
|
|
|
export async function readEphemeralServerDataEligibility(db: Db, options: { purpose?: "migration" | "startup" } = {}) {
|
|
const blockers: string[] = [];
|
|
const [missingFiles] = await db.select({ id: agents.id }).from(agents).leftJoin(agentFileRoots,
|
|
and(eq(agentFileRoots.companyId, agents.companyId), eq(agentFileRoots.agentId, agents.id)))
|
|
.where(isNull(agentFileRoots.agentId)).limit(1);
|
|
if (missingFiles) blockers.push("agent_files_require_migration");
|
|
if (await hasUnsettledLocalInstructionCopies(db)) blockers.push("local_instruction_copies_require_settlement");
|
|
// Only inspect configuration here; filesystem reads must never adopt an
|
|
// empty directory on the replacement machine as the old server's content.
|
|
const configured = await db.select({ id: agents.id, companyId: agents.companyId, name: agents.name, adapterConfig: agents.adapterConfig }).from(agents);
|
|
if (configured.some(agent => agentInstructionsBundleMode(agent) === "external")) blockers.push("external_agent_instructions");
|
|
const [localWorkspace] = await db.select({ id: executionWorkspaces.id }).from(executionWorkspaces)
|
|
.where(localWorkspaceDependencyWhere(db)).limit(1);
|
|
if (localWorkspace) blockers.push("workspaces_require_migration");
|
|
const [localLog] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
|
eq(heartbeatRuns.logStore, "local_file"), isNotNull(heartbeatRuns.logRef),
|
|
)).limit(1);
|
|
if (localLog) blockers.push("run_logs_require_migration");
|
|
const [localTrace] = await db.select({ id: providerTraceRecords.id }).from(providerTraceRecords).where(and(
|
|
isNull(providerTraceRecords.deletedAt), sql`${providerTraceRecords.traceRef} not like 's3-segments-v1:%'`,
|
|
)).limit(1);
|
|
if (localTrace) blockers.push("provider_traces_require_migration");
|
|
if (await hasLocalNativeHarnessObligations(db)) blockers.push("native_harness_backups_require_migration");
|
|
const controller = await findLocalNativeControllerObligation(db);
|
|
if (controller) blockers.push("native_controllers_require_migration");
|
|
if (await hasLocalRuntimeSkillObligations(db)) blockers.push(RUNTIME_SKILL_ASSETS_BLOCKER);
|
|
const skills = skillInstallationMaintenance(db);
|
|
let after: string | undefined;
|
|
for (;;) {
|
|
const page = await skills.eligibility({ after, limit: 50, purpose: options.purpose });
|
|
if (page.results.some(skill => skill.status !== "version_backed")) { blockers.push("skills_require_migration"); break; }
|
|
if (page.inspected < 50 || !page.after) break;
|
|
after = page.after;
|
|
}
|
|
return { eligible: blockers.length === 0, blockers, controller };
|
|
}
|