diff --git a/server/src/__tests__/ephemeral-migration.test.ts b/server/src/__tests__/ephemeral-migration.test.ts index 93b8f1e51ee..5577b2f4ff7 100644 --- a/server/src/__tests__/ephemeral-migration.test.ts +++ b/server/src/__tests__/ephemeral-migration.test.ts @@ -27,7 +27,12 @@ import { companySkillService } from "../services/company-skills.js"; import { skillPackageStore } from "../services/skill-package-store.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { migrateLocalSkillPackageObjects } from "../services/skill-package-storage-migration.js"; -import { companySkills, companySkillRepositorySnapshots } from "@paperclipai/db"; +import { companySkills, companySkillRepositorySnapshots, nativeRunFinalizations } from "@paperclipai/db"; +import { materializeAsset } from "../services/native-runtime/runtime-context.js"; +import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js"; +import { canonicalNativeRuntimeContextDigest, parseNativeExecutionInput } from "../vendor/paperclip-runner/index.js"; +import { snapshotFile } from "../services/skill-snapshot.js"; +import { hasLocalRuntimeSkillObligations } from "../services/native-runtime/runtime-skill-preservation.js"; // Explicit real-dependency qualification. Never contacts off-host storage in // the default test path; use a disposable S3-compatible service and PostgreSQL. @@ -300,4 +305,42 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll .where(eq(agentInstructionWorkingCopies.runId, run!.id)); expect((await migrate()).eligibility.eligible).toBe(true); }); + // The original trigger: a retained runtime skill bundle whose bytes match a + // package snapshot still on the volume's local disk. Package phases must move + // that snapshot first, or the bundle's save reuses a local-disk receipt. + it("saves a retained runtime skill bundle that matches a local-disk package snapshot", async () => { + const owner = await agent(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Old native run" }); + const content = Buffer.from("# legacy admitted skill\n"); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "local_disk"); + await skillPackageStore(db, getStorageService()).prepare(companyId, [snapshotFile("SKILL.md", "skill", content)], true); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + const bundle = await materializeAsset([{ path: "SKILL.md", content, mode: 0o400 }]); + const context = nativeRuntimeContextFixture(); + context.skills = [{ key: "legacy", runtimeName: "legacy", versionId: null, bundle }]; + context.aggregateDigest = canonicalNativeRuntimeContextDigest(context); + const execution = parseNativeExecutionInput({ schema: "paperclip.native-execution-input.v3", + binding: { companyId, runId, agentId: owner.id, issueId, executionWorkspaceId: runId }, + task: { identifier: "MIG-1", title: "Old admission", description: null, prompt: "Use the skill", workMode: "standard" }, + workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: null, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + provider: { kind: "codex", model: null }, + completionContract: { id: "contract", sha256: "saved", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Use the skill", criteria: [{ id: "use", requirement: "Use skill" }] } }, + interactionResponses: [], credentialBindings: [], executionMode: "default", planningContext: null, runtimeContext: context }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId: owner.id, nativeIssueId: issueId, runtimeMode: "native", + status: "failed", finishedAt: new Date(), runnerProfileJson: { nativeExecutionInput: execution } }); + await db.insert(nativeRunFinalizations).values({ companyId, runId, issueId, phase: "terminal_failure", recoveryState: "blocked" }); + try { + expect(await hasLocalRuntimeSkillObligations(db)).toBe(true); + expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { runtimeSkillAssets: 1 } }); + expect(await hasLocalRuntimeSkillObligations(db)).toBe(false); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const receipt = run!.runnerProfileJson!.nativeRuntimeSkillPreservation as { assets: Array<{ packageSnapshotId: string }> }; + expect((await skillPackageStore(db).descriptor(companyId, receipt.assets[0]!.packageSnapshotId)).storage) + .toMatchObject({ provider: "s3", durable: true }); + } finally { + await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + } + }, 120_000); }); diff --git a/server/src/__tests__/ephemeral-server-runtime-skill-startup.test.ts b/server/src/__tests__/ephemeral-server-runtime-skill-startup.test.ts new file mode 100644 index 00000000000..f76d8e88498 --- /dev/null +++ b/server/src/__tests__/ephemeral-server-runtime-skill-startup.test.ts @@ -0,0 +1,198 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, expect, it, vi } from "vitest"; +import { activityLog, agentFileRoots, agents, companies, heartbeatRuns, issues, companySkillRepositorySnapshots as snapshots, + nativeRunFinalizations, createDb } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import * as storageModule from "../storage/index.js"; +import { createStorageService } from "../storage/service.js"; +import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; +import { materializeAsset } from "../services/native-runtime/runtime-context.js"; +import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js"; +import { canonicalNativeRuntimeContextDigest, parseNativeExecutionInput } from "../vendor/paperclip-runner/index.js"; +import { hasLocalRuntimeSkillObligations, preserveLocalRuntimeSkillObligations } from "../services/native-runtime/runtime-skill-preservation.js"; +import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; +import { assertEphemeralServerData } from "../services/ephemeral-server-contract.js"; +import { volumeFreeControllerStateOwned } from "../services/native-runtime/volume-free-controller-state.js"; + +vi.mock("../services/native-runtime/native-harness-backup-migration.js", () => ({ + hasLocalNativeHarnessObligations: vi.fn(async () => false) })); +vi.mock("../services/native-runtime/volume-free-controller-state.js", async importOriginal => ({ + ...await importOriginal(), + volumeFreeControllerStateOwned: vi.fn(), +})); + +let database: Awaited>; +let db: ReturnType; +let home: string; +const previousHome = process.env.PAPERCLIP_HOME; +beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("ephemeral-runtime-skill-startup-"); + db = createDb(database.connectionString, { maxConnections: 6 }); + home = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-runtime-skill-startup-")); + process.env.PAPERCLIP_HOME = home; + const storage = createStorageService({ ...createLocalDiskStorageProvider(path.join(home, "objects")), id: "s3" }); + vi.spyOn(storageModule, "getStorageService").mockReturnValue(storage); +}, 90_000); +afterEach(async () => { + vi.unstubAllEnvs(); + vi.mocked(volumeFreeControllerStateOwned).mockReset(); + for (const table of [activityLog, nativeRunFinalizations, heartbeatRuns, issues, agentFileRoots, agents, snapshots, companies]) await db.delete(table); +}); +afterAll(async () => { + vi.restoreAllMocks(); + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + await db?.$client.end({ timeout: 1 }); + await database?.cleanup(); + if (home) await removeRuntimeAssetTree(home); +}); + +/** A native run admitted by a release that did not yet save runtime skill + * bundles at admission: the bundle exists only on the admitting server's disk. */ +async function retainedUnsavedRuntimeSkill(options: { pins?: Array } = {}) { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Runtime skill startup", issuePrefix: `S${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Legacy admission" }); + await db.insert(agentFileRoots).values({ companyId, agentId, generation: randomUUID(), entries: [] }); + await db.insert(issues).values({ id: issueId, companyId, title: "Old native run" }); + const bundle = await materializeAsset([{ path: "SKILL.md", content: Buffer.from(`# legacy ${runId}\n`), mode: 0o400 }, + { path: "notes.md", content: Buffer.from("notes\n"), mode: 0o400 }]); + const context = nativeRuntimeContextFixture(); + context.skills = (options.pins ?? [null]).map((versionId, index) => ({ key: `legacy-${index}`, runtimeName: `legacy-${index}`, versionId, bundle })); + context.aggregateDigest = canonicalNativeRuntimeContextDigest(context); + const execution = parseNativeExecutionInput({ schema: "paperclip.native-execution-input.v3", + binding: { companyId, runId, agentId, issueId, executionWorkspaceId: runId }, + task: { identifier: "LEGACY-1", title: "Old admission", description: null, prompt: "Use the skill", workMode: "standard" }, + workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: null, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + provider: { kind: "codex", model: null }, + completionContract: { id: "contract", sha256: "saved", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Use the skill", criteria: [{ id: "use", requirement: "Use skill" }] } }, + interactionResponses: [], credentialBindings: [], executionMode: "default", planningContext: null, runtimeContext: context }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, runtimeMode: "native", + status: "failed", finishedAt: new Date(), runnerProfileJson: { nativeExecutionInput: execution } }); + await db.insert(nativeRunFinalizations).values({ companyId, runId, issueId, phase: "terminal_failure", recoveryState: "blocked" }); + return { runId, bundle }; +} +const lose = (bundle: { rootPath: string }) => removeRuntimeAssetTree(bundle.rootPath); +const saved = async (runId: string) => { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + return Object.hasOwn(run!.runnerProfileJson ?? {}, "nativeRuntimeSkillPreservation"); +}; +const volumeFreeServer = () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); +}; + +// Staging, 2026-11-18: a stack ran volume-free for twelve days on +// a release that kept runtime skill bundles only on its container disk. When +// that container stopped, every newer release refused to start with +// runtime_skill_assets_require_migration, and nothing could clear it. +it("starts a volume-free server over runtime skill bundles a lost disk held", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(true); + const { bundle } = await retainedUnsavedRuntimeSkill(); + await removeRuntimeAssetTree(path.dirname(path.dirname(bundle.rootPath))); + await expect(assertEphemeralServerData(db)).resolves.toBeUndefined(); +}); + +it("saves bundles still on this disk to durable storage before starting", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(false); + const { runId } = await retainedUnsavedRuntimeSkill(); + expect(await hasLocalRuntimeSkillObligations(db)).toBe(true); + await expect(assertEphemeralServerData(db)).resolves.toBeUndefined(); + expect(await hasLocalRuntimeSkillObligations(db)).toBe(false); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(run!.runnerProfileJson).toHaveProperty("nativeRuntimeSkillPreservation"); + expect(volumeFreeControllerStateOwned).not.toHaveBeenCalled(); +}); + +it("still refuses a missing bundle on a filesystem with no volume-free owner marker", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(false); + const { bundle } = await retainedUnsavedRuntimeSkill(); + await removeRuntimeAssetTree(path.dirname(path.dirname(bundle.rootPath))); + await expect(assertEphemeralServerData(db)).rejects.toThrow("runtime_skill_assets_require_migration"); +}); + +it("saves the bundles still here and starts over the lost ones on a volume-free filesystem", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(true); + const present = await retainedUnsavedRuntimeSkill(), lost = await retainedUnsavedRuntimeSkill(); + await lose(lost.bundle); + await expect(assertEphemeralServerData(db)).resolves.toBeUndefined(); + expect(await saved(present.runId)).toBe(true); + expect(await saved(lost.runId)).toBe(false); +}); + +it("treats a bundle missing one listed file as lost, not changed", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(true); + const { bundle } = await retainedUnsavedRuntimeSkill(); + await fs.chmod(bundle.rootPath, 0o700); + await fs.rm(path.join(bundle.rootPath, "notes.md")); + await expect(assertEphemeralServerData(db)).resolves.toBeUndefined(); +}); + +it("does not hold startup on a non-null pin that is not a version ID, which preservation never copies", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(false); + await retainedUnsavedRuntimeSkill({ pins: ["legacy-pin"] }); + expect(await hasLocalRuntimeSkillObligations(db)).toBe(false); + await expect(assertEphemeralServerData(db)).resolves.toBeUndefined(); +}); + +it("saves the null pin of a run that mixes it with a non-version pin, and settles the run", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(false); + const { runId } = await retainedUnsavedRuntimeSkill({ pins: ["legacy-pin", null] }); + expect(await preserveLocalRuntimeSkillObligations(db)).toEqual({ bundles: 1, unsaved: [] }); + expect(await saved(runId)).toBe(true); + expect(await hasLocalRuntimeSkillObligations(db)).toBe(false); +}); + +it("recaptures a bundle still here when its receipt's package is no longer durable", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(false); + const { runId } = await retainedUnsavedRuntimeSkill(); + await preserveLocalRuntimeSkillObligations(db); + await db.update(snapshots).set({ storage: null }); + expect(await hasLocalRuntimeSkillObligations(db)).toBe(true); + await expect(assertEphemeralServerData(db)).resolves.toBeUndefined(); + expect(await hasLocalRuntimeSkillObligations(db)).toBe(false); + expect(await saved(runId)).toBe(true); +}); + +it("refuses a storage error that looks like a missing file, even on a volume-free filesystem", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(true); + await retainedUnsavedRuntimeSkill(); + const missing = Object.assign(new Error("temp file vanished"), { code: "ENOENT" }); + const put = vi.spyOn(storageModule.getStorageService(), "putFile").mockRejectedValue(missing); + try { await expect(assertEphemeralServerData(db)).rejects.toThrow("runtime_skill_assets_require_migration"); } + finally { put.mockRestore(); } +}); + +it("refuses when saving a bundle that is here fails, even on a volume-free filesystem", async () => { + volumeFreeServer(); + vi.mocked(volumeFreeControllerStateOwned).mockResolvedValue(true); + const { runId } = await retainedUnsavedRuntimeSkill(); + const put = vi.spyOn(storageModule.getStorageService(), "putFile").mockRejectedValue(new Error("storage offline")); + try { await expect(assertEphemeralServerData(db)).rejects.toThrow("runtime_skill_assets_require_migration"); } + finally { put.mockRestore(); } + expect(await saved(runId)).toBe(false); +}); + +// The offline enrollment phase is this pass with a hard refusal on any unsaved +// run; the full enrollment suite needs a real S3 endpoint (ephemeral-migration.test.ts). +it("reports exactly the runs it could not save, for enrollment to refuse", async () => { + const present = await retainedUnsavedRuntimeSkill(), lost = await retainedUnsavedRuntimeSkill(); + await lose(lost.bundle); + expect(await preserveLocalRuntimeSkillObligations(db)).toEqual({ bundles: 1, unsaved: [lost.runId] }); + expect(await saved(present.runId)).toBe(true); + expect(await preserveLocalRuntimeSkillObligations(db)).toEqual({ bundles: 0, unsaved: [lost.runId] }); +}); diff --git a/server/src/index.ts b/server/src/index.ts index 3488284e080..fa076227efc 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -796,8 +796,9 @@ async function startServerWithDatabaseTeardown( // then abandons no work. It used to run minutes later, after startup had // already dispatched queued runs. No startup step it skipped could make an // ineligible database eligible: the migrations that clear its blockers are - // maintenance commands, and the background passes startup starts were never - // awaited before it. + // maintenance commands (the gate itself copies only retained runtime skill + // bundles still on this disk), and the background passes startup starts + // were never awaited before it. if (deploymentListenUrl !== null) await assertEphemeralServerData(db as any); startupTimeline.mark("volume_free_data_gate"); diff --git a/server/src/maintenance/ephemeral-migration-data.ts b/server/src/maintenance/ephemeral-migration-data.ts index 462de768097..3087f6be256 100644 --- a/server/src/maintenance/ephemeral-migration-data.ts +++ b/server/src/maintenance/ephemeral-migration-data.ts @@ -6,7 +6,7 @@ import { heartbeatRuns, type Db } from "@paperclipai/db"; import { offlineMigrationExecutionWhere } from "../services/offline-migration-execution.js"; import { getStorageService } from "../storage/index.js"; import { migrateAgentFiles, verifyAgentFileObjects } from "../services/agent-file-maintenance.js"; -import { readEphemeralServerDataEligibility } from "../services/ephemeral-server-contract.js"; +import { readEphemeralServerDataEligibility, RUNTIME_SKILL_ASSETS_BLOCKER } from "../services/ephemeral-server-contract.js"; import { migrateRunEvidence } from "../services/run-evidence-migration.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { skillPackageMaintenance } from "../services/skill-package-maintenance.js"; @@ -16,6 +16,7 @@ import { reconcileOfflineWorkspaceOperations } from "../services/offline-workspa import { migrateWorkspaceTrees } from "../services/remote-workspace-migration-batch.js"; import { migrateNativeHarnessBackups } from "../services/native-runtime/native-harness-backup-migration.js"; import { migrateNativeControllers } from "../services/native-runtime/native-controller-migration.js"; +import { preserveLocalRuntimeSkillObligations } from "../services/native-runtime/runtime-skill-preservation.js"; /** Runs only in the maintenance process after Cloud has confirmed all legacy * deployments stopped. Each owner retains its own migration/publication rules; @@ -34,6 +35,7 @@ export async function migrateEphemeralServerData(db: Db, input: { "agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration", "workspaces_require_migration", "native_harness_backups_require_migration", "native_controllers_require_migration", + RUNTIME_SKILL_ASSETS_BLOCKER, ].includes(blocker)); if (unsupported.length) throw new Error(unsupported[0]); @@ -103,6 +105,13 @@ export async function migrateEphemeralServerData(db: Db, input: { after = page.after; } } + // After the package phases: a bundle whose bytes match a package snapshot + // reuses that snapshot, which must already live in shared storage. + phase("runtime_skill_assets"); + // The volume is still attached here, so every retained bundle must be on it. + const runtimeSkills = await preserveLocalRuntimeSkillObligations(db, { signal }, storage); + if (runtimeSkills.unsaved.length) throw new Error("runtime_skill_asset_migration_blocked"); + counts.runtimeSkillAssets = runtimeSkills.bundles; phase("skill_installations"); after = undefined; for (;;) { diff --git a/server/src/services/ephemeral-server-contract.ts b/server/src/services/ephemeral-server-contract.ts index 6274346f788..955ef14acf8 100644 --- a/server/src/services/ephemeral-server-contract.ts +++ b/server/src/services/ephemeral-server-contract.ts @@ -7,7 +7,7 @@ import { skillInstallationMaintenance } from "./skill-installation-maintenance.j import { hasLocalNativeHarnessObligations } from "./native-runtime/native-harness-backup-migration.js"; import { findLocalNativeControllerObligation } from "./native-runtime/native-controller-migration.js"; import { hasUnsettledLocalInstructionCopies } from "./agent-instruction-copy-preservation.js"; -import { hasLocalRuntimeSkillObligations } from "./native-runtime/runtime-skill-preservation.js"; +import { hasLocalRuntimeSkillObligations, preserveLocalRuntimeSkillObligations } from "./native-runtime/runtime-skill-preservation.js"; import { localWorkspaceDependencyWhere } from "./local-workspace-dependencies.js"; import { ephemeralServerEnabled } from "./ephemeral-server-mode.js"; @@ -45,20 +45,51 @@ export function assertEphemeralServerConfig(config: Pick = new Set([ "native_harness_backups_require_migration", "local_instruction_copies_require_settlement", ]); -/** A readiness gate, not a periodic scan or an implicit migration. Existing - * volume data is adopted by the owning maintenance commands before removal. */ +export const RUNTIME_SKILL_ASSETS_BLOCKER = "runtime_skill_assets_require_migration"; + +/** Runtime skill bundles admitted before Core saved them at admission + * (Nov 2026) can still be retained by old runs. Copy the ones this filesystem + * holds into durable storage now. A bundle whose files are not here cannot be + * saved by refusing to start: on a filesystem with the volume-free owner + * marker there is no volume to read it from, and a volume-free stack never + * gets its volume back. Refusing would keep the stack down on every restart + * (staging, 2026-11-18: twelve days of failed deploys, then an outage). */ +async function settleRuntimeSkillObligations(db: Db, owned: () => Promise) { + let outcome: Awaited>; + try { outcome = await preserveLocalRuntimeSkillObligations(db); } + catch (error) { + logger.warn({ err: error }, "Volume-free server could not save retained runtime skill bundles to durable storage"); + return false; + } + if (outcome.bundles) logger.info({ bundles: outcome.bundles }, "Volume-free server saved retained runtime skill bundles to durable storage"); + // The copy pass visited every obligation, so its result is the answer. + if (!outcome.unsaved.length) return true; + if (!await owned()) return false; + logger.warn({ blockers: [RUNTIME_SKILL_ASSETS_BLOCKER], runs: outcome.unsaved.length }, + "Volume-free server is starting over runtime skill bundles that never reached durable storage; restart recovery owns those runs"); + return true; +} + +/** A readiness gate, not a periodic scan. Existing volume data is adopted by + * the owning maintenance commands before removal; the one copy it makes + * itself is of retained runtime skill bundles still on this filesystem. */ export async function assertEphemeralServerData(db: Db) { if (!ephemeralServerEnabled()) return; const eligibility = await readEphemeralServerDataEligibility(db, { purpose: "startup" }); const { controller } = eligibility; let blockers = eligibility.blockers; + let ownedMarker: Promise | undefined; + const owned = () => ownedMarker ??= volumeFreeControllerStateOwned(); + if (blockers.includes(RUNTIME_SKILL_ASSETS_BLOCKER) && await settleRuntimeSkillObligations(db, owned)) { + blockers = blockers.filter(blocker => blocker !== RUNTIME_SKILL_ASSETS_BLOCKER); + } // Run state that never reached durable storage needs migration only while // it can still be on a volume. Once this filesystem carries the volume-free // owner marker there is no volume: the state was written by a volume-free @@ -67,7 +98,7 @@ export async function assertEphemeralServerData(db: Db) { // starting again. Restart recovery owns those runs; start and say so. // Configuration that would write new data to the local disk stays refused. const interrupted = blockers.filter(blocker => interruptedRunBlockers.has(blocker)); - if (interrupted.length && await volumeFreeControllerStateOwned()) { + if (interrupted.length && await owned()) { blockers = blockers.filter(blocker => !interruptedRunBlockers.has(blocker)); logger.warn({ blockers: interrupted }, "Volume-free server is starting over run state that never reached durable storage; restart recovery owns those runs"); @@ -104,7 +135,7 @@ export async function readEphemeralServerDataEligibility(db: Db, options: { purp if (await hasLocalNativeHarnessObligations(db)) blockers.push("native_harness_backups_require_migration"); const controller = await findLocalNativeControllerObligation(db); if (controller) blockers.push("native_controllers_require_migration"); - if (await hasLocalRuntimeSkillObligations(db)) blockers.push("runtime_skill_assets_require_migration"); + if (await hasLocalRuntimeSkillObligations(db)) blockers.push(RUNTIME_SKILL_ASSETS_BLOCKER); const skills = skillInstallationMaintenance(db); let after: string | undefined; for (;;) { diff --git a/server/src/services/file-manifest-cache.ts b/server/src/services/file-manifest-cache.ts index 97c360701a3..4f9f0d187f6 100644 --- a/server/src/services/file-manifest-cache.ts +++ b/server/src/services/file-manifest-cache.ts @@ -28,6 +28,12 @@ export function relocateFileManifestVerification(from: string, to: string, files } } export async function verifyFileManifest(directory: string, files: readonly FileManifestEntry[]): Promise { + return await inspectFileManifest(directory, files) === "verified"; +} + +/** Like verifyFileManifest, and says why it failed: "missing" when the + * directory or a listed file is gone, "changed" for any other difference. */ +export async function inspectFileManifest(directory: string, files: readonly FileManifestEntry[]): Promise<"verified" | "missing" | "changed"> { const expected = new Map(files.map(file => [file.path, file])); const seen = new Set(); async function walk(relative: string): Promise { @@ -69,6 +75,13 @@ export async function verifyFileManifest(directory: string, files: readonly File } return true; } - try { return await walk("") && seen.size === expected.size; } - catch (error) { if (["ENOENT", "ENOTDIR", "ELOOP"].includes((error as NodeJS.ErrnoException).code ?? "")) return false; throw error; } + try { + if (!await walk("")) return "changed"; + return seen.size === expected.size ? "verified" : "missing"; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code ?? ""; + if (["ENOENT", "ENOTDIR"].includes(code)) return "missing"; + if (code === "ELOOP") return "changed"; + throw error; + } } diff --git a/server/src/services/native-runtime/runtime-skill-preservation-reference.ts b/server/src/services/native-runtime/runtime-skill-preservation-reference.ts index a09a0fd5042..c796decff11 100644 --- a/server/src/services/native-runtime/runtime-skill-preservation-reference.ts +++ b/server/src/services/native-runtime/runtime-skill-preservation-reference.ts @@ -49,14 +49,17 @@ export function readRuntimeSkillPreservation(row: Parameters '{nativeExecutionInput,runtimeContext,skills}'`; return sql`${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.runnerProfileJson} #> '{nativeExecutionInput,runtimeContext}' is not null and (jsonb_typeof(${skills}) is distinct from 'array' or exists ( select 1 from jsonb_array_elements(case when jsonb_typeof(${skills}) = 'array' then ${skills} else '[]'::jsonb end) s - where s->>'versionId' is null or s->>'versionId' !~* '^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$'))`; + where s->>'versionId' is null))`; } /** JSON receipts are references just like version FKs. Unknown receipt shapes diff --git a/server/src/services/native-runtime/runtime-skill-preservation.ts b/server/src/services/native-runtime/runtime-skill-preservation.ts index 04855854af2..df8ad79216c 100644 --- a/server/src/services/native-runtime/runtime-skill-preservation.ts +++ b/server/src/services/native-runtime/runtime-skill-preservation.ts @@ -10,7 +10,7 @@ import { getStorageService } from "../../storage/index.js"; import { skillPackageStore, withSkillPackageReadScope } from "../skill-package-store.js"; import { assertSkillSnapshotPath, snapshotFile, skillSnapshotHash } from "../skill-snapshot.js"; import { ensureSkillCacheDirectory, skillRunNeedsFilesSql } from "../skill-version-cache.js"; -import { verifyFileManifest } from "../file-manifest-cache.js"; +import { inspectFileManifest, verifyFileManifest } from "../file-manifest-cache.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { withRuntimeAssetUse } from "./runtime-asset-retention.js"; import { admittedRuntimeSkills, hasUnversionedRuntimeSkillInputSql, readRuntimeSkillPreservation, @@ -41,13 +41,38 @@ function manifestFiles(asset: NativeRuntimeAssetReference, text: string): Manife if (hash(JSON.stringify(files)) !== asset.digest || files.reduce((n, f) => n + f.size, 0) !== asset.totalBytes) throw new Error("runtime_skill_manifest_invalid"); return files; } +/** The admitted bundle's source is not on this filesystem: its directory, + * manifest or a listed file is gone, or it was written under another instance + * root. Only the source readers throw it; every other failure is real. */ +export class RuntimeSkillSourceAbsent extends Error { + constructor(readonly reason: "path_mismatch" | "missing") { super(`runtime_skill_source_absent:${reason}`); } +} +const missingPath = (error: unknown) => ["ENOENT", "ENOTDIR"].includes((error as NodeJS.ErrnoException | undefined)?.code ?? ""); +async function sourceRead(read: () => Promise): Promise { + try { return await read(); } + catch (error) { throw missingPath(error) ? new RuntimeSkillSourceAbsent("missing") : error; } +} +type PackageRow = { contentHash: string; storage: typeof snapshots.$inferSelect.storage } | undefined; +/** The one durability test for a preserved asset. "metadata" trusts the + * package row; "object" also reads the stored package back. A storage error + * during the object read propagates: it is not evidence either way. */ +function preservedAssetRowDurable(asset: PreservedRuntimeSkill, row: PackageRow) { + try { manifestFiles(asset.bundle, asset.manifestText); } catch { return false; } + return row?.storage?.provider === "s3" && row.storage.durable === true && row.contentHash === asset.packageContentHash; +} +async function preservedAssetDurable(store: ReturnType, companyId: string, asset: PreservedRuntimeSkill, + verify: "metadata" | "object", signal?: AbortSignal) { + const row = await store.descriptor(companyId, asset.packageSnapshotId).catch(() => undefined); + if (!preservedAssetRowDurable(asset, row)) return false; + return verify === "metadata" || await store.verifyDurable(companyId, asset.packageSnapshotId, signal); +} async function readLocalManifest(asset: NativeRuntimeAssetReference) { const root = path.join(resolvePaperclipInstanceRoot(), "runtime-context-assets"); const expected = path.join(root, "bundles", asset.digest); - if (asset.rootPath !== expected) throw new Error("runtime_skill_source_path_mismatch"); - await ensureSkillCacheDirectory(expected); - await ensureSkillCacheDirectory(path.join(root, "manifests")); - const handle = await fs.open(path.join(root, "manifests", `${asset.digest}.json`), constants.O_RDONLY | constants.O_NOFOLLOW); + if (asset.rootPath !== expected) throw new RuntimeSkillSourceAbsent("path_mismatch"); + await sourceRead(() => ensureSkillCacheDirectory(expected)); + await sourceRead(() => ensureSkillCacheDirectory(path.join(root, "manifests"))); + const handle = await sourceRead(() => fs.open(path.join(root, "manifests", `${asset.digest}.json`), constants.O_RDONLY | constants.O_NOFOLLOW)); let text: string; try { const stat = await handle.stat(); @@ -59,10 +84,12 @@ async function readLocalManifest(asset: NativeRuntimeAssetReference) { } async function readSource(asset: NativeRuntimeAssetReference) { const { expected, manifestText, entries } = await readLocalManifest(asset); - if (!await verifyFileManifest(expected, entries.map(f => ({ ...f, executable: Boolean(f.mode & 0o111) })))) throw new Error("runtime_skill_source_changed"); + const inspected = await inspectFileManifest(expected, entries.map(f => ({ ...f, executable: Boolean(f.mode & 0o111) }))); + if (inspected === "missing") throw new RuntimeSkillSourceAbsent("missing"); + if (inspected === "changed") throw new Error("runtime_skill_source_changed"); const files = []; for (const entry of entries) { - const handle = await fs.open(path.join(expected, entry.path), constants.O_RDONLY | constants.O_NOFOLLOW); + const handle = await sourceRead(() => fs.open(path.join(expected, entry.path), constants.O_RDONLY | constants.O_NOFOLLOW)); try { const stat = await handle.stat(); if (!stat.isFile() || stat.nlink !== 1 || stat.size !== entry.size || (stat.mode & 0o777) !== entry.mode) throw new Error("runtime_skill_source_changed"); @@ -90,8 +117,7 @@ async function captureSkills(db: Db, companyId: string, admitted: ReturnType a.packageSnapshotId.localeCompare(b.packageSnapshotId))) { const [row] = await db.select().from(snapshots).where(and(eq(snapshots.companyId, companyId), eq(snapshots.id, asset.packageSnapshotId))).for("update"); - if (!row?.storage || row.storage.provider !== "s3" || row.storage.durable !== true - || row.contentHash !== asset.packageContentHash) throw new Error("runtime_skill_package_not_durable"); + if (!preservedAssetRowDurable(asset, row)) throw new Error("runtime_skill_package_not_durable"); } } @@ -116,9 +142,7 @@ export async function stageNativeRuntimeSkillAssets(db: Db, input: Admission & { if (receipt) { const store = skillPackageStore(db, shared); for (const asset of receipt.assets) { - manifestFiles(asset.bundle, asset.manifestText); - const row = await store.descriptor(input.companyId, asset.packageSnapshotId); - if (row.storage?.provider !== "s3" || row.storage.durable !== true || row.contentHash !== asset.packageContentHash) throw new Error("runtime_skill_package_not_durable"); + if (!await preservedAssetDurable(store, input.companyId, asset, "metadata")) throw new Error("runtime_skill_package_not_durable"); } } else receipt = { schema: RUNTIME_SKILL_PRESERVATION_SCHEMA, inputDigest: admitted.inputDigest, bindingDigest: admitted.bindingDigest, assets: await captureSkills(db, input.companyId, admitted, shared, input.signal) }; @@ -144,23 +168,24 @@ export async function preserveRuntimeSkillAssets(db: Db, input: { companyId: str const admitted = admittedRuntimeSkills(source); const store = skillPackageStore(db, storage); const existing = readRuntimeSkillPreservation(source); - if (source.runnerProfileJson?.[RUNTIME_SKILL_PRESERVATION_KEY] != null && !existing) throw new Error("runtime_skill_preservation_receipt_invalid"); if (existing) { - for (const asset of existing.assets) { - manifestFiles(asset.bundle, asset.manifestText); - const row = await store.descriptor(input.companyId, asset.packageSnapshotId); - if (row.storage?.provider !== "s3" || row.storage.durable !== true || row.contentHash !== asset.packageContentHash - || !await store.verifyDurable(input.companyId, asset.packageSnapshotId, input.signal)) throw new Error("runtime_skill_package_not_durable"); - } - return { preserved: existing.assets.length }; + let durable = true; + for (const asset of existing.assets) if (durable) durable = await preservedAssetDurable(store, input.companyId, asset, "object", input.signal); + if (durable) return { preserved: existing.assets.length }; } + // No receipt, an invalid one, or one whose package is not durable: copy the + // admitted bytes again and replace it. A missing source throws RuntimeSkillSourceAbsent. + const previousReceipt = JSON.stringify(source.runnerProfileJson?.[RUNTIME_SKILL_PRESERVATION_KEY] ?? null); const assets = await captureSkills(db, input.companyId, admitted, storage, input.signal); if (!assets.length) return { preserved: 0 }; const publications: ActivityPublication[] = []; await db.transaction(async tx => { const [current] = await tx.select().from(heartbeatRuns).where(where).for("update"); if (!current || admittedRuntimeSkills(current).inputDigest !== admitted.inputDigest || current.status !== source.status - || current.finishedAt?.getTime() !== source.finishedAt?.getTime()) throw new Error("runtime_skill_preservation_binding_changed"); + || current.finishedAt?.getTime() !== source.finishedAt?.getTime() + || JSON.stringify(current.runnerProfileJson?.[RUNTIME_SKILL_PRESERVATION_KEY] ?? null) !== previousReceipt) { + throw new Error("runtime_skill_preservation_binding_changed"); + } await lockPackages(tx as unknown as Db, input.companyId, assets); await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...current.runnerProfileJson, [RUNTIME_SKILL_PRESERVATION_KEY]: { schema: RUNTIME_SKILL_PRESERVATION_SCHEMA, inputDigest: admitted.inputDigest, @@ -173,9 +198,21 @@ export async function preserveRuntimeSkillAssets(db: Db, input: { companyId: str return { preserved: assets.length }; } -export async function hasLocalRuntimeSkillObligations(db: Db): Promise { +/** Retained runs whose admitted local skill bundles have no durable receipt. */ +async function* localRuntimeSkillObligations(db: Db): AsyncGenerator<{ id: string; companyId: string }> { let after: string | undefined; - const checkedPackages = new Map(); + const checkedPackages = new Map>(); + // Metadata only: this scan runs before every volume-free start. + const durable = async (companyId: string, asset: PreservedRuntimeSkill) => { + const key = `${companyId}:${asset.packageSnapshotId}`; + let snapshot = checkedPackages.get(key); + if (!snapshot) { + [snapshot] = await db.select({ contentHash: snapshots.contentHash, storage: snapshots.storage }).from(snapshots) + .where(and(eq(snapshots.companyId, companyId), eq(snapshots.id, asset.packageSnapshotId))); + if (snapshot) checkedPackages.set(key, snapshot); + } + return preservedAssetRowDurable(asset, snapshot); + }; for (;;) { const rows = await db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, nativeIssueId: heartbeatRuns.nativeIssueId, runnerProfileJson: sql>`jsonb_build_object( @@ -184,26 +221,45 @@ export async function hasLocalRuntimeSkillObligations(db: Db): Promise .from(heartbeatRuns).where(and(retainedRuntimeSkillsSql(), after ? gt(heartbeatRuns.id, after) : undefined)) .orderBy(asc(heartbeatRuns.id)).limit(100); for (const row of rows) { - let receipt; - try { receipt = readRuntimeSkillPreservation(row); } catch { return true; } - if (!receipt) return true; - for (const asset of receipt.assets) { - try { manifestFiles(asset.bundle, asset.manifestText); } catch { return true; } - const key = `${row.companyId}:${asset.packageSnapshotId}`; - let snapshot = checkedPackages.get(key); - if (!snapshot) { - [snapshot] = await db.select({ hash: snapshots.contentHash, storage: snapshots.storage }).from(snapshots) - .where(and(eq(snapshots.companyId, row.companyId), eq(snapshots.id, asset.packageSnapshotId))); - if (snapshot) checkedPackages.set(key, snapshot); - } - if (snapshot?.storage?.provider !== "s3" || snapshot.storage.durable !== true || snapshot.hash !== asset.packageContentHash) return true; - } + let receipt: RuntimeSkillPreservation | null; + try { receipt = readRuntimeSkillPreservation(row); } catch { receipt = null; } + let settled = receipt !== null; + for (const asset of receipt?.assets ?? []) if (settled) settled = await durable(row.companyId, asset); + if (!settled) yield { id: row.id, companyId: row.companyId }; } - if (rows.length < 100) return false; + if (rows.length < 100) return; after = rows.at(-1)!.id; } } +export async function hasLocalRuntimeSkillObligations(db: Db): Promise { + for await (const _ of localRuntimeSkillObligations(db)) return true; + return false; +} + +/** Copies every retained bundle this filesystem still holds into durable + * storage: the same copy the skill package maintenance command makes. A run + * whose bundle source is not here is reported as unsaved, never guessed at. + * Any other failure (a changed source, storage) propagates to the caller. */ +export async function preserveLocalRuntimeSkillObligations(db: Db, options: { signal?: AbortSignal } = {}, storage?: StorageService) { + let bundles = 0; + const unsaved: string[] = []; + await withSkillPackageReadScope(db, async () => { + for await (const run of localRuntimeSkillObligations(db)) { + options.signal?.throwIfAborted(); + try { + const { preserved } = await preserveRuntimeSkillAssets(db, { companyId: run.companyId, runId: run.id, signal: options.signal }, storage); + if (preserved) bundles += preserved; + else unsaved.push(run.id); + } catch (error) { + if (!(error instanceof RuntimeSkillSourceAbsent)) throw error; + unsaved.push(run.id); + } + } + }); + return { bundles, unsaved }; +} + /** Existing bounded skill maintenance owns this optional online conversion; * there is no scanner, new execution admission or controller takeover. */ export async function migrateRuntimeSkillAssets(db: Db, options: { companyId?: string; after?: string; @@ -231,9 +287,7 @@ export async function rehydratePreservedRuntimeSkill(db: Db, row: Parameters ({ ...file, executable: Boolean(file.mode & 0o111) }))); } catch (error) { - if (["ENOENT", "ENOTDIR"].includes((error as NodeJS.ErrnoException).code ?? "") + if (error instanceof RuntimeSkillSourceAbsent || missingPath(error) || error instanceof SyntaxError || (error instanceof Error && error.message.startsWith("runtime_skill_manifest_"))) return false; throw error; }