From 3e82aeaa6098de841b1e57730a76ca5a9cd1dc4f Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 22:54:22 -0600 Subject: [PATCH] fix(server): archive failed legacy controllers without inventing recovery authority Co-Authored-By: Paperclip --- doc/DATABASE.md | 11 ++++ .../maintenance/ephemeral-migration-data.ts | 3 +- .../native-controller-archive.ts | 40 +++++++++++++ ...e-controller-migration.integration.test.ts | 56 ++++++++++++++++++- .../native-controller-migration.ts | 34 ++++++++--- .../native-harness-backup-storage.ts | 8 ++- 6 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 server/src/services/native-runtime/native-controller-archive.ts diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 00e16bd69be..c8ec16af380 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -27,6 +27,17 @@ files stay intact, canonical database checkpoints take precedence, and missing source files are reported separately rather than fabricated. This migration does not recover provider state that was already lost before enrollment. +For an already-failed run, a valid legacy controller can lack proof of the +original Runner artifact after its old bootstrap tickets were pruned. The +migration preserves those exact controller and supervisor bytes as an encrypted +`paperclip.legacy-controller-archive.v1` object through the same backup ledger. +Its scope is the hash of its manifest and is separate from the provider session +scope. Publication requires a terminal-failure coordinator with no live owner. +The archive does not become an executable Runner checkpoint. Task pointers, +failed results and recovery blockers stay unchanged. A successful retained +session without artifact proof still blocks migration. The migration receipt +reports failed-session archives separately from imported recovery authorities. + ### Proactive agent chat messages Migration `0419_slippery_impossible_man.sql` adds an agent-scoped unique constraint diff --git a/server/src/maintenance/ephemeral-migration-data.ts b/server/src/maintenance/ephemeral-migration-data.ts index b6e705016c4..44acf0a3635 100644 --- a/server/src/maintenance/ephemeral-migration-data.ts +++ b/server/src/maintenance/ephemeral-migration-data.ts @@ -35,10 +35,11 @@ export async function migrateEphemeralServerData(db: Db, input: { const storage = getStorageService(); if (storage.provider !== "s3") throw new Error("shared_object_storage_required"); phase("native_controllers"); - const controllers = await migrateNativeControllers(db, { writersStopped: true, signal }); + const controllers = await migrateNativeControllers(db, { writersStopped: true, signal, storage }); counts.nativeControllers = controllers.imported; counts.nativeControllersUnchanged = controllers.unchanged; counts.nativeControllerSourcesAlreadyAbsent = controllers.sourceAlreadyAbsent; + counts.nativeControllersArchivedFailed = controllers.archivedFailed; phase("native_harness_backups"); counts.nativeHarnessBackups = await migrateNativeHarnessBackups(db, { storage, writersStopped: true, signal }); phase("workspaces"); diff --git a/server/src/services/native-runtime/native-controller-archive.ts b/server/src/services/native-runtime/native-controller-archive.ts new file mode 100644 index 00000000000..d0602ce24f1 --- /dev/null +++ b/server/src/services/native-runtime/native-controller-archive.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Db } from "@paperclipai/db"; +import type { StorageService } from "../../storage/types.js"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; +import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; + +type Source = { companyId: string; issueId: string; runId: string; controller: string; supervisor?: string }; +const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); + +export function legacyNativeControllerArchive(source: Source) { + const manifest = JSON.stringify({ schema: "paperclip.legacy-controller-archive.v1", companyId: source.companyId, + issueId: source.issueId, runId: source.runId, reason: "original_runner_artifact_unproven", + files: { "control-plane-state.json": sha256(source.controller), + ...(source.supervisor === undefined ? {} : { "supervisor.json": sha256(source.supervisor) }) } }); + return { manifest, scopeDigest: sha256(manifest), manifestSha256: `sha256:${sha256(manifest)}` }; +} + +export async function hasArchivedLegacyNativeController(db: Db, input: Source & { storage: StorageService; signal?: AbortSignal }) { + return verifyDurableNativeHarnessBackup({ db, storage: input.storage, companyId: input.companyId, + ...legacyNativeControllerArchive(input), signal: input.signal }); +} + +export async function archiveFailedLegacyNativeController(db: Db, input: Source & { + storage: StorageService; writersStopped: true; signal: AbortSignal; +}) { + if (await hasArchivedLegacyNativeController(db, input)) return; + const descriptor = legacyNativeControllerArchive(input); + const root = await mkdtemp(join(tmpdir(), "pc-legacy-controller-archive-")); + try { + await writeFile(join(root, "manifest.json"), descriptor.manifest, { mode: 0o600 }); + await writeFile(join(root, "control-plane-state.json"), input.controller, { mode: 0o600 }); + if (input.supervisor !== undefined) await writeFile(join(root, "supervisor.json"), input.supervisor, { mode: 0o600 }); + await persistNativeHarnessBackup({ db, storage: input.storage, root, scopeDigest: descriptor.scopeDigest, signal: input.signal, + owner: { kind: "offline_migration", writersStopped: input.writersStopped, terminalFailureOnly: true, + companyId: input.companyId, issueId: input.issueId, runId: input.runId } }); + } finally { await removeNativeHarnessBackup(root); } +} diff --git a/server/src/services/native-runtime/native-controller-migration.integration.test.ts b/server/src/services/native-runtime/native-controller-migration.integration.test.ts index c0b7829072b..ea5d334e0ec 100644 --- a/server/src/services/native-runtime/native-controller-migration.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-migration.integration.test.ts @@ -3,7 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { eq } from "drizzle-orm"; import { afterEach, expect, it, vi } from "vitest"; -import { agentTaskSessions, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db"; +import { agentTaskSessions, heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db"; import { DurablePrpControlPlane, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; @@ -13,11 +13,15 @@ import { resolveNativeControllerRecovery } from "./native-controller-recovery.js import { resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; import { nativeSha256 } from "./canonical.js"; +import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; +import { createStorageService } from "../../storage/service.js"; +import { legacyNativeControllerArchive } from "./native-controller-archive.js"; +import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; -describeEmbeddedPostgres("offline native controller migration", () => { +describeEmbeddedPostgres.each(["import", "archive", "success"])("offline native controller migration (%s)", scenario => { const environment = nativeRestartFixture(); afterEach(() => vi.unstubAllEnvs()); - it("imports an exact settled authority once, preserves queued work, and resumes from an empty home", async () => { + async function fixture() { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32)); const f = await environment.seedRun(randomUUID()); const { companyId, agentId, paperclipHome, runtimeRoot } = environment.current; @@ -71,6 +75,11 @@ describeEmbeddedPostgres("offline native controller migration", () => { await f.db.update(issues).set({ executionRunId: queuedId }).where(eq(issues.id, f.issueId)); const [session] = await f.db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: "paperclip_runner", taskKey: f.issueId, lastRunId: f.runId }).returning(); + return { ...f, companyId, agentId, paperclipHome, runtimeRoot, paths, file, source, profile, session, queuedId, identity, snapshot }; + } + if (scenario === "import") it("imports an exact settled authority once, preserves queued work, and resumes from an empty home", async () => { + const f = await fixture(); + const { companyId, agentId, paperclipHome, runtimeRoot, profile, file, source, session, queuedId, identity, snapshot } = f; expect(await hasLocalNativeControllerObligations(f.db)).toBe(true); await f.db.update(heartbeatRuns).set({ runnerProfileJson: { ...profile, nativeWorkspaceSync: { ...profile.nativeWorkspaceSync, leaseId: randomUUID() } } }).where(eq(heartbeatRuns.id, f.runId)); @@ -103,4 +112,45 @@ describeEmbeddedPostgres("offline native controller migration", () => { expect(JSON.parse(await readFile(join(replacement, "control-plane/control-plane-state.json"), "utf8")).identity).toEqual(identity); expect(recovered.snapshot).toEqual(snapshot); }); + if (scenario === "archive") it("preserves an already-failed legacy controller without inventing executable recovery authority", async () => { + const f = await fixture(); + await rm(join(f.paths.root, "provider-pack-authority.json")); + await f.db.update(nativeRunFinalizations).set({ recoveryState: "blocked", failureCode: "native_provider_terminal_failed" }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const [originalRun] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [originalOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects"))); + const options = { storage, writersStopped: true as const, signal: AbortSignal.timeout(30_000) }; + const put = vi.spyOn(storage, "putFile").mockRejectedValue(new Error("storage unavailable")); + await expect(migrateNativeControllers(f.db, options)).rejects.toThrow("storage unavailable"); + put.mockRestore(); + expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true); + expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 }); + expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 }); + expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(false); + expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0); + expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(originalRun); + expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(originalOwner); + expect((await f.db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionRunId).toBe(f.queuedId); + expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, f.session.id)))[0].lastRunId).toBe(f.runId); + await rm(f.paperclipHome, { recursive: true, force: true }); + const descriptor = legacyNativeControllerArchive({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, controller: f.source }); + const replacement = join(f.runtimeRoot, "archived-inspection"); + expect(await restoreDurableNativeHarnessBackup({ db: f.db, storage, companyId: f.companyId, + scopeDigest: descriptor.scopeDigest, backupRoot: replacement })).toBe(true); + expect(await readFile(join(replacement, "current", "control-plane-state.json"), "utf8")).toBe(f.source); + const recovery = await resolveNativeControllerRecovery(f.db, { companyId: f.companyId, agentId: f.agentId, + issueId: f.issueId, normalizedSessionId: f.native.normalizedSessionId, currentRunId: f.queuedId }); + expect(recovery.kind).not.toBe("recoverable"); + }); + if (scenario === "success") it("does not substitute a failure archive for a successful retained session", async () => { + const f = await fixture(); + await rm(join(f.paths.root, "provider-pack-authority.json")); + await f.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId)); + const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects"))); + await expect(migrateNativeControllers(f.db, { storage, writersStopped: true, signal: AbortSignal.timeout(30_000) })) + .rejects.toThrow("native_runner_checkpoint_artifact_unproven"); + expect(await f.db.select().from(nativeHarnessBackups)).toHaveLength(0); + expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true); + }); }); diff --git a/server/src/services/native-runtime/native-controller-migration.ts b/server/src/services/native-runtime/native-controller-migration.ts index 830e837b855..005d4af3a48 100644 --- a/server/src/services/native-runtime/native-controller-migration.ts +++ b/server/src/services/native-runtime/native-controller-migration.ts @@ -13,6 +13,9 @@ import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js"; import { providerPackWorkspaceId, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; import { nativeSha256 } from "./canonical.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; +import type { StorageService } from "../../storage/types.js"; +import { durableNativeHarnessStorage } from "./native-harness-backup-storage.js"; +import { archiveFailedLegacyNativeController, hasArchivedLegacyNativeController } from "./native-controller-archive.js"; type Selected = { id: string; companyId: string; agentId: string; issueId: string; sessionId: string; createdAt: Date }; @@ -65,7 +68,7 @@ async function localPaths(source: NonNullable { +export async function hasLocalNativeControllerObligations(db: Db, storage = durableNativeHarnessStorage()): Promise { const roots = await readdir(nativeHarnessStateBase()).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return [] as string[]; throw error; }); @@ -78,15 +81,21 @@ export async function hasLocalNativeControllerObligations(db: Db): Promise { if (error.code === "ENOENT") return undefined; throw error; }); + if (present && source.status === "failed" && storage) { + const controller = await readPrivateSource(paths.controlFile, 256 * 1024 * 1024); + const supervisor = paths.execution.session.lifecyclePolicy.mode === "warm" + ? await readPrivateSource(paths.supervisorPath, 4 * 1024 * 1024) : undefined; + if (controller !== undefined && await hasArchivedLegacyNativeController(db, { ...binding, storage, controller, supervisor })) continue; + } if (present || await hasNativeRunnerCheckpoint(db, binding)) return true; } return false; } -export async function migrateNativeControllers(db: Db, input: { writersStopped: true; signal: AbortSignal }) { +export async function migrateNativeControllers(db: Db, input: { writersStopped: true; signal: AbortSignal; storage?: StorageService }) { const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1); if (!input.writersStopped || active) throw new Error("native_controller_migration_requires_offline_writers"); - const counts = { inspected: 0, imported: 0, unchanged: 0, sourceAlreadyAbsent: 0 }; + const counts = { inspected: 0, imported: 0, unchanged: 0, sourceAlreadyAbsent: 0, archivedFailed: 0 }; for await (const { selected, source } of retainedSources(db, input.signal)) { counts.inspected++; const binding = { companyId: selected.companyId, issueId: selected.issueId, runId: source.id }; @@ -129,10 +138,21 @@ export async function migrateNativeControllers(db: Db, input: { writersStopped: const supervisor = paths.execution.session.lifecyclePolicy.mode === "warm" ? await readPrivateSource(paths.supervisorPath, 4 * 1024 * 1024) : undefined; input.signal.throwIfAborted(); - const result = await importNativeControllerRecovery(db, { ...binding, agentId: selected.agentId, - normalizedSessionId: selected.sessionId, expectedConfigDigest: paths.configDigest, - supervisor: supervisor === undefined ? undefined : JSON.parse(supervisor), - offline: { writersStopped: true, legacyControllerState, legacyRunnerArtifact } }); + let result: Awaited>; + try { + result = await importNativeControllerRecovery(db, { ...binding, agentId: selected.agentId, + normalizedSessionId: selected.sessionId, expectedConfigDigest: paths.configDigest, + supervisor: supervisor === undefined ? undefined : JSON.parse(supervisor), + offline: { writersStopped: true, legacyControllerState, legacyRunnerArtifact } }); + } catch (error) { + const storage = input.storage ?? durableNativeHarnessStorage(); + if (!(error instanceof Error) || error.message !== "native_runner_checkpoint_artifact_unproven" + || source.status !== "failed" || legacyControllerState === undefined || !storage) throw error; + await archiveFailedLegacyNativeController(db, { ...binding, storage, controller: legacyControllerState, supervisor, + writersStopped: true, signal: input.signal }); + counts.archivedFailed++; + continue; + } if (result.kind !== "imported" && result.kind !== "unchanged") { throw new Error(`native_controller_migration_blocked:${source.id}:${"reason" in result ? result.reason : result.kind}`); } diff --git a/server/src/services/native-runtime/native-harness-backup-storage.ts b/server/src/services/native-runtime/native-harness-backup-storage.ts index 629389ea0a6..2754f7d156f 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.ts @@ -24,7 +24,8 @@ type Reference = Binding & { material: Record; manifestSha256: string; }; type Row = typeof nativeHarnessBackups.$inferSelect; -type OfflineOwner = { kind: "offline_migration"; companyId: string; issueId: string; runId: string; writersStopped: true }; +type OfflineOwner = { kind: "offline_migration"; companyId: string; issueId: string; runId: string; + writersStopped: true; terminalFailureOnly?: true }; const fail = (reason: string): never => { throw new Error(`native_harness_storage_${reason}`); }; const aad = (binding: Binding) => Buffer.from(JSON.stringify([SCHEMA, binding.companyId, binding.scopeDigest, binding.id])); const scope = (binding: Pick) => and( @@ -99,11 +100,12 @@ async function fence(tx: Parameters[0]>[0], owner: eq(heartbeatRuns.id, owner.runId), eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.nativeIssueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "native"))).for("share"); const [coordinator] = await tx.select({ leaseOwner: nativeRunFinalizations.leaseOwner, - leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }) + leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, phase: nativeRunFinalizations.phase }) .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, owner.runId)).for("share"); const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`); if (!run?.finishedAt || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(run.status) - || !coordinator || coordinator.leaseOwner && (!coordinator.leaseExpiresAt || coordinator.leaseExpiresAt > new Date(clock.now))) { + || !coordinator || coordinator.leaseOwner && (!coordinator.leaseExpiresAt || coordinator.leaseExpiresAt > new Date(clock.now)) + || owner.terminalFailureOnly && (run.status !== "failed" || coordinator.phase !== "terminal_failure")) { fail("offline_owner_unsettled"); } return;