fix(server): archive failed legacy controllers without inventing recovery authority

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/maintenance/ephemeral-migration-data.ts

@@ -35,10 +35,11 @@ export async function migrateEphemeralServerData(db: Db, input: {
   const storage = getStorageService();
   if (storage.provider !== "s3") throw new Error("shared_object_storage_required");
   phase("native_controllers");
-  const controllers = await migrateNativeControllers(db, { writersStopped: true, signal });
+  const controllers = await migrateNativeControllers(db, { writersStopped: true, signal, storage });
   counts.nativeControllers = controllers.imported;
   counts.nativeControllersUnchanged = controllers.unchanged;
   counts.nativeControllerSourcesAlreadyAbsent = controllers.sourceAlreadyAbsent;
+  counts.nativeControllersArchivedFailed = controllers.archivedFailed;
   phase("native_harness_backups");
   counts.nativeHarnessBackups = await migrateNativeHarnessBackups(db, { storage, writersStopped: true, signal });
   phase("workspaces");

server/src/services/native-runtime/native-controller-archive.ts

@@ -0,0 +1,40 @@
+import { createHash } from "node:crypto";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { Db } from "@paperclipai/db";
+import type { StorageService } from "../../storage/types.js";
+import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js";
+import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";
+
+type Source = { companyId: string; issueId: string; runId: string; controller: string; supervisor?: string };
+const sha256 = (value: string) => createHash("sha256").update(value).digest("hex");
+
+export function legacyNativeControllerArchive(source: Source) {
+  const manifest = JSON.stringify({ schema: "paperclip.legacy-controller-archive.v1", companyId: source.companyId,
+    issueId: source.issueId, runId: source.runId, reason: "original_runner_artifact_unproven",
+    files: { "control-plane-state.json": sha256(source.controller),
+      ...(source.supervisor === undefined ? {} : { "supervisor.json": sha256(source.supervisor) }) } });
+  return { manifest, scopeDigest: sha256(manifest), manifestSha256: `sha256:${sha256(manifest)}` };
+}
+
+export async function hasArchivedLegacyNativeController(db: Db, input: Source & { storage: StorageService; signal?: AbortSignal }) {
+  return verifyDurableNativeHarnessBackup({ db, storage: input.storage, companyId: input.companyId,
+    ...legacyNativeControllerArchive(input), signal: input.signal });
+}
+
+export async function archiveFailedLegacyNativeController(db: Db, input: Source & {
+  storage: StorageService; writersStopped: true; signal: AbortSignal;
+}) {
+  if (await hasArchivedLegacyNativeController(db, input)) return;
+  const descriptor = legacyNativeControllerArchive(input);
+  const root = await mkdtemp(join(tmpdir(), "pc-legacy-controller-archive-"));
+  try {
+    await writeFile(join(root, "manifest.json"), descriptor.manifest, { mode: 0o600 });
+    await writeFile(join(root, "control-plane-state.json"), input.controller, { mode: 0o600 });
+    if (input.supervisor !== undefined) await writeFile(join(root, "supervisor.json"), input.supervisor, { mode: 0o600 });
+    await persistNativeHarnessBackup({ db, storage: input.storage, root, scopeDigest: descriptor.scopeDigest, signal: input.signal,
+      owner: { kind: "offline_migration", writersStopped: input.writersStopped, terminalFailureOnly: true,
+        companyId: input.companyId, issueId: input.issueId, runId: input.runId } });
+  } finally { await removeNativeHarnessBackup(root); }
+}

server/src/services/native-runtime/native-controller-migration.integration.test.ts

@@ -3,7 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { eq } from "drizzle-orm";
 import { afterEach, expect, it, vi } from "vitest";
-import { agentTaskSessions, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
+import { agentTaskSessions, heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
 import { DurablePrpControlPlane, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
 import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js";
 import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
@@ -13,11 +13,15 @@ import { resolveNativeControllerRecovery } from "./native-controller-recovery.js
 import { resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js";
 import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js";
 import { nativeSha256 } from "./canonical.js";
+import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js";
+import { createStorageService } from "../../storage/service.js";
+import { legacyNativeControllerArchive } from "./native-controller-archive.js";
+import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";
 
-describeEmbeddedPostgres("offline native controller migration", () => {
+describeEmbeddedPostgres.each(["import", "archive", "success"])("offline native controller migration (%s)", scenario => {
   const environment = nativeRestartFixture();
   afterEach(() => vi.unstubAllEnvs());
-  it("imports an exact settled authority once, preserves queued work, and resumes from an empty home", async () => {
+  async function fixture() {
     vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32));
     const f = await environment.seedRun(randomUUID());
     const { companyId, agentId, paperclipHome, runtimeRoot } = environment.current;
@@ -71,6 +75,11 @@ describeEmbeddedPostgres("offline native controller migration", () => {
     await f.db.update(issues).set({ executionRunId: queuedId }).where(eq(issues.id, f.issueId));
     const [session] = await f.db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: "paperclip_runner",
       taskKey: f.issueId, lastRunId: f.runId }).returning();
+    return { ...f, companyId, agentId, paperclipHome, runtimeRoot, paths, file, source, profile, session, queuedId, identity, snapshot };
+  }
+  if (scenario === "import") it("imports an exact settled authority once, preserves queued work, and resumes from an empty home", async () => {
+    const f = await fixture();
+    const { companyId, agentId, paperclipHome, runtimeRoot, profile, file, source, session, queuedId, identity, snapshot } = f;
     expect(await hasLocalNativeControllerObligations(f.db)).toBe(true);
     await f.db.update(heartbeatRuns).set({ runnerProfileJson: { ...profile,
       nativeWorkspaceSync: { ...profile.nativeWorkspaceSync, leaseId: randomUUID() } } }).where(eq(heartbeatRuns.id, f.runId));
@@ -103,4 +112,45 @@ describeEmbeddedPostgres("offline native controller migration", () => {
     expect(JSON.parse(await readFile(join(replacement, "control-plane/control-plane-state.json"), "utf8")).identity).toEqual(identity);
     expect(recovered.snapshot).toEqual(snapshot);
   });
+  if (scenario === "archive") it("preserves an already-failed legacy controller without inventing executable recovery authority", async () => {
+    const f = await fixture();
+    await rm(join(f.paths.root, "provider-pack-authority.json"));
+    await f.db.update(nativeRunFinalizations).set({ recoveryState: "blocked", failureCode: "native_provider_terminal_failed" })
+      .where(eq(nativeRunFinalizations.runId, f.runId));
+    const [originalRun] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
+    const [originalOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
+    const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects")));
+    const options = { storage, writersStopped: true as const, signal: AbortSignal.timeout(30_000) };
+    const put = vi.spyOn(storage, "putFile").mockRejectedValue(new Error("storage unavailable"));
+    await expect(migrateNativeControllers(f.db, options)).rejects.toThrow("storage unavailable");
+    put.mockRestore();
+    expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true);
+    expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 });
+    expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 });
+    expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(false);
+    expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0);
+    expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(originalRun);
+    expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(originalOwner);
+    expect((await f.db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionRunId).toBe(f.queuedId);
+    expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, f.session.id)))[0].lastRunId).toBe(f.runId);
+    await rm(f.paperclipHome, { recursive: true, force: true });
+    const descriptor = legacyNativeControllerArchive({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, controller: f.source });
+    const replacement = join(f.runtimeRoot, "archived-inspection");
+    expect(await restoreDurableNativeHarnessBackup({ db: f.db, storage, companyId: f.companyId,
+      scopeDigest: descriptor.scopeDigest, backupRoot: replacement })).toBe(true);
+    expect(await readFile(join(replacement, "current", "control-plane-state.json"), "utf8")).toBe(f.source);
+    const recovery = await resolveNativeControllerRecovery(f.db, { companyId: f.companyId, agentId: f.agentId,
+      issueId: f.issueId, normalizedSessionId: f.native.normalizedSessionId, currentRunId: f.queuedId });
+    expect(recovery.kind).not.toBe("recoverable");
+  });
+  if (scenario === "success") it("does not substitute a failure archive for a successful retained session", async () => {
+    const f = await fixture();
+    await rm(join(f.paths.root, "provider-pack-authority.json"));
+    await f.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId));
+    const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects")));
+    await expect(migrateNativeControllers(f.db, { storage, writersStopped: true, signal: AbortSignal.timeout(30_000) }))
+      .rejects.toThrow("native_runner_checkpoint_artifact_unproven");
+    expect(await f.db.select().from(nativeHarnessBackups)).toHaveLength(0);
+    expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true);
+  });
 });

server/src/services/native-runtime/native-controller-migration.ts

@@ -13,6 +13,9 @@ import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js";
 import { providerPackWorkspaceId, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js";
 import { nativeSha256 } from "./canonical.js";
 import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
+import type { StorageService } from "../../storage/types.js";
+import { durableNativeHarnessStorage } from "./native-harness-backup-storage.js";
+import { archiveFailedLegacyNativeController, hasArchivedLegacyNativeController } from "./native-controller-archive.js";
 
 type Selected = { id: string; companyId: string; agentId: string; issueId: string; sessionId: string; createdAt: Date };
 
@@ -65,7 +68,7 @@ async function localPaths(source: NonNullable<Awaited<ReturnType<typeof findNati
   return { ...descriptor, execution, controlFile: join(control, "control-plane-state.json") };
 }
 
-export async function hasLocalNativeControllerObligations(db: Db): Promise<boolean> {
+export async function hasLocalNativeControllerObligations(db: Db, storage = durableNativeHarnessStorage()): Promise<boolean> {
   const roots = await readdir(nativeHarnessStateBase()).catch((error: NodeJS.ErrnoException) => {
     if (error.code === "ENOENT") return [] as string[]; throw error;
   });
@@ -78,15 +81,21 @@ export async function hasLocalNativeControllerObligations(db: Db): Promise<boole
     const present = await lstat(paths.controlFile).catch((error: NodeJS.ErrnoException) => {
       if (error.code === "ENOENT") return undefined; throw error;
     });
+    if (present && source.status === "failed" && storage) {
+      const controller = await readPrivateSource(paths.controlFile, 256 * 1024 * 1024);
+      const supervisor = paths.execution.session.lifecyclePolicy.mode === "warm"
+        ? await readPrivateSource(paths.supervisorPath, 4 * 1024 * 1024) : undefined;
+      if (controller !== undefined && await hasArchivedLegacyNativeController(db, { ...binding, storage, controller, supervisor })) continue;
+    }
     if (present || await hasNativeRunnerCheckpoint(db, binding)) return true;
   }
   return false;
 }
 
-export async function migrateNativeControllers(db: Db, input: { writersStopped: true; signal: AbortSignal }) {
+export async function migrateNativeControllers(db: Db, input: { writersStopped: true; signal: AbortSignal; storage?: StorageService }) {
   const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1);
   if (!input.writersStopped || active) throw new Error("native_controller_migration_requires_offline_writers");
-  const counts = { inspected: 0, imported: 0, unchanged: 0, sourceAlreadyAbsent: 0 };
+  const counts = { inspected: 0, imported: 0, unchanged: 0, sourceAlreadyAbsent: 0, archivedFailed: 0 };
   for await (const { selected, source } of retainedSources(db, input.signal)) {
     counts.inspected++;
     const binding = { companyId: selected.companyId, issueId: selected.issueId, runId: source.id };
@@ -129,10 +138,21 @@ export async function migrateNativeControllers(db: Db, input: { writersStopped:
     const supervisor = paths.execution.session.lifecyclePolicy.mode === "warm"
       ? await readPrivateSource(paths.supervisorPath, 4 * 1024 * 1024) : undefined;
     input.signal.throwIfAborted();
-    const result = await importNativeControllerRecovery(db, { ...binding, agentId: selected.agentId,
-      normalizedSessionId: selected.sessionId, expectedConfigDigest: paths.configDigest,
-      supervisor: supervisor === undefined ? undefined : JSON.parse(supervisor),
-      offline: { writersStopped: true, legacyControllerState, legacyRunnerArtifact } });
+    let result: Awaited<ReturnType<typeof importNativeControllerRecovery>>;
+    try {
+      result = await importNativeControllerRecovery(db, { ...binding, agentId: selected.agentId,
+        normalizedSessionId: selected.sessionId, expectedConfigDigest: paths.configDigest,
+        supervisor: supervisor === undefined ? undefined : JSON.parse(supervisor),
+        offline: { writersStopped: true, legacyControllerState, legacyRunnerArtifact } });
+    } catch (error) {
+      const storage = input.storage ?? durableNativeHarnessStorage();
+      if (!(error instanceof Error) || error.message !== "native_runner_checkpoint_artifact_unproven"
+        || source.status !== "failed" || legacyControllerState === undefined || !storage) throw error;
+      await archiveFailedLegacyNativeController(db, { ...binding, storage, controller: legacyControllerState, supervisor,
+        writersStopped: true, signal: input.signal });
+      counts.archivedFailed++;
+      continue;
+    }
     if (result.kind !== "imported" && result.kind !== "unchanged") {
       throw new Error(`native_controller_migration_blocked:${source.id}:${"reason" in result ? result.reason : result.kind}`);
     }

server/src/services/native-runtime/native-harness-backup-storage.ts

@@ -24,7 +24,8 @@ type Reference = Binding & {
   material: Record<string, unknown>; manifestSha256: string;
 };
 type Row = typeof nativeHarnessBackups.$inferSelect;
-type OfflineOwner = { kind: "offline_migration"; companyId: string; issueId: string; runId: string; writersStopped: true };
+type OfflineOwner = { kind: "offline_migration"; companyId: string; issueId: string; runId: string;
+  writersStopped: true; terminalFailureOnly?: true };
 const fail = (reason: string): never => { throw new Error(`native_harness_storage_${reason}`); };
 const aad = (binding: Binding) => Buffer.from(JSON.stringify([SCHEMA, binding.companyId, binding.scopeDigest, binding.id]));
 const scope = (binding: Pick<Binding, "companyId" | "scopeDigest">) => and(
@@ -99,11 +100,12 @@ async function fence(tx: Parameters<Parameters<Db["transaction"]>[0]>[0], owner:
       eq(heartbeatRuns.id, owner.runId), eq(heartbeatRuns.companyId, owner.companyId),
       eq(heartbeatRuns.nativeIssueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "native"))).for("share");
     const [coordinator] = await tx.select({ leaseOwner: nativeRunFinalizations.leaseOwner,
-      leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt })
+      leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, phase: nativeRunFinalizations.phase })
       .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, owner.runId)).for("share");
     const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
     if (!run?.finishedAt || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(run.status)
-      || !coordinator || coordinator.leaseOwner && (!coordinator.leaseExpiresAt || coordinator.leaseExpiresAt > new Date(clock.now))) {
+      || !coordinator || coordinator.leaseOwner && (!coordinator.leaseExpiresAt || coordinator.leaseExpiresAt > new Date(clock.now))
+      || owner.terminalFailureOnly && (run.status !== "failed" || coordinator.phase !== "terminal_failure")) {
       fail("offline_owner_unsettled");
     }
     return;