test(server): qualify native backup publication and retention recovery

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/native-runtime/native-harness-backup-expiration.ts

@@ -764,7 +764,7 @@ export function nativeHarnessBackupExpirationService(
     }));
     if ("reason" in blocker) return null;
     const retentionMs = Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS;
-    return { id: scopeDigest, ...owner, dueAt: Math.max(stat!.mtimeMs, blocker.closedAt) + retentionMs + 1 };
+    return { id: scopeDigest, ...owner, dueAt: Math.max(stat!.mtime.getTime(), blocker.closedAt) + retentionMs + 1 };
   }
 
   async function backupScopes(after?: string, limit = 100) {

server/src/services/native-runtime/native-harness-backup-publication.test.ts

@@ -0,0 +1,44 @@
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { onNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js";
+import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js";
+
+describe("native harness backup publication", () => {
+  let backupRoot = "";
+  let unsubscribe: (() => void) | undefined;
+  afterEach(async () => {
+    unsubscribe?.(); unsubscribe = undefined;
+    if (backupRoot) await fs.rm(backupRoot, { recursive: true, force: true });
+  });
+  async function fixture() {
+    backupRoot = await fs.mkdtemp(path.join(os.tmpdir(), "native-backup-publication-"));
+    const current = path.join(backupRoot, "current"), pendingRoot = path.join(backupRoot, "pending");
+    await fs.mkdir(current); await fs.mkdir(pendingRoot);
+    await fs.writeFile(path.join(current, "manifest.json"), "saved");
+    await fs.writeFile(path.join(pendingRoot, "manifest.json"), "replacement");
+    return { db: {}, companyId: "company", scopeDigest: "scope", backupRoot, pendingRoot, current };
+  }
+  it("restores the existing backup when its lease stamp fails without publishing a false change", async () => {
+    const input = await fixture(); const changes: unknown[] = [];
+    unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change));
+    await expect(publishNativeHarnessBackup({ ...input, stamp: async current => {
+      expect(await fs.readFile(path.join(current, "manifest.json"), "utf8")).toBe("replacement");
+      throw new Error("stamp transaction rejected");
+    } })).rejects.toThrow("stamp transaction rejected");
+    expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("saved");
+    expect(await fs.readdir(backupRoot)).toEqual(["current"]); expect(changes).toEqual([]);
+  });
+  it("publishes the completed replacement once and removes its predecessor", async () => {
+    const input = await fixture(); const changes: unknown[] = [];
+    unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change));
+    await publishNativeHarnessBackup({ ...input, stamp: async current => {
+      expect(changes).toEqual([]);
+      expect(await fs.readFile(path.join(current, "manifest.json"), "utf8")).toBe("replacement");
+    } });
+    expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("replacement");
+    expect(await fs.readdir(backupRoot)).toEqual(["current"]);
+    expect(changes).toEqual([{ companyId: input.companyId, scopeDigest: input.scopeDigest }]);
+  });
+});

server/src/services/native-runtime/native-harness-backup-publication.ts

@@ -0,0 +1,28 @@
+import { existsSync, renameSync } from "node:fs";
+import { resolve } from "node:path";
+import { noteNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js";
+import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js";
+
+export async function publishNativeHarnessBackup(input: {
+  db: object; companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string;
+  stamp?: (currentRoot: string) => Promise<void>;
+}) {
+  const currentRoot = resolve(input.backupRoot, "current");
+  const previousRoot = resolve(input.backupRoot, "previous");
+  removeNativeHarnessBackup(previousRoot);
+  let movedCurrent = false;
+  if (existsSync(currentRoot)) {
+    renameSync(currentRoot, previousRoot);
+    movedCurrent = true;
+  }
+  try {
+    renameSync(input.pendingRoot, currentRoot);
+    await input.stamp?.(currentRoot);
+  } catch (error) {
+    if (existsSync(currentRoot)) removeNativeHarnessBackup(currentRoot);
+    if (movedCurrent && existsSync(previousRoot) && !existsSync(currentRoot)) renameSync(previousRoot, currentRoot);
+    throw error;
+  }
+  removeNativeHarnessBackup(previousRoot);
+  noteNativeHarnessBackupChange(input.db, { companyId: input.companyId, scopeDigest: input.scopeDigest });
+}

server/src/services/native-runtime/native-session-executor.ts

@@ -1,4 +1,4 @@
-import { noteNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js";
+import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js";
 import { createNativeUsageDelivery } from "./native-usage-delivery.js";
 import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js";
 export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js";
@@ -13277,42 +13277,15 @@ async function createRunnerdBackendWithinSessionClaim(
                 });
                 renameSync(temporaryManifest, manifestPath);
 
-                const currentRoot = resolve(backupRoot, "current");
-                const previousRoot = resolve(backupRoot, "previous");
-                removeNativeHarnessBackup(previousRoot);
-                let movedCurrent = false;
-                if (existsSync(currentRoot)) {
-                  renameSync(currentRoot, previousRoot);
-                  movedCurrent = true;
-                }
-                try {
-                  renameSync(pendingRoot, currentRoot);
-                  if (
-                    remoteTarget?.transport === "sandbox" &&
-                    remoteTarget.leaseId
-                  ) {
+                await publishNativeHarnessBackup({
+                  db: input.db, companyId: input.execution.binding.companyId,
+                  scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), backupRoot, pendingRoot,
+                  stamp: remoteTarget?.transport === "sandbox" && remoteTarget.leaseId ? async currentRoot => {
                     await recordHarnessBackupStampForCurrentLease({
-                      root: currentRoot,
-                      manifest,
-                      bytes: backupSpanAttributes.bytesTransferred,
+                      root: currentRoot, manifest, bytes: backupSpanAttributes.bytesTransferred,
                     });
-                  }
-                } catch (error) {
-                  if (existsSync(currentRoot)) {
-                    removeNativeHarnessBackup(currentRoot);
-                  }
-                  if (
-                    movedCurrent &&
-                    existsSync(previousRoot) &&
-                    !existsSync(currentRoot)
-                  ) {
-                    renameSync(previousRoot, currentRoot);
-                  }
-                  throw error;
-                }
-                removeNativeHarnessBackup(previousRoot);
-                noteNativeHarnessBackupChange(input.db, { companyId: input.execution.binding.companyId,
-                  scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex") });
+                  } : undefined,
+                });
               } finally {
                 removeNativeHarnessBackup(pendingRoot);
               }