PaperClipAI/server/src/services/native-runtime/native-controller-migration.integration.test.ts

import { createHash, randomUUID } from "node:crypto";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { eq } from "drizzle-orm";
import { afterEach, expect, it, vi } from "vitest";
import { agentTaskSessions, environmentLeases, environments, heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
import { captureLegacyDurablePrpCheckpoint, DurablePrpControlPlane, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js";
import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
import { nativeControllerMigrationDescriptor } from "./native-session-executor.js";
import { hasLocalNativeControllerObligations, migrateNativeControllers } from "./native-controller-migration.js";
import { insertMigratedNativeRunnerCheckpoint, loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
import { createNativeControllerRecovery, resolveNativeControllerRecovery } from "./native-controller-recovery.js";
import { resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js";
import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js";
import { createReusableLeaseHandoff } from "../environment-lease-handoff.js";
import { remoteTerminationReceipt } from "../remote-execution-termination.js";
import { nativeSha256 } from "./canonical.js";
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js";
import { createStorageService } from "../../storage/service.js";
import { legacyNativeControllerArchive } from "./native-controller-archive.js";
import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";

describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable_failed",
  "stale_confirmed", "stale_foreign", "stale_changed", "stale_queued", "stale_overlap", "stale_success", "stale_busy", "stale_durable", "stale_durable_absent",
  "stale_durable_runner", "stale_durable_session", "stale_durable_snapshot",
  "controller_confirmed", "controller_foreign", "controller_thread", "controller_runner", "controller_success", "controller_busy", "controller_invalid", "controller_durable",
  "controller_observed_stopped", "controller_observed_stop_before_finish", "controller_observed_missing", "controller_observed_foreign",
  "controller_observed_live", "controller_observed_result", "controller_observed_retry", "controller_observed_task", "controller_observed_old_receipt", "controller_observed_revoked",
  "controller_observed_handoff", "controller_observed_handoff_stopped", "controller_observed_handoff_broken", "controller_observed_handoff_fork"])("offline native controller migration (%s)", scenario => {
  const environment = nativeRestartFixture();
  afterEach(() => vi.unstubAllEnvs());
  async function fixture(commandCount = 1, warm = false) {
    vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32));
    const f = await environment.seedRun(randomUUID());
    const { companyId, agentId, paperclipHome, runtimeRoot } = environment.current;
    const binding = { companyId, agentId, issueId: f.issueId, runId: f.runId, executionWorkspaceId: f.native.environmentLeaseId };
    const execution = parseNativeExecutionInput({
      schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, binding,
      task: { identifier: "MIG-1", title: "Retained work", description: null, prompt: "Continue the work.", workMode: "standard" },
      workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
      session: { normalizedSessionId: f.native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1,
        lifecyclePolicy: warm ? { mode: "warm", idleTimeoutMs: 60_000 } : { mode: "per_turn", idleTimeoutMs: null } },
      completionContract: { id: randomUUID(), sha256: "fixture", schemaVersion: "paperclip.completion-contract.v1",
        contract: { revision: "1", objective: "Continue", criteria: [{ id: "objective", requirement: "Save work." }] } },
      credentialBindings: [], interactionResponses: [],
    });
    const fingerprint = nativeToolContractFingerprintForTarget("remote", "codex");
    const paths = nativeControllerMigrationDescriptor(execution, fingerprint);
    const identity = { runId: f.runId, runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: f.native.normalizedSessionId,
      environmentLeaseId: f.native.environmentLeaseId, turnId: f.native.turnId, itemId: f.native.itemId };
    const core = new DurablePrpControlPlane({ stateDirectory: join(paths.root, "control-plane"), identity,
      expectedRunnerVersion: "migration-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
    core.issueBootstrapTicket();
    for (let index = 0; index < commandCount; index++) core.queueCommand("session.snapshot",
      { diagnostic: "x".repeat(commandCount > 1 ? 6000 : 0) }, `retained-command-${index}`);
    await core.stop();
    const file = join(paths.root, "control-plane/control-plane-state.json");
    const state = JSON.parse(await readFile(file, "utf8"));
    state.tickets = {};
    const source = JSON.stringify(state); await writeFile(file, source, { mode: 0o600 });
    const packBinding = { companyId, agentId, workspaceId: binding.executionWorkspaceId,
      sessionId: f.native.normalizedSessionId, runnerInstanceId: f.native.runnerInstanceId,
      environmentLeaseId: f.native.environmentLeaseId, remoteCwd: "/remote/workspace", providerDigest: nativeSha256(execution.provider) };
    const payload = {
      runnerSourceRevision: "a".repeat(40), target: { platform: "linux", architecture: "x64" },
      pins: { opencode: "1.18.34" }, artifacts: {
        nodeCommand: { path: "node", sha256: `sha256:${"1".repeat(64)}` },
        opencodeCommand: { path: "opencode", sha256: `sha256:${"2".repeat(64)}` },
      },
    };
    const pack = resolveRemoteProviderPackAuthority({ stateDirectory: paths.root, reattach: false, binding: packBinding,
      current: () => ({ schema: "paperclip-runner/remote-provider-pack/v1", payload,
        digest: `sha256:${createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex")}` }),
      runnerArtifact: () => ({ version: "migration-fixture", digest: `sha256:${"a".repeat(64)}` }) });
    pack.verified();
    const snapshot = { backendKind: "mock" as const, sessionId: f.native.normalizedSessionId,
      identity: { companyId, agentId, issueId: f.issueId, runId: f.runId, sessionId: f.native.normalizedSessionId } };
    const profile = { ...f.preparedProfile, nativeExecutionInput: execution, nativeToolContractFingerprint: fingerprint, sessionCheckpoint: snapshot,
      nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v1", state: "prepared", descriptorSha256: "a".repeat(64),
        baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: binding.executionWorkspaceId, leaseId: randomUUID(),
        providerLeaseId: randomUUID(), remoteCwd: "/remote/workspace", resourceDisposition: "destroy" } };
    await f.db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
    await f.db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null }).where(eq(nativeRunFinalizations.runId, f.runId));
    const queuedId = randomUUID();
    await f.db.insert(heartbeatRuns).values({ id: queuedId, companyId, agentId, issueId: f.issueId, status: "queued" });
    await f.db.update(issues).set({ executionRunId: queuedId }).where(eq(issues.id, f.issueId));
    const [session] = await f.db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: "paperclip_runner",
      taskKey: f.issueId, lastRunId: f.runId }).returning();
    return { ...f, companyId, agentId, paperclipHome, runtimeRoot, paths, file, source, profile, session, queuedId, identity, snapshot };
  }
  if (scenario === "import") it("imports an exact settled authority once, preserves queued work, and resumes from an empty home", async () => {
    const f = await fixture();
    const { companyId, agentId, paperclipHome, runtimeRoot, profile, file, source, session, queuedId, identity, snapshot } = f;
    expect(await hasLocalNativeControllerObligations(f.db)).toBe(true);
    await f.db.update(heartbeatRuns).set({ runnerProfileJson: { ...profile,
      nativeWorkspaceSync: { ...profile.nativeWorkspaceSync, workspaceId: randomUUID() } } }).where(eq(heartbeatRuns.id, f.runId));
    await expect(migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) }))
      .rejects.toThrow("native_controller_migration_artifact_binding");
    await f.db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
    expect(profile.nativeWorkspaceSync.leaseId).not.toBe(identity.environmentLeaseId);
    expect(profile.nativeWorkspaceSync.workspaceId).toBe(identity.environmentLeaseId);
    const foreignProtocol = JSON.parse(source);
    foreignProtocol.identity.environmentLeaseId = randomUUID();
    await writeFile(file, JSON.stringify(foreignProtocol), { mode: 0o600 });
    await expect(migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) }))
      .rejects.toThrow("saved provider pack authority is invalid or does not match this execution");
    expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0);
    await writeFile(file, source, { mode: 0o600 });
    await f.db.update(nativeRunFinalizations).set({ leaseOwner: "unsettled-owner" }).where(eq(nativeRunFinalizations.runId, f.runId));
    await expect(migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) })).rejects.toThrow("normal_ownership_recovery_required");
    expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0);
    await f.db.update(nativeRunFinalizations).set({ leaseOwner: "expired-owner", leaseExpiresAt: new Date(0),
      phase: "retryable_failure", recoveryState: "resuming_session" }).where(eq(nativeRunFinalizations.runId, f.runId));
    const [preservedOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
    expect(await migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) }))
      .toMatchObject({ imported: 1, sourceAlreadyAbsent: 0 });
    expect(await migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) })).toMatchObject({ unchanged: 1, imported: 0 });
    expect(await hasLocalNativeControllerObligations(f.db)).toBe(false);
    expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(preservedOwner);
    expect(await readFile(file, "utf8")).toBe(source);
    expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, session.id)))[0].lastRunId).toBe(f.runId);
    expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queuedId)))[0].status).toBe("queued");
    await rm(paperclipHome, { recursive: true, force: true });
    await f.db.update(heartbeatRuns).set({ nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId,
      status: "running", runtimeMode: "native" }).where(eq(heartbeatRuns.id, queuedId));
    const recovered = await resolveNativeControllerRecovery(f.db, { companyId, agentId, issueId: f.issueId,
      normalizedSessionId: f.native.normalizedSessionId, currentRunId: queuedId });
    expect(recovered.kind).toBe("recoverable");
    if (recovered.kind !== "recoverable") throw new Error(JSON.stringify(recovered));
    const replacement = join(runtimeRoot, "replacement"); await mkdir(replacement, { mode: 0o700 });
    recovered.checkpoint.restoreSessionDirectory(replacement);
    expect(JSON.parse(await readFile(join(replacement, "control-plane/control-plane-state.json"), "utf8")).identity).toEqual(identity);
    expect(recovered.snapshot).toEqual(snapshot);
  });
  if (scenario === "batch") it("imports a large checkpoint with bounded writes and rolls back every batch on a late conflict", async () => {
    const f = await fixture(160);
    const binding = { companyId: f.companyId, issueId: f.issueId, runId: f.runId };
    const checkpoint = captureLegacyDurablePrpCheckpoint(f.source, { runId: f.runId,
      runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: f.native.normalizedSessionId },
    { version: "migration-fixture", digest: `sha256:${"a".repeat(64)}` });
    const last = [...checkpoint.records].at(-1)!;
    await f.db.transaction(tx => insertMigratedNativeRunnerCheckpoint(tx, binding, {
      ...checkpoint, records: new Map([last]),
    }));
    const before = await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId));
    const parameters: number[] = [];
    const payloadBytes: number[] = [];
    const previousDebug = f.db.$client.options.debug;
    f.db.$client.options.debug = (_connection, query, values) => {
      if (query.startsWith('insert into "native_runner_checkpoints"')) {
        parameters.push(values.length);
        payloadBytes.push(values.reduce((sum, value) => sum + Buffer.byteLength(JSON.stringify(value)), 0));
      }
    };
    try {
      await expect(f.db.transaction(tx => insertMigratedNativeRunnerCheckpoint(tx, binding, checkpoint)))
        .rejects.toThrow();
      expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toEqual(before);
      expect(parameters.length).toBeGreaterThan(1);
      expect(Math.max(...parameters)).toBeLessThanOrEqual(320);
      expect(Math.max(...payloadBytes)).toBeLessThanOrEqual(512 * 1024);
      await f.db.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId));
      expect(await migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) }))
        .toMatchObject({ imported: 1 });
      const rows = await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId));
      const cipher = createLocalEncryptedCipher("read");
      for (const row of rows) expect(JSON.parse(cipher.decrypt(row.material))).toEqual({
        ...binding, part: row.part, contents: checkpoint.records.get(row.part),
      });
      const restored = await loadNativeRunnerCheckpoint(f.db, binding);
      expect(restored?.records).toEqual(checkpoint.records);
      expect(await readFile(f.file, "utf8")).toBe(f.source);
    } finally { f.db.$client.options.debug = previousDebug; }
  }, 60_000);
  if (scenario === "durable_failed") it("preserves a failed durable checkpoint without inventing a missing session snapshot", async () => {
    const f = await fixture();
    const binding = { companyId: f.companyId, issueId: f.issueId, runId: f.runId };
    const checkpoint = captureLegacyDurablePrpCheckpoint(f.source, { runId: f.runId,
      runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: f.native.normalizedSessionId },
    { version: "migration-fixture", digest: `sha256:${"a".repeat(64)}` });
    await f.db.transaction(tx => insertMigratedNativeRunnerCheckpoint(tx, binding, checkpoint));
    await f.db.update(heartbeatRuns).set({ processPid: 12345, runnerProfileJson: { ...f.profile, sessionCheckpoint: null } })
      .where(eq(heartbeatRuns.id, f.runId));
    const options = { writersStopped: true as const, signal: AbortSignal.timeout(30_000) };
    await f.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId));
    await expect(migrateNativeControllers(f.db, options)).rejects.toThrow("stale_supervisor_snapshot");
    await f.db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.runId));
    const [run] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
    const [owner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
    const rows = await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId));
    const foreign = JSON.parse(f.source);
    foreign.identity.normalizedSessionId = randomUUID();
    const foreignCheckpoint = captureLegacyDurablePrpCheckpoint(JSON.stringify(foreign), { runId: f.runId,
      runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: foreign.identity.normalizedSessionId },
    { version: "migration-fixture", digest: `sha256:${"a".repeat(64)}` });
    await f.db.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId));
    await f.db.transaction(tx => insertMigratedNativeRunnerCheckpoint(tx, binding, foreignCheckpoint));
    expect(await hasLocalNativeControllerObligations(f.db)).toBe(true);
    await expect(migrateNativeControllers(f.db, options)).rejects.toThrow("stale_supervisor_snapshot");
    await f.db.delete(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId));
    await f.db.insert(nativeRunnerCheckpoints).values(rows);
    expect(await migrateNativeControllers(f.db, options)).toMatchObject({ preservedFailed: 1, imported: 0 });
    expect(await hasLocalNativeControllerObligations(f.db)).toBe(false);
    expect(await migrateNativeControllers(f.db, options)).toMatchObject({ preservedFailed: 1, imported: 0 });
    expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toEqual(rows);
    expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(run);
    expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(owner);
    expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, f.session.id)))[0].lastRunId).toBe(f.runId);
    expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.queuedId)))[0].status).toBe("queued");
    await rm(f.paperclipHome, { recursive: true, force: true });
    expect((await loadNativeRunnerCheckpoint(f.db, binding))?.records).toEqual(checkpoint.records);
    const recovery = await resolveNativeControllerRecovery(f.db, { companyId: f.companyId, agentId: f.agentId,
      issueId: f.issueId, normalizedSessionId: f.native.normalizedSessionId, currentRunId: f.queuedId });
    expect(recovery.kind).not.toBe("recoverable");
  });
  if (scenario === "archive") it("preserves an already-failed legacy controller without inventing executable recovery authority", async () => {
    const f = await fixture();
    await rm(join(f.paths.root, "provider-pack-authority.json"));
    await f.db.update(nativeRunFinalizations).set({ recoveryState: "blocked", failureCode: "native_provider_terminal_failed" })
      .where(eq(nativeRunFinalizations.runId, f.runId));
    const [originalRun] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
    const [originalOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
    const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects")));
    const options = { storage, writersStopped: true as const, signal: AbortSignal.timeout(30_000) };
    const put = vi.spyOn(storage, "putFile").mockRejectedValue(new Error("storage unavailable"));
    await expect(migrateNativeControllers(f.db, options)).rejects.toThrow("storage unavailable");
    put.mockRestore();
    expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true);
    expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 });
    expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 });
    expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(false);
    expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0);
    expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(originalRun);
    expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(originalOwner);
    expect((await f.db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionRunId).toBe(f.queuedId);
    expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, f.session.id)))[0].lastRunId).toBe(f.runId);
    await rm(f.paperclipHome, { recursive: true, force: true });
    const descriptor = legacyNativeControllerArchive({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, controller: f.source });
    const replacement = join(f.runtimeRoot, "archived-inspection");
    expect(await restoreDurableNativeHarnessBackup({ db: f.db, storage, companyId: f.companyId,
      scopeDigest: descriptor.scopeDigest, backupRoot: replacement })).toBe(true);
    expect(await readFile(join(replacement, "current", "control-plane-state.json"), "utf8")).toBe(f.source);
    const recovery = await resolveNativeControllerRecovery(f.db, { companyId: f.companyId, agentId: f.agentId,
      issueId: f.issueId, normalizedSessionId: f.native.normalizedSessionId, currentRunId: f.queuedId });
    expect(recovery.kind).not.toBe("recoverable");
  });
  if (scenario.startsWith("stale_")) it("archives a failed source only when its older supervisor matches scoped history", async () => {
    const f = await fixture(1, true), priorId = randomUUID();
    const priorSnapshot = { ...f.snapshot, identity: { ...f.snapshot.identity, runId: priorId } };
    if (scenario === "stale_foreign") priorSnapshot.identity.agentId = randomUUID();
    const priorProfile = { ...f.profile, nativeExecutionInput: { ...f.profile.nativeExecutionInput,
      binding: { ...f.profile.nativeExecutionInput.binding, runId: priorId } }, sessionCheckpoint: priorSnapshot };
    await f.db.insert(heartbeatRuns).values({ id: priorId, companyId: f.companyId, agentId: f.agentId,
      nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId, runnerInstanceId: f.native.runnerInstanceId,
      runtimeMode: "native", status: "succeeded", createdAt: new Date(Date.now() - 60_000),
      finishedAt: new Date(Date.now() - 50_000), runnerProfileJson: priorProfile });
    if (scenario === "stale_queued" || scenario === "stale_overlap") {
      const now = Date.now();
      await f.db.update(heartbeatRuns).set({ createdAt: new Date(now - 45_000),
        startedAt: new Date(now - (scenario === "stale_queued" ? 30_000 : 43_000)) }).where(eq(heartbeatRuns.id, f.runId));
      await f.db.update(heartbeatRuns).set({ finishedAt: new Date(now - 40_000) }).where(eq(heartbeatRuns.id, priorId));
    }
    const supervisor = JSON.stringify({ schema: "paperclip.native-session-supervisor.v1",
      configDigest: f.paths.configDigest, snapshot: priorSnapshot });
    await mkdir(dirname(f.paths.supervisorPath), { recursive: true });
    await writeFile(f.paths.supervisorPath, supervisor, { mode: 0o600 });
    if (scenario === "stale_changed") await f.db.update(heartbeatRuns).set({ runnerProfileJson: {
      ...priorProfile, sessionCheckpoint: { ...priorSnapshot, sessionId: "different-provider-thread" },
    } }).where(eq(heartbeatRuns.id, priorId));
    if (scenario === "stale_success") await f.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId));
    if (scenario === "stale_busy") await f.db.update(nativeRunFinalizations).set({ leaseOwner: "live-owner",
      leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId));
    if (scenario.startsWith("stale_durable")) {
      const state = JSON.parse(f.source);
      if (scenario === "stale_durable_runner") state.identity.runnerInstanceId = randomUUID();
      if (scenario === "stale_durable_session") state.identity.normalizedSessionId = randomUUID();
      const checkpoint = captureLegacyDurablePrpCheckpoint(JSON.stringify(state), { runId: f.runId,
        runnerInstanceId: state.identity.runnerInstanceId, normalizedSessionId: state.identity.normalizedSessionId },
      { version: "migration-fixture", digest: `sha256:${"a".repeat(64)}` });
      await f.db.transaction(tx => insertMigratedNativeRunnerCheckpoint(tx,
        { companyId: f.companyId, issueId: f.issueId, runId: f.runId }, checkpoint));
      if (scenario === "stale_durable_absent") await rm(f.paths.supervisorPath);
      if (scenario === "stale_durable_snapshot") await f.db.update(heartbeatRuns).set({ runnerProfileJson: {
        ...f.profile, sessionCheckpoint: { ...f.snapshot, identity: { ...f.snapshot.identity, agentId: randomUUID() } },
      } }).where(eq(heartbeatRuns.id, f.runId));
    }
    const [originalRun] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
    const [originalOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
    const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects")));
    const options = { storage, writersStopped: true as const, signal: AbortSignal.timeout(30_000) };
    if (scenario === "stale_durable" || scenario === "stale_durable_absent") {
      const binding = { companyId: f.companyId, issueId: f.issueId, runId: f.runId };
      const before = await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId));
      expect(await migrateNativeControllers(f.db, options)).toMatchObject({ imported: 1, archivedFailed: 0 });
      expect(await migrateNativeControllers(f.db, options)).toMatchObject({ unchanged: 1, archivedFailed: 0 });
      expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(false);
      expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toEqual(before);
      expect(await f.db.select().from(nativeHarnessBackups)).toHaveLength(0);
      const [afterRun] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
      expect(afterRun.runnerProfileJson?.sessionCheckpoint).toEqual(originalRun.runnerProfileJson?.sessionCheckpoint);
      expect(afterRun).toEqual({ ...originalRun, runnerProfileJson: afterRun.runnerProfileJson, updatedAt: afterRun.updatedAt });
      expect(afterRun.runnerProfileJson).toEqual({ ...originalRun.runnerProfileJson,
        nativeControllerRecovery: expect.objectContaining({ snapshotSha256: nativeSha256(f.snapshot),
          checkpoint: { runId: f.runId, runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: f.native.normalizedSessionId } }) });
      expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(originalOwner);
      await rm(f.paperclipHome, { recursive: true, force: true });
      expect((await loadNativeRunnerCheckpoint(f.db, binding))?.records.size).toBe(before.length);
      await f.db.update(heartbeatRuns).set({ nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId,
        status: "running", runtimeMode: "native" }).where(eq(heartbeatRuns.id, f.queuedId));
      expect((await resolveNativeControllerRecovery(f.db, { companyId: f.companyId, agentId: f.agentId,
        issueId: f.issueId, normalizedSessionId: f.native.normalizedSessionId, currentRunId: f.queuedId })).kind).toBe("recoverable");
      return;
    }
    if (!["stale_confirmed", "stale_queued"].includes(scenario)) {
      await expect(migrateNativeControllers(f.db, options)).rejects.toThrow();
      expect(await f.db.select().from(nativeHarnessBackups)).toHaveLength(0);
      expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true);
      expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(originalRun);
      expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(originalOwner);
      return;
    }
    const put = vi.spyOn(storage, "putFile").mockRejectedValue(new Error("storage unavailable"));
    await expect(migrateNativeControllers(f.db, options)).rejects.toThrow("storage unavailable");
    put.mockRestore();
    expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 });
    expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 });
    expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(false);
    expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0);
    expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(originalRun);
    expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(originalOwner);
    expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, f.session.id)))[0].lastRunId).toBe(f.runId);
    await rm(f.paperclipHome, { recursive: true, force: true });
    const descriptor = legacyNativeControllerArchive({ companyId: f.companyId, issueId: f.issueId,
      runId: f.runId, controller: f.source, supervisor });
    const replacement = join(f.runtimeRoot, "archived-stale-supervisor");
    expect(await restoreDurableNativeHarnessBackup({ db: f.db, storage, companyId: f.companyId,
      scopeDigest: descriptor.scopeDigest, backupRoot: replacement })).toBe(true);
    expect(await readFile(join(replacement, "current", "control-plane-state.json"), "utf8")).toBe(f.source);
    expect(await readFile(join(replacement, "current", "supervisor.json"), "utf8")).toBe(supervisor);
    expect((await resolveNativeControllerRecovery(f.db, { companyId: f.companyId, agentId: f.agentId,
      issueId: f.issueId, normalizedSessionId: f.native.normalizedSessionId, currentRunId: f.queuedId })).kind).not.toBe("recoverable");
  });
  if (scenario.startsWith("controller_")) it("preserves failed legacy files only with proven older controller lineage", async () => {
    const f = await fixture(), priorId = randomUUID();
    const priorSnapshot = { ...f.snapshot, identity: { ...f.snapshot.identity, runId: priorId } };
    if (scenario === "controller_foreign") priorSnapshot.identity.agentId = randomUUID();
    const priorProfile = { ...f.profile, nativeExecutionInput: { ...f.profile.nativeExecutionInput,
      binding: { ...f.profile.nativeExecutionInput.binding, runId: priorId } }, sessionCheckpoint: priorSnapshot };
    await f.db.insert(heartbeatRuns).values({ id: priorId, companyId: f.companyId,
      agentId: f.agentId,
      nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId,
      runnerInstanceId: scenario === "controller_runner" ? randomUUID() : f.native.runnerInstanceId,
      runtimeMode: "native", status: "succeeded", createdAt: new Date(Date.now() - 60_000),
      finishedAt: new Date(Date.now() - 50_000), runnerProfileJson: scenario === "controller_thread"
        ? { ...priorProfile, sessionCheckpoint: { ...priorSnapshot, sessionId: "foreign-provider-thread" } } : priorProfile });
    const state = JSON.parse(f.source);
    state.identity = { ...state.identity, runId: priorId, turnId: `turn-${priorId}`, itemId: `item-${priorId}` };
    if (scenario === "controller_invalid") state.schema = "unrecognized";
    const source = JSON.stringify(state);
    await writeFile(f.file, source, { mode: 0o600 });
    if (scenario === "controller_success") await f.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId));
    if (scenario === "controller_busy") await f.db.update(nativeRunFinalizations).set({ leaseOwner: "live-owner",
      leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId));
    if (scenario === "controller_durable") {
      const checkpoint = captureLegacyDurablePrpCheckpoint(f.source, f.identity,
        { version: "migration-fixture", digest: `sha256:${"a".repeat(64)}` });
      await f.db.transaction(tx => insertMigratedNativeRunnerCheckpoint(tx,
        { companyId: f.companyId, issueId: f.issueId, runId: f.runId }, checkpoint));
      await f.db.update(heartbeatRuns).set({ runnerProfileJson: { ...f.profile, nativeControllerRecovery:
        createNativeControllerRecovery(f.snapshot, { configDigest: f.paths.configDigest, processConfigDigest: null },
          { runId: f.runId, runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: f.native.normalizedSessionId }) } })
        .where(eq(heartbeatRuns.id, f.runId));
    }
    if (scenario.startsWith("controller_observed_")) {
      await f.db.update(nativeRunFinalizations).set({ phase: scenario === "controller_observed_result" ? "result_accepted" : "observed",
        leaseOwner: "retired-controller", leaseExpiresAt: new Date(Date.now() + (scenario === "controller_observed_live" ? 60_000 : -60_000)),
        nextAttemptAt: scenario === "controller_observed_retry" ? new Date(Date.now() + 60_000) : null,
      }).where(eq(nativeRunFinalizations.runId, f.runId));
      if (scenario === "controller_observed_task") await f.db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issueId));
      if (scenario !== "controller_observed_missing") {
        const lease = { id: randomUUID(), companyId: f.companyId, heartbeatRunId: f.runId, issueId: f.issueId,
          provider: "daytona", providerLeaseId: randomUUID(), status: "released", cleanupStatus: "success", releasedAt: new Date() };
        const receipt = remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "stopped" })!;
        if (scenario === "controller_observed_stop_before_finish") {
          const now = Date.now();
          await f.db.update(heartbeatRuns).set({ createdAt: new Date(now - 40_000), startedAt: new Date(now - 30_000) })
            .where(eq(heartbeatRuns.id, f.runId));
          receipt.confirmedAt = new Date(now - 10_000).toISOString();
          lease.releasedAt = new Date(now - 10_000);
        }
        if (scenario === "controller_observed_foreign") receipt.runId = randomUUID();
        if (scenario === "controller_observed_old_receipt") receipt.confirmedAt = new Date(0).toISOString();
        await f.db.insert(environmentLeases).values({ ...lease, metadata: { remoteExecutionTermination: receipt } });
        if (scenario.startsWith("controller_observed_handoff")) {
          const environmentId = randomUUID(), successorRunId = randomUUID(), handoffAt = new Date(Date.now() - 1000);
          await f.db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" });
          await f.db.insert(heartbeatRuns).values({ id: successorRunId, companyId: f.companyId, agentId: f.agentId,
            nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId, runtimeMode: "native", status: "succeeded", finishedAt: new Date() });
          const [previous] = await f.db.update(environmentLeases).set({ environmentId, leasePolicy: "reuse_by_environment",
            status: "expired", acquiredAt: new Date(handoffAt.getTime() - 1000), releasedAt: handoffAt,
            metadata: { agentId: f.agentId } }).where(eq(environmentLeases.id, lease.id)).returning();
          const successor = { ...previous, id: randomUUID(), heartbeatRunId: successorRunId, acquiredAt: handoffAt,
            status: "released", releasedAt: new Date(), metadata: { agentId: f.agentId } };
          const handoff = createReusableLeaseHandoff(previous, successor);
          const termination = remoteTerminationReceipt(successor, { providerLeaseId: successor.providerLeaseId,
            state: scenario === "controller_observed_handoff_stopped" ? "stopped" : "destroyed" });
          await f.db.update(environmentLeases).set({ metadata: { agentId: f.agentId, reusableLeaseSuccessor: handoff } })
            .where(eq(environmentLeases.id, previous.id));
          await f.db.insert(environmentLeases).values({ ...successor, metadata: { agentId: f.agentId,
            reusableLeasePredecessor: scenario === "controller_observed_handoff_broken" ? { ...handoff, handoffId: randomUUID() } : handoff,
            remoteExecutionTermination: termination } });
          if (scenario === "controller_observed_handoff_fork") await f.db.insert(environmentLeases).values({ ...previous,
            id: randomUUID(), heartbeatRunId: f.queuedId, status: "active", releasedAt: null, metadata: { agentId: f.agentId } });
        }
      }
    }
    const [run] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
    const [owner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
    const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects")));
    const options = { storage, writersStopped: true as const, signal: AbortSignal.timeout(30_000) };
    if (scenario === "controller_observed_revoked") {
      const put = storage.putFile.bind(storage);
      vi.spyOn(storage, "putFile").mockImplementation(async args => {
        const result = await put(args);
        await f.db.update(environmentLeases).set({ metadata: {} }).where(eq(environmentLeases.heartbeatRunId, f.runId));
        return result;
      });
    }
    if (scenario === "controller_durable") {
      expect(await migrateNativeControllers(f.db, options)).toMatchObject({ unchanged: 1, archivedFailed: 0 });
    } else if (!["controller_confirmed", "controller_observed_stopped", "controller_observed_stop_before_finish", "controller_observed_handoff"].includes(scenario)) {
      await expect(migrateNativeControllers(f.db, options)).rejects.toThrow();
      const archives = await f.db.select().from(nativeHarnessBackups);
      if (scenario === "controller_observed_revoked") {
        expect(storage.putFile).toHaveBeenCalledOnce();
        expect(archives).toHaveLength(1);
        expect(archives[0].status).toBe("pending");
      }
      else expect(archives).toHaveLength(0);
    } else {
      const put = vi.spyOn(storage, "putFile").mockRejectedValue(new Error("storage unavailable"));
      await expect(migrateNativeControllers(f.db, options)).rejects.toThrow("storage unavailable");
      put.mockRestore();
      expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 });
      expect(await migrateNativeControllers(f.db, options)).toMatchObject({ archivedFailed: 1, imported: 0 });
      expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(false);
      expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0);
      await rm(f.paperclipHome, { recursive: true, force: true });
      const descriptor = legacyNativeControllerArchive({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, controller: source });
      const replacement = join(f.runtimeRoot, "archived-older-controller");
      expect(await restoreDurableNativeHarnessBackup({ db: f.db, storage, companyId: f.companyId,
        scopeDigest: descriptor.scopeDigest, backupRoot: replacement })).toBe(true);
      expect(await readFile(join(replacement, "current", "control-plane-state.json"), "utf8")).toBe(source);
      expect((await resolveNativeControllerRecovery(f.db, { companyId: f.companyId, agentId: f.agentId,
        issueId: f.issueId, normalizedSessionId: f.native.normalizedSessionId, currentRunId: f.queuedId })).kind).not.toBe("recoverable");
    }
    expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(run);
    expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(owner);
    expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, f.session.id)))[0].lastRunId).toBe(f.runId);
    expect((await f.db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionRunId)
      .toBe(scenario === "controller_observed_task" ? f.runId : f.queuedId);
  });
  if (scenario === "success") it("does not substitute a failure archive for a successful retained session", async () => {
    const f = await fixture();
    await rm(join(f.paths.root, "provider-pack-authority.json"));
    await f.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId));
    const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects")));
    await expect(migrateNativeControllers(f.db, { storage, writersStopped: true, signal: AbortSignal.timeout(30_000) }))
      .rejects.toThrow("native_runner_checkpoint_artifact_unproven");
    expect(await f.db.select().from(nativeHarnessBackups)).toHaveLength(0);
    expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true);
  });
});