fix(runtime): use execution start for archival lifecycle proofs

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md

@@ -148,6 +148,11 @@ executable recovery authority. Foreign, invalid and active sources remain blocke
 For a finished failed source with a stale observed coordinator, evidence archival
 also requires an expired controller lease, no accepted result or retry deadline,
 and no current task ownership. Every remote lease must carry the exact host-owned
-termination receipt, confirmed after the failed finish. Recheck those receipts
+termination receipt, confirmed after execution started. Recheck those receipts
 before publication. Preserve the coordinator as recorded; this proof permits
 evidence storage, not execution or a claim that the coordinator has settled.
+
+Compare retained turn completion to the follow-up execution start, not its queue
+creation time. A queued follow-up can predate the prior turn completion. A remote
+stop can also predate a later logical failure caused by recovery timeout. These
+proofs use the recorded execution start and never rewrite lifecycle timestamps.

server/src/services/native-runtime/native-controller-migration.integration.test.ts

@@ -22,9 +22,9 @@ import { legacyNativeControllerArchive } from "./native-controller-archive.js";
 import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";
 
 describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable_failed",
-  "stale_confirmed", "stale_foreign", "stale_changed", "stale_success", "stale_busy", "stale_durable",
+  "stale_confirmed", "stale_foreign", "stale_changed", "stale_queued", "stale_overlap", "stale_success", "stale_busy", "stale_durable",
   "controller_confirmed", "controller_foreign", "controller_thread", "controller_runner", "controller_success", "controller_busy", "controller_invalid", "controller_durable",
-  "controller_observed_stopped", "controller_observed_missing", "controller_observed_foreign",
+  "controller_observed_stopped", "controller_observed_stop_before_finish", "controller_observed_missing", "controller_observed_foreign",
   "controller_observed_live", "controller_observed_result", "controller_observed_retry", "controller_observed_task", "controller_observed_old_receipt", "controller_observed_revoked"])("offline native controller migration (%s)", scenario => {
   const environment = nativeRestartFixture();
   afterEach(() => vi.unstubAllEnvs());
@@ -253,6 +253,12 @@ describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable
       nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId, runnerInstanceId: f.native.runnerInstanceId,
       runtimeMode: "native", status: "succeeded", createdAt: new Date(Date.now() - 60_000),
       finishedAt: new Date(Date.now() - 50_000), runnerProfileJson: priorProfile });
+    if (scenario === "stale_queued" || scenario === "stale_overlap") {
+      const now = Date.now();
+      await f.db.update(heartbeatRuns).set({ createdAt: new Date(now - 45_000),
+        startedAt: new Date(now - (scenario === "stale_queued" ? 30_000 : 43_000)) }).where(eq(heartbeatRuns.id, f.runId));
+      await f.db.update(heartbeatRuns).set({ finishedAt: new Date(now - 40_000) }).where(eq(heartbeatRuns.id, priorId));
+    }
     const supervisor = JSON.stringify({ schema: "paperclip.native-session-supervisor.v1",
       configDigest: f.paths.configDigest, snapshot: priorSnapshot });
     await mkdir(dirname(f.paths.supervisorPath), { recursive: true });
@@ -274,7 +280,7 @@ describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable
     const [originalOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
     const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects")));
     const options = { storage, writersStopped: true as const, signal: AbortSignal.timeout(30_000) };
-    if (scenario !== "stale_confirmed") {
+    if (!["stale_confirmed", "stale_queued"].includes(scenario)) {
       await expect(migrateNativeControllers(f.db, options)).rejects.toThrow();
       expect(await f.db.select().from(nativeHarnessBackups)).toHaveLength(0);
       expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true);
@@ -344,6 +350,13 @@ describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable
         const lease = { id: randomUUID(), companyId: f.companyId, heartbeatRunId: f.runId, issueId: f.issueId,
           provider: "daytona", providerLeaseId: randomUUID(), status: "released", cleanupStatus: "success", releasedAt: new Date() };
         const receipt = remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "stopped" })!;
+        if (scenario === "controller_observed_stop_before_finish") {
+          const now = Date.now();
+          await f.db.update(heartbeatRuns).set({ createdAt: new Date(now - 40_000), startedAt: new Date(now - 30_000) })
+            .where(eq(heartbeatRuns.id, f.runId));
+          receipt.confirmedAt = new Date(now - 10_000).toISOString();
+          lease.releasedAt = new Date(now - 10_000);
+        }
         if (scenario === "controller_observed_foreign") receipt.runId = randomUUID();
         if (scenario === "controller_observed_old_receipt") receipt.confirmedAt = new Date(0).toISOString();
         await f.db.insert(environmentLeases).values({ ...lease, metadata: { remoteExecutionTermination: receipt } });
@@ -363,7 +376,7 @@ describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable
     }
     if (scenario === "controller_durable") {
       expect(await migrateNativeControllers(f.db, options)).toMatchObject({ unchanged: 1, archivedFailed: 0 });
-    } else if (!["controller_confirmed", "controller_observed_stopped"].includes(scenario)) {
+    } else if (!["controller_confirmed", "controller_observed_stopped", "controller_observed_stop_before_finish"].includes(scenario)) {
       await expect(migrateNativeControllers(f.db, options)).rejects.toThrow();
       const archives = await f.db.select().from(nativeHarnessBackups);
       if (scenario === "controller_observed_revoked") {

server/src/services/native-runtime/native-controller-migration.ts

@@ -43,15 +43,15 @@ async function confirmedOlderSuccessfulSnapshot(db: Db, selected: Selected,
       eq(heartbeatRuns.agentId, selected.agentId), eq(heartbeatRuns.nativeIssueId, selected.issueId),
       eq(heartbeatRuns.nativeSessionId, selected.sessionId), eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "succeeded")));
   const priorSnapshot = record(prior?.snapshot), priorIdentity = record(priorSnapshot.identity);
-  if (!prior?.finishedAt || prior.createdAt >= source.createdAt || prior.finishedAt > source.createdAt
+  if (!prior?.finishedAt || prior.createdAt >= source.createdAt
     || !source.runnerInstanceId || prior.runnerInstanceId !== source.runnerInstanceId
     || priorIdentity.runId !== olderRunId || Object.entries(scope).some(([key, value]) => priorIdentity[key] !== value)
     || priorSnapshot.sessionId !== snapshot.sessionId || priorSnapshot.backendKind !== snapshot.backendKind) return null;
-  const [failed] = await db.select({ finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns).where(and(
+  const [failed] = await db.select({ finishedAt: heartbeatRuns.finishedAt, startedAt: heartbeatRuns.startedAt }).from(heartbeatRuns).where(and(
     eq(heartbeatRuns.id, source.id), eq(heartbeatRuns.companyId, selected.companyId), eq(heartbeatRuns.agentId, selected.agentId),
     eq(heartbeatRuns.nativeIssueId, selected.issueId), eq(heartbeatRuns.nativeSessionId, selected.sessionId),
     eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "failed")));
-  return failed?.finishedAt ? priorSnapshot : null;
+  return failed?.finishedAt && prior.finishedAt <= (failed.startedAt ?? source.createdAt) ? priorSnapshot : null;
 }
 
 async function hasConfirmedOlderSupervisor(db: Db, selected: Selected,

server/src/services/native-runtime/native-harness-backup-storage.ts

@@ -99,7 +99,7 @@ async function fence(tx: Parameters<Parameters<Db["transaction"]>[0]>[0], owner:
     if (!owner.writersStopped) fail("offline_writers_required");
     const [active] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1);
     if (active) fail("offline_writers_required");
-    const [run] = await tx.select({ finishedAt: heartbeatRuns.finishedAt, status: heartbeatRuns.status }).from(heartbeatRuns).where(and(
+    const [run] = await tx.select({ finishedAt: heartbeatRuns.finishedAt, startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt, status: heartbeatRuns.status }).from(heartbeatRuns).where(and(
       eq(heartbeatRuns.id, owner.runId), eq(heartbeatRuns.companyId, owner.companyId),
       eq(heartbeatRuns.nativeIssueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "native"))).for("share");
     const [coordinator] = await tx.select({ leaseOwner: nativeRunFinalizations.leaseOwner,
@@ -116,7 +116,7 @@ async function fence(tx: Parameters<Parameters<Db["transaction"]>[0]>[0], owner:
       if (coordinator.phase !== "terminal_failure") {
         // Evidence archival does not settle or resume this stale coordinator.
         // Lease expiry alone is insufficient: every remote writer needs its
-        // exact host-owned stop receipt, recorded after the failed finish.
+        // exact host-owned stop receipt, recorded after execution began.
         if (coordinator.phase !== "observed" || coordinator.resultId || coordinator.nextAttemptAt
           || task.executionRunId === owner.runId) fail("offline_owner_unsettled");
         const leases = await tx.select().from(environmentLeases).where(and(
@@ -125,7 +125,7 @@ async function fence(tx: Parameters<Parameters<Db["transaction"]>[0]>[0], owner:
           const receipt = lease.metadata?.remoteExecutionTermination as { confirmedAt?: unknown } | undefined;
           const confirmedAt = Date.parse(String(receipt?.confirmedAt));
           return lease.issueId !== owner.issueId || !hasRemoteTerminationReceipt(lease)
-            || !Number.isFinite(confirmedAt) || confirmedAt < run.finishedAt!.getTime() || confirmedAt > new Date(clock.now).getTime();
+            || !Number.isFinite(confirmedAt) || confirmedAt < (run.startedAt ?? run.createdAt).getTime() || confirmedAt > new Date(clock.now).getTime();
         })) fail("offline_owner_unsettled");
       }
     }