diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 97b22d1da5b..777db388d05 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -148,6 +148,11 @@ executable recovery authority. Foreign, invalid and active sources remain blocke For a finished failed source with a stale observed coordinator, evidence archival also requires an expired controller lease, no accepted result or retry deadline, and no current task ownership. Every remote lease must carry the exact host-owned -termination receipt, confirmed after the failed finish. Recheck those receipts +termination receipt, confirmed after execution started. Recheck those receipts before publication. Preserve the coordinator as recorded; this proof permits evidence storage, not execution or a claim that the coordinator has settled. + +Compare retained turn completion to the follow-up execution start, not its queue +creation time. A queued follow-up can predate the prior turn completion. A remote +stop can also predate a later logical failure caused by recovery timeout. These +proofs use the recorded execution start and never rewrite lifecycle timestamps. diff --git a/server/src/services/native-runtime/native-controller-migration.integration.test.ts b/server/src/services/native-runtime/native-controller-migration.integration.test.ts index 7e280a12fc6..1e71c94b132 100644 --- a/server/src/services/native-runtime/native-controller-migration.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-migration.integration.test.ts @@ -22,9 +22,9 @@ import { legacyNativeControllerArchive } from "./native-controller-archive.js"; import { restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable_failed", - "stale_confirmed", "stale_foreign", "stale_changed", "stale_success", "stale_busy", "stale_durable", + "stale_confirmed", "stale_foreign", "stale_changed", "stale_queued", "stale_overlap", "stale_success", "stale_busy", "stale_durable", "controller_confirmed", "controller_foreign", "controller_thread", "controller_runner", "controller_success", "controller_busy", "controller_invalid", "controller_durable", - "controller_observed_stopped", "controller_observed_missing", "controller_observed_foreign", + "controller_observed_stopped", "controller_observed_stop_before_finish", "controller_observed_missing", "controller_observed_foreign", "controller_observed_live", "controller_observed_result", "controller_observed_retry", "controller_observed_task", "controller_observed_old_receipt", "controller_observed_revoked"])("offline native controller migration (%s)", scenario => { const environment = nativeRestartFixture(); afterEach(() => vi.unstubAllEnvs()); @@ -253,6 +253,12 @@ describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId, runnerInstanceId: f.native.runnerInstanceId, runtimeMode: "native", status: "succeeded", createdAt: new Date(Date.now() - 60_000), finishedAt: new Date(Date.now() - 50_000), runnerProfileJson: priorProfile }); + if (scenario === "stale_queued" || scenario === "stale_overlap") { + const now = Date.now(); + await f.db.update(heartbeatRuns).set({ createdAt: new Date(now - 45_000), + startedAt: new Date(now - (scenario === "stale_queued" ? 30_000 : 43_000)) }).where(eq(heartbeatRuns.id, f.runId)); + await f.db.update(heartbeatRuns).set({ finishedAt: new Date(now - 40_000) }).where(eq(heartbeatRuns.id, priorId)); + } const supervisor = JSON.stringify({ schema: "paperclip.native-session-supervisor.v1", configDigest: f.paths.configDigest, snapshot: priorSnapshot }); await mkdir(dirname(f.paths.supervisorPath), { recursive: true }); @@ -274,7 +280,7 @@ describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable const [originalOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); const storage = createStorageService(createLocalDiskStorageProvider(join(f.runtimeRoot, "durable-objects"))); const options = { storage, writersStopped: true as const, signal: AbortSignal.timeout(30_000) }; - if (scenario !== "stale_confirmed") { + if (!["stale_confirmed", "stale_queued"].includes(scenario)) { await expect(migrateNativeControllers(f.db, options)).rejects.toThrow(); expect(await f.db.select().from(nativeHarnessBackups)).toHaveLength(0); expect(await hasLocalNativeControllerObligations(f.db, storage)).toBe(true); @@ -344,6 +350,13 @@ describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable const lease = { id: randomUUID(), companyId: f.companyId, heartbeatRunId: f.runId, issueId: f.issueId, provider: "daytona", providerLeaseId: randomUUID(), status: "released", cleanupStatus: "success", releasedAt: new Date() }; const receipt = remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "stopped" })!; + if (scenario === "controller_observed_stop_before_finish") { + const now = Date.now(); + await f.db.update(heartbeatRuns).set({ createdAt: new Date(now - 40_000), startedAt: new Date(now - 30_000) }) + .where(eq(heartbeatRuns.id, f.runId)); + receipt.confirmedAt = new Date(now - 10_000).toISOString(); + lease.releasedAt = new Date(now - 10_000); + } if (scenario === "controller_observed_foreign") receipt.runId = randomUUID(); if (scenario === "controller_observed_old_receipt") receipt.confirmedAt = new Date(0).toISOString(); await f.db.insert(environmentLeases).values({ ...lease, metadata: { remoteExecutionTermination: receipt } }); @@ -363,7 +376,7 @@ describeEmbeddedPostgres.each(["import", "archive", "success", "batch", "durable } if (scenario === "controller_durable") { expect(await migrateNativeControllers(f.db, options)).toMatchObject({ unchanged: 1, archivedFailed: 0 }); - } else if (!["controller_confirmed", "controller_observed_stopped"].includes(scenario)) { + } else if (!["controller_confirmed", "controller_observed_stopped", "controller_observed_stop_before_finish"].includes(scenario)) { await expect(migrateNativeControllers(f.db, options)).rejects.toThrow(); const archives = await f.db.select().from(nativeHarnessBackups); if (scenario === "controller_observed_revoked") { diff --git a/server/src/services/native-runtime/native-controller-migration.ts b/server/src/services/native-runtime/native-controller-migration.ts index af15143a400..9dbc9f5a240 100644 --- a/server/src/services/native-runtime/native-controller-migration.ts +++ b/server/src/services/native-runtime/native-controller-migration.ts @@ -43,15 +43,15 @@ async function confirmedOlderSuccessfulSnapshot(db: Db, selected: Selected, eq(heartbeatRuns.agentId, selected.agentId), eq(heartbeatRuns.nativeIssueId, selected.issueId), eq(heartbeatRuns.nativeSessionId, selected.sessionId), eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "succeeded"))); const priorSnapshot = record(prior?.snapshot), priorIdentity = record(priorSnapshot.identity); - if (!prior?.finishedAt || prior.createdAt >= source.createdAt || prior.finishedAt > source.createdAt + if (!prior?.finishedAt || prior.createdAt >= source.createdAt || !source.runnerInstanceId || prior.runnerInstanceId !== source.runnerInstanceId || priorIdentity.runId !== olderRunId || Object.entries(scope).some(([key, value]) => priorIdentity[key] !== value) || priorSnapshot.sessionId !== snapshot.sessionId || priorSnapshot.backendKind !== snapshot.backendKind) return null; - const [failed] = await db.select({ finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns).where(and( + const [failed] = await db.select({ finishedAt: heartbeatRuns.finishedAt, startedAt: heartbeatRuns.startedAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, source.id), eq(heartbeatRuns.companyId, selected.companyId), eq(heartbeatRuns.agentId, selected.agentId), eq(heartbeatRuns.nativeIssueId, selected.issueId), eq(heartbeatRuns.nativeSessionId, selected.sessionId), eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "failed"))); - return failed?.finishedAt ? priorSnapshot : null; + return failed?.finishedAt && prior.finishedAt <= (failed.startedAt ?? source.createdAt) ? priorSnapshot : null; } async function hasConfirmedOlderSupervisor(db: Db, selected: Selected, diff --git a/server/src/services/native-runtime/native-harness-backup-storage.ts b/server/src/services/native-runtime/native-harness-backup-storage.ts index e6b704dbc66..eb5d13e3562 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.ts @@ -99,7 +99,7 @@ async function fence(tx: Parameters[0]>[0], owner: if (!owner.writersStopped) fail("offline_writers_required"); const [active] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1); if (active) fail("offline_writers_required"); - const [run] = await tx.select({ finishedAt: heartbeatRuns.finishedAt, status: heartbeatRuns.status }).from(heartbeatRuns).where(and( + const [run] = await tx.select({ finishedAt: heartbeatRuns.finishedAt, startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt, status: heartbeatRuns.status }).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, owner.runId), eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.nativeIssueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "native"))).for("share"); const [coordinator] = await tx.select({ leaseOwner: nativeRunFinalizations.leaseOwner, @@ -116,7 +116,7 @@ async function fence(tx: Parameters[0]>[0], owner: if (coordinator.phase !== "terminal_failure") { // Evidence archival does not settle or resume this stale coordinator. // Lease expiry alone is insufficient: every remote writer needs its - // exact host-owned stop receipt, recorded after the failed finish. + // exact host-owned stop receipt, recorded after execution began. if (coordinator.phase !== "observed" || coordinator.resultId || coordinator.nextAttemptAt || task.executionRunId === owner.runId) fail("offline_owner_unsettled"); const leases = await tx.select().from(environmentLeases).where(and( @@ -125,7 +125,7 @@ async function fence(tx: Parameters[0]>[0], owner: const receipt = lease.metadata?.remoteExecutionTermination as { confirmedAt?: unknown } | undefined; const confirmedAt = Date.parse(String(receipt?.confirmedAt)); return lease.issueId !== owner.issueId || !hasRemoteTerminationReceipt(lease) - || !Number.isFinite(confirmedAt) || confirmedAt < run.finishedAt!.getTime() || confirmedAt > new Date(clock.now).getTime(); + || !Number.isFinite(confirmedAt) || confirmedAt < (run.startedAt ?? run.createdAt).getTime() || confirmedAt > new Date(clock.now).getTime(); })) fail("offline_owner_unsettled"); } }