From 16df54ba90e4daaa4de3755b9859d425bd971c78 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 11:23:03 -0500 Subject: [PATCH] Restore suspended remote authority and narrow snapshot checkpoint reads Co-Authored-By: Paperclip --- ...026-10-30-native-controller-checkpoints.md | 17 +++++++++++-- .../native-runtime/native-session-executor.ts | 7 ++++++ .../paperclip-control-plane-port.ts | 25 ++++++++++++++----- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/doc/plans/2026-10-30-native-controller-checkpoints.md b/doc/plans/2026-10-30-native-controller-checkpoints.md index 17911af4e84..81223312d9d 100644 --- a/doc/plans/2026-10-30-native-controller-checkpoints.md +++ b/doc/plans/2026-10-30-native-controller-checkpoints.md @@ -388,8 +388,10 @@ test command and additional warm compatibility cases are running. The paired 300-controller/10,000-task workload now includes actual snapshot persistence. Its initial host pilot missed throughput gates; an isolated Linux client/database pilot is underway. Database cost acceptance, distinct-release application startup, -compatible rollback, and native Daytona qualification remain open. Daytona needs -an immutable qualification image; an asynchronous question requests that fixture. +compatible rollback, and native Daytona qualification remain open. The published Daytona qualification image was independently resolved from successful +workflow 40709784515 to `sha256:b38ed051d00fed409092d354eea9ae1b01fb232bcb7d5a187ed763382dda2d0b` +(source `8fe5d5defcdfb55a1b9049a801cfa5498629cebc`). The exact Linux Runner was +extracted from that image. A single five-minute outage Product E2E attempt is running. No production deployment or complete zero-downtime acceptance is claimed. Failures retained in the implementation record: sandbox listener denial and host @@ -415,3 +417,14 @@ Next action: complete the remaining native qualification and compatibility cells retain reproducible measurements, and prepare the reviewed PR. Full-home application acceptance remains dependent on workspace/backup durability; no controller-only test can waive that gate. + +Qualification update: the legacy and changed-process-configuration warm cases +exposed a marker left by suspension before authority rotation. Restoring the +committed PRP checkpoint before constructing the replacement backend fixes both; +both real-process cases pass. The initial rerun used the wrong fixture executable +path and failed before provider startup; its log is retained. All 59 control-plane +port tests pass after narrowing the snapshot read to the binding and profile +columns it actually needs. The initial paired 300-controller ABBA comparison +completed every offered operation with no ownership losses, but candidate latency +and CPU increased above the investigation threshold. It is not performance +acceptance. A new paired run will measure the narrower snapshot projection. diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 71d887736f4..614dca7f0cb 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -9265,6 +9265,13 @@ async function executePaperclipNativeSessionWithinScope( persistedWarmSession = null; } else { await recovered.session.close({ reason: "recovered warm session requires current process configuration" }); + // Closing an externally persisted controller leaves an identity marker, + // not its attachment template. The next backend must rotate the saved + // PRP authority from the committed suspension checkpoint. + const suspendedController = await restoreNativeRunnerCheckpoint(input.db, + remoteWarmRecovery.previousExecution.binding, scopedRunnerdStateRoot(input.execution), + nativeSessionKey(input.execution)); + if (!suspendedController) throw new Error("native_runner_checkpoint_missing_after_suspension"); persistWarmNativeCheckpoint(remoteWarmRecovery.previousExecution, nativeSessionConfigDigest(remoteWarmRecovery.previousExecution, "remote"), await recovered.session.snapshot()); persistedWarmSession = loadWarmNativeCheckpoint(input.execution, warmConfigDigest!, "remote"); diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index e15af14b1a6..18ac88f54e6 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -32,7 +32,6 @@ import { publishLiveEvent } from "../live-events.js"; import { nativeSha256 } from "./canonical.js"; import { redactRunnerApiEvent, redactRunnerApiPayloads } from "./runner-api-log-redaction.js"; import { createNativeControllerRecovery, type NativeControllerCompatibility } from "./native-controller-recovery.js"; -import { getTableColumns } from "drizzle-orm"; import { lockNativeRunnerCheckpointOwner, validateNativeRunnerCheckpointOwner, inspectNativeRunnerUpgradeHeader, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js"; export interface PaperclipControlPlaneBinding { @@ -130,7 +129,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort { options?.signal.throwIfAborted(); } - #matchesPersistedBinding(run: typeof heartbeatRuns.$inferSelect): boolean { + #matchesPersistedBinding(run: Pick): boolean { return run.companyId === this.#binding.companyId && run.agentId === this.#binding.agentId && run.runtimeMode === "native" @@ -210,13 +211,25 @@ export class PaperclipControlPlanePort implements ControlPlanePort { await this.#db.transaction(async (tx) => { const recovery = this.#nativeRecovery; const owner = recovery ? await lockNativeRunnerCheckpointOwner(tx, recovery.owner) : null; - const lockedRun = tx.$with("snapshot_run").as(tx.select().from(heartbeatRuns) + const lockedRun = tx.$with("snapshot_run").as(tx.select({ + companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, + runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId, + nativeSessionId: heartbeatRuns.nativeSessionId, runnerInstanceId: heartbeatRuns.runnerInstanceId, + completionContractId: heartbeatRuns.completionContractId, + completionContractSha256: heartbeatRuns.completionContractSha256, + runnerProfileJson: heartbeatRuns.runnerProfileJson, + }).from(heartbeatRuns) .where(eq(heartbeatRuns.id, this.#binding.runId)).for("update")); // Project the locking CTE's tuples, including any update committed while // the lock waited. A second scan of heartbeat_runs could see an old MVCC tuple. - const runColumns = Object.fromEntries(Object.keys(getTableColumns(heartbeatRuns)).map(key => - [key, lockedRun[key as keyof typeof heartbeatRuns.$inferSelect]])) as unknown as ReturnType>; - const selected = await tx.with(lockedRun).select({ run: runColumns, now: sql`clock_timestamp()`, ...(recovery ? { + const selected = await tx.with(lockedRun).select({ run: { + companyId: lockedRun.companyId, agentId: lockedRun.agentId, + runtimeMode: lockedRun.runtimeMode, nativeIssueId: lockedRun.nativeIssueId, + nativeSessionId: lockedRun.nativeSessionId, runnerInstanceId: lockedRun.runnerInstanceId, + completionContractId: lockedRun.completionContractId, + completionContractSha256: lockedRun.completionContractSha256, + runnerProfileJson: lockedRun.runnerProfileJson, + }, now: sql`clock_timestamp()`, ...(recovery ? { checkpointHeader: sql | null>`(select material from ${nativeRunnerCheckpoints} where ${nativeRunnerCheckpoints.runId} = ${this.#binding.runId} and ${nativeRunnerCheckpoints.companyId} = ${this.#binding.companyId}