Restore suspended remote authority and narrow snapshot checkpoint reads

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 11:54:10 -05:00
1 parent 59ff89a18f
commit 16df54ba90
3 files changed
+41 -8

No files matched your search

@@ -388,8 +388,10 @@ test command and additional warm compatibility cases are running. The paired
300-controller/10,000-task workload now includes actual snapshot persistence.
Its initial host pilot missed throughput gates; an isolated Linux client/database
pilot is underway. Database cost acceptance, distinct-release application startup,
compatible rollback, and native Daytona qualification remain open. Daytona needs
an immutable qualification image; an asynchronous question requests that fixture.
compatible rollback, and native Daytona qualification remain open. The published Daytona qualification image was independently resolved from successful
workflow 40709784515 to `sha256:b38ed051d00fed409092d354eea9ae1b01fb232bcb7d5a187ed763382dda2d0b`
(source `8fe5d5defcdfb55a1b9049a801cfa5498629cebc`). The exact Linux Runner was
extracted from that image. A single five-minute outage Product E2E attempt is running.
No production deployment or complete zero-downtime acceptance is claimed.
Failures retained in the implementation record: sandbox listener denial and host
@@ -415,3 +417,14 @@ Next action: complete the remaining native qualification and compatibility cells
retain reproducible measurements, and prepare the reviewed PR. Full-home application
acceptance remains dependent on workspace/backup durability; no controller-only
test can waive that gate.
Qualification update: the legacy and changed-process-configuration warm cases
exposed a marker left by suspension before authority rotation. Restoring the
committed PRP checkpoint before constructing the replacement backend fixes both;
both real-process cases pass. The initial rerun used the wrong fixture executable
path and failed before provider startup; its log is retained. All 59 control-plane
port tests pass after narrowing the snapshot read to the binding and profile
columns it actually needs. The initial paired 300-controller ABBA comparison
completed every offered operation with no ownership losses, but candidate latency
and CPU increased above the investigation threshold. It is not performance
acceptance. A new paired run will measure the narrower snapshot projection.
@@ -9265,6 +9265,13 @@ async function executePaperclipNativeSessionWithinScope(
persistedWarmSession = null;
} else {
await recovered.session.close({ reason: "recovered warm session requires current process configuration" });
// Closing an externally persisted controller leaves an identity marker,
// not its attachment template. The next backend must rotate the saved
// PRP authority from the committed suspension checkpoint.
const suspendedController = await restoreNativeRunnerCheckpoint(input.db,
remoteWarmRecovery.previousExecution.binding, scopedRunnerdStateRoot(input.execution),
nativeSessionKey(input.execution));
if (!suspendedController) throw new Error("native_runner_checkpoint_missing_after_suspension");
persistWarmNativeCheckpoint(remoteWarmRecovery.previousExecution,
nativeSessionConfigDigest(remoteWarmRecovery.previousExecution, "remote"), await recovered.session.snapshot());
persistedWarmSession = loadWarmNativeCheckpoint(input.execution, warmConfigDigest!, "remote");
@@ -32,7 +32,6 @@ import { publishLiveEvent } from "../live-events.js";
import { nativeSha256 } from "./canonical.js";
import { redactRunnerApiEvent, redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
import { createNativeControllerRecovery, type NativeControllerCompatibility } from "./native-controller-recovery.js";
import { getTableColumns } from "drizzle-orm";
import { lockNativeRunnerCheckpointOwner, validateNativeRunnerCheckpointOwner, inspectNativeRunnerUpgradeHeader, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js";
export interface PaperclipControlPlaneBinding {
@@ -130,7 +129,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
options?.signal.throwIfAborted();
}
#matchesPersistedBinding(run: typeof heartbeatRuns.$inferSelect): boolean {
#matchesPersistedBinding(run: Pick<typeof heartbeatRuns.$inferSelect,
"companyId" | "agentId" | "runtimeMode" | "nativeIssueId" | "nativeSessionId" |
"runnerInstanceId" | "completionContractId" | "completionContractSha256">): boolean {
return run.companyId === this.#binding.companyId
&& run.agentId === this.#binding.agentId
&& run.runtimeMode === "native"
@@ -210,13 +211,25 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
await this.#db.transaction(async (tx) => {
const recovery = this.#nativeRecovery;
const owner = recovery ? await lockNativeRunnerCheckpointOwner(tx, recovery.owner) : null;
const lockedRun = tx.$with("snapshot_run").as(tx.select().from(heartbeatRuns)
const lockedRun = tx.$with("snapshot_run").as(tx.select({
companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId,
nativeSessionId: heartbeatRuns.nativeSessionId, runnerInstanceId: heartbeatRuns.runnerInstanceId,
completionContractId: heartbeatRuns.completionContractId,
completionContractSha256: heartbeatRuns.completionContractSha256,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
}).from(heartbeatRuns)
.where(eq(heartbeatRuns.id, this.#binding.runId)).for("update"));
// Project the locking CTE's tuples, including any update committed while
// the lock waited. A second scan of heartbeat_runs could see an old MVCC tuple.
const runColumns = Object.fromEntries(Object.keys(getTableColumns(heartbeatRuns)).map(key =>
[key, lockedRun[key as keyof typeof heartbeatRuns.$inferSelect]])) as unknown as ReturnType<typeof getTableColumns<typeof heartbeatRuns>>;
const selected = await tx.with(lockedRun).select({ run: runColumns, now: sql<Date>`clock_timestamp()`, ...(recovery ? {
const selected = await tx.with(lockedRun).select({ run: {
companyId: lockedRun.companyId, agentId: lockedRun.agentId,
runtimeMode: lockedRun.runtimeMode, nativeIssueId: lockedRun.nativeIssueId,
nativeSessionId: lockedRun.nativeSessionId, runnerInstanceId: lockedRun.runnerInstanceId,
completionContractId: lockedRun.completionContractId,
completionContractSha256: lockedRun.completionContractSha256,
runnerProfileJson: lockedRun.runnerProfileJson,
}, now: sql<Date>`clock_timestamp()`, ...(recovery ? {
checkpointHeader: sql<Record<string, unknown> | null>`(select material from ${nativeRunnerCheckpoints}
where ${nativeRunnerCheckpoints.runId} = ${this.#binding.runId}
and ${nativeRunnerCheckpoints.companyId} = ${this.#binding.companyId}