diff --git a/doc/plans/2026-10-30-native-controller-checkpoints.md b/doc/plans/2026-10-30-native-controller-checkpoints.md index 8ba19ba1c61..a1178fa340c 100644 --- a/doc/plans/2026-10-30-native-controller-checkpoints.md +++ b/doc/plans/2026-10-30-native-controller-checkpoints.md @@ -515,3 +515,58 @@ the next replacement still continues the same provider history. Prepared-task resumed and failed-read/retry cases also pass for Codex and Claude fixtures. These are scripted-provider component results with remote workspace storage available, not live Daytona or distinct-release acceptance. + + +### Bounded qualification campaign on October 30 + +The follow-up campaign began at 18:33 UTC with an approximately one-hour budget. +Three GPT-6.1 Sol workers own live Daytona, database load, and distinct-reader +replacement/rollback; the coordinator owns integration, artifact builds and review. +The exact native implementation under test is `a0ef3ce7493c8bb76c4766ff007e8a390118d145`. +Qualification commit `da8c7f6e1837ddd896dfe83a7524812fee813487` adds measurement and +installed-journey admission only; its native runtime, Runner, shared and database +production files are byte-identical to that implementation. + +The independent Linux database campaign uses an eight-vCPU, 32 GiB disposable +host, with PostgreSQL and its Node workload each limited to two CPUs and 1536 MiB. +It preserves the existing 10/2/2 request/renewal/checkpoint pool caps per logical +server. Steady and aligned-burst ABBA comparisons passed the existing absolute, +correctness and 10% relative regression gates with 300 controllers and 10,000 +historical tasks. Neither changes snapshot/PRP transaction counts. The separate +claim-capacity probe waits for natural lease expiry, invokes the production +claim path for 300 synthetic executions, restores real encrypted checkpoints +under empty controller directories, and measures current-owner checkpoint and +renewal work. Synthetic provider references do not establish sandbox existence +or authenticated provider reattachment. + +The closed, sanitized measurements and limits are recorded in +`2026-10-30-native-controller-qualification-evidence.json`. The earlier failed, +contended-host measurements remain in `2026-10-30-native-controller-load-evidence.json`. +Lock measurements use calibrated PostgreSQL wait-start timestamps and sampled +duration bounds. Observer traffic contributes to the measured database cost; +short unsampled waits and full HTTP latency remain outside that measurement. + +Live qualification uses protected campaign +[41108162900](https://github.com/paperclipai/paperclip/actions/runs/41108162900). +Its initial catalog timeout occurred before provider allocation after Azure +reported a dependency-cache egress limit. A scoped exact-entry cache invalidation +and targeted retry forced a successful cold install. Only this campaign's +ephemeral image publisher received a larger, delete-on-termination disk; image +identity, source admission and provider authorization gates remain in force. + +Provider-free package builds pin first reader `59ff89a18fe1312b5f0bb908439bbdb1237b0252` +and candidate `a0ef3ce7493c8bb76c4766ff007e8a390118d145` in runs +[41108342354](https://github.com/paperclipai/paperclip/actions/runs/41108342354) and +[41108352138](https://github.com/paperclipai/paperclip/actions/runs/41108352138). +The source-process crash probe independently observed B waiting for A's live +lease and claiming the same execution after natural expiry, with no lease edits. +Its initial all-turns/single-provider-process assertion also crossed completion +contract revision 1 to revision 2. The existing Runner attachment policy can +retire a provider at a changed semantic contract while preserving its conversation; +phase-specific observations distinguish that boundary from same-contract rollback. +Earlier strict failures remain retained until that distinction is demonstrated. + +The final live, reader, installed-artifact, cleanup and CI outcomes are maintained +in [PR #18343](https://github.com/paperclipai/paperclip/pull/18343), with their exact +revisions and evidence limits. Campaign registration, successful artifact builds +and this plan entry do not themselves certify that a live journey passed. diff --git a/doc/plans/2026-10-30-native-controller-claim.patch b/doc/plans/2026-10-30-native-controller-claim.patch new file mode 100644 index 00000000000..0a3fe495bba --- /dev/null +++ b/doc/plans/2026-10-30-native-controller-claim.patch @@ -0,0 +1,170 @@ +--- scripts/benchmark-native-controller-recovery.ts ++++ scripts/benchmark-native-controller-recovery.ts +@@ -14 +14 @@ +-const repoRoot = fileURLToPath(new URL("../", import.meta.url)); ++const repoRoot = path.resolve(required("BENCH_CANDIDATE_ROOT")); +@@ -241,0 +242,112 @@ ++async function claimCapacityPhase(db: any, u: string, items: any[], monitor: any) { ++ const { claimNativeRestartRecoveries } = await load(repoRoot, "server/src/services/native-runtime/native-restart-recovery.ts"); ++ const { loadNativeRunnerCheckpoint } = await load(repoRoot, "server/src/services/native-runtime/native-runner-checkpoint.ts"); ++ const replacement = createDb(u, { maxConnections: 10, applicationName: "benchmark-claim-replacement" }); ++ const successor = { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }; ++ const result: any = { schema: "paperclip.native-controller-claim-capacity/v1", ...provenance, ++ scope: "Production DB claim function against synthetic prepared-workspace references and real persisted snapshots/PRP records. Natural lease expiry; two server database clients. No provider attachment or external sandbox existence claim.", ++ errors: [], claims: [], reads: [], samples: [], sqlByClient: {}, renewedOwnerLosses: 0, restoredCheckpoints: 0, ++ retainedOldControllerLeaseMs: 60000, remoteReconnectGraceMs: 420000, checkpointPoolPerServer: 2 }; ++ const clients = [ ++ ["old_request", db.$client], ["old_liveness", livenessDb(db).$client], ["old_checkpoint", checkpointDb(db).$client], ++ ["new_request", replacement.$client], ["new_liveness", livenessDb(replacement).$client], ["new_checkpoint", checkpointDb(replacement).$client], ++ ]; ++ for (const [name, client] of clients) { result.sqlByClient[name] = 0; client.options.debug = () => { result.sqlByClient[name]++; }; } ++ // Data is persisted by the ordinary snapshot port during the preconditioning phase. ++ const validity = await db.$client.unsafe("select count(*)::int as valid from heartbeat_runs r join native_run_finalizations f on f.run_id=r.id where r.status='running' and r.runner_profile_json->'nativeWorkspaceSync'->>'state'='prepared' and r.runner_profile_json->'sessionCheckpoint'->'identity'->>'runId'=r.id::text and exists(select 1 from native_runner_checkpoints c where c.run_id=r.id)"); ++ assert.equal(validity[0].valid, 300, "All claim inputs need real snapshots/checkpoints and prepared references"); ++ const before = pgResources(), started = performance.now(), requests = new Set>(), renewals: any[] = []; ++ const observed = observeLockWaits(monitor); ++ let tick = 0, offered = 0, completed = 0; ++ const traffic = setInterval(() => { ++ for (let j = 0; j < 20; j++) { ++ const index = tick++, item = items[index % items.length], client = index % 2 ? replacement : db; ++ const at = performance.now(); offered++; ++ const task = loadAgentRunAuthentication(client, item).then(() => { completed++; result.reads.push(performance.now() - at); }) ++ .catch((e: any) => result.errors.push({ source: "request", message: e.message })).finally(() => requests.delete(task)); ++ requests.add(task); ++ } ++ }, 100); // 200 database authentication requests/second, split evenly between servers. ++ let sampling: Promise | null = null; ++ const sample = setInterval(() => { ++ if (sampling) return; ++ sampling = monitor`select application_name,count(*)::int as connections,count(*) filter(where state='active')::int as active from pg_stat_activity where datname=current_database() and application_name!='benchmark-observer' group by application_name` ++ .then((rows: any) => result.samples.push({ elapsedMs: performance.now() - started, rows, postgresMemoryBytes: pgResources().memoryBytes })) ++ .catch((e: any) => result.errors.push({ source: "observer", message: e.message })).finally(() => { sampling = null; }); ++ }, 1000); ++ try { ++ const [lease] = await db.$client.unsafe("select greatest(0,extract(epoch from(max(lease_expires_at)-clock_timestamp()))*1000)::float8 as remaining from native_run_finalizations where lease_owner is not null"); ++ result.naturalExpiryWaitMs = Number(lease.remaining); ++ await sleep(result.naturalExpiryWaitMs + 100); ++ const [expired] = await db.$client.unsafe("select count(*)::int as expired from native_run_finalizations where lease_owner is not null and lease_expires_at<=clock_timestamp()"); ++ assert.equal(expired.expired, 300, "Never rewrite or shorten a controller lease for the probe"); ++ const claimStarted = performance.now(); ++ const dispositions = await claimNativeRestartRecoveries({ db: replacement, controller: successor, restartKind: "hard", ++ runIds: items.map(item => item.runId), limit: 300, ++ onClaim: (claim: any, scope: any) => { ++ result.claims.push({ runId: claim.runId, kind: claim.kind, elapsedMs: performance.now() - claimStarted, generation: claim.controllerGeneration }); ++ renewals.push(startNativeSessionExecutionLeaseRenewal({ db: replacement, ++ companyId: scope.companyId, issueId: scope.issueId, runId: claim.runId, leaseOwner: claim.leaseOwner, ++ attempt: claim.providerAttempt, controller: successor, remote: true, ++ onLost: async () => { result.renewedOwnerLosses++; } })); ++ }, ++ }); ++ result.claimDrainMs = performance.now() - claimStarted; ++ assert.equal(dispositions.length, 300); ++ assert(dispositions.every((claim: any) => claim.kind === "reattach_remote_runner"), "Every DB claim must independently validate its evidence"); ++ // Restore persisted records into empty replacement directories, then persist a real new control and snapshot. ++ const work = [...dispositions]; ++ await Promise.all(Array.from({ length: 4 }, async () => { ++ while (work.length) { ++ const claim: any = work.shift(), item = items.find(i => i.runId === claim.runId); ++ const checkpoint = await loadNativeRunnerCheckpoint(replacement, { companyId: item.companyId, issueId: item.issueId, runId: item.runId }); ++ assert(checkpoint); ++ const directory = root + "/replacement-states/" + item.runId; ++ checkpoint.restoreSessionDirectory(directory); ++ const header = JSON.parse(checkpoint.records.get("header")!); ++ const core = new CandidateCore({ stateDirectory: directory, identity: header.state.identity, ++ expectedRunnerVersion: checkpoint.runnerVersion, expectedRunnerDigest: checkpoint.runnerDigest }); ++ const binding = { companyId: item.companyId, issueId: item.issueId, runId: item.runId, ++ authorityRunId: item.runId, bootId: successor.bootId, generation: claim.controllerGeneration, leaseOwner: claim.leaseOwner }; ++ core.setCheckpointPersistence(CandidateWriter(replacement, binding)); ++ core.queueCommand("semantic_tool.result", { callId: "replacement-capacity", operationId: "fixture_read", result: { body: "x".repeat(4096) }, isError: false }, "replacement-control"); ++ await core.flushCheckpoint(); ++ const identity = item.snapshot.identity; ++ const port = new CandidatePort(replacement, { ...identity, sourceInstanceId: checkpoint.recoveryEvidence.runnerInstanceId, ++ controlPlaneSourceInstanceId: "replacement-controller", completionContractId: item.completionContractId, completionContractSha256: item.completionContractSha256 }, { nativeRecovery: { ++ compatibility: { configDigest: "sha256:" + "a".repeat(64), processConfigDigest: null }, owner: binding } }); ++ await port.openRun({ identity, backendKind: "codex", sourceInstanceId: checkpoint.recoveryEvidence.runnerInstanceId }); ++ await port.checkpointSession(item.snapshot); ++ result.restoredCheckpoints++; ++ await core.stop(); ++ } ++ })); ++ result.claimAndRestoreDrainMs = performance.now() - claimStarted; ++ // Observe one natural 20-second renewal from the last replacement claim. ++ await sleep(Math.max(0, 21000 - (performance.now() - claimStarted - result.claimDrainMs))); ++ const [saved] = await replacement.$client.unsafe("select count(*)::int as owners,min(controller_generation)::int as min_generation,max(controller_generation)::int as max_generation,min(extract(epoch from(lease_expires_at-clock_timestamp()))*1000)::float8 as minimum_remaining_ms from native_run_finalizations where controller_boot_id=$1 and lease_owner is not null", [successor.bootId]); ++ result.postRenewalOwners = saved; ++ assert.equal(saved.owners, 300); assert.equal(saved.min_generation, 2); assert.equal(saved.max_generation, 2); ++ assert(saved.minimum_remaining_ms > 40000, "Replacement leases should have naturally renewed from the database clock"); ++ } catch (e: any) { result.errors.push({ source: "claim-or-restore", message: e.message, stack: e.stack }); } ++ finally { ++ clearInterval(traffic); clearInterval(sample); ++ await Promise.allSettled([...requests]); await sampling; ++ await Promise.allSettled(renewals.map(r => r.stop())); ++ result.lockWait = await observed.stop(); ++ } ++ result.elapsedMs = performance.now() - started; ++ result.requestsOffered = offered; result.requestsCompleted = completed; result.requestLatency = summary(result.reads); delete result.reads; ++ result.postgresCpuUsec = pgResources().cpuUsec - before.cpuUsec; ++ const totalSql = () => Object.values(result.sqlByClient).reduce((a: number, b: any) => a + Number(b), 0); ++ const idleStart = totalSql(); await sleep(1000); result.settledIdleSql = totalSql() - idleStart; ++ result.passed = result.errors.length === 0 && result.lockWait.errors.length === 0 && result.claims.length === 300 ++ && result.restoredCheckpoints === 300 && completed === offered && result.renewedOwnerLosses === 0 ++ && result.claimAndRestoreDrainMs < 420000 && result.requestLatency.p99 < 250 && result.settledIdleSql === 0; ++ await writeFile(root + "/claim-capacity.json", JSON.stringify(result, null, 2) + "\n"); ++ console.log(JSON.stringify({ phase: "claim-capacity-complete", passed: result.passed, claims: result.claims.length, ++ claimDrainMs: result.claimDrainMs, claimAndRestoreDrainMs: result.claimAndRestoreDrainMs, ++ requests: result.requestLatency, errors: result.errors.slice(0, 3) })); ++ if (!result.passed) process.exitCode = 1; ++} ++ +@@ -429,0 +542 @@ ++ runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: randomUUID(), remoteCwd: "/workspace", resourceDisposition: "destroy" } }, +@@ -544,0 +658,2 @@ ++ completionContractId: contractId, ++ completionContractSha256: "fixture", +@@ -905,0 +1021 @@ ++ await claimCapacityPhase(db, u, items, monitor); +@@ -917,6 +1033 @@ +- const profile = [ +- ["baseline", 300], +- ["candidate", 300], +- ["candidate", 300], +- ["baseline", 300], +- ] as const; ++ const profile = [["candidate", 300]] as const; +@@ -953,35 +1064 @@ +- const perOperation = (r: any, key: string) => +- r[key] / (r.normalCompleted + r.normalWritesCompleted + r.controls + r.admissionsCompleted); +- const average = (values: number[]) => values.reduce((a, b) => a + b, 0) / values.length; +- const comparable = +- result.every((r) => r.correctnessPassed) && +- result.every( +- (r) => +- r.controls === result[0].controls && +- r.normalCompleted === result[0].normalCompleted && +- r.normalWritesCompleted === result[0].normalWritesCompleted, +- ); +- const relative = {}; +- for (const key of ["postgresCpuUsec", "walBytes", "requestP99"]) { +- const value = (r: any) => (key === "requestP99" ? r.normalReadSummary.p99 : perOperation(r, key)); +- const baseline = average(result.filter((r) => r.variant === "baseline").map(value)); +- const candidate = average(result.filter((r) => r.variant === "candidate").map(value)); +- relative[key] = { +- baseline, +- candidate, +- increasePercent: comparable ? 100 * (candidate / baseline - 1) : null, +- passed: comparable ? candidate <= baseline * 1.1 : null, +- }; +- } +- Object.assign(report, { +- relative, +- comparable, +- passed: result.every((r) => r.passed) && Object.values(relative).every((r: any) => r.passed), +- measurementLimits: [ +- "Database request latency is not HTTP latency", +- "Pool queue includes client encoding; BEGIN response includes network time", +- "pg_stat_statements execution includes lock waits; lock episodes report sampled duration bounds, with short waits potentially missed", +- "Peak memory is sampled once a second", +- "Use an otherwise idle cluster and host; WAL and cgroup CPU are cluster-wide", +- ], +- }); ++ Object.assign(report, { passed: result.every(r => r.passed), scope: "Preconditioning only for the separate claim-capacity.json report" }); diff --git a/doc/plans/2026-10-30-native-controller-overlap.patch b/doc/plans/2026-10-30-native-controller-overlap.patch new file mode 100644 index 00000000000..4963f5045e2 --- /dev/null +++ b/doc/plans/2026-10-30-native-controller-overlap.patch @@ -0,0 +1,76 @@ +--- scripts/benchmark-native-controller-recovery.ts ++++ scripts/benchmark-native-controller-recovery.ts +@@ -0,0 +1 @@ ++// Temporary qualification probe derived from frozen benchmark SHA256 abaa29d50eb95013836b54b8dacc94c72e080edf24744cc5d291cc5eb1ff2818. +@@ -14 +15 @@ +-const repoRoot = fileURLToPath(new URL("../", import.meta.url)); ++const repoRoot = path.resolve(required("BENCH_CANDIDATE_ROOT")); +@@ -187 +188 @@ +- const byPool = Object.fromEntries(["benchmark-application", "paperclip-controller-lease", "paperclip-runner-checkpoint"].map(pool => { ++ const byPool = Object.fromEntries(["benchmark-application", "benchmark-replacement-application", "paperclip-controller-lease", "paperclip-runner-checkpoint"].map(pool => { +@@ -250,0 +252 @@ ++ const replacementDb = createDb(u, { maxConnections: 10, applicationName: "benchmark-replacement-application" }); +@@ -256,0 +259 @@ ++ const replacementOwner = { ...owner, bootId: randomUUID() }; +@@ -299,0 +303,3 @@ ++ ["replacement_request", replacementDb.$client], ++ ["replacement_liveness", livenessDb(replacementDb).$client], ++ ["replacement_checkpoint", checkpointDb(replacementDb).$client], +@@ -369,0 +376 @@ ++ replacementDb.$client.options.debug = db.$client.options.debug; +@@ -386,0 +394,2 @@ ++ livenessDb(replacementDb).$client.options.debug = livenessDb(db).$client.options.debug; ++ checkpointDb(replacementDb).$client.options.debug = checkpointDb(db).$client.options.debug; +@@ -389,0 +399,2 @@ ++ const controllerDb = i % 2 ? replacementDb : db; ++ const controllerOwner = i % 2 ? replacementOwner : owner; +@@ -443,3 +454,3 @@ +- controllerBootId: owner.bootId, +- controllerPid: owner.pid, +- controllerProcessStartedAt: owner.processStartedAt, ++ controllerBootId: controllerOwner.bootId, ++ controllerPid: controllerOwner.pid, ++ controllerProcessStartedAt: controllerOwner.processStartedAt, +@@ -467 +478 @@ +- const writer = (variant === "candidate" ? CandidateWriter : BaselineWriter)(db, { ++ const writer = (variant === "candidate" ? CandidateWriter : BaselineWriter)(controllerDb, { +@@ -472 +483 @@ +- bootId: owner.bootId, ++ bootId: controllerOwner.bootId, +@@ -511 +522 @@ +- db, ++ controllerDb, +@@ -531 +542 @@ +- bootId: owner.bootId, ++ bootId: controllerOwner.bootId, +@@ -544,0 +556,2 @@ ++ db: controllerDb, ++ owner: controllerOwner, +@@ -592 +605 @@ +- for (let i = 0; i < 100; i++) await loadAgentRunAuthentication(db, items[i % n]); ++ for (let i = 0; i < 100; i++) await loadAgentRunAuthentication(items[i % n].db, items[i % n]); +@@ -639 +652 @@ +- const input = { db, ...item, attempt: 1, controller: owner }; ++ const input = { ...item, attempt: 1, controller: item.owner }; +@@ -664 +677 @@ +- const rows = await loadAgentRunAuthentication(db, item); ++ const rows = await loadAgentRunAuthentication(item.db, item); +@@ -700 +713 @@ +- await db.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, item.issueId)); ++ await item.db.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, item.issueId)); +@@ -713 +726 @@ +- await db.transaction(async (tx: any) => { ++ await (metrics.admissionsOffered % 2 ? replacementDb : db).transaction(async (tx: any) => { +@@ -917,6 +930 @@ +- const profile = [ +- ["baseline", 300], +- ["candidate", 300], +- ["candidate", 300], +- ["baseline", 300], +- ] as const; ++ const profile = [["baseline", 300], ["candidate", 300], ["candidate", 300], ["baseline", 300]] as const; +@@ -929 +937 @@ +- "Paired baseline/candidate, including real PaperclipControlPlanePort snapshot persistence on every control checkpoint. Database/control-plane component stress; real production checkpoint writer and natural remote lease timers. Same synthetic DB request load. Not full HTTP, harness or hosted capacity qualification.", ++ "Two logical servers in one client process, 150 controllers each, separate request/liveness/checkpoint pools at unchanged caps; same aggregate aligned-burst workload. Matched baseline/candidate ABBA pool-overlap component qualification; no takeover latency or provider reattachment claim. Includes real PaperclipControlPlanePort snapshot persistence on every control checkpoint. Database/control-plane component stress; real production checkpoint writer and natural remote lease timers. Same synthetic DB request load. Not full HTTP, harness or hosted capacity qualification.", +@@ -930,0 +939 @@ ++ logicalServers: 2, diff --git a/doc/plans/2026-10-30-native-controller-qualification-evidence.json b/doc/plans/2026-10-30-native-controller-qualification-evidence.json new file mode 100644 index 00000000000..1b8e6f50444 --- /dev/null +++ b/doc/plans/2026-10-30-native-controller-qualification-evidence.json @@ -0,0 +1,3628 @@ +{ + "schema": "paperclip.native-controller-qualification-evidence/v1", + "recordedAtUtc": "2026-10-30T19:08:26.343965+00:00", + "qualification": "Passed isolated PostgreSQL steady, aligned-burst and matched two-logical-server ABBA correctness, absolute and relative gates, plus production database recovery-claim/restore capacity. These are component measurements.", + "baselineRevision": "59e191c368cb3119ff3206621fefa9c5d4ad3110", + "candidateRevision": "a0ef3ce7493c8bb76c4766ff007e8a390118d145", + "measurementHarnessRevision": "da8c7f6e1837ddd896dfe83a7524812fee813487", + "measurementHarnessSha256": "abaa29d50eb95013836b54b8dacc94c72e080edf24744cc5d291cc5eb1ff2818", + "overlapProbeSha256": "fae5f15053eab0eeac15f4689035c7225488e2b8fd9cf1920e3543b71187f25e", + "sourceInputs": [ + { + "path": "server/src/services/native-runtime/native-runner-checkpoint.ts", + "candidateSha256": "d79f0e3ab44c2cea158571d9f0235aba1f9985e1e3c30ae4b347eb3e15772acf", + "baselineSha256": "d798cd0859ca363ae3be33c607f10b4755420d77d796824d1ee9570d0f7a253c" + }, + { + "path": "server/src/services/native-runtime/paperclip-control-plane-port.ts", + "candidateSha256": "b9143eea07733b67eab80e7e000cf1018202b047c755c47533a6679b67d57c51", + "baselineSha256": "a1f807c5d8fbdcb30e966ebf8a4789974f6a8a7086a0056d9351da6e9817e5f2" + }, + { + "path": "packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts", + "candidateSha256": "a8a8bed373cdd18397a762b9f67b04c789d9e5d63208f06c5ebe000faff090ac", + "baselineSha256": "a8a8bed373cdd18397a762b9f67b04c789d9e5d63208f06c5ebe000faff090ac" + }, + { + "path": "server/src/services/native-runtime/native-execution-lease.ts", + "candidateSha256": "03e70c33342ce505011ff883f8f67892e3ac5f53c58eb758316b67b4b0b2b386", + "baselineSha256": "03e70c33342ce505011ff883f8f67892e3ac5f53c58eb758316b67b4b0b2b386" + } + ], + "environment": { + "host": "One isolated disposable AWS m7i.2xlarge Linux x86_64 VM", + "hostVcpus": 8, + "hostMemoryGiB": 32, + "hostSwapMiB": 0, + "nodeVersion": "v24.22.2", + "postgresMajor": 17, + "pnpmVersion": "9.15.4", + "postgresCpuLimit": 2, + "postgresMemoryLimitMiB": 1536, + "nodeCpuLimit": 2, + "nodeMemoryLimitMiB": 1536, + "perServerPoolLimits": { + "request": 10, + "liveness": 2, + "checkpoint": 2 + }, + "observerConnectionLimit": 1, + "leaseDurationMs": 60000, + "remoteReconnectGraceMs": 420000 + }, + "fixture": { + "historicalTasks": 10000, + "controllers": 300, + "phaseDurationSeconds": 65, + "controlIntervalSeconds": 5, + "controlPayloadBytes": 4096, + "snapshotsPerControl": 1, + "mainProfileOrder": [ + "baseline", + "candidate", + "candidate", + "baseline" + ], + "baselineAndCandidateUseIdenticalDependencyBytes": true, + "normalRequestAndLeaseRenewalImplementationsAreCommonControls": true + }, + "profiles": { + "steady": { + "schema": "paperclip.normal-recovery-load/v1", + "scope": "Paired baseline/candidate, including real PaperclipControlPlanePort snapshot persistence on every control checkpoint. Database/control-plane component stress; real production checkpoint writer and natural remote lease timers. Same synthetic DB request load. Not full HTTP, harness or hosted capacity qualification.", + "hardware": { + "postgresCpuLimit": 2, + "postgresMemoryLimitMiB": 1536, + "mainPool": 10, + "livenessPool": 2, + "candidateCheckpointPool": 2 + }, + "workload": { + "mode": "steady", + "normalReadsPerSecond": 500, + "normalWritesPerSecond": 100, + "databaseAdmissionEverySeconds": 5, + "checkpointControlEverySeconds": 5, + "controlPayloadBytes": 4096, + "phaseSeconds": 65, + "snapshotTransactionsPerControl": 1, + "baselineCheckpointPool": 2, + "baselineRenewalIntervalSeconds": 20, + "candidateRenewalIntervalSeconds": 20, + "historicalIssues": 10000 + }, + "passed": true, + "comparable": true, + "measurementLimits": [ + "Database request latency is not HTTP latency", + "Pool queue includes client encoding; BEGIN response includes network time", + "pg_stat_statements execution includes lock waits; lock episodes report sampled duration bounds, with short waits potentially missed", + "Peak memory is sampled once a second", + "Use an otherwise idle cluster and host; WAL and cgroup CPU are cluster-wide" + ], + "sourceReport": "/tmp/native-controller-load-qualification-20261030/remote/evidence/steady/report.json", + "relative": { + "postgresCpuUsec": { + "baseline": 656.5912310954723, + "candidate": 664.3645165800573, + "increasePercent": 1.183885059143397, + "passed": true + }, + "walBytes": { + "baseline": 1532.2154125789389, + "candidate": 1559.9179735744415, + "increasePercent": 1.8080069400212562, + "passed": true + }, + "requestP99": { + "baseline": 91.71616899999935, + "candidate": 93.46717100001115, + "increasePercent": 1.9091530087915087, + "passed": true + } + }, + "results": [ + { + "label": "01_baseline_300", + "variant": "baseline", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:43:31.121Z", + "endedAt": "2026-10-30T18:44:36.144Z", + "normalOffered": 32500, + "normalCompleted": 32500, + "normalWritesOffered": 6500, + "normalWritesCompleted": 6500, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 54687, + "livenessSqlCount": 4502, + "checkpointSqlCount": 15601, + "walBytes": 67048632, + "postgresCpuUsec": 28159533, + "postgresPeakSampledMemoryBytes": 917352448, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 19.461555999994744, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 32500, + "p50": 23.630726999996114, + "p95": 104.30216500000097, + "p99": 107.72847300001013, + "max": 111.46659400001226 + }, + "normalWriteSummary": { + "n": 6500, + "p50": 28.412395999999717, + "p95": 109.15322800001013, + "p99": 112.54311900000903, + "max": 117.06165799999872 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 14.350610333334771, + "p95": 21.62447166666243, + "p99": 42.645023666656925, + "max": 92.58285566666018 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 4.567198000004282, + "p95": 6.6246809999938705, + "p99": 11.499447000001965, + "max": 34.80159999999887 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 4.406659000000218, + "p95": 8.00715200000559, + "p99": 12.106628999998065, + "max": 81.07146300000022 + }, + "poolQueueByPool": { + "request": { + "n": 3913, + "p50": 0.031578000001900364, + "p95": 3.2165359999999055, + "p99": 6.9602370000066, + "max": 65.06650499999887 + }, + "liveness": { + "n": 900, + "p50": 222.9600139999966, + "p95": 408.67597899999964, + "p99": 436.3592290000015, + "max": 446.41857600000367 + }, + "checkpoint": { + "n": 3900, + "p50": 0.03176600000006147, + "p95": 0.044882999995024875, + "p99": 0.18860399999903166, + "max": 17.319057000000612 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 3913, + "p50": 4.37169300000096, + "p95": 8.198765000001004, + "p99": 12.514188999994076, + "max": 81.02461400000175 + }, + "liveness": { + "n": 900, + "p50": 226.25227900000027, + "p95": 412.00253099999827, + "p99": 440.3486919999996, + "max": 448.6205390000032 + }, + "checkpoint": { + "n": 3900, + "p50": 4.4467920000024606, + "p95": 6.499621000002662, + "p99": 11.388421999999991, + "max": 34.62909299999956 + } + }, + "sqlAllApplicationPools": 74790, + "sampledPeakDatabaseActivityRows": 15, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2563, + "gapsMs": { + "n": 2563, + "p50": 25.121846000001824, + "p95": 26.47124900000199, + "p99": 28.41609900000185, + "max": 86.58039899999858 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2563, + "total_exec_time": 222.99437100000054, + "rows": 0 + } + ], + "nodeCpu": { + "user": 20498304, + "system": 1761522 + }, + "nodeCpuAverageOneCorePercent": 34.23372797056988, + "eventLoopDelay": { + "p95Ms": 11.337727, + "p99Ms": 12.173311, + "maxMs": 70.975487 + }, + "peakSampledNodeRssBytes": 829227008, + "peakSampledNodeHeapBytes": 579761248, + "postgresStatementExecutionMs": 7781.774002, + "lockObserverExecutionMs": 222.99437100000054 + }, + { + "label": "02_candidate_300", + "variant": "candidate", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:44:42.986Z", + "endedAt": "2026-10-30T18:45:48.008Z", + "normalOffered": 32500, + "normalCompleted": 32500, + "normalWritesOffered": 6500, + "normalWritesCompleted": 6500, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 54687, + "livenessSqlCount": 4502, + "checkpointSqlCount": 15601, + "walBytes": 67046184, + "postgresCpuUsec": 28544913, + "postgresPeakSampledMemoryBytes": 901398528, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 18.60484000001452, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 32500, + "p50": 52.137395999990986, + "p95": 73.59120199998142, + "p99": 80.18543700000737, + "max": 160.65012499998556 + }, + "normalWriteSummary": { + "n": 6500, + "p50": 56.79107799999474, + "p95": 79.06722500000615, + "p99": 124.89146799998707, + "max": 164.84574299998349 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 14.875081999998656, + "p95": 22.424880666658282, + "p99": 59.884479666667175, + "max": 174.85218533332227 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 4.532260000007227, + "p95": 6.7100529999879654, + "p99": 14.624643999995897, + "max": 70.33155800000532 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 4.792860999994446, + "p95": 8.380057000002125, + "p99": 14.55555699999968, + "max": 96.5399219999963 + }, + "poolQueueByPool": { + "request": { + "n": 3913, + "p50": 0.03197399998316541, + "p95": 3.4958060000208206, + "p99": 8.42655800000648, + "max": 86.68744899999001 + }, + "liveness": { + "n": 900, + "p50": 226.07839499998954, + "p95": 631.9207260000112, + "p99": 687.6313799999916, + "max": 700.4564780000073 + }, + "checkpoint": { + "n": 3900, + "p50": 0.03252500000235159, + "p95": 0.04554400000779424, + "p99": 1.5021660000056727, + "max": 30.859597000002395 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 3913, + "p50": 4.757719000001089, + "p95": 8.515767000004416, + "p99": 14.811911999990116, + "max": 96.50402399999439 + }, + "liveness": { + "n": 900, + "p50": 228.28790900000604, + "p95": 634.5756129999936, + "p99": 689.5229259999905, + "max": 703.2544249999919 + }, + "checkpoint": { + "n": 3900, + "p50": 4.425221999990754, + "p95": 6.599863999988884, + "p99": 14.496323999992455, + "max": 70.18957200000295 + } + }, + "sqlAllApplicationPools": 74790, + "sampledPeakDatabaseActivityRows": 14, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2564, + "gapsMs": { + "n": 2564, + "p50": 25.082806999998866, + "p95": 26.16317899999558, + "p99": 28.29036499999347, + "max": 96.75551200000336 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2564, + "total_exec_time": 222.22588199999979, + "rows": 0 + } + ], + "nodeCpu": { + "user": 21447274, + "system": 1827953 + }, + "nodeCpuAverageOneCorePercent": 35.79597645613477, + "eventLoopDelay": { + "p95Ms": 10.903551, + "p99Ms": 12.877823, + "maxMs": 70.778879 + }, + "peakSampledNodeRssBytes": 861179904, + "peakSampledNodeHeapBytes": 612108352, + "postgresStatementExecutionMs": 7869.057230000005, + "lockObserverExecutionMs": 222.22588199999979 + }, + { + "label": "03_candidate_300", + "variant": "candidate", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:45:55.253Z", + "endedAt": "2026-10-30T18:47:00.271Z", + "normalOffered": 32500, + "normalCompleted": 32500, + "normalWritesOffered": 6500, + "normalWritesCompleted": 6500, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 54687, + "livenessSqlCount": 4502, + "checkpointSqlCount": 15601, + "walBytes": 66835336, + "postgresCpuUsec": 28474836, + "postgresPeakSampledMemoryBytes": 890142720, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 14.864391999988584, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 32500, + "p50": 70.75364599999739, + "p95": 101.21604299999308, + "p99": 106.74890500001493, + "max": 184.49763100000564 + }, + "normalWriteSummary": { + "n": 6500, + "p50": 75.31148000000394, + "p95": 106.2007309999899, + "p99": 112.02943600001163, + "max": 189.8124850000022 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 14.700123333343072, + "p95": 22.93642300000647, + "p99": 74.96976966666989, + "max": 167.43050899999798 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 4.546646000002511, + "p95": 6.599335000006249, + "p99": 16.56861200000276, + "max": 66.67637499998091 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 4.787251000001561, + "p95": 9.346571000001859, + "p99": 17.659020999999484, + "max": 89.22574900000473 + }, + "poolQueueByPool": { + "request": { + "n": 3913, + "p50": 0.032805000024382025, + "p95": 3.5023479999799747, + "p99": 7.330482000019401, + "max": 22.141313000000082 + }, + "liveness": { + "n": 900, + "p50": 229.3470420000085, + "p95": 658.2537230000016, + "p99": 692.6467640000046, + "max": 699.1289490000054 + }, + "checkpoint": { + "n": 3900, + "p50": 0.034085999999661, + "p95": 0.048119000013684854, + "p99": 4.801466999982949, + "max": 22.820584999979474 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 3913, + "p50": 4.7497089999960735, + "p95": 9.576310999982525, + "p99": 17.61802799999714, + "max": 89.19793199998094 + }, + "liveness": { + "n": 900, + "p50": 232.34173700000974, + "p95": 665.4881870000099, + "p99": 694.4492489999975, + "max": 701.2261050000088 + }, + "checkpoint": { + "n": 3900, + "p50": 4.438666999980342, + "p95": 6.487386000022525, + "p99": 16.5198870000022, + "max": 66.55283199998667 + } + }, + "sqlAllApplicationPools": 74790, + "sampledPeakDatabaseActivityRows": 14, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2560, + "gapsMs": { + "n": 2560, + "p50": 25.087980000011157, + "p95": 26.18886699998984, + "p99": 29.31080700000166, + "max": 87.73278699998627 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2560, + "total_exec_time": 228.186316, + "rows": 0 + } + ], + "nodeCpu": { + "user": 21036535, + "system": 1821778 + }, + "nodeCpuAverageOneCorePercent": 35.15690455164375, + "eventLoopDelay": { + "p95Ms": 11.026431, + "p99Ms": 13.180927, + "maxMs": 74.579967 + }, + "peakSampledNodeRssBytes": 867966976, + "peakSampledNodeHeapBytes": 609559232, + "postgresStatementExecutionMs": 7841.367742000012, + "lockObserverExecutionMs": 228.186316 + }, + { + "label": "04_baseline_300", + "variant": "baseline", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:47:06.695Z", + "endedAt": "2026-10-30T18:48:11.712Z", + "normalOffered": 32500, + "normalCompleted": 32500, + "normalWritesOffered": 6500, + "normalWritesCompleted": 6500, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 54687, + "livenessSqlCount": 4502, + "checkpointSqlCount": 15601, + "walBytes": 64455288, + "postgresCpuUsec": 28193066, + "postgresPeakSampledMemoryBytes": 876048384, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 14.899754000012763, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 32500, + "p50": 44.329154999984894, + "p95": 69.53714800003218, + "p99": 75.70386499998858, + "max": 144.0518920000177 + }, + "normalWriteSummary": { + "n": 6500, + "p50": 49.25532100000419, + "p95": 75.08631300000707, + "p99": 122.02264599999762, + "max": 148.6692589999875 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 14.625869666662766, + "p95": 21.669017333333613, + "p99": 61.56695066665998, + "max": 185.71976100001484 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 4.66447999997763, + "p95": 6.80966199998511, + "p99": 16.252097999997204, + "max": 80.11221200000728 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 4.480548000021372, + "p95": 8.228377999999793, + "p99": 14.50026900001103, + "max": 103.96219200000633 + }, + "poolQueueByPool": { + "request": { + "n": 3913, + "p50": 0.032313000003341585, + "p95": 3.0398300000233576, + "p99": 7.722563000017544, + "max": 90.98564000002807 + }, + "liveness": { + "n": 900, + "p50": 225.77165700000478, + "p95": 577.020279999997, + "p99": 636.1102529999916, + "max": 644.8716770000174 + }, + "checkpoint": { + "n": 3900, + "p50": 0.033404999994672835, + "p95": 0.04764499998418614, + "p99": 3.443251999997301, + "max": 24.120947999996133 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 3913, + "p50": 4.446676999999909, + "p95": 8.55906799994409, + "p99": 14.462981000018772, + "max": 103.91490999999223 + }, + "liveness": { + "n": 900, + "p50": 228.31017700000666, + "p95": 580.3901179999812, + "p99": 639.0160339999711, + "max": 646.8926890000002 + }, + "checkpoint": { + "n": 3900, + "p50": 4.548077000014018, + "p95": 6.678080999990925, + "p99": 15.536747000005562, + "max": 79.98216100002173 + } + }, + "sqlAllApplicationPools": 74790, + "sampledPeakDatabaseActivityRows": 14, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2555, + "gapsMs": { + "n": 2555, + "p50": 25.13867300003767, + "p95": 26.488039000018034, + "p99": 28.65206400002353, + "max": 90.49584500002675 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2555, + "total_exec_time": 226.8500740000005, + "rows": 0 + } + ], + "nodeCpu": { + "user": 21150924, + "system": 1781032 + }, + "nodeCpuAverageOneCorePercent": 35.27067846892734, + "eventLoopDelay": { + "p95Ms": 11.354111, + "p99Ms": 12.230655, + "maxMs": 70.909951 + }, + "peakSampledNodeRssBytes": 869142528, + "peakSampledNodeHeapBytes": 618486624, + "postgresStatementExecutionMs": 7824.61821599999, + "lockObserverExecutionMs": 226.8500740000005 + } + ] + }, + "burst": { + "schema": "paperclip.normal-recovery-load/v1", + "scope": "Paired baseline/candidate, including real PaperclipControlPlanePort snapshot persistence on every control checkpoint. Database/control-plane component stress; real production checkpoint writer and natural remote lease timers. Same synthetic DB request load. Not full HTTP, harness or hosted capacity qualification.", + "hardware": { + "postgresCpuLimit": 2, + "postgresMemoryLimitMiB": 1536, + "mainPool": 10, + "livenessPool": 2, + "candidateCheckpointPool": 2 + }, + "workload": { + "mode": "burst", + "normalReadsPerSecond": 100, + "normalWritesPerSecond": 20, + "databaseAdmissionEverySeconds": 5, + "checkpointControlEverySeconds": 5, + "controlPayloadBytes": 4096, + "phaseSeconds": 65, + "snapshotTransactionsPerControl": 1, + "baselineCheckpointPool": 2, + "baselineRenewalIntervalSeconds": 20, + "candidateRenewalIntervalSeconds": 20, + "historicalIssues": 10000 + }, + "passed": true, + "comparable": true, + "measurementLimits": [ + "Database request latency is not HTTP latency", + "Pool queue includes client encoding; BEGIN response includes network time", + "pg_stat_statements execution includes lock waits; lock episodes report sampled duration bounds, with short waits potentially missed", + "Peak memory is sampled once a second", + "Use an otherwise idle cluster and host; WAL and cgroup CPU are cluster-wide" + ], + "sourceReport": "/tmp/native-controller-load-qualification-20261030/remote/evidence/burst/report.json", + "relative": { + "postgresCpuUsec": { + "baseline": 1212.0604029710578, + "candidate": 1253.290446512422, + "increasePercent": 3.4016492445672952, + "passed": true + }, + "walBytes": { + "baseline": 4214.911636643046, + "candidate": 4322.979253820542, + "increasePercent": 2.563935533973982, + "passed": true + }, + "requestP99": { + "baseline": 99.50419749999128, + "candidate": 103.2992319999903, + "increasePercent": 3.813944130346214, + "passed": true + } + }, + "results": [ + { + "label": "01_baseline_300", + "variant": "baseline", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:48:31.444Z", + "endedAt": "2026-10-30T18:49:37.203Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23487, + "livenessSqlCount": 4502, + "checkpointSqlCount": 15601, + "walBytes": 50551328, + "postgresCpuUsec": 14274208, + "postgresPeakSampledMemoryBytes": 987574272, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 754.7449020000058, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 74.4481569999989, + "p95": 98.20989800000098, + "p99": 101.2616299999936, + "max": 169.48861500000203 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 76.8531340000045, + "p95": 100.53923999999824, + "p99": 103.82765900000231, + "max": 171.57443800000328 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 504.43135499999335, + "p95": 904.9732890000014, + "p99": 987.5739109999995, + "max": 1058.7055260000052 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 464.0462219999972, + "p95": 860.9761490000092, + "p99": 942.788827999997, + "max": 998.9880149999954 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 5.402625000002445, + "p95": 9.567354999999225, + "p99": 32.887894999999844, + "max": 80.82659199999762 + }, + "poolQueueByPool": { + "request": { + "n": 3913, + "p50": 0.01833500000066124, + "p95": 0.03541399999812711, + "p99": 1.1884140000038315, + "max": 35.858127999992575 + }, + "liveness": { + "n": 900, + "p50": 577.4616719999976, + "p95": 935.9824230000013, + "p99": 967.3116440000013, + "max": 986.1326500000141 + }, + "checkpoint": { + "n": 3900, + "p50": 458.1738780000014, + "p95": 856.6399959999981, + "p99": 936.7588849999993, + "max": 995.3760920000059 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 3913, + "p50": 5.375102999998489, + "p95": 9.639744000000064, + "p99": 39.09893100000045, + "max": 90.62846400000126 + }, + "liveness": { + "n": 900, + "p50": 582.0743040000016, + "p95": 940.3565129999988, + "p99": 971.0303759999952, + "max": 991.3499540000048 + }, + "checkpoint": { + "n": 3900, + "p50": 463.99203199999465, + "p95": 860.9391810000088, + "p99": 942.7536680000012, + "max": 998.9510200000077 + } + }, + "sqlAllApplicationPools": 43590, + "sampledPeakDatabaseActivityRows": 14, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2555, + "gapsMs": { + "n": 2555, + "p50": 25.1262850000021, + "p95": 26.189194999999017, + "p99": 62.625012000000424, + "max": 86.79922200000146 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2555, + "total_exec_time": 216.60417100000035, + "rows": 0 + } + ], + "nodeCpu": { + "user": 18625390, + "system": 871929 + }, + "nodeCpuAverageOneCorePercent": 29.649551657605677, + "eventLoopDelay": { + "p95Ms": 10.977279, + "p99Ms": 12.943359, + "maxMs": 70.647807 + }, + "peakSampledNodeRssBytes": 809156608, + "peakSampledNodeHeapBytes": 509942048, + "postgresStatementExecutionMs": 3523.919157, + "lockObserverExecutionMs": 216.60417100000035 + }, + { + "label": "02_candidate_300", + "variant": "candidate", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:49:43.659Z", + "endedAt": "2026-10-30T18:50:49.402Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23487, + "livenessSqlCount": 4502, + "checkpointSqlCount": 15601, + "walBytes": 50596552, + "postgresCpuUsec": 14685071, + "postgresPeakSampledMemoryBytes": 990670848, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 739.15390200002, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 66.31109799999103, + "p95": 99.20572600000014, + "p99": 102.6184629999916, + "max": 152.73279199999524 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 69.19254099999671, + "p95": 101.30469299998367, + "p99": 151.40168099998846, + "max": 156.07804199999373 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 545.065419999999, + "p95": 943.5618349999859, + "p99": 1053.6890769999882, + "max": 1114.0595559999929 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 503.06268100001034, + "p95": 899.0765079999983, + "p99": 1003.1476519999997, + "max": 1050.1976159999904 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 5.609040000010282, + "p95": 10.175828000006732, + "p99": 66.40549299999839, + "max": 77.22545900000841 + }, + "poolQueueByPool": { + "request": { + "n": 3913, + "p50": 0.018979999993462116, + "p95": 0.03818100001080893, + "p99": 1.3919429999950808, + "max": 60.06078600001638 + }, + "liveness": { + "n": 900, + "p50": 618.575686000011, + "p95": 1052.3832659999898, + "p99": 1074.6600569999719, + "max": 1082.593469000014 + }, + "checkpoint": { + "n": 3900, + "p50": 498.0827250000002, + "p95": 893.5146509999904, + "p99": 996.6987400000071, + "max": 1045.0579469999939 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 3913, + "p50": 5.584780999997747, + "p95": 10.301430999999866, + "p99": 66.71016699999745, + "max": 79.74655799999891 + }, + "liveness": { + "n": 900, + "p50": 625.6686590000027, + "p95": 1054.6769740000018, + "p99": 1076.872750999988, + "max": 1084.9572240000125 + }, + "checkpoint": { + "n": 3900, + "p50": 503.0248439999996, + "p95": 899.0387380000029, + "p99": 1003.1150250000064, + "max": 1050.135020000016 + } + }, + "sqlAllApplicationPools": 43590, + "sampledPeakDatabaseActivityRows": 14, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2542, + "gapsMs": { + "n": 2542, + "p50": 25.161112000001594, + "p95": 26.290963999999803, + "p99": 69.67558000001009, + "max": 85.49304400000256 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2542, + "total_exec_time": 216.69468099999958, + "rows": 0 + } + ], + "nodeCpu": { + "user": 17900804, + "system": 947711 + }, + "nodeCpuAverageOneCorePercent": 28.670043958722392, + "eventLoopDelay": { + "p95Ms": 11.100159, + "p99Ms": 12.918783, + "maxMs": 71.368703 + }, + "peakSampledNodeRssBytes": 870387712, + "peakSampledNodeHeapBytes": 576073168, + "postgresStatementExecutionMs": 3486.1393980000007, + "lockObserverExecutionMs": 216.69468099999958 + }, + { + "label": "03_candidate_300", + "variant": "candidate", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:50:56.635Z", + "endedAt": "2026-10-30T18:52:02.382Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23487, + "livenessSqlCount": 4502, + "checkpointSqlCount": 15601, + "walBytes": 50673560, + "postgresCpuUsec": 14674511, + "postgresPeakSampledMemoryBytes": 973410304, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 744.6864390000119, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 65.08340400000452, + "p95": 101.02233700000215, + "p99": 103.98000099998899, + "max": 108.83414200000698 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 75.7316179999907, + "p95": 103.83602499999688, + "p99": 107.78453299999819, + "max": 111.01445499999681 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 496.9523610000033, + "p95": 920.7745010000071, + "p99": 1049.187115000008, + "max": 1091.9349159999983 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 457.1240420000104, + "p95": 873.8922820000153, + "p99": 995.4707909999997, + "max": 1029.6669490000058 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 5.59725800002343, + "p95": 9.960343999991892, + "p99": 63.98731100000441, + "max": 81.85869700001786 + }, + "poolQueueByPool": { + "request": { + "n": 3913, + "p50": 0.019010999996680766, + "p95": 0.034908000001451, + "p99": 1.545111999992514, + "max": 38.025777999981074 + }, + "liveness": { + "n": 900, + "p50": 624.3187389999803, + "p95": 1033.8756909999938, + "p99": 1092.2892289999872, + "max": 1099.7822960000194 + }, + "checkpoint": { + "n": 3900, + "p50": 451.83978900001966, + "p95": 868.8492180000176, + "p99": 990.5199409999768, + "max": 1025.1646750000073 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 3913, + "p50": 5.570027000008849, + "p95": 10.061665000015637, + "p99": 64.13720699999249, + "max": 81.82990699997754 + }, + "liveness": { + "n": 900, + "p50": 630.1176869999908, + "p95": 1036.091023999994, + "p99": 1094.1707950000127, + "max": 1101.9002290000208 + }, + "checkpoint": { + "n": 3900, + "p50": 457.09078199998476, + "p95": 873.8558170000033, + "p99": 995.4385109999857, + "max": 1029.6382080000185 + } + }, + "sqlAllApplicationPools": 43590, + "sampledPeakDatabaseActivityRows": 14, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2544, + "gapsMs": { + "n": 2544, + "p50": 25.161667999986093, + "p95": 26.483623999985866, + "p99": 65.51908900000853, + "max": 86.48127200000454 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 1, + "observedLowerMs": { + "n": 1, + "p50": 0.694, + "p95": 0.694, + "p99": 0.694, + "max": 0.694 + }, + "estimatedUpperMs": { + "n": 1, + "p50": 26.656120999978157, + "p95": 26.656120999978157, + "p99": 26.656120999978157, + "max": 26.656120999978157 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 2, + "observedLowerMs": { + "n": 2, + "p50": 3.427, + "p95": 3.427, + "p99": 3.427, + "max": 3.427 + }, + "estimatedUpperMs": { + "n": 2, + "p50": 81.2373620000144, + "p95": 81.2373620000144, + "p99": 81.2373620000144, + "max": 81.2373620000144 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2544, + "total_exec_time": 218.81579299999981, + "rows": 3 + } + ], + "nodeCpu": { + "user": 18483524, + "system": 895970 + }, + "nodeCpuAverageOneCorePercent": 29.475632426669986, + "eventLoopDelay": { + "p95Ms": 11.198463, + "p99Ms": 12.804095, + "maxMs": 74.317823 + }, + "peakSampledNodeRssBytes": 871841792, + "peakSampledNodeHeapBytes": 566295128, + "postgresStatementExecutionMs": 3610.8802799999994, + "lockObserverExecutionMs": 218.81579299999981 + }, + { + "label": "04_baseline_300", + "variant": "baseline", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:52:08.252Z", + "endedAt": "2026-10-30T18:53:14.009Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23487, + "livenessSqlCount": 4502, + "checkpointSqlCount": 15601, + "walBytes": 48187192, + "postgresCpuUsec": 14119519, + "postgresPeakSampledMemoryBytes": 961527808, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 753.365232000011, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 70.38637200000812, + "p95": 94.09619499999098, + "p99": 97.74676499998895, + "max": 185.41928999999072 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 72.84908000001451, + "p95": 96.762837999966, + "p99": 105.79805099999066, + "max": 190.5021940000006 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 472.87740499997744, + "p95": 851.2413069999893, + "p99": 927.7114349999465, + "max": 1017.7665159999742 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 425.16228799999226, + "p95": 785.2056810000213, + "p99": 846.040737000003, + "max": 879.5945269999793 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 5.294340000022203, + "p95": 9.09644999995362, + "p99": 13.077560000005178, + "max": 79.4648959999904 + }, + "poolQueueByPool": { + "request": { + "n": 3913, + "p50": 0.01867399999173358, + "p95": 0.03246000001672655, + "p99": 1.0189679999893997, + "max": 38.69638399995165 + }, + "liveness": { + "n": 900, + "p50": 450.48504699999467, + "p95": 818.5160719999694, + "p99": 846.6014129999967, + "max": 852.2024820000515 + }, + "checkpoint": { + "n": 3900, + "p50": 420.15820399997756, + "p95": 780.1903570000432, + "p99": 842.1725940000033, + "max": 876.1292810000014 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 3913, + "p50": 5.267995000001974, + "p95": 9.177220999990823, + "p99": 13.902025000017602, + "max": 80.02995799999917 + }, + "liveness": { + "n": 900, + "p50": 455.3887499999837, + "p95": 824.6480000000156, + "p99": 848.961249999993, + "max": 854.1895970000187 + }, + "checkpoint": { + "n": 3900, + "p50": 425.0832320000045, + "p95": 785.1541730000172, + "p99": 845.993524999998, + "max": 879.5596040000091 + } + }, + "sqlAllApplicationPools": 43590, + "sampledPeakDatabaseActivityRows": 14, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2567, + "gapsMs": { + "n": 2567, + "p50": 25.124033000000054, + "p95": 26.08253999997396, + "p99": 50.32741100003477, + "max": 183.68320700002369 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 1, + "observedLowerMs": { + "n": 1, + "p50": 0.264, + "p95": 0.264, + "p99": 0.264, + "max": 0.264 + }, + "estimatedUpperMs": { + "n": 1, + "p50": 25.689116000023205, + "p95": 25.689116000023205, + "p99": 25.689116000023205, + "max": 25.689116000023205 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2567, + "total_exec_time": 212.36981699999987, + "rows": 1 + } + ], + "nodeCpu": { + "user": 17769478, + "system": 879767 + }, + "nodeCpuAverageOneCorePercent": 28.36105586844359, + "eventLoopDelay": { + "p95Ms": 10.936319, + "p99Ms": 12.083199, + "maxMs": 177.602559 + }, + "peakSampledNodeRssBytes": 872235008, + "peakSampledNodeHeapBytes": 588085240, + "postgresStatementExecutionMs": 3513.288256999997, + "lockObserverExecutionMs": 212.36981699999987 + } + ] + }, + "overlap-burst": { + "schema": "paperclip.normal-recovery-load/v1", + "scope": "Two logical servers in one client process, 150 controllers each, separate request/liveness/checkpoint pools at unchanged caps; same aggregate aligned-burst workload. Component overlap capacity only; no takeover latency or provider reattachment claim. Includes real PaperclipControlPlanePort snapshot persistence on every control checkpoint. Database/control-plane component stress; real production checkpoint writer and natural remote lease timers. Same synthetic DB request load. Not full HTTP, harness or hosted capacity qualification.", + "hardware": { + "logicalServers": 2, + "postgresCpuLimit": 2, + "postgresMemoryLimitMiB": 1536, + "mainPool": 10, + "livenessPool": 2, + "candidateCheckpointPool": 2 + }, + "workload": { + "mode": "burst", + "normalReadsPerSecond": 100, + "normalWritesPerSecond": 20, + "databaseAdmissionEverySeconds": 5, + "checkpointControlEverySeconds": 5, + "controlPayloadBytes": 4096, + "phaseSeconds": 65, + "snapshotTransactionsPerControl": 1, + "baselineCheckpointPool": 2, + "baselineRenewalIntervalSeconds": 20, + "candidateRenewalIntervalSeconds": 20, + "historicalIssues": 10000 + }, + "passed": true, + "comparable": false, + "measurementLimits": [ + "Two logical server clients share one two-CPU Node process; this is a bounded component pool-overlap probe", + "Each logical server retains its own 10 request, 2 liveness, and 2 checkpoint connection limits", + "Same aggregate workload as the one-server aligned burst; no actual takeover or provider reattachment latency", + "Lock wait uses sampled PostgreSQL waitstart ages; waits shorter than polling gaps may be missed", + "Observer nominally adds 40 queries/second, included in database CPU", + "Database request latency is not HTTP latency; PostgreSQL CPU/WAL are cluster-wide" + ], + "sourceReport": "/tmp/native-controller-load-qualification-20261030/remote/evidence/overlap-burst/report.json", + "relative": null, + "results": [ + { + "label": "01_candidate_300", + "variant": "candidate", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:54:12.653Z", + "endedAt": "2026-10-30T18:55:18.280Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23494, + "livenessSqlCount": 4504, + "checkpointSqlCount": 15602, + "walBytes": 58953856, + "postgresCpuUsec": 16207490, + "postgresPeakSampledMemoryBytes": 1184854016, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 622.7349329999997, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 71.72571099999186, + "p95": 99.64835199999652, + "p99": 103.41825999999128, + "max": 167.0082839999959 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 73.87743999999657, + "p95": 101.96136399998795, + "p99": 106.03463099998771, + "max": 167.02244699999574 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 510.7785539999895, + "p95": 847.7751439999993, + "p99": 955.7120819999982, + "max": 1019.8632959999959 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 464.80313800000295, + "p95": 799.8753710000019, + "p99": 903.230091999998, + "max": 973.0980429999981 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 8.885602000002109, + "p95": 18.187949999999546, + "p99": 75.0605080000023, + "max": 103.59090300000025 + }, + "poolQueueByPool": { + "request": { + "n": 1956, + "p50": 0.01652999999350868, + "p95": 0.032654000009642914, + "p99": 0.5533529999956954, + "max": 32.65080599999783 + }, + "liveness": { + "n": 450, + "p50": 679.932338999999, + "p95": 983.9729049999951, + "p99": 1005.0673800000004, + "max": 1008.1818359999961 + }, + "checkpoint": { + "n": 1950, + "p50": 455.7207139999955, + "p95": 789.6575360000061, + "p99": 879.0804459999927, + "max": 927.088631999999 + }, + "replacement_request": { + "n": 1957, + "p50": 0.016338000001269393, + "p95": 0.030588000001444016, + "p99": 0.04346399999485584, + "max": 34.07667899999069 + }, + "replacement_liveness": { + "n": 450, + "p50": 680.664604000005, + "p95": 985.0139530000015, + "p99": 1007.0503020000033, + "max": 1013.1946360000002 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 451.09980499999074, + "p95": 795.8284449999992, + "p99": 905.3483259999994, + "max": 968.1059260000038 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 1956, + "p50": 8.849019000001135, + "p95": 18.71429999999964, + "p99": 79.22411199999988, + "max": 103.53132499999992 + }, + "liveness": { + "n": 450, + "p50": 690.4811160000027, + "p95": 989.9459690000003, + "p99": 1008.2045910000015, + "max": 1011.9318149999963 + }, + "checkpoint": { + "n": 1950, + "p50": 465.5724979999941, + "p95": 798.1944409999996, + "p99": 891.0923289999992, + "max": 935.9049690000029 + }, + "replacement_request": { + "n": 1957, + "p50": 8.899836000011419, + "p95": 20.206529000002774, + "p99": 72.87767399999575, + "max": 96.72780800001055 + }, + "replacement_liveness": { + "n": 450, + "p50": 690.242895000003, + "p95": 991.1967210000003, + "p99": 1010.9223639999982, + "max": 1015.3685380000024 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 461.74945899999875, + "p95": 805.4827579999983, + "p99": 914.226066999996, + "max": 973.0625819999987 + } + }, + "sqlAllApplicationPools": 43600, + "sampledPeakDatabaseActivityRows": 26, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2532, + "gapsMs": { + "n": 2532, + "p50": 25.151506999998674, + "p95": 26.541017999999895, + "p99": 66.61091300000408, + "max": 87.0732219999918 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "benchmark-replacement-application": { + "observedEpisodes": 2, + "observedLowerMs": { + "n": 2, + "p50": 1.992, + "p95": 1.992, + "p99": 1.992, + "max": 1.992 + }, + "estimatedUpperMs": { + "n": 2, + "p50": 77.9257440000074, + "p95": 77.9257440000074, + "p99": 77.9257440000074, + "max": 77.9257440000074 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 4, + "observedLowerMs": { + "n": 4, + "p50": 2.681, + "p95": 3.012, + "p99": 3.012, + "max": 3.012 + }, + "estimatedUpperMs": { + "n": 4, + "p50": 89.75422199998866, + "p95": 90.08522199999425, + "p99": 90.08522199999425, + "max": 90.08522199999425 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 6, + "observedLowerMs": { + "n": 6, + "p50": 1.226, + "p95": 3.516, + "p99": 3.516, + "max": 3.516 + }, + "estimatedUpperMs": { + "n": 6, + "p50": 28.208660000003874, + "p95": 64.30090599999676, + "p99": 64.30090599999676, + "max": 64.30090599999676 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2532, + "total_exec_time": 228.71010199999986, + "rows": 12 + } + ], + "nodeCpu": { + "user": 18403801, + "system": 860810 + }, + "nodeCpuAverageOneCorePercent": 29.35486475249997, + "eventLoopDelay": { + "p95Ms": 11.182079, + "p99Ms": 13.328383, + "maxMs": 72.482815 + }, + "peakSampledNodeRssBytes": 835289088, + "peakSampledNodeHeapBytes": 527752512, + "postgresStatementExecutionMs": 4084.128306999997, + "lockObserverExecutionMs": 228.71010199999986 + } + ] + }, + "overlap-abba": { + "schema": "paperclip.normal-recovery-load/v1", + "scope": "Two logical servers in one client process, 150 controllers each, separate request/liveness/checkpoint pools at unchanged caps; same aggregate aligned-burst workload. Matched baseline/candidate ABBA pool-overlap component qualification; no takeover latency or provider reattachment claim. Includes real PaperclipControlPlanePort snapshot persistence on every control checkpoint. Database/control-plane component stress; real production checkpoint writer and natural remote lease timers. Same synthetic DB request load. Not full HTTP, harness or hosted capacity qualification.", + "hardware": { + "logicalServers": 2, + "postgresCpuLimit": 2, + "postgresMemoryLimitMiB": 1536, + "mainPool": 10, + "livenessPool": 2, + "candidateCheckpointPool": 2 + }, + "workload": { + "mode": "burst", + "normalReadsPerSecond": 100, + "normalWritesPerSecond": 20, + "databaseAdmissionEverySeconds": 5, + "checkpointControlEverySeconds": 5, + "controlPayloadBytes": 4096, + "phaseSeconds": 65, + "snapshotTransactionsPerControl": 1, + "baselineCheckpointPool": 2, + "baselineRenewalIntervalSeconds": 20, + "candidateRenewalIntervalSeconds": 20, + "historicalIssues": 10000 + }, + "passed": true, + "comparable": true, + "relative": { + "postgresCpuUsec": { + "baseline": 1334.8748826090668, + "candidate": 1376.3816699393838, + "increasePercent": 3.109414063525584, + "passed": true + }, + "walBytes": { + "baseline": 4209.21437718774, + "candidate": 4317.105779902672, + "increasePercent": 2.5632194762913674, + "passed": true + }, + "requestP99": { + "baseline": 106.66748450000887, + "candidate": 116.98804850000306, + "increasePercent": 9.675454566469433, + "passed": true + } + }, + "measurementLimits": [ + "Database request latency is not HTTP latency", + "Pool queue includes client encoding; BEGIN response includes network time", + "pg_stat_statements execution includes lock waits; lock episodes report sampled duration bounds, with short waits potentially missed", + "Peak memory is sampled once a second", + "Use an otherwise idle cluster and host; WAL and cgroup CPU are cluster-wide" + ], + "results": [ + { + "label": "01_baseline_300", + "variant": "baseline", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T18:59:52.122Z", + "endedAt": "2026-10-30T19:00:57.905Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23496, + "livenessSqlCount": 4504, + "checkpointSqlCount": 15602, + "walBytes": 50462184, + "postgresCpuUsec": 15645449, + "postgresPeakSampledMemoryBytes": 1343967232, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 721.3357259999902, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 18.673452000002726, + "p95": 101.14544200000091, + "p99": 105.69386800000211, + "max": 219.4722890000121 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 22.73070000000007, + "p95": 103.43886099999872, + "p99": 116.90596599999844, + "max": 224.30324400000973 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 487.1431490000032, + "p95": 804.2522070000123, + "p99": 901.6204700000017, + "max": 941.495895 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 436.6414150000055, + "p95": 746.4600769999961, + "p99": 826.2692409999945, + "max": 891.541333000001 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 8.546736999996938, + "p95": 15.356733000000531, + "p99": 75.99347599998873, + "max": 98.56840399999783 + }, + "poolQueueByPool": { + "request": { + "n": 1956, + "p50": 0.016741999999794643, + "p95": 0.03154400000494206, + "p99": 0.6992969999992056, + "max": 5.09585099999822 + }, + "liveness": { + "n": 450, + "p50": 501.6249909999897, + "p95": 876.1304410000012, + "p99": 913.067286999998, + "max": 919.2936289999998 + }, + "checkpoint": { + "n": 1950, + "p50": 427.4311180000004, + "p95": 738.933901999997, + "p99": 813.4696679999979, + "max": 886.4332880000002 + }, + "replacement_request": { + "n": 1957, + "p50": 0.0168159999884665, + "p95": 0.02985600000829436, + "p99": 0.04265299999678973, + "max": 10.915440000000672 + }, + "replacement_liveness": { + "n": 450, + "p50": 503.20186999999714, + "p95": 886.2206920000026, + "p99": 913.0074840000016, + "max": 918.3099900000016 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 426.9944880000112, + "p95": 739.3993829999963, + "p99": 821.3317409999981, + "max": 878.8103770000016 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 1956, + "p50": 8.524627999999211, + "p95": 15.395046999998158, + "p99": 76.11285200000202, + "max": 97.21485499999835 + }, + "liveness": { + "n": 450, + "p50": 510.662817000004, + "p95": 886.2594030000037, + "p99": 916.8632429999998, + "max": 921.3872209999972 + }, + "checkpoint": { + "n": 1950, + "p50": 436.56327500000043, + "p95": 745.776893000002, + "p99": 822.4703049999989, + "max": 891.5069800000056 + }, + "replacement_request": { + "n": 1957, + "p50": 8.534450999999535, + "p95": 15.9994200000001, + "p99": 75.55317300000024, + "max": 98.5201530000013 + }, + "replacement_liveness": { + "n": 450, + "p50": 513.2629840000009, + "p95": 890.6694930000012, + "p99": 915.7612549999976, + "max": 920.3067900000024 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 436.6130089999933, + "p95": 746.4048050000129, + "p99": 830.9584389999982, + "max": 887.7181599999967 + } + }, + "nodeCpu": { + "user": 17806507, + "system": 879516 + }, + "nodeCpuAverageOneCorePercent": 28.40549223176189, + "eventLoopDelay": { + "p95Ms": 11.010047, + "p99Ms": 12.697599, + "maxMs": 183.107583 + }, + "sqlAllApplicationPools": 43602, + "sampledPeakDatabaseActivityRows": 29, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2551, + "gapsMs": { + "n": 2551, + "p50": 25.145184999997582, + "p95": 26.409488999997848, + "p99": 59.03993599999376, + "max": 229.8140570000105 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "benchmark-replacement-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 10, + "observedLowerMs": { + "n": 10, + "p50": 2.481, + "p95": 11.224, + "p99": 11.224, + "max": 11.224 + }, + "estimatedUpperMs": { + "n": 10, + "p50": 27.13945600000443, + "p95": 36.74406199999794, + "p99": 36.74406199999794, + "max": 36.74406199999794 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2551, + "total_exec_time": 230.25003400000065, + "rows": 10 + } + ], + "peakSampledNodeRssBytes": 815190016, + "peakSampledNodeHeapBytes": 503697472, + "postgresStatementExecutionMs": 4126.207232999998, + "lockObserverExecutionMs": 230.25003400000065 + }, + { + "label": "02_candidate_300", + "variant": "candidate", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T19:01:04.413Z", + "endedAt": "2026-10-30T19:02:10.241Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23496, + "livenessSqlCount": 4504, + "checkpointSqlCount": 15602, + "walBytes": 50562024, + "postgresCpuUsec": 16161232, + "postgresPeakSampledMemoryBytes": 1324052480, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 825.5109480000101, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 14.214422999997623, + "p95": 103.00589499998023, + "p99": 109.18438099999912, + "max": 124.26887600000191 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 16.232166000001598, + "p95": 104.91053900000406, + "p99": 111.67462099998374, + "max": 124.31452200000058 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 497.3481239999819, + "p95": 844.0066550000047, + "p99": 960.1377219999995, + "max": 998.2879429999884 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 452.44012500000827, + "p95": 791.6675080000132, + "p99": 904.2310109999962, + "max": 948.914775000012 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 8.975842999992892, + "p95": 16.999474000011105, + "p99": 80.54993900000409, + "max": 92.77781900000991 + }, + "poolQueueByPool": { + "request": { + "n": 1956, + "p50": 0.01796499999181833, + "p95": 0.03485300000465941, + "p99": 0.5488950000144541, + "max": 16.1574099999998 + }, + "liveness": { + "n": 450, + "p50": 634.4087819999986, + "p95": 971.200656999994, + "p99": 989.703067999988, + "max": 996.0718149999884 + }, + "checkpoint": { + "n": 1950, + "p50": 443.02296800000477, + "p95": 778.6681580000004, + "p99": 884.4173180000071, + "max": 918.9185089999955 + }, + "replacement_request": { + "n": 1957, + "p50": 0.017980999997234903, + "p95": 0.03247500001452863, + "p99": 0.04606999998213723, + "max": 39.24437300002319 + }, + "replacement_liveness": { + "n": 450, + "p50": 634.364161000005, + "p95": 969.9561950000061, + "p99": 989.6641479999962, + "max": 995.097293999992 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 440.39501599999494, + "p95": 786.2747190000082, + "p99": 904.2403229999909, + "max": 943.5718220000126 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 1956, + "p50": 8.969991000019945, + "p95": 17.73811299999943, + "p99": 80.50918799999636, + "max": 92.72936200001277 + }, + "liveness": { + "n": 450, + "p50": 645.302377, + "p95": 976.2641570000123, + "p99": 993.3791540000093, + "max": 1001.4772759999905 + }, + "checkpoint": { + "n": 1950, + "p50": 453.5094059999974, + "p95": 786.6211449999828, + "p99": 893.5732319999952, + "max": 926.4708950000058 + }, + "replacement_request": { + "n": 1957, + "p50": 8.951632000011159, + "p95": 17.27179700000852, + "p99": 79.54755400000431, + "max": 92.341597000006 + }, + "replacement_liveness": { + "n": 450, + "p50": 645.1912089999969, + "p95": 975.069325000004, + "p99": 992.4281219999975, + "max": 997.0403169999918 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 451.6021180000098, + "p95": 793.450920000003, + "p99": 914.5407790000027, + "max": 948.8798110000062 + } + }, + "nodeCpu": { + "user": 18799633, + "system": 889825 + }, + "nodeCpuAverageOneCorePercent": 29.910693792544862, + "eventLoopDelay": { + "p95Ms": 11.124735, + "p99Ms": 12.779519, + "maxMs": 80.478207 + }, + "sqlAllApplicationPools": 43602, + "sampledPeakDatabaseActivityRows": 28, + "postgresThrottledUsec": 117766, + "lockWait": { + "intervalMs": 25, + "samples": 2548, + "gapsMs": { + "n": 2548, + "p50": 25.149598000003607, + "p95": 26.418762000001152, + "p99": 73.46682900001178, + "max": 90.0133790000109 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 2, + "observedLowerMs": { + "n": 2, + "p50": 57.415, + "p95": 57.415, + "p99": 57.415, + "max": 57.415 + }, + "estimatedUpperMs": { + "n": 2, + "p50": 78.82680399998208, + "p95": 78.82680399998208, + "p99": 78.82680399998208, + "max": 78.82680399998208 + } + }, + "benchmark-replacement-application": { + "observedEpisodes": 1, + "observedLowerMs": { + "n": 1, + "p50": 0.817, + "p95": 0.817, + "p99": 0.817, + "max": 0.817 + }, + "estimatedUpperMs": { + "n": 1, + "p50": 25.092009999993024, + "p95": 25.092009999993024, + "p99": 25.092009999993024, + "max": 25.092009999993024 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 11, + "observedLowerMs": { + "n": 11, + "p50": 3.97, + "p95": 58.82, + "p99": 58.82, + "max": 58.82 + }, + "estimatedUpperMs": { + "n": 11, + "p50": 30.69183900000644, + "p95": 83.27365099999588, + "p99": 83.27365099999588, + "max": 83.27365099999588 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2548, + "total_exec_time": 234.671226, + "rows": 14 + } + ], + "peakSampledNodeRssBytes": 870621184, + "peakSampledNodeHeapBytes": 593590232, + "postgresStatementExecutionMs": 4403.789213000001, + "lockObserverExecutionMs": 234.671226 + }, + { + "label": "03_candidate_300", + "variant": "candidate", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T19:02:16.491Z", + "endedAt": "2026-10-30T19:03:22.313Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23496, + "livenessSqlCount": 4504, + "checkpointSqlCount": 15602, + "walBytes": 50570496, + "postgresCpuUsec": 16081885, + "postgresPeakSampledMemoryBytes": 1332752384, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 818.6222430000198, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 39.5862119999947, + "p95": 74.14522999999463, + "p99": 124.791716000007, + "max": 129.85977700000512 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 41.993144000007305, + "p95": 77.44222500000615, + "p99": 125.49473299999954, + "max": 132.81708999999682 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 487.02540600000066, + "p95": 836.5916340000113, + "p99": 955.1296190000139, + "max": 1000.8564950000145 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 444.6453029999975, + "p95": 790.974338, + "p99": 903.6869510000106, + "max": 944.6195900000166 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 8.99197100001038, + "p95": 17.247062000009464, + "p99": 77.24802900000941, + "max": 89.03845799999544 + }, + "poolQueueByPool": { + "request": { + "n": 1956, + "p50": 0.01789499999722466, + "p95": 0.03461500001139939, + "p99": 1.2706449999823235, + "max": 7.25288700000965 + }, + "liveness": { + "n": 450, + "p50": 668.1329050000058, + "p95": 943.7231139999931, + "p99": 964.8324849999917, + "max": 971.0895420000015 + }, + "checkpoint": { + "n": 1950, + "p50": 436.23111700001755, + "p95": 776.24320300002, + "p99": 884.2146380000049, + "max": 935.5753669999831 + }, + "replacement_request": { + "n": 1957, + "p50": 0.017906000022776425, + "p95": 0.03174499998567626, + "p99": 0.04958200000692159, + "max": 37.47016100000474 + }, + "replacement_liveness": { + "n": 450, + "p50": 660.5174439999973, + "p95": 939.0108659999969, + "p99": 963.677406999981, + "max": 968.687052000023 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 436.32736499997554, + "p95": 788.8090800000064, + "p99": 902.0589020000189, + "max": 938.6039999999921 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 1956, + "p50": 8.946074000006774, + "p95": 17.429038000002038, + "p99": 76.37251200000173, + "max": 89.003899000003 + }, + "liveness": { + "n": 450, + "p50": 678.8731129999796, + "p95": 945.945057999983, + "p99": 967.5097349999996, + "max": 973.1065839999937 + }, + "checkpoint": { + "n": 1950, + "p50": 444.8119090000109, + "p95": 784.2202169999946, + "p99": 893.1187290000089, + "max": 943.4892260000051 + }, + "replacement_request": { + "n": 1957, + "p50": 9.007467999996152, + "p95": 18.345773000008194, + "p99": 78.94358199997805, + "max": 89.00447099999292 + }, + "replacement_liveness": { + "n": 450, + "p50": 668.120407999988, + "p95": 941.9229499999783, + "p99": 966.5471529999922, + "max": 971.1461609999824 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 444.2886780000117, + "p95": 798.4288989999914, + "p99": 912.1802920000046, + "max": 944.5919719999947 + } + }, + "nodeCpu": { + "user": 18563401, + "system": 892958 + }, + "nodeCpuAverageOneCorePercent": 29.559203827490894, + "eventLoopDelay": { + "p95Ms": 11.214847, + "p99Ms": 13.647871, + "maxMs": 80.478207 + }, + "sqlAllApplicationPools": 43602, + "sampledPeakDatabaseActivityRows": 28, + "postgresThrottledUsec": 0, + "lockWait": { + "intervalMs": 25, + "samples": 2541, + "gapsMs": { + "n": 2541, + "p50": 25.158021000010194, + "p95": 26.61356500000693, + "p99": 70.55964399999357, + "max": 90.20055800001137 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 3, + "observedLowerMs": { + "n": 3, + "p50": 2.624, + "p95": 42.948, + "p99": 42.948, + "max": 42.948 + }, + "estimatedUpperMs": { + "n": 3, + "p50": 68.97033699997701, + "p95": 72.40636000002269, + "p99": 72.40636000002269, + "max": 72.40636000002269 + } + }, + "benchmark-replacement-application": { + "observedEpisodes": 2, + "observedLowerMs": { + "n": 2, + "p50": 2.623, + "p95": 2.623, + "p99": 2.623, + "max": 2.623 + }, + "estimatedUpperMs": { + "n": 2, + "p50": 59.07080300000962, + "p95": 59.07080300000962, + "p99": 59.07080300000962, + "max": 59.07080300000962 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 4, + "observedLowerMs": { + "n": 4, + "p50": 1.948, + "p95": 4.972, + "p99": 4.972, + "max": 4.972 + }, + "estimatedUpperMs": { + "n": 4, + "p50": 56.89580300002126, + "p95": 76.26736000002711, + "p99": 76.26736000002711, + "max": 76.26736000002711 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 8, + "observedLowerMs": { + "n": 8, + "p50": 2.616, + "p95": 62.06, + "p99": 62.06, + "max": 62.06 + }, + "estimatedUpperMs": { + "n": 8, + "p50": 28.899388999998337, + "p95": 87.05610200000228, + "p99": 87.05610200000228, + "max": 87.05610200000228 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2541, + "total_exec_time": 230.91749600000003, + "rows": 17 + } + ], + "peakSampledNodeRssBytes": 887791616, + "peakSampledNodeHeapBytes": 633808520, + "postgresStatementExecutionMs": 4428.595989999998, + "lockObserverExecutionMs": 230.91749600000003 + }, + { + "label": "04_baseline_300", + "variant": "baseline", + "n": 300, + "durationMs": 65000, + "startedAt": "2026-10-30T19:03:29.319Z", + "endedAt": "2026-10-30T19:04:35.119Z", + "normalOffered": 6500, + "normalCompleted": 6500, + "normalWritesOffered": 1300, + "normalWritesCompleted": 1300, + "admissionsOffered": 13, + "admissionsCompleted": 13, + "controls": 3900, + "checkpointTransactions": 3900, + "snapshotTransactions": 3900, + "changedRows": 7800, + "changedBytes": 17371500, + "sqlCount": 23496, + "livenessSqlCount": 4504, + "checkpointSqlCount": 15602, + "walBytes": 48142872, + "postgresCpuUsec": 15625330, + "postgresPeakSampledMemoryBytes": 1307955200, + "renewalLosses": 0, + "dropped": 0, + "errors": [], + "drainMs": 796.7394680000143, + "correctnessPassed": true, + "absoluteGates": { + "requestP99": true, + "drain": true, + "throughput": true, + "postgresCpu": true, + "postgresMemory": true + }, + "passed": true, + "normalReadSummary": { + "n": 6500, + "p50": 14.284052000002703, + "p95": 102.27245000004768, + "p99": 107.64110100001562, + "max": 141.18229500000598 + }, + "normalWriteSummary": { + "n": 1300, + "p50": 16.601036000007298, + "p95": 104.65098200005013, + "p99": 110.24340700003086, + "max": 141.02557800000068 + }, + "controlLatencySummary": { + "n": 3900, + "p50": 511.11750200000824, + "p95": 835.1623750000144, + "p99": 932.1108870000462, + "max": 966.9618110000156 + }, + "checkpointLatencySummary": { + "n": 3900, + "p50": 466.6611930000363, + "p95": 785.0629789999803, + "p99": 882.4489049999975, + "max": 911.663005999988 + }, + "snapshotLatencySummary": { + "n": 3900, + "p50": 8.741485000005923, + "p95": 21.27701399999205, + "p99": 79.73378300003242, + "max": 98.61222199999611 + }, + "poolQueueByPool": { + "request": { + "n": 1956, + "p50": 0.017884999979287386, + "p95": 0.035715999954845756, + "p99": 0.7316219999920577, + "max": 6.9568530000397 + }, + "liveness": { + "n": 450, + "p50": 612.479364000028, + "p95": 931.2895360000257, + "p99": 968.4246090000088, + "max": 974.2110239999893 + }, + "checkpoint": { + "n": 1950, + "p50": 457.2952529999893, + "p95": 777.7845369999995, + "p99": 872.6880790000141, + "max": 904.4663130000117 + }, + "replacement_request": { + "n": 1957, + "p50": 0.017825999995693564, + "p95": 0.032011000032071024, + "p99": 0.05147999996552244, + "max": 58.91463199999998 + }, + "replacement_liveness": { + "n": 450, + "p50": 611.0748399999866, + "p95": 924.6075030000065, + "p99": 966.4776800000109, + "max": 972.2507760000008 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 458.15926700001, + "p95": 773.1846900000237, + "p99": 871.5540899999905, + "max": 906.4964309999486 + } + }, + "transactionLatencyByPool": { + "request": { + "n": 1956, + "p50": 8.748108999978285, + "p95": 21.241075000027195, + "p99": 79.69776200002525, + "max": 96.44556800002465 + }, + "liveness": { + "n": 450, + "p50": 623.5618810000014, + "p95": 938.7270220000064, + "p99": 971.3052120000066, + "max": 976.3884749999852 + }, + "checkpoint": { + "n": 1950, + "p50": 466.64239300001645, + "p95": 786.580037000007, + "p99": 882.414477000013, + "max": 910.3289480000385 + }, + "replacement_request": { + "n": 1957, + "p50": 8.698626999976113, + "p95": 22.4105860000127, + "p99": 80.60295999998925, + "max": 98.5777620000008 + }, + "replacement_liveness": { + "n": 450, + "p50": 620.2892739999807, + "p95": 935.5929790000082, + "p99": 969.3374020000047, + "max": 975.1367830000236 + }, + "replacement_checkpoint": { + "n": 1950, + "p50": 466.62874600000214, + "p95": 782.9779160000035, + "p99": 880.8073159999913, + "max": 911.626213999989 + } + }, + "nodeCpu": { + "user": 19222778, + "system": 829326 + }, + "nodeCpuAverageOneCorePercent": 30.474186242549273, + "eventLoopDelay": { + "p95Ms": 10.977279, + "p99Ms": 14.262271, + "maxMs": 82.444287 + }, + "sqlAllApplicationPools": 43602, + "sampledPeakDatabaseActivityRows": 28, + "postgresThrottledUsec": 86175, + "lockWait": { + "intervalMs": 25, + "samples": 2544, + "gapsMs": { + "n": 2544, + "p50": 25.112813999992795, + "p95": 26.291275000025053, + "p99": 71.21967900000163, + "max": 94.01448299997719 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "benchmark-replacement-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 7, + "observedLowerMs": { + "n": 7, + "p50": 3.134, + "p95": 62.973, + "p99": 62.973, + "max": 62.973 + }, + "estimatedUpperMs": { + "n": 7, + "p50": 29.69778799993219, + "p95": 91.62701000005472, + "p99": 91.62701000005472, + "max": 91.62701000005472 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + }, + "lockObserverSql": [ + { + "calls": 2544, + "total_exec_time": 225.15488799999974, + "rows": 7 + } + ], + "peakSampledNodeRssBytes": 900505600, + "peakSampledNodeHeapBytes": 621405448, + "postgresStatementExecutionMs": 4218.09364, + "lockObserverExecutionMs": 225.15488799999974 + } + ], + "sourceReport": "/tmp/native-controller-load-qualification-20261030/remote/evidence/overlap-abba/report.json" + } + }, + "lockObserverCalibration": { + "intentionalHoldMs": 250, + "releasedAtMs": 249.4750299999996, + "transactionElapsedMs": 304.0121600000002, + "observed": { + "intervalMs": 25, + "samples": 12, + "gapsMs": { + "n": 12, + "p50": 25.226912000000084, + "p95": 35.78078900000037, + "p99": 35.78078900000037, + "max": 35.78078900000037 + }, + "errors": [], + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 1, + "observedLowerMs": { + "n": 1, + "p50": 229.153, + "p95": 229.153, + "p99": 229.153, + "max": 229.153 + }, + "estimatedUpperMs": { + "n": 1, + "p50": 254.2638969999989, + "p95": 254.2638969999989, + "p99": 254.2638969999989, + "max": 254.2638969999989 + } + } + }, + "episodes": [ + { + "pid": 88, + "application_name": "paperclip-runner-checkpoint", + "query_start": "2026-10-30 18:42:31.6856+00", + "wait_event": "transactionid", + "wait_start": "2026-10-30 18:42:31.685612+00", + "wait_age_ms": 229.153, + "firstMs": 12704.136085, + "lastMs": 12899.644581, + "precedingGapMs": 35.78078900000037, + "observedLowerMs": 229.153, + "estimatedUpperMs": 254.2638969999989, + "rightCensored": false + } + ], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + } + }, + "overlapInterpretation": { + "scope": "Two logical server clients share one two-CPU Node process; each has its own bounded request/liveness/checkpoint pools. Same aggregate workload as one-server aligned burst.", + "postgresCpuSeconds": 16.20749, + "oneServerCandidateBurstMeanPostgresCpuSeconds": 14.679791, + "increasePercent": 10.406817099780241, + "interpretation": "Capacity passed. This additional CPU cost comes from a different topology and one sample; it is not an improvement claim or a repeatable baseline/candidate regression finding." + }, + "retainedFailures": { + "earlierReport": "doc/plans/2026-10-30-native-controller-load-evidence.json", + "earlierReportUnchanged": true, + "freshAttemptLedger": "/tmp/native-controller-load-qualification-20261030/failure-ledger.json", + "localFailedPilot": { + "requestP99Ms": 1211.193594999997, + "droppedOperations": 33, + "postgresThrottledUsec": 12294711, + "outcome": "Ineligible for acceptance; local scheduling pressure and unmatched work. A subsequent identical local pilot passed, demonstrating temporal host variability." + }, + "remoteSetupFailures": [ + "Abbreviated Git revision fetch rejected; exact full revision fetched.", + "Missing built plugin SDK export; compiled from locked installed dependencies.", + "Observer database missing pg_stat_statements; harness now creates the extension in the owned observer database." + ], + "allFailedAttemptsAccountedFor": true + }, + "limits": [ + "Database authentication/read latency is not HTTP latency or full application latency.", + "The baseline supplies writer, snapshot port and PRP controller; normal request and lease-renewal paths are common controls.", + "PostgreSQL CPU and WAL are cluster-wide and include observer overhead and PostgreSQL background work.", + "Lock observer nominally adds 40 queries/second; actual calls and SQL execution time are included in each result.", + "Lock episodes use PostgreSQL waitstart identities and server-reported ages. Short waits between polls can be missed, and client scheduling/network delay widens estimated upper bounds.", + "Pool admission includes client encoding; SQL execution statistics include lock waits.", + "Memory and database activity are sampled once per second. Activity-row counts include non-client database backends, so they are not exact client-connection counts.", + "The two-server overlap probe uses two logical clients in one Node process and does not measure real process replacement or authenticated provider reattachment.", + "The separate claim-capacity probe uses synthetic prepared-workspace references. Provider authentication and external sandbox existence are outside its scope." + ], + "rawEvidence": { + "root": "/tmp/native-controller-load-qualification-20261030", + "initialArchive": "/tmp/native-controller-load-qualification-20261030/remote-evidence-initial.tar.gz", + "initialArchiveSha256": "d1f2da9429e2ae1944df35d40da77c0333c506336fed4f78dcfd1d13447403f8", + "cloudInstanceCleanup": "Owned benchmark containers stopped and owned disposable VM confirmed terminated with no attached root volumes.", + "finalDeltaArchive": "/tmp/native-controller-load-qualification-20261030/remote-evidence-final-delta.tar.gz", + "finalDeltaArchiveSha256": "1e77edbfd3ee9e05bd5de6a9ecefb82fdb12c89e52e575f9e8b48ea0c94d26ed", + "cleanupReceipt": "/tmp/native-controller-load-qualification-20261030/ec2-final-state.json" + }, + "claimCapacity": { + "schema": "paperclip.native-controller-claim-capacity/v1", + "scope": "Production DB claim function against synthetic prepared-workspace references and real persisted snapshots/PRP records. Natural lease expiry; two server database clients. No provider attachment or external sandbox existence claim.", + "passed": true, + "harnessSha256": "b083e2e34eccf21a1921e3b32a434ac7d5b1dec24c903b0b3b5052f9f48adf2e", + "candidateRevision": "a0ef3ce7493c8bb76c4766ff007e8a390118d145", + "baselineRevision": "59e191c368cb3119ff3206621fefa9c5d4ad3110", + "errors": [], + "naturalExpiryWaitMs": 54026.14, + "claimDrainMs": 2838.7143720000167, + "claimAndRestoreDrainMs": 8742.97213000001, + "restoredCheckpoints": 300, + "renewedOwnerLosses": 0, + "postRenewalOwners": { + "owners": 300, + "min_generation": 2, + "max_generation": 2, + "minimum_remaining_ms": 56198.788 + }, + "elapsedMs": 77971.21296299998, + "requestsOffered": 15400, + "requestsCompleted": 15400, + "requestLatency": { + "n": 15400, + "p50": 2.684827999997651, + "p95": 5.376435999991372, + "p99": 9.759766000002855, + "max": 64.86076799999864 + }, + "postgresCpuUsec": 11280308, + "sqlByClient": { + "old_request": 7703, + "old_liveness": 0, + "old_checkpoint": 0, + "new_request": 12512, + "new_liveness": 1502, + "new_checkpoint": 1502 + }, + "settledIdleSql": 0, + "retainedOldControllerLeaseMs": 60000, + "remoteReconnectGraceMs": 420000, + "checkpointPoolPerServer": 2, + "claimsByKind": { + "reattach_remote_runner": 300 + }, + "distinctClaimedRuns": 300, + "claimGenerations": [ + 2 + ], + "fixtureDatabaseRequestsPerSecond": 200, + "sourceCode": "/tmp/native-controller-load-qualification-20261030/benchmark-native-controller-claim-capacity.ts", + "sourceReport": "/tmp/native-controller-load-qualification-20261030/remote/evidence/claim-capacity/claim-capacity.json", + "sourceReportSha256": "cd9fc360f1cc1df0b6e82ca50694f6b2ad4c3ff3f638381637dcc99dd3385bc2", + "peakSampledDatabaseActivityRows": 25, + "lockWait": { + "samples": 2987, + "gapsMs": { + "n": 2987, + "p50": 25.607273999994504, + "p95": 29.01517500000773, + "p99": 43.6128069999977, + "max": 85.42577500001062 + }, + "byPool": { + "benchmark-application": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-controller-lease": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + }, + "paperclip-runner-checkpoint": { + "observedEpisodes": 0, + "observedLowerMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + }, + "estimatedUpperMs": { + "n": 0, + "p50": 0, + "p95": 0, + "p99": 0, + "max": 0 + } + } + }, + "errors": [], + "limits": "Episodes use PostgreSQL pg_locks.waitstart identity. Lower bounds are server-reported wait age at the last observation; estimated upper bounds add the next client polling gap. Waits shorter than polling gaps can be missed; network/event-loop delay widens upper bounds. Observer nominally adds 40 queries/second, included in database CPU. SQL execution statistics include lock time and are not lock-exclusive." + } + }, + "matchedOverlapAbba": { + "status": "passed", + "sourceCode": "/tmp/native-controller-load-qualification-20261030/benchmark-native-controller-overlap-abba.ts", + "sourceSha256": "2cc52afee21c34efab7f65af88bbd911e6efdd424094f7020bfccd4a16b4d441", + "requestP99IncreasePercent": 9.675454566469433, + "interpretation": "All relative gates passed; request p99 increase is close to the 10 percent threshold." + }, + "reproduction": { + "baseHarnessSha256": "abaa29d50eb95013836b54b8dacc94c72e080edf24744cc5d291cc5eb1ff2818", + "patches": { + "overlap": { + "path": "doc/plans/2026-10-30-native-controller-overlap.patch", + "patchSha256": "e8e14ddbdbed289f47e57fa0fe8ae2be337e77f2919c755d7003afd73150faf4", + "reconstructedSourceSha256": "2cc52afee21c34efab7f65af88bbd911e6efdd424094f7020bfccd4a16b4d441", + "verifiedPatchReconstruction": true + }, + "claim": { + "path": "doc/plans/2026-10-30-native-controller-claim.patch", + "patchSha256": "1fcbf028654499562c5f6b789f36e7c4681ec5d28c3dec6d13e2a51650496e2e", + "reconstructedSourceSha256": "b083e2e34eccf21a1921e3b32a434ac7d5b1dec24c903b0b3b5052f9f48adf2e", + "verifiedPatchReconstruction": true + } + }, + "instructions": [ + "Copy scripts/benchmark-native-controller-recovery.ts to a new probe source file.", + "Apply the corresponding dated patch with: patch /absolute/path/to/probe.ts /absolute/path/to/patch.", + "Set existing benchmark inputs plus BENCH_CANDIDATE_ROOT to the repository root; use the pinned baseline and identical installed dependencies.", + "Run Node 24 with the repository tsx loader and the patched probe file followed by burst. Each report directory must be fresh." + ], + "requiredInputs": [ + "BENCH_DISPOSABLE_POSTGRES=1", + "BENCH_DATABASE_URL", + "BENCH_BASELINE_ROOT", + "BENCH_REPORT_ROOT", + "BENCH_POSTGRES_CGROUP", + "BENCH_CANDIDATE_ROOT" + ] + }, + "streamingSqlIndependence": { + "status": "passed", + "sourceRevision": "da8c7f6e1837ddd896dfe83a7524812fee813487", + "sourceReport": "/tmp/native-controller-release-qualification-20261030/stream-cost-current.json", + "sourceLog": "/tmp/native-controller-release-qualification-20261030/stream-cost-current.log", + "samples": [ + { + "chunks": 8, + "streamSql": 0, + "streamTransactions": 0, + "streamChangedRows": 0, + "idleSql": 0, + "checkpointWrites": 8, + "checkpointTransactions": 7, + "changedRows": 12 + }, + { + "chunks": 512, + "streamSql": 0, + "streamTransactions": 0, + "streamChangedRows": 0, + "idleSql": 0, + "checkpointWrites": 8, + "checkpointTransactions": 7, + "changedRows": 12 + }, + { + "chunks": 8192, + "streamSql": 0, + "streamTransactions": 0, + "streamChangedRows": 0, + "idleSql": 0, + "checkpointWrites": 8, + "checkpointTransactions": 7, + "changedRows": 12 + } + ], + "positiveCalibrationPassed": true, + "existingRenewalSqlFor8192ChunkCase": 66, + "limits": [ + "Separate real Runner fixture executed by the release qualification agent.", + "Concurrent owned fixtures may affect its cluster-wide WAL and elapsed diagnostics; those values are excluded from performance claims." + ] + } +}