mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 13:01:05 +01:00
perf(server): stop re-reading and re-decrypting unchanged checkpoint headers on every upgrade readiness probe (#21130)
Cloud repeats the upgrade readiness probe. Every probe loaded, decrypted and parsed the checkpoint header of each active native run and of each retained task session's last run, even when nothing had changed. The statement that reads a run now also reads its header's row revision (xmin, ctid, updated_at, stored size) and returns the material only for a revision the process has not inspected. A bounded memo in the checkpoint owner keeps the four identifiers the answer compares. It reuses an answer only for the same binding, row revision and secrets key, loads the key again for each answer it serves, never remembers a header that failed to decrypt, and forgets a header the latest probe did not inspect. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
1 parent
f2e449f8f6
commit
2174f2dfa5
6 files changed
+783
-24
No files matched your search
@@ -1,40 +1,55 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { inspectNativeRunnerUpgradeHeader, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
||||
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES } from "./native-runner-checkpoint.js";
|
||||
import { nativeRunnerCheckpointRevisionSql, nativeRunnerUpgradeHeaderSql, directNativeRunnerUpgradeHeaderRead, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
||||
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES, type NativeRunnerUpgradeHeaderRead } from "./native-runner-checkpoint.js";
|
||||
|
||||
const RECOVERY_METADATA_SCHEMA = "paperclip.native-controller-recovery.v1";
|
||||
|
||||
/** An explicit, paginated diagnostic. Never called by an idle/background scan.
|
||||
* This inventories selected retained task-session pointers, not all history.
|
||||
* Header evidence is preliminary: full snapshot/journal validation still occurs
|
||||
* only when ordinary admission selects its exact continuation source. */
|
||||
export async function readNativeControllerRecoveryInventory(db: Db, input: { afterId?: string; limit?: number } = {}) {
|
||||
export async function readNativeControllerRecoveryInventory(db: Db, input: {
|
||||
afterId?: string; limit?: number;
|
||||
/** The caller finishes this read. Without one, every inspected header is decrypted. */
|
||||
headers?: NativeRunnerUpgradeHeaderRead;
|
||||
} = {}) {
|
||||
if (input.limit !== undefined && !Number.isFinite(input.limit)) throw new Error("native_recovery_inventory_limit_invalid");
|
||||
const limit = Math.min(128, Math.max(1, Math.floor(input.limit ?? 128)));
|
||||
const headers = input.headers ?? directNativeRunnerUpgradeHeaderRead();
|
||||
// The rows whose header the loop below inspects. The row after the page only shows that more exist.
|
||||
const header = nativeRunnerUpgradeHeaderSql(headers, { companyId: sql`q."companyId"`, issueId: sql`q."issueId"`,
|
||||
runId: sql`q."runId"`, revision: sql`q."headerRevision"`, material: sql`case when q.position <= ${limit} and not q.pending
|
||||
and q."runId" is not null and q."issueId" is not null and q."sessionId" is not null
|
||||
and q.metadata->>'schema' = ${RECOVERY_METADATA_SCHEMA} then q.material end` });
|
||||
const rows = await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local statement_timeout = '2s'`);
|
||||
return tx.execute<{
|
||||
id: string; runId: string | null; companyId: string; issueId: string | null; sessionId: string | null;
|
||||
runnerId: string | null; metadata: Record<string, unknown> | null; material: Record<string, unknown> | null;
|
||||
pending: boolean;
|
||||
runnerId: string | null; metadata: Record<string, unknown> | null; headerRevision: string | null;
|
||||
material: Record<string, unknown> | null; pending: boolean;
|
||||
}>(sql`with selected as (
|
||||
select id, company_id, agent_id, last_run_id from agent_task_sessions
|
||||
select id, company_id, agent_id, last_run_id, row_number() over (order by id) as position from agent_task_sessions
|
||||
where adapter_type = 'paperclip_runner' ${input.afterId ? sql`and id > ${input.afterId}::uuid` : sql``}
|
||||
order by id limit ${limit + 1}
|
||||
) select s.id, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
|
||||
) select q.id, q."companyId", q."runId", q."issueId", q."sessionId", q."runnerId", q.metadata, q.pending,
|
||||
${header.revision} as "headerRevision", ${header.material} as material
|
||||
from (select s.id, s.position, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
|
||||
r.native_session_id as "sessionId", r.runner_instance_id as "runnerId",
|
||||
case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096
|
||||
then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata,
|
||||
case when pg_column_size(c.material) <= ${NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES}
|
||||
and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES}
|
||||
then c.material else null end as material,
|
||||
${nativeRunnerCheckpointRevisionSql(sql`c`)} as "headerRevision",
|
||||
(r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out')
|
||||
or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null
|
||||
or f.phase not in ('committed', 'terminal_failure')) is true as pending
|
||||
from selected s left join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id
|
||||
left join native_run_finalizations f on f.run_id = r.id and f.company_id = s.company_id
|
||||
left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = s.company_id
|
||||
and c.issue_id = r.native_issue_id and c.part = 'header'
|
||||
order by s.id`);
|
||||
and c.issue_id = r.native_issue_id and c.part = 'header') q ${header.join}
|
||||
order by q.id`);
|
||||
}, { accessMode: "read only" });
|
||||
const truncated = rows.length > limit;
|
||||
const page = rows.slice(0, limit);
|
||||
@@ -44,10 +59,11 @@ export async function readNativeControllerRecoveryInventory(db: Db, input: { aft
|
||||
let reason: string | null = null;
|
||||
if (row.pending) { outstandingObligations++; reason = "persistence_or_handoff_pending"; }
|
||||
else if (!row.runId || !row.issueId || !row.sessionId) reason = "selected_binding_missing";
|
||||
else if (row.metadata?.schema !== "paperclip.native-controller-recovery.v1") reason = "metadata_unsupported_or_missing";
|
||||
else if (row.metadata?.schema !== RECOVERY_METADATA_SCHEMA) reason = "metadata_unsupported_or_missing";
|
||||
else {
|
||||
const checkpoint = row.metadata.checkpoint as Record<string, unknown> | null;
|
||||
const header = row.material ? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.runId }, row.material) : null;
|
||||
const header = row.headerRevision ? headers.inspect({ companyId: row.companyId, issueId: row.issueId, runId: row.runId },
|
||||
row.headerRevision, row.material) : null;
|
||||
if (!checkpoint || checkpoint.runId !== row.runId || checkpoint.runnerInstanceId !== row.runnerId
|
||||
|| checkpoint.normalizedSessionId !== row.sessionId || !header
|
||||
|| header.runnerInstanceId !== row.runnerId || header.normalizedSessionId !== row.sessionId) reason = "checkpoint_binding_missing";
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { readNativeCheckpointRetirement } from "./native-runner-checkpoint-retirement.js";
|
||||
import { NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js";
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql } from "drizzle-orm";
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql, type SQL } from "drizzle-orm";
|
||||
import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
|
||||
import { inspectDurablePrpCheckpointHeader, inspectUnauthenticatedBootstrapCheckpoint, inspectUnclaimedBootstrapCheckpoint, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
|
||||
import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret } from "../../secrets/local-encrypted-provider.js";
|
||||
import { legacyControllerBootId } from "../legacy-controller-lease.js";
|
||||
import { assertNativeSourceCheckpointConsumer, lockSourceRecoveryCoordinators } from "./native-source-recovery-ownership.js";
|
||||
import { runnerdStateHasNoProviderTurn, runnerdStateIsIncompleteBootstrap } from "./native-bootstrap-proof.js";
|
||||
@@ -396,10 +397,137 @@ export async function listExpiredNativeRunnerAuthorities(db: Db, now: Date, runI
|
||||
return rows.map(row => row.runId);
|
||||
}
|
||||
|
||||
/** Only a header that decrypted has a repeatable answer: a key or material failure may not recur. */
|
||||
function inspectUpgradeHeaderMaterial(binding: Binding, material: Record<string, unknown>) {
|
||||
let contents: string;
|
||||
try { contents = decryptPart(createLocalEncryptedCipher("read"), binding, "header", material); }
|
||||
catch { return { decrypted: false as const, identity: null }; }
|
||||
return { decrypted: true as const, identity: inspectDurablePrpCheckpointHeader(contents) };
|
||||
}
|
||||
|
||||
/** Inspect only the bounded encrypted header. Protocol interpretation stays in Runner. */
|
||||
export function inspectNativeRunnerUpgradeHeader(binding: Binding, material: Record<string, unknown>) {
|
||||
return inspectUpgradeHeaderMaterial(binding, material).identity;
|
||||
}
|
||||
|
||||
/** The only header fields that rollout evidence compares, and that a memo keeps. */
|
||||
export type NativeRunnerUpgradeHeaderIdentity = Pick<DurableRecoveryIdentity,
|
||||
"runId" | "runnerInstanceId" | "normalizedSessionId" | "environmentLeaseId">;
|
||||
const upgradeHeaderIdentity = (identity: DurableRecoveryIdentity | null): NativeRunnerUpgradeHeaderIdentity | null => identity && {
|
||||
runId: identity.runId, runnerInstanceId: identity.runnerInstanceId,
|
||||
normalizedSessionId: identity.normalizedSessionId, environmentLeaseId: identity.environmentLeaseId };
|
||||
|
||||
/** One read of header evidence. `known` is a JSON object from binding to the row
|
||||
* revision this read answers without the material. Call `finish` after a completed read. */
|
||||
export type NativeRunnerUpgradeHeaderRead = {
|
||||
readonly known: string;
|
||||
inspect(binding: Binding, revision: string, material: Record<string, unknown> | null): NativeRunnerUpgradeHeaderIdentity | null;
|
||||
finish(): void;
|
||||
};
|
||||
|
||||
/** PostgreSQL writes a new row version, with a new xmin or ctid, for every change to
|
||||
* a row, so an unchanged revision means unchanged material. The write time and stored
|
||||
* size are included so that no single field carries that guarantee. */
|
||||
export function nativeRunnerCheckpointRevisionSql(checkpoint: SQL) {
|
||||
return sql`${checkpoint}.xmin::text || ':' || ${checkpoint}.ctid::text
|
||||
|| ':' || (extract(epoch from ${checkpoint}.updated_at) * 1000000)::bigint::text
|
||||
|| ':' || pg_column_size(${checkpoint}.material)::text`;
|
||||
}
|
||||
|
||||
/** `material` must already be null for a row the reader will not inspect. The statement
|
||||
* that reads a run sends its header's material only for a revision the read does not know. */
|
||||
export function nativeRunnerUpgradeHeaderSql(read: Pick<NativeRunnerUpgradeHeaderRead, "known">,
|
||||
row: { companyId: SQL; issueId: SQL; runId: SQL; revision: SQL; material: SQL }) {
|
||||
return {
|
||||
join: sql`left join jsonb_each_text(${read.known}::jsonb) as known_header(binding, revision)
|
||||
on known_header.binding = ${row.companyId}::text || '/' || ${row.issueId}::text || '/' || ${row.runId}::text`,
|
||||
revision: sql`case when ${row.material} is not null then ${row.revision} end`,
|
||||
material: sql`case when ${row.revision} is distinct from known_header.revision then ${row.material} end`,
|
||||
};
|
||||
}
|
||||
|
||||
/** A read that remembers nothing: every inspected header is sent and decrypted. */
|
||||
export function directNativeRunnerUpgradeHeaderRead(): NativeRunnerUpgradeHeaderRead {
|
||||
return { known: "{}", finish() {},
|
||||
inspect: (binding, _revision, material) => material ? upgradeHeaderIdentity(inspectNativeRunnerUpgradeHeader(binding, material)) : null };
|
||||
}
|
||||
|
||||
/** A digest of a value derived from the secrets key for this purpose alone. It opens nothing. */
|
||||
function upgradeHeaderKeyIdentity(): string | null {
|
||||
try {
|
||||
const contents = decryptPart(createLocalEncryptedCipher("read"), binding, "header", material);
|
||||
return inspectDurablePrpCheckpointHeader(contents);
|
||||
return createHash("sha256").update(deriveLocalEncryptedInstanceSecret("native-runner-upgrade-header-memo",
|
||||
process.env, { allowCreate: false })).digest("hex");
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
/** Remembers the answer for a header that decrypted, so an unchanged header is not
|
||||
* sent or decrypted again. An answer is reused only for the same binding, row revision
|
||||
* and secrets key; a header that did not decrypt is read and tried on every read.
|
||||
* An entry holds four identifiers, entries are bounded in number and bytes, and a
|
||||
* read forgets every header it did not inspect, such as that of a run that ended. */
|
||||
export function createNativeRunnerUpgradeHeaderMemo(limits: { maxEntries?: number; maxBytes?: number } = {}) {
|
||||
const maxEntries = limits.maxEntries ?? 1_200, maxBytes = limits.maxBytes ?? 1024 * 1024;
|
||||
type Answer = { revision: string; identity: NativeRunnerUpgradeHeaderIdentity | null };
|
||||
const entries = new Map<string, Answer & { keyIdentity: string; bytes: number }>();
|
||||
let bytes = 0;
|
||||
const forget = (key: string) => {
|
||||
const entry = entries.get(key);
|
||||
if (entry && entries.delete(key)) bytes -= entry.bytes;
|
||||
};
|
||||
return {
|
||||
get size() { return entries.size; },
|
||||
get bytes() { return bytes; },
|
||||
read(): NativeRunnerUpgradeHeaderRead {
|
||||
let keyIdentity = entries.size ? upgradeHeaderKeyIdentity() : undefined;
|
||||
const usable = new Map([...entries].filter(([, entry]) => entry.keyIdentity === keyIdentity));
|
||||
const reused = new Set<string>(), decrypted = new Map<string, Answer>();
|
||||
let keyChanged = false;
|
||||
// A direct read loads the key for each header, after the query. An answer from memory
|
||||
// is as conservative only if the key is loaded and unchanged when the answer is used.
|
||||
const sameKey = () => {
|
||||
const now = upgradeHeaderKeyIdentity();
|
||||
if (keyIdentity === undefined) keyIdentity = now;
|
||||
const same = now !== null && now === keyIdentity;
|
||||
if (!same) keyChanged = true;
|
||||
return same;
|
||||
};
|
||||
return {
|
||||
known: JSON.stringify(Object.fromEntries([...usable].map(([key, entry]) => [key, entry.revision]))),
|
||||
inspect(binding, revision, material) {
|
||||
const key = `${binding.companyId}/${binding.issueId}/${binding.runId}`;
|
||||
if (!material) {
|
||||
// The query withholds material only for a revision this read offered.
|
||||
const entry = usable.get(key);
|
||||
if (entry?.revision !== revision || !sameKey()) return null;
|
||||
reused.add(key);
|
||||
return entry.identity;
|
||||
}
|
||||
const before = sameKey();
|
||||
const result = inspectUpgradeHeaderMaterial(binding, material);
|
||||
const identity = upgradeHeaderIdentity(result.identity);
|
||||
if (result.decrypted && before && sameKey()) decrypted.set(key, { revision, identity });
|
||||
else decrypted.delete(key);
|
||||
return identity;
|
||||
},
|
||||
finish() {
|
||||
// A read that saw the key missing or changed keeps and remembers nothing.
|
||||
const current = keyChanged || !keyIdentity ? null : keyIdentity;
|
||||
for (const [key, entry] of entries) {
|
||||
if (!reused.has(key) || entry.keyIdentity !== current) forget(key);
|
||||
}
|
||||
for (const [key, answer] of current ? decrypted : []) {
|
||||
const size = 2 * (key.length + answer.revision.length + current!.length
|
||||
+ Object.values(answer.identity ?? {}).reduce((sum, value) => sum + value.length, 0));
|
||||
forget(key);
|
||||
entries.set(key, { ...answer, keyIdentity: current!, bytes: size });
|
||||
bytes += size;
|
||||
}
|
||||
for (const key of entries.keys()) {
|
||||
if (entries.size <= maxEntries && bytes <= maxBytes) break;
|
||||
forget(key);
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user