mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-20 18:26:40 +01:00
Cloud repeats the upgrade readiness probe. Every probe loaded, decrypted and parsed the checkpoint header of each active native run and of each retained task session's last run, even when nothing had changed. The statement that reads a run now also reads its header's row revision (xmin, ctid, updated_at, stored size) and returns the material only for a revision the process has not inspected. A bounded memo in the checkpoint owner keeps the four identifiers the answer compares. It reuses an answer only for the same binding, row revision and secrets key, loads the key again for each answer it serves, never remembers a header that failed to decrypt, and forgets a header the latest probe did not inspect. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
80 lines
5.6 KiB
TypeScript
80 lines
5.6 KiB
TypeScript
import type { Db } from "@paperclipai/db";
|
|
import { sql } from "drizzle-orm";
|
|
import { nativeRunnerCheckpointRevisionSql, nativeRunnerUpgradeHeaderSql, directNativeRunnerUpgradeHeaderRead, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
|
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES, type NativeRunnerUpgradeHeaderRead } from "./native-runner-checkpoint.js";
|
|
|
|
const RECOVERY_METADATA_SCHEMA = "paperclip.native-controller-recovery.v1";
|
|
|
|
/** An explicit, paginated diagnostic. Never called by an idle/background scan.
|
|
* This inventories selected retained task-session pointers, not all history.
|
|
* Header evidence is preliminary: full snapshot/journal validation still occurs
|
|
* only when ordinary admission selects its exact continuation source. */
|
|
export async function readNativeControllerRecoveryInventory(db: Db, input: {
|
|
afterId?: string; limit?: number;
|
|
/** The caller finishes this read. Without one, every inspected header is decrypted. */
|
|
headers?: NativeRunnerUpgradeHeaderRead;
|
|
} = {}) {
|
|
if (input.limit !== undefined && !Number.isFinite(input.limit)) throw new Error("native_recovery_inventory_limit_invalid");
|
|
const limit = Math.min(128, Math.max(1, Math.floor(input.limit ?? 128)));
|
|
const headers = input.headers ?? directNativeRunnerUpgradeHeaderRead();
|
|
// The rows whose header the loop below inspects. The row after the page only shows that more exist.
|
|
const header = nativeRunnerUpgradeHeaderSql(headers, { companyId: sql`q."companyId"`, issueId: sql`q."issueId"`,
|
|
runId: sql`q."runId"`, revision: sql`q."headerRevision"`, material: sql`case when q.position <= ${limit} and not q.pending
|
|
and q."runId" is not null and q."issueId" is not null and q."sessionId" is not null
|
|
and q.metadata->>'schema' = ${RECOVERY_METADATA_SCHEMA} then q.material end` });
|
|
const rows = await db.transaction(async tx => {
|
|
await tx.execute(sql`set local statement_timeout = '2s'`);
|
|
return tx.execute<{
|
|
id: string; runId: string | null; companyId: string; issueId: string | null; sessionId: string | null;
|
|
runnerId: string | null; metadata: Record<string, unknown> | null; headerRevision: string | null;
|
|
material: Record<string, unknown> | null; pending: boolean;
|
|
}>(sql`with selected as (
|
|
select id, company_id, agent_id, last_run_id, row_number() over (order by id) as position from agent_task_sessions
|
|
where adapter_type = 'paperclip_runner' ${input.afterId ? sql`and id > ${input.afterId}::uuid` : sql``}
|
|
order by id limit ${limit + 1}
|
|
) select q.id, q."companyId", q."runId", q."issueId", q."sessionId", q."runnerId", q.metadata, q.pending,
|
|
${header.revision} as "headerRevision", ${header.material} as material
|
|
from (select s.id, s.position, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
|
|
r.native_session_id as "sessionId", r.runner_instance_id as "runnerId",
|
|
case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096
|
|
then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata,
|
|
case when pg_column_size(c.material) <= ${NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES}
|
|
and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES}
|
|
then c.material else null end as material,
|
|
${nativeRunnerCheckpointRevisionSql(sql`c`)} as "headerRevision",
|
|
(r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out')
|
|
or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null
|
|
or f.phase not in ('committed', 'terminal_failure')) is true as pending
|
|
from selected s left join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id
|
|
left join native_run_finalizations f on f.run_id = r.id and f.company_id = s.company_id
|
|
left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = s.company_id
|
|
and c.issue_id = r.native_issue_id and c.part = 'header') q ${header.join}
|
|
order by q.id`);
|
|
}, { accessMode: "read only" });
|
|
const truncated = rows.length > limit;
|
|
const page = rows.slice(0, limit);
|
|
const reasons: Record<string, number> = {};
|
|
let selectedRetainedEvidence = 0, outstandingObligations = 0;
|
|
for (const row of page) {
|
|
let reason: string | null = null;
|
|
if (row.pending) { outstandingObligations++; reason = "persistence_or_handoff_pending"; }
|
|
else if (!row.runId || !row.issueId || !row.sessionId) reason = "selected_binding_missing";
|
|
else if (row.metadata?.schema !== RECOVERY_METADATA_SCHEMA) reason = "metadata_unsupported_or_missing";
|
|
else {
|
|
const checkpoint = row.metadata.checkpoint as Record<string, unknown> | null;
|
|
const header = row.headerRevision ? headers.inspect({ companyId: row.companyId, issueId: row.issueId, runId: row.runId },
|
|
row.headerRevision, row.material) : null;
|
|
if (!checkpoint || checkpoint.runId !== row.runId || checkpoint.runnerInstanceId !== row.runnerId
|
|
|| checkpoint.normalizedSessionId !== row.sessionId || !header
|
|
|| header.runnerInstanceId !== row.runnerId || header.normalizedSessionId !== row.sessionId) reason = "checkpoint_binding_missing";
|
|
else selectedRetainedEvidence++;
|
|
}
|
|
if (reason) reasons[reason] = (reasons[reason] ?? 0) + 1;
|
|
}
|
|
return { version: 1 as const, observedAt: new Date().toISOString(), inspected: page.length,
|
|
coverage: truncated || input.afterId ? "partial" as const : "selected_task_session_pointers" as const,
|
|
truncated, nextAfterId: truncated ? page.at(-1)!.id : null,
|
|
selectedRetainedEvidence, outstandingObligations, reasons,
|
|
fullPayloadsValidated: false as const };
|
|
}
|